@gmessier/nitro-speech 0.3.3 → 0.4.0

package/ios/Engines/AnalyzerEngine.swift

@@ -0,0 +1,241 @@
+import Foundation
+import Speech
+import AVFoundation
+
+@available(iOS 26.0, *)
+final class AnalyzerEngine: RecognizerEngine {
+    private var inputSequence: AsyncStream<AnalyzerInput>?
+    private var inputBuilder: AsyncStream<AnalyzerInput>.Continuation?
+    private var outputContinuation: AsyncStream<AVAudioPCMBuffer>.Continuation?
+    private var analyzer: SpeechAnalyzer?
+    private let transcriber: TranscriberRuntime
+    
+    private var audioProducerTask: Task<Void, Never>?
+    private var recognizerTask: Task<(), Error>?
+    private var lastBatchStartTime: Float64? = nil
+    private var resultBatches: [String] = []
+    
+    init(backend: RecognizerBackend, locale: Locale, delegate: RecognizerDelegate) {
+        if backend == .speechTranscriber {
+            transcriber = SpeechRuntime(with: locale)
+        } else {
+            transcriber = DictationRuntime(with: locale)
+        }
+        super.init(locale: locale, delegate: delegate)
+    }
+    
+    override func stop() {
+        super.stop()
+        inputBuilder?.finish()
+        
+        Task { [weak self] in
+            guard let self = self else { return }
+            
+            do {
+                try await self.analyzer?.finalizeAndFinishThroughEndOfInput()
+            } catch {
+                self.reportFailure(
+                    from: "stop.finalizeAndFinishThroughEndOfInput",
+                    message: "Failed to finalize the end of input",
+                    type: .onSession
+                )
+                await self.analyzer?.cancelAndFinishNow()
+            }
+            
+            self.cleanup(from: "stopListening")
+        }
+    }
+    
+    override func prewarm(for type: FailureType) async {
+        await super.prewarm(for: type)
+        do {
+            // Create transcriber and install assets
+            try await transcriber.create(config: self.recognizerDelegate?.config)
+        }
+        catch {
+            self.reportFailure(
+                from: "prewarm.assets",
+                message: "Failed to create transcriber",
+                type: type
+            )
+        }
+    }
+    
+    override func startSession() async {
+        await super.startSession()
+        
+        // Prepares transcriber and handles errors.
+        // On failure, reportFailure triggers cleanup + engine reselection.
+        await prewarm(for: .start)
+        
+        // 3. Input sequence
+        (inputSequence, inputBuilder) = AsyncStream.makeStream(of: AnalyzerInput.self)
+        
+        let modules = transcriber.getModules()
+        // 4. Analyzer
+        guard let audioFormat = await SpeechAnalyzer.bestAvailableAudioFormat(
+            compatibleWith: modules
+        ) else {
+            self.reportFailure(
+                from: "startRecognition.SpeechAnalyzer.bestAvailableAudioFormat",
+                message: "Failed to find SpeechAnalyzer audio format",
+                type: .start
+            )
+            return
+        }
+        
+        analyzer = SpeechAnalyzer(modules: modules)
+        
+        // 5. Supply audio
+        audioProducerTask = Task {
+            self.startAudioEngine(
+                onBuffer: { [weak self] buffer in
+                    self?.outputContinuation?.yield(buffer)
+                }
+            )
+            guard let hardwareFormat else { return }
+            let stream = AsyncStream(
+                AVAudioPCMBuffer.self,
+                bufferingPolicy: .unbounded
+            ) { continuation in
+                outputContinuation = continuation
+            }
+            
+            let needsConversion =
+                hardwareFormat.commonFormat != audioFormat.commonFormat ||
+                hardwareFormat.sampleRate != audioFormat.sampleRate ||
+                hardwareFormat.channelCount != audioFormat.channelCount
+            do {
+                guard let converter = AVAudioConverter(
+                    from: hardwareFormat,
+                    to: audioFormat
+                ) else {
+                    throw NSError()
+                }
+                for await pcmBuffer in stream {
+                    if Task.isCancelled { break }
+                    
+                    let bufferForAnalyzer: AVAudioPCMBuffer
+                    if needsConversion {
+                        // Skip analyzing for empty buffers and
+                        // Throw error if buffers are inconvertable
+                        guard let convertedBuffer = try AudioBufferConverter.convertBuffer(
+                            converter: converter,
+                            audioFormat: audioFormat,
+                            pcmBuffer: pcmBuffer
+                        ) else {
+                            continue
+                        }
+                        bufferForAnalyzer = convertedBuffer
+                    } else {
+                        bufferForAnalyzer = pcmBuffer
+                    }
+                    
+                    let input = AnalyzerInput(buffer: bufferForAnalyzer)
+                    inputBuilder?.yield(input)
+                }
+            } catch {
+                if Task.isCancelled || self.isStopping {
+                    return
+                }
+                self.reportFailure(
+                    from: "startRecognition.audioProducerTask",
+                    message: "Failed to convert audio format",
+                    type: .start
+                )
+                return
+            }
+        }
+        
+        // 7. Handle the results
+        recognizerTask = Task {
+            do {
+                try await transcriber.handleResults(
+                    onResult: { [weak self] result in
+                        guard let self else { return }
+                        self.handleBatch(
+                            attrString: result.text,
+                            rangeStart: result.rangeStart,
+                            isFinal: result.isFinal
+                        )
+                    }
+                )
+            } catch {
+                if self.isStopping || error is CancellationError {
+                    return
+                }
+                self.reportFailure(
+                    from: "startRecognition.recognizerTask",
+                    message: "Failed to retrieve transcriber result",
+                    type: .onSession
+                )
+            }
+        }
+        
+        do {
+            if let inputSequence, let analyzer {
+                if let contextualStrings = self.recognizerDelegate?.config?.contextualStrings {
+                    let context = AnalysisContext()
+                    context.contextualStrings = [
+                        AnalysisContext.ContextualStringsTag.general: contextualStrings
+                    ]
+                    try await analyzer.setContext(context)
+                }
+                try await analyzer.start(inputSequence: inputSequence)
+            }
+        } catch {
+            self.reportFailure(
+                from: "startRecognition.analyzerStart",
+                message: "Failed to start analyze input sequence",
+                type: .start
+            )
+            return
+        }
+
+        self.sendFeedbackOnStart()
+    }
+    
+    override func cleanup(from: String) {
+        super.cleanup(from: "overridden.\(from)")
+
+        inputSequence = nil
+        inputBuilder = nil
+        outputContinuation?.finish()
+        outputContinuation = nil
+        analyzer = nil
+        transcriber.clean()
+        audioProducerTask?.cancel()
+        audioProducerTask = nil
+        recognizerTask?.cancel()
+        recognizerTask = nil
+        lastBatchStartTime = nil
+        resultBatches = []
+    }
+    
+    private func handleBatch(attrString: AttributedString, rangeStart: CMTime, isFinal: Bool) {
+        var newBatch = String(attrString.characters)
+        // Ignore all batches without A-z0-9
+        if !newBatch.contains(/\w+/) {
+            return
+        }
+        // Track only when transcription is coming
+        self.trackPartialActivity()
+        
+        let disableRepeatingFilter = self.recognizerDelegate?.config?.disableRepeatingFilter ?? false
+        if !disableRepeatingFilter {
+            newBatch = Utils.repeatingFilter(newBatch)
+        }
+        Log.log("[1] lastBatch: \(self.resultBatches.last ?? "") | newBatch: \(newBatch)")
+        if self.resultBatches.isEmpty {
+            self.resultBatches.append(newBatch)
+        } else if CMTimeGetSeconds(rangeStart) == self.lastBatchStartTime || isFinal {
+            Log.log("[2] replace, isFinal: \(isFinal)")
+            self.resultBatches[self.resultBatches.count - 1] = newBatch
+        } else {
+            Log.log("[2] add new batch")
+            self.resultBatches.append(newBatch)
+        }
+        self.lastBatchStartTime = CMTimeGetSeconds(rangeStart)
+        self.recognizerDelegate?.result(batches: self.resultBatches)
+    }
+}

package/ios/Engines/DictationRuntime.swift

@@ -0,0 +1,67 @@
+import Foundation
+import Speech
+
+@available(iOS 26.0, *)
+final class DictationRuntime: TranscriberRuntime {
+    let locale: Locale
+    private var transcriber: DictationTranscriber?
+    
+    init(with locale: Locale) {
+        self.locale = locale
+    }
+    
+    func create(config: SpeechRecognitionConfig?) async throws {
+        var dictationTranscriptionOptions: Set<DictationTranscriber.TranscriptionOption> = [
+            .punctuation
+        ]
+        if config?.maskOffensiveWords == true {
+            dictationTranscriptionOptions.insert(.etiquetteReplacements)
+        }
+        if config?.iosAddPunctuation == false
+            || config?.iosPreset == IosPreset.shortform {
+            dictationTranscriptionOptions.remove(.punctuation)
+        }
+        var contentHints: Set<DictationTranscriber.ContentHint> = [
+            .shortForm,
+            .farField,
+        ]
+        if config?.iosAtypicalSpeech == true {
+            contentHints.insert(.atypicalSpeech)
+        }
+        transcriber = DictationTranscriber(
+            locale: locale,
+            contentHints: contentHints,
+            transcriptionOptions: dictationTranscriptionOptions,
+            reportingOptions: [.frequentFinalization, .volatileResults],
+            attributeOptions: [.audioTimeRange]
+        )
+
+        if let transcriber, let installationRequest = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) {
+            try await installationRequest.downloadAndInstall()
+        }
+    }
+    
+    func getModules() -> [any SpeechModule] {
+        guard let transcriber else { return [] }
+        return [transcriber]
+    }
+    
+    func handleResults(
+        onResult: @escaping (TranscriberResult) -> Void
+    ) async throws {
+        if let transcriber {
+            for try await result in transcriber.results {
+                onResult(
+                    TranscriberResult(
+                        text: result.text,
+                        rangeStart: result.range.start,
+                        isFinal: result.isFinal)
+                )
+            }
+        }
+    }
+    
+    func clean() {
+        transcriber = nil
+    }
+}

package/ios/Engines/RecognizerEngine.swift

@@ -0,0 +1,312 @@
+import Foundation
+import Speech
+import AVFoundation
+
+// No practical diff between "system" and "onSession" for now.
+// For future: send the level of error to RN
+// "onSession" is less critical level, since the session has been started successfully
+enum FailureType {
+    case system
+    case start
+    case prewarm
+    case onSession
+}
+
+class RecognizerEngine {
+    var isActive = false
+    var isStopping = false
+    var hardwareFormat: AVAudioFormat?
+    weak var recognizerDelegate: RecognizerDelegate?
+    
+    private let audioLevelTracker: AudioLevelTracker
+    private var appStateObserver: AppStateObserver?
+    private var audioEngine: AVAudioEngine?
+    private var autoStopper: AutoStopper?
+    private let lg = Lg(prefix: "RecognizerEngine")
+    
+    let locale: Locale
+    
+    init(locale: Locale, delegate: RecognizerDelegate) {
+        self.locale = locale
+        self.recognizerDelegate = delegate
+        self.audioLevelTracker = AudioLevelTracker(
+            resetAutoFinishVoiceSensitivity: delegate.config?.resetAutoFinishVoiceSensitivity
+        )
+    }
+    
+    // MARK: - Recognizer Methods
+    
+    func prewarm(for: FailureType) async {
+        self.prepareAudioEngine()
+        // for SpeechTranscriber: .isAvailable and async assets
+        // for Dictation: only async assets
+        // for legacy SF: only sync .isAvailable
+    }
+    
+    func start() {
+        guard let recognizerDelegate, !isActive else { return }
+        
+        Permissions(
+            onGranted: self.startSession,
+            onDenied: recognizerDelegate.permissionDenied,
+            onError: recognizerDelegate.error
+        ).requestAuthorization()
+    }
+    
+    func stop() {
+        guard isActive, !isStopping else { return }
+        isStopping = true
+        HapticImpact.trigger(with: self.recognizerDelegate?.config?.stopHapticFeedbackStyle)
+    }
+    
+    func startSession() async {
+        lg.log("[startSession.startSession]")
+        // Init everything
+        isStopping = false
+        isActive = true
+        
+        initAutoStop()
+        lg.log("[startSession.initAutoStop]")
+        startAppStateObserver()
+        lg.log("[startSession.startAppStateObserver]")
+        startAudioSession()
+        lg.log("[startSession.startAudioSession]")
+    }
+    
+    func startAudioEngine(
+        onBuffer: @escaping (AVAudioPCMBuffer) -> Void
+    ) {
+        lg.log("[startAudioEngine]")
+        guard let audioEngine, let hardwareFormat else { return }
+        audioEngine.inputNode.installTap(
+            onBus: 0,
+            bufferSize: 1024,
+            format: hardwareFormat
+        ) { [weak self] buffer, _ in
+            guard let self, let recognizerDelegate = self.recognizerDelegate else { return }
+            if let sample = self.audioLevelTracker.process(buffer) {
+                // Send buffer volume data
+                recognizerDelegate.volumeChange(
+                    event:
+                        VolumeChangeEvent(
+                            smoothedVolume: sample.smoothed,
+                            rawVolume: sample.raw,
+                            db: sample.db
+                        )
+                )
+                if sample.resetTimer {
+                    self.autoStopper?.resetTimer(from: "rms threshold")
+                }
+            }
+            onBuffer(buffer)
+        }
+        lg.log("[startAudioEngine.installTap]")
+        do {
+            audioEngine.prepare()
+            lg.log("[startAudioEngine.prepare]")
+            try audioEngine.start()
+            lg.log("[startAudioEngine.start]")
+        } catch {
+            self.reportFailure(
+                from: "Audio Engine",
+                message: "Audio Engine failed to start",
+                // RecognizerEngine-agnostic Error
+                type: .system
+            )
+        }
+    }
+    
+    func sendFeedbackOnStart() {
+        guard let recognizerDelegate else { return }
+        lg.log("[sendFeedbackOnStart]")
+        HapticImpact.trigger(with: recognizerDelegate.config?.startHapticFeedbackStyle)
+        autoStopper?.resetTimer(from: "startListening.sendFeedbackOnStart")
+        recognizerDelegate.readyForSpeech()
+        recognizerDelegate.result(batches: [])
+    }
+    
+    func updateSession(
+        newConfig: MutableSpeechRecognitionConfig? = nil,
+        addMsToTimer: Double? = nil,
+        resetTimer: Bool? = nil
+    ) {
+        guard let recognizerDelegate, isActive, !isStopping else { return }
+        let currentConfig = recognizerDelegate.config
+        // Update AutoFinish time
+        if let newAutoFinish = newConfig?.autoFinishRecognitionMs,
+           newAutoFinish != currentConfig?.autoFinishRecognitionMs {
+            autoStopper?.updateThreshold(
+                newAutoFinish,
+                from: "updateSession"
+            )
+        }
+        // Update AutoFinish progress interval
+        if let newInterval = newConfig?.autoFinishProgressIntervalMs,
+           newInterval != currentConfig?.autoFinishProgressIntervalMs {
+            autoStopper?.updateProgressInterval(
+                newInterval,
+                from: "updateSession"
+            )
+        }
+        // Update AutoFinish reset voice sensitivity interval
+        if let newSensitivity = newConfig?.resetAutoFinishVoiceSensitivity, 
+            newSensitivity != currentConfig?.resetAutoFinishVoiceSensitivity {
+            audioLevelTracker.updateResetAutoFinishVoiceSensitivity(
+                newValue: newSensitivity
+            )
+        }
+        if let addMsToTimer {
+            // Add time to the timer once
+            autoStopper?.addMsOnce(
+                addMsToTimer,
+                from: "updateSession"
+            )
+        } else if resetTimer == true {
+            // Reset to current baseline threshold.
+            autoStopper?.resetTimer(from: "updateSession")
+        }
+        // Only update new non-nil values in the config
+        recognizerDelegate.softlyUpdateConfig(newConfig: newConfig)
+    }
+    
+    func cleanup(from: String) {
+        lg.log("[cleanup]: \(from)")
+        let wasActive = isActive
+        deinitAutoStop()
+        stopAppStateObserver()
+        stopAudioSession()
+        audioLevelTracker.reset()
+        
+        if let audioEngine, audioEngine.isRunning {
+            audioEngine.stop()
+        }
+        audioEngine?.inputNode.removeTap(onBus: 0)
+        
+        audioEngine = nil
+        isActive = false
+        isStopping = false
+        self.recognizerDelegate?.volumeChange(
+            event:
+                VolumeChangeEvent(
+                    smoothedVolume: 0,
+                    rawVolume: 0,
+                    db: nil
+                )
+        )
+        if wasActive {
+            self.recognizerDelegate?.recordingStopped()
+        }
+    }
+    
+    func reportFailure(from: String, message: String, type: FailureType) {
+        // Log message
+        lg.log("[Failure] type: \(type), message: \(message)")
+        
+        // Cleanup on engine level anyway
+        self.cleanup(from: from)
+        
+        switch type {
+            // Try to reselect engine and try again
+            case .prewarm, .start:
+                let isPrewarm = type == .prewarm
+                self.recognizerDelegate?.reselectEngine(forPrewarm: isPrewarm)
+            // System level issue: send onError with description and clean
+            // Session has already started: send onError and cleanup
+            case .system, .onSession:
+                self.recognizerDelegate?.error(message: message)
+        }
+    }
+    
+    func trackPartialActivity() {
+        if !self.isStopping {
+            self.autoStopper?.resetTimer(from: "Partial results")
+        }
+    }
+    
+    // MARK: - AudioEngine heavy prepare
+    
+    private func prepareAudioEngine() {
+        lg.log("[prewarm.start]")
+        audioEngine = AVAudioEngine()
+        guard let audioEngine else {
+            self.reportFailure(
+                from: "Audio Engine",
+                message: "Audio Engine failed to initiate",
+                // RecognizerEngine-agnostic Error
+                type: .system
+            )
+            return
+        }
+        lg.log("[prewarm.audioEngine]")
+        // heavy first hardwareFormat retrieval
+        if hardwareFormat == nil {
+            hardwareFormat = audioEngine.inputNode.outputFormat(forBus: 0)
+            lg.log("[prewarm.hardwareFormat]")
+        }
+    }
+    
+    // MARK: - AutoStopper
+    
+    private func initAutoStop() {
+        let config = self.recognizerDelegate?.config
+        autoStopper = AutoStopper(
+            silenceThresholdMs: config?.autoFinishRecognitionMs,
+            progressIntervalMs: config?.autoFinishProgressIntervalMs,
+            onProgress: { [weak self] timeLeftMs in
+                guard let self else { return }
+                self.recognizerDelegate?.autoFinishProgress(
+                    timeLeftMs: timeLeftMs
+                )
+            },
+            onTimeout: { [weak self] in
+                self?.stop()
+            }
+        )
+    }
+    private func deinitAutoStop() {
+        autoStopper?.stop()
+        autoStopper = nil
+    }
+    
+    // MARK: - App State Observer
+    
+    private func startAppStateObserver() {
+        appStateObserver = AppStateObserver { [weak self] in
+            guard let self, self.isActive else { return }
+            self.stop()
+        }
+    }
+    
+    private func stopAppStateObserver() {
+        appStateObserver?.stop()
+        appStateObserver = nil
+    }
+    
+    // MARK: - Audio Session
+    
+    private func startAudioSession() {
+        do {
+            let audioSession = AVAudioSession.sharedInstance()
+            try audioSession.setCategory(.record, mode: .measurement, options: .duckOthers)
+            // Required for haptic feedback
+            try audioSession.setAllowHapticsAndSystemSoundsDuringRecording(true)
+            try audioSession.setActive(true, options: .notifyOthersOnDeactivation)
+        } catch {
+            self.reportFailure(
+                from: "startAudioSession",
+                message: "Failed to activate audio session: \(error.localizedDescription)",
+                // RecognizerEngine-agnostic Error
+                type: .system
+            )
+        }
+    }
+    private func stopAudioSession() {
+        do {
+            // TODO: check unduck
+            try AVAudioSession.sharedInstance().setActive(false)
+        } catch {
+            // Just log and no-op - not critical
+            lg.log("Failed to deactivate audio session: \(error.localizedDescription)")
+        }
+    }
+}