@gmessier/nitro-speech 0.3.2 → 0.4.0

package/ios/Engines/SFSpeechEngine.swift

@@ -0,0 +1,119 @@
+import Foundation
+import Speech
+import AVFoundation
+
+final class SFSpeechEngine: RecognizerEngine {
+    private var recognitionRequest: SFSpeechAudioBufferRecognitionRequest?
+    private var recognitionTask: SFSpeechRecognitionTask?
+    private var speechRecognizer: SFSpeechRecognizer?
+
+    private let lg = Lg(prefix: "SFSpeechEngine")
+
+    override func stop() {
+        super.stop()
+        recognitionRequest?.endAudio()
+        recognitionTask?.finish()
+    }
+    
+    override func prewarm(for type: FailureType) async {
+        speechRecognizer = SFSpeechRecognizer(
+            locale: Locale(identifier: self.recognizerDelegate?.config?.locale ?? "en-US")
+        )
+        if speechRecognizer?.isAvailable != true {
+            self.reportFailure(
+                from: "prewarm",
+                message: "SFSpeechRecognizer is not available",
+                type: type
+            )
+        }
+        await super.prewarm(for: type)
+    }
+    
+    override func startSession() async {
+        await super.startSession()
+        lg.log("[startSession.startSession]")
+        
+        await prewarm(for: .start)
+        lg.log("[startSession.prewarm]")
+        guard let speechRecognizer else { return }
+        
+        recognitionRequest = createRecognitionRequest()
+        lg.log("[startSession.createRecognitionRequest]")
+        guard let recognitionRequest else { return }
+        
+        recognitionTask = speechRecognizer.recognitionTask(
+            with: recognitionRequest
+        ) { [weak self] result, error in
+            guard let self else { return }
+            
+            if let result = result {
+                var transcription = result.bestTranscription.formattedString
+                if !transcription.isEmpty {
+                    // Track only when transcription is coming
+                    self.trackPartialActivity()
+                    
+                    let disableRepeatingFilter = self.recognizerDelegate?.config?.disableRepeatingFilter ?? false
+                    if !disableRepeatingFilter {
+                        transcription = Utils.repeatingFilter(transcription)
+                    }
+                    // Legacy transcriber collects everything into one batch
+                    self.recognizerDelegate?.result(batches: [transcription])
+                }
+                
+                if result.isFinal {
+                    self.cleanup(from: "startRecognition.recognitionTask.final")
+                }
+            }
+            
+            if let error = error {
+                if !self.isStopping {
+                    self.reportFailure(
+                        from: "startSession.recognitionTask.error",
+                        message: "Recognition Error: \(error.localizedDescription)",
+                        type: .onSession
+                    )
+                } else {
+                    self.cleanup(from: "startRecognition.recognitionTask.manualStop")
+                }
+            }
+        }
+        lg.log("[startSession.recognitionTask]")
+        
+        self.startAudioEngine(
+            onBuffer: { [weak self] buffer in
+                self?.recognitionRequest?.append(buffer)
+            }
+        )
+        lg.log("[startSession.startAudioEngine]")
+        
+        self.sendFeedbackOnStart()
+        lg.log("[startSession.sendFeedbackOnStart]")
+    }
+    
+    override func cleanup(from: String) {
+        super.cleanup(from: "overridden.\(from)")
+        recognitionRequest = nil
+        recognitionTask = nil
+        speechRecognizer = nil
+    }
+    
+    private func createRecognitionRequest() -> SFSpeechAudioBufferRecognitionRequest {
+        let request = SFSpeechAudioBufferRecognitionRequest()
+        request.shouldReportPartialResults = true
+        
+        if let contextualStrings = self.recognizerDelegate?.config?.contextualStrings,
+           !contextualStrings.isEmpty {
+            request.contextualStrings = contextualStrings
+        }
+        
+        if #available(iOS 16, *) {
+            if self.recognizerDelegate?.config?.iosAddPunctuation == false {
+                request.addsPunctuation = false
+            } else {
+                request.addsPunctuation = true
+            }
+        }
+        
+        return request
+    }
+}

package/ios/Engines/SpeechRuntime.swift

@@ -0,0 +1,58 @@
+import Foundation
+import Speech
+
+@available(iOS 26.0, *)
+final class SpeechRuntime: TranscriberRuntime {
+    let locale: Locale
+    private var transcriber: SpeechTranscriber?
+    
+    init(with locale: Locale) {
+        self.locale = locale
+    }
+    
+    func create(config: SpeechRecognitionConfig?) async throws {
+        if !SpeechTranscriber.isAvailable {
+            throw NSError()
+        }
+        var speechTranscriptionOptions: Set<SpeechTranscriber.TranscriptionOption> = []
+        if config?.maskOffensiveWords == true {
+            speechTranscriptionOptions.insert(.etiquetteReplacements)
+        }
+        transcriber = SpeechTranscriber(
+            locale: locale,
+            transcriptionOptions: speechTranscriptionOptions,
+            reportingOptions: [.volatileResults, .fastResults],
+            attributeOptions: [.audioTimeRange]
+        )
+        
+        
+
+        if let transcriber, let installationRequest = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) {
+            try await installationRequest.downloadAndInstall()
+        }
+    }
+    
+    func getModules() -> [any SpeechModule] {
+        guard let transcriber else { return [] }
+        return [transcriber]
+    }
+    
+    func handleResults(
+        onResult: @escaping (TranscriberResult) -> Void
+    ) async throws {
+        if let transcriber {
+            for try await result in transcriber.results {
+                onResult(
+                    TranscriberResult(
+                        text: result.text,
+                        rangeStart: result.range.start,
+                        isFinal: result.isFinal)
+                )
+            }
+        }
+    }
+    
+    func clean() {
+        transcriber = nil
+    }
+}

package/ios/Engines/TranscriberRuntimeProtocol.swift

@@ -0,0 +1,21 @@
+import Foundation
+import Speech
+
+struct TranscriberResult {
+    let text: AttributedString
+    let rangeStart: CMTime
+    let isFinal: Bool
+}
+
+@available(iOS 26.0, *)
+protocol TranscriberRuntime {
+    var locale: Locale { get }
+    
+    func create(config: SpeechRecognitionConfig?) async throws
+    
+    func getModules() -> [any SpeechModule]
+    
+    func handleResults(onResult: @escaping (TranscriberResult) -> Void) async throws
+    
+    func clean() -> Void
+}

package/ios/HybridNitroSpeech.swift

@@ -2,14 +2,5 @@ import Foundation
 import NitroModules
 
 class HybridNitroSpeech : HybridNitroSpeechSpec {
-    var recognizer: any HybridRecognizerSpec
-
-    override init() {
-        if #available(iOS 26.0, *) {
-            recognizer = AnalyzerTranscriber()
-        } else {
-            recognizer = LegacySpeechRecognizer()
-        }
-        super.init()
-    }
+    var recognizer: HybridRecognizerSpec = HybridRecognizer()
 }

package/ios/HybridRecognizer.swift

@@ -1,13 +1,8 @@
 import Foundation
-import Speech
 import NitroModules
-import os.log
-import AVFoundation
 
-class HybridRecognizer: HybridRecognizerSpec {
-    internal let logger = Logger(subsystem: "com.margelo.nitro.nitrospeech", category: "Recognizer")
-    internal static let defaultAutoFinishRecognitionMs = 8000.0
-    internal static let speechRmsThreshold: Float = 0.005623
+class HybridRecognizer: HybridRecognizerSpec  {
+    var config: SpeechRecognitionConfig?
     
     var onReadyForSpeech: (() -> Void)?
     var onRecordingStopped: (() -> Void)?
@@ -15,228 +10,176 @@ class HybridRecognizer: HybridRecognizerSpec {
     var onAutoFinishProgress: ((Double) -> Void)?
     var onError: ((String) -> Void)?
     var onPermissionDenied: (() -> Void)?
-    var onVolumeChange: ((Double) -> Void)?
+    var onVolumeChange: ((VolumeChangeEvent) -> Void)?
     
-    internal var audioEngine: AVAudioEngine?
+    private let coordinator = Coordinator()
+    private var paramsHash: String?
+    private var engine: RecognizerEngine?
     
-    internal var autoStopper: AutoStopper?
-    internal var appStateObserver: AppStateObserver?
-    internal var isActive: Bool = false
-    internal var isStopping: Bool = false
-    internal var config: SpeechToTextParams?
-    internal var levelSmoothed: Float = 0
-    
-    func getIsActive() -> Bool {
-        return self.isActive
+    override init() {
+        super.init()
+        self.coordinator.recognizerDelegate = self
     }
     
-    func startListening(params: SpeechToTextParams) {
-        if isActive {
-            return
-        }
-        
-        SFSpeechRecognizer.requestAuthorization { [weak self] authStatus in
-            Task { @MainActor in
-                guard let self = self else { return }
-                
-                self.config = params
-                
-                switch authStatus {
-                case .authorized:
-                    self.requestMicrophonePermission()
-                case .denied, .restricted:
-                    self.onPermissionDenied?()
-                case .notDetermined:
-                    self.onError?("Speech recognition not determined")
-                @unknown default:
-                    self.onError?("Unknown authorization status")
-                }
-            }
+    private let lg = Lg(prefix: "HybridRecognizer")
+    
+    @discardableResult
+    func prewarm(defaultParams: SpeechRecognitionConfig?) -> Promise<Void> {
+        return Promise.async(.userInitiated) { [weak self] in
+            // Ensure correct engine is selected based on params and ios version
+            await self?.ensureEngine(params: defaultParams)
+            // try to preload assets and check if speech engine is available on OS level
+            await self?.engine?.prewarm(for: .prewarm)
         }
     }
-
-    func dispose() {
-        stopListening()
+    
+    func startListening(params: SpeechRecognitionConfig?) {
+        Task {
+            // Ensure correct engine is selected based on params and ios version
+            await ensureEngine(params: params)
+            engine?.start()
+        }
     }
     
     func stopListening() {
-        guard isActive, !isStopping else { return }
-        isStopping = true
-        
-        self.stopHapticFeedback()
+        engine?.stop()
     }
     
-    internal func handleInternalStopTrigger() {
-        self.stopListening()
+    func resetAutoFinishTime() {
+        engine?.updateSession(resetTimer: true)
     }
     
     func addAutoFinishTime(additionalTimeMs: Double?) {
-        guard isActive, !isStopping else { return }
-        
-        autoStopper?.indicateRecordingActivity(
-            from: "refreshAutoFinish",
-            addMsToThreshold: additionalTimeMs
+        if let additionalTimeMs {
+            engine?.updateSession(addMsToTimer: additionalTimeMs)
+        } else {
+            // Reset timer to original baseline.
+            engine?.updateSession(resetTimer: true)
+        }
+    }
+    
+    func updateConfig(newConfig: MutableSpeechRecognitionConfig?, resetAutoFinishTime: Bool?) {
+        engine?.updateSession(
+            newConfig: newConfig,
+            resetTimer: resetAutoFinishTime
         )
     }
+
+    func getIsActive() -> Bool {
+        engine?.isActive ?? false
+    }
     
-    func updateAutoFinishTime(newTimeMs: Double, withRefresh: Bool?) {
-        guard isActive, !isStopping else { return }
-        
-        autoStopper?.updateSilenceThreshold(newThresholdMs: newTimeMs)
-        
-        if withRefresh == true {
-            autoStopper?.indicateRecordingActivity(
-                from: "updateAutoFinishTime",
-                addMsToThreshold: nil
-            )
+    func getSupportedLocalesIOS() -> [String] {
+        return self.coordinator.getSupportedLocales()
+    }
+
+    private func ensureEngine(params: SpeechRecognitionConfig?) async {
+        // Remember new params
+        config = params
+        let hash = Utils.hashParams(params)
+        if engine != nil && hash == paramsHash {
+            lg.log("Reuse Engine")
+            // Engine is already correct
+            return
+        }
+        if hash != paramsHash {
+            // Initialize when trying to select new engine with new params
+            await coordinator.initialize()
+            paramsHash = hash
+        }
+        lg.log("hash: \(hash)")
+        // Try to select new engine
+        engine = coordinator.getEngine()
+        if engine == nil {
+            // Only wrong locale can wipe out all candidates
+            self.onError?("No recognition engine available for the requested locale")
+            return
         }
     }
+}
 
-    internal func requestMicrophonePermission() {}
-    
-    internal func startRecognitionSetup() -> Bool {
-        isStopping = false
-        isActive = true
-        
-        initAutoStop()
-        monitorAppState()
-        guard startAudioSession() else {
-            cleanup(from: "startRecognitionSetup")
-            return false
+protocol RecognizerDelegate: AnyObject {
+    var config: SpeechRecognitionConfig? { get }
+    func softlyUpdateConfig(newConfig: MutableSpeechRecognitionConfig?)
+    func reselectEngine(forPrewarm: Bool)
+    func readyForSpeech()
+    func recordingStopped()
+    func result (batches: [String])
+    func autoFinishProgress (timeLeftMs: Double)
+    func error (message: String)
+    func permissionDenied ()
+    func volumeChange (event: VolumeChangeEvent)
+}
+
+extension HybridRecognizer: RecognizerDelegate {
+    func softlyUpdateConfig(newConfig: MutableSpeechRecognitionConfig?) {
+        if let newConfig {
+            config = SpeechRecognitionConfig(
+                locale: config?.locale,
+                contextualStrings: config?.contextualStrings,
+                maskOffensiveWords: config?.maskOffensiveWords,
+                autoFinishRecognitionMs: newConfig.autoFinishRecognitionMs ?? config?.autoFinishRecognitionMs,
+                autoFinishProgressIntervalMs: newConfig.autoFinishProgressIntervalMs ?? config?.autoFinishProgressIntervalMs,
+                resetAutoFinishVoiceSensitivity: newConfig.resetAutoFinishVoiceSensitivity ?? config?.resetAutoFinishVoiceSensitivity,
+                disableRepeatingFilter: newConfig.disableRepeatingFilter ?? config?.disableRepeatingFilter,
+                startHapticFeedbackStyle: newConfig.startHapticFeedbackStyle ?? config?.startHapticFeedbackStyle,
+                stopHapticFeedbackStyle: newConfig.stopHapticFeedbackStyle ?? config?.stopHapticFeedbackStyle,
+                androidFormattingPreferQuality: config?.androidFormattingPreferQuality,
+                androidUseWebSearchModel: config?.androidUseWebSearchModel,
+                androidDisableBatchHandling: config?.androidDisableBatchHandling,
+                iosAddPunctuation: config?.iosAddPunctuation,
+                iosPreset: config?.iosPreset,
+                iosAtypicalSpeech: config?.iosAtypicalSpeech
+            )
         }
-        
-        return true
     }
     
-    internal func startRecognitionFeedback() {
-        self.startHapticFeedback()
-        autoStopper?.indicateRecordingActivity(
-            from: "startListening",
-            addMsToThreshold: nil
-        )
-        onReadyForSpeech?()
-        onResult?([])
+    func readyForSpeech() {
+        self.lg.log("[HR -> onReadyForSpeech]")
+        self.onReadyForSpeech?()
     }
     
-    internal func startRecognition() {}
-    internal func startRecognition() async {}
-    
-    internal func cleanup(from: String) {
-        logger.info("cleanup called from: \(from)")
-        deinitAutoStop()
-        stopMonitorAppState()
-        stopAudioSession()
-        stopAudioEngine()
-        levelSmoothed = 0
-        isActive = false
-        isStopping = false
-        onVolumeChange?(0)
+    func recordingStopped() {
+        self.lg.log("[onRecordingStopped]")
+        self.onRecordingStopped?()
     }
     
-    internal func stopAudioEngine() {
-        if let audioEngine = audioEngine, audioEngine.isRunning {
-            audioEngine.stop()
-        }
-        audioEngine?.inputNode.removeTap(onBus: 0)
-        audioEngine = nil
+    func result(batches: [String]) {
+        self.lg.log("[onResult] \(batches)")
+        self.onResult?(batches)
     }
     
-    internal func monitorAppState() {
-        appStateObserver = AppStateObserver { [weak self] in
-            guard let self = self, self.isActive else { return }
-            self.handleInternalStopTrigger()
-        }
-    }
-    internal func stopMonitorAppState () {
-        appStateObserver?.stop()
-        appStateObserver = nil
+    func autoFinishProgress(timeLeftMs: Double) {
+        self.lg.log("[onAutoFinishProgress] \(timeLeftMs)ms")
+        self.onAutoFinishProgress?(timeLeftMs)
     }
     
-    internal func initAutoStop() {
-        autoStopper = AutoStopper(
-            silenceThresholdMs: config?.autoFinishRecognitionMs ?? Self.defaultAutoFinishRecognitionMs,
-            onProgress: { [weak self] timeLeftMs in
-                self?.onAutoFinishProgress?(timeLeftMs)
-            },
-            onTimeout: { [weak self] in
-                self?.handleInternalStopTrigger()
-            }
-        )
-    }
-    internal func deinitAutoStop () {
-        autoStopper?.stop()
-        autoStopper = nil
-    }
-    
-    internal func startAudioSession() -> Bool {
-        do {
-            let audioSession = AVAudioSession.sharedInstance()
-            try audioSession.setCategory(.record, mode: .measurement, options: .duckOthers)
-            // Without this, iOS may suppress haptics while recording.
-            try audioSession.setAllowHapticsAndSystemSoundsDuringRecording(true)
-            try audioSession.setActive(true, options: .notifyOthersOnDeactivation)
-            return true
-        } catch {
-            onError?("Failed to activate audio session: \(error.localizedDescription)")
-            return false
-        }
-    }
-    internal func stopAudioSession () {
-        do {
-            try AVAudioSession.sharedInstance().setActive(false)
-        } catch {
-            logger.info("Failed to deactivate audio session: \(error.localizedDescription)")
-            return
-        }
+    func error(message: String) {
+        self.lg.log("[onError]")
+        self.onError?(message)
     }
     
-    internal func startHapticFeedback() {
-        if let hapticStyle = config?.startHapticFeedbackStyle {
-            HapticImpact(style: hapticStyle).trigger()
-        } else {
-            HapticImpact(style: .medium).trigger()
-        }
-    }
-    internal func stopHapticFeedback () {
-        if let hapticStyle = config?.stopHapticFeedbackStyle {
-            HapticImpact(style: hapticStyle).trigger()
-        } else {
-            HapticImpact(style: .medium).trigger()
-        }
+    func permissionDenied() {
+        self.lg.log("[onPermissionDenied]")
+        self.onPermissionDenied?()
     }
     
-    internal func trackPartialActivity() {
-        if !self.isStopping {
-            self.autoStopper?.indicateRecordingActivity(
-                from: "partial results",
-                addMsToThreshold: nil
-            )
-        }
+    func volumeChange(event: VolumeChangeEvent) {
+        // self.lg.log("[onVolumeChange] \(event.rawVolume)")
+        self.onVolumeChange?(event)
     }
-
-    internal func repeatingFilter(text: String) -> String {
-        var subStrings = text.split { $0.isWhitespace }.map { String($0) }
-        var joiner = ""
-        // 10 - arbitrary number of last substrings that is still unstable
-        // and needs to be filtered. Prev substrings were handled earlier.
-        if subStrings.count >= 10 {
-            joiner = subStrings.prefix(subStrings.count - 9).joined(separator: " ")
-            subStrings = Array(subStrings.suffix(10))
+    
+    func reselectEngine(forPrewarm: Bool) {
+        // Remove failed engine from candidates
+        coordinator.reportEngineFailure()
+        // Reset active engine
+        engine = nil
+        // Try to prewarm with another candidate
+        if forPrewarm {
+            self.prewarm(defaultParams: config)
         } else {
-            joiner = subStrings.first ?? ""
-        }
-        for i in subStrings.indices {
-            if i == 0 { continue }
-            // Always add number-contained strings
-            if #available(iOS 16.0, *), subStrings[i].contains(/\d+/) {
-                joiner += " \(subStrings[i])"
-                continue
-            }
-            
-            // Skip consecutive duplicate strings
-            if subStrings[i] == subStrings[i-1] { continue }
-            joiner += " \(subStrings[i])"
+            // Try to start with another candidate
+            self.startListening(params: config)
         }
-        return joiner
     }
 }

package/ios/LocaleManager.swift

@@ -0,0 +1,73 @@
+import Foundation
+import Speech
+
+final class LocaleManager {
+    private let sfSpeechLocales = SFSpeechRecognizer.supportedLocales().map { $0.identifier }
+    private var speechLocales: [String]
+    private var dictationLocales: [String]
+    var supportedLocales: [String]
+    var SFLocale: Locale?
+    var speechLocale: Locale?
+    var dictationLocale: Locale?
+    
+    private var equivalentsCountedFor: String?
+    
+    init() async {
+        self.speechLocales = []
+        self.dictationLocales = []
+        self.supportedLocales = sfSpeechLocales
+        
+        if #available(iOS 26.0, *) {
+            self.speechLocales = await SpeechTranscriber.supportedLocales.map {
+                $0.identifier
+            }
+            self.dictationLocales = await DictationTranscriber.supportedLocales.map {
+                $0.identifier
+            }
+            Log.log("[Coordinator] sfSpeechLocales: \(self.sfSpeechLocales)")
+            Log.log("[Coordinator] speechLocales: \(self.speechLocales)")
+            Log.log("[Coordinator] dictationLocales: \(self.dictationLocales)")
+            self.supportedLocales = Array(
+                Set(sfSpeechLocales)
+                .union(Set(speechLocales))
+                .union(Set(dictationLocales))
+            )
+        }
+    }
+
+    func ensureLocale(localeString: String?) async {
+        let identifier = localeString ?? "en-US"
+        if self.equivalentsCountedFor == identifier {
+            // All locales has been counted already, might be nil, but use them
+            Log.log("[Coordinator] ensureLocale: \(identifier) -> Already counted ")
+            return
+        }
+        if #available(iOS 26.0, *) {
+            let speechEquivalent = await SpeechTranscriber.supportedLocale(
+                equivalentTo: Locale(identifier: identifier)
+            )?.identifier
+            if let speechEquivalent, speechLocales.contains(speechEquivalent) {
+                self.speechLocale = Locale(identifier: speechEquivalent)
+            } else {
+                self.speechLocale = nil
+            }
+            
+            let dictationEquivalent = await DictationTranscriber.supportedLocale(
+                equivalentTo: Locale(identifier: identifier)
+            )?.identifier
+            if let dictationEquivalent, self.dictationLocales.contains(dictationEquivalent) {
+                self.dictationLocale = Locale(identifier: dictationEquivalent)
+            } else {
+                self.dictationLocale = nil
+            }
+        }
+        if sfSpeechLocales.contains(identifier) {
+            self.SFLocale = Locale(identifier: identifier)
+        } else {
+            self.SFLocale = nil
+        }
+        self.equivalentsCountedFor = identifier
+        Log.log("[Coordinator] equivalents: speechLocale: \(self.speechLocale?.identifier), dictationLocale: \(self.dictationLocale?.identifier), SFLocale: \(self.SFLocale?.identifier)")
+        Log.log("[Coordinator] ensureLocale: \(identifier) -> New")
+    }
+}

package/ios/{AppStateObserver.swift → Shared/AppStateObserver.swift}

@@ -1,7 +1,7 @@
 import Foundation
 import UIKit
 
-class AppStateObserver {
+final class AppStateObserver {
     private var observer: NSObjectProtocol?
     private let onResignActive: () -> Void
     
@@ -28,4 +28,3 @@ class AppStateObserver {
         stop()
     }
 }
-