@gmessier/nitro-speech 0.3.3 → 0.4.1

package/ios/HybridRecognizer.swift

@@ -1,13 +1,8 @@
 import Foundation
-import Speech
 import NitroModules
-import os.log
-import AVFoundation
 
-class HybridRecognizer: HybridRecognizerSpec {
-    internal let logger = Logger(subsystem: "com.margelo.nitro.nitrospeech", category: "Recognizer")
-    internal static let defaultAutoFinishRecognitionMs = 8000.0
-    internal static let speechRmsThreshold: Float = 0.005623
+class HybridRecognizer: HybridRecognizerSpec  {
+    var config: SpeechRecognitionConfig?
     
     var onReadyForSpeech: (() -> Void)?
     var onRecordingStopped: (() -> Void)?
@@ -15,228 +10,184 @@ class HybridRecognizer: HybridRecognizerSpec {
     var onAutoFinishProgress: ((Double) -> Void)?
     var onError: ((String) -> Void)?
     var onPermissionDenied: (() -> Void)?
-    var onVolumeChange: ((Double) -> Void)?
+    var onVolumeChange: ((VolumeChangeEvent) -> Void)?
     
-    internal var audioEngine: AVAudioEngine?
+    private let coordinator = Coordinator()
+    private var paramsHash: String?
+    private var engine: RecognizerEngine?
     
-    internal var autoStopper: AutoStopper?
-    internal var appStateObserver: AppStateObserver?
-    internal var isActive: Bool = false
-    internal var isStopping: Bool = false
-    internal var config: SpeechToTextParams?
-    internal var levelSmoothed: Float = 0
-    
-    func getIsActive() -> Bool {
-        return self.isActive
+    override init() {
+        super.init()
+        self.coordinator.recognizerDelegate = self
     }
     
-    func startListening(params: SpeechToTextParams) {
-        if isActive {
-            return
-        }
-        
-        SFSpeechRecognizer.requestAuthorization { [weak self] authStatus in
-            Task { @MainActor in
-                guard let self = self else { return }
-                
-                self.config = params
-                
-                switch authStatus {
-                case .authorized:
-                    self.requestMicrophonePermission()
-                case .denied, .restricted:
-                    self.onPermissionDenied?()
-                case .notDetermined:
-                    self.onError?("Speech recognition not determined")
-                @unknown default:
-                    self.onError?("Unknown authorization status")
-                }
-            }
+    private let lg = Lg(prefix: "HybridRecognizer")
+    
+    @discardableResult
+    func prewarm(defaultParams: SpeechRecognitionConfig?) -> Promise<Void> {
+        return Promise.async(.userInitiated) { [weak self] in
+            // Ensure correct engine is selected based on params and ios version
+            await self?.ensureEngine(params: defaultParams)
+            // try to preload assets and check if speech engine is available on OS level
+            await self?.engine?.prewarm(for: .prewarm)
         }
     }
-
-    func dispose() {
-        stopListening()
+    
+    func startListening(params: SpeechRecognitionConfig?) {
+        Task {
+            // Ensure correct engine is selected based on params and ios version
+            await ensureEngine(params: params)
+            engine?.start()
+        }
     }
     
     func stopListening() {
-        guard isActive, !isStopping else { return }
-        isStopping = true
-        
-        self.stopHapticFeedback()
+        engine?.stop()
     }
     
-    internal func handleInternalStopTrigger() {
-        self.stopListening()
+    func resetAutoFinishTime() {
+        engine?.updateSession(resetTimer: true)
     }
     
     func addAutoFinishTime(additionalTimeMs: Double?) {
-        guard isActive, !isStopping else { return }
-        
-        autoStopper?.indicateRecordingActivity(
-            from: "refreshAutoFinish",
-            addMsToThreshold: additionalTimeMs
-        )
+        if let additionalTimeMs {
+            engine?.updateSession(addMsToTimer: additionalTimeMs)
+        } else {
+            // Reset timer to original baseline.
+            engine?.updateSession(resetTimer: true)
+        }
     }
     
-    func updateAutoFinishTime(newTimeMs: Double, withRefresh: Bool?) {
-        guard isActive, !isStopping else { return }
-        
-        autoStopper?.updateSilenceThreshold(newThresholdMs: newTimeMs)
-        
-        if withRefresh == true {
-            autoStopper?.indicateRecordingActivity(
-                from: "updateAutoFinishTime",
-                addMsToThreshold: nil
-            )
-        }
+    func updateConfig(newConfig: MutableSpeechRecognitionConfig?, resetAutoFinishTime: Bool?) {
+        engine?.updateSession(
+            newConfig: newConfig,
+            resetTimer: resetAutoFinishTime
+        )
     }
 
-    internal func requestMicrophonePermission() {}
-    
-    internal func startRecognitionSetup() -> Bool {
-        isStopping = false
-        isActive = true
-        
-        initAutoStop()
-        monitorAppState()
-        guard startAudioSession() else {
-            cleanup(from: "startRecognitionSetup")
-            return false
-        }
-        
-        return true
+    func getIsActive() -> Bool {
+        engine?.isActive ?? false
     }
-    
-    internal func startRecognitionFeedback() {
-        self.startHapticFeedback()
-        autoStopper?.indicateRecordingActivity(
-            from: "startListening",
-            addMsToThreshold: nil
+
+    func getVoiceInputVolume() -> VolumeChangeEvent {
+        return engine?.getVoiceInputVolume() ?? VolumeChangeEvent(
+            smoothedVolume: 0,
+            rawVolume: 0,
+            db: nil
         )
-        onReadyForSpeech?()
-        onResult?([])
     }
     
-    internal func startRecognition() {}
-    internal func startRecognition() async {}
-    
-    internal func cleanup(from: String) {
-        logger.info("cleanup called from: \(from)")
-        deinitAutoStop()
-        stopMonitorAppState()
-        stopAudioSession()
-        stopAudioEngine()
-        levelSmoothed = 0
-        isActive = false
-        isStopping = false
-        onVolumeChange?(0)
+    func getSupportedLocalesIOS() -> [String] {
+        return self.coordinator.getSupportedLocales()
     }
-    
-    internal func stopAudioEngine() {
-        if let audioEngine = audioEngine, audioEngine.isRunning {
-            audioEngine.stop()
+
+    private func ensureEngine(params: SpeechRecognitionConfig?) async {
+        // Remember new params
+        config = params
+        let hash = Utils.hashParams(params)
+        if engine != nil && hash == paramsHash {
+            lg.log("Reuse Engine")
+            // Engine is already correct
+            return
+        }
+        if hash != paramsHash {
+            // Initialize when trying to select new engine with new params
+            await coordinator.initialize()
+            paramsHash = hash
+        }
+        lg.log("hash: \(hash)")
+        // Try to select new engine
+        engine = coordinator.getEngine()
+        if engine == nil {
+            // Only wrong locale can wipe out all candidates
+            self.onError?("No recognition engine available for the requested locale")
+            return
         }
-        audioEngine?.inputNode.removeTap(onBus: 0)
-        audioEngine = nil
     }
-    
-    internal func monitorAppState() {
-        appStateObserver = AppStateObserver { [weak self] in
-            guard let self = self, self.isActive else { return }
-            self.handleInternalStopTrigger()
+}
+
+protocol RecognizerDelegate: AnyObject {
+    var config: SpeechRecognitionConfig? { get }
+    func softlyUpdateConfig(newConfig: MutableSpeechRecognitionConfig?)
+    func reselectEngine(forPrewarm: Bool)
+    func readyForSpeech()
+    func recordingStopped()
+    func result (batches: [String])
+    func autoFinishProgress (timeLeftMs: Double)
+    func error (message: String)
+    func permissionDenied ()
+    func volumeChange (event: VolumeChangeEvent)
+}
+
+extension HybridRecognizer: RecognizerDelegate {
+    func softlyUpdateConfig(newConfig: MutableSpeechRecognitionConfig?) {
+        if let newConfig {
+            config = SpeechRecognitionConfig(
+                locale: config?.locale,
+                contextualStrings: config?.contextualStrings,
+                maskOffensiveWords: config?.maskOffensiveWords,
+                autoFinishRecognitionMs: newConfig.autoFinishRecognitionMs ?? config?.autoFinishRecognitionMs,
+                autoFinishProgressIntervalMs: newConfig.autoFinishProgressIntervalMs ?? config?.autoFinishProgressIntervalMs,
+                resetAutoFinishVoiceSensitivity: newConfig.resetAutoFinishVoiceSensitivity ?? config?.resetAutoFinishVoiceSensitivity,
+                disableRepeatingFilter: newConfig.disableRepeatingFilter ?? config?.disableRepeatingFilter,
+                startHapticFeedbackStyle: newConfig.startHapticFeedbackStyle ?? config?.startHapticFeedbackStyle,
+                stopHapticFeedbackStyle: newConfig.stopHapticFeedbackStyle ?? config?.stopHapticFeedbackStyle,
+                androidFormattingPreferQuality: config?.androidFormattingPreferQuality,
+                androidUseWebSearchModel: config?.androidUseWebSearchModel,
+                androidDisableBatchHandling: config?.androidDisableBatchHandling,
+                iosAddPunctuation: config?.iosAddPunctuation,
+                iosPreset: config?.iosPreset,
+                iosAtypicalSpeech: config?.iosAtypicalSpeech
+            )
         }
     }
-    internal func stopMonitorAppState () {
-        appStateObserver?.stop()
-        appStateObserver = nil
+    
+    func readyForSpeech() {
+        self.lg.log("[HR -> onReadyForSpeech]")
+        self.onReadyForSpeech?()
     }
     
-    internal func initAutoStop() {
-        autoStopper = AutoStopper(
-            silenceThresholdMs: config?.autoFinishRecognitionMs ?? Self.defaultAutoFinishRecognitionMs,
-            onProgress: { [weak self] timeLeftMs in
-                self?.onAutoFinishProgress?(timeLeftMs)
-            },
-            onTimeout: { [weak self] in
-                self?.handleInternalStopTrigger()
-            }
-        )
+    func recordingStopped() {
+        self.lg.log("[onRecordingStopped]")
+        self.onRecordingStopped?()
     }
-    internal func deinitAutoStop () {
-        autoStopper?.stop()
-        autoStopper = nil
-    }
-    
-    internal func startAudioSession() -> Bool {
-        do {
-            let audioSession = AVAudioSession.sharedInstance()
-            try audioSession.setCategory(.record, mode: .measurement, options: .duckOthers)
-            // Without this, iOS may suppress haptics while recording.
-            try audioSession.setAllowHapticsAndSystemSoundsDuringRecording(true)
-            try audioSession.setActive(true, options: .notifyOthersOnDeactivation)
-            return true
-        } catch {
-            onError?("Failed to activate audio session: \(error.localizedDescription)")
-            return false
-        }
+    
+    func result(batches: [String]) {
+        self.lg.log("[onResult] \(batches)")
+        self.onResult?(batches)
     }
-    internal func stopAudioSession () {
-        do {
-            try AVAudioSession.sharedInstance().setActive(false)
-        } catch {
-            logger.info("Failed to deactivate audio session: \(error.localizedDescription)")
-            return
-        }
+    
+    func autoFinishProgress(timeLeftMs: Double) {
+        self.lg.log("[onAutoFinishProgress] \(timeLeftMs)ms")
+        self.onAutoFinishProgress?(timeLeftMs)
     }
     
-    internal func startHapticFeedback() {
-        if let hapticStyle = config?.startHapticFeedbackStyle {
-            HapticImpact(style: hapticStyle).trigger()
-        } else {
-            HapticImpact(style: .medium).trigger()
-        }
+    func error(message: String) {
+        self.lg.log("[onError]")
+        self.onError?(message)
     }
-    internal func stopHapticFeedback () {
-        if let hapticStyle = config?.stopHapticFeedbackStyle {
-            HapticImpact(style: hapticStyle).trigger()
-        } else {
-            HapticImpact(style: .medium).trigger()
-        }
+    
+    func permissionDenied() {
+        self.lg.log("[onPermissionDenied]")
+        self.onPermissionDenied?()
     }
     
-    internal func trackPartialActivity() {
-        if !self.isStopping {
-            self.autoStopper?.indicateRecordingActivity(
-                from: "partial results",
-                addMsToThreshold: nil
-            )
-        }
+    func volumeChange(event: VolumeChangeEvent) {
+        // self.lg.log("[onVolumeChange] \(event.rawVolume)")
+        self.onVolumeChange?(event)
     }
-
-    internal func repeatingFilter(text: String) -> String {
-        var subStrings = text.split { $0.isWhitespace }.map { String($0) }
-        var joiner = ""
-        // 10 - arbitrary number of last substrings that is still unstable
-        // and needs to be filtered. Prev substrings were handled earlier.
-        if subStrings.count >= 10 {
-            joiner = subStrings.prefix(subStrings.count - 9).joined(separator: " ")
-            subStrings = Array(subStrings.suffix(10))
+    
+    func reselectEngine(forPrewarm: Bool) {
+        // Remove failed engine from candidates
+        coordinator.reportEngineFailure()
+        // Reset active engine
+        engine = nil
+        // Try to prewarm with another candidate
+        if forPrewarm {
+            self.prewarm(defaultParams: config)
         } else {
-            joiner = subStrings.first ?? ""
-        }
-        for i in subStrings.indices {
-            if i == 0 { continue }
-            // Always add number-contained strings
-            if #available(iOS 16.0, *), subStrings[i].contains(/\d+/) {
-                joiner += " \(subStrings[i])"
-                continue
-            }
-            
-            // Skip consecutive duplicate strings
-            if subStrings[i] == subStrings[i-1] { continue }
-            joiner += " \(subStrings[i])"
+            // Try to start with another candidate
+            self.startListening(params: config)
         }
-        return joiner
     }
 }

package/ios/LocaleManager.swift

@@ -0,0 +1,73 @@
+import Foundation
+import Speech
+
+final class LocaleManager {
+    private let sfSpeechLocales = SFSpeechRecognizer.supportedLocales().map { $0.identifier }
+    private var speechLocales: [String]
+    private var dictationLocales: [String]
+    var supportedLocales: [String]
+    var SFLocale: Locale?
+    var speechLocale: Locale?
+    var dictationLocale: Locale?
+    
+    private var equivalentsCountedFor: String?
+    
+    init() async {
+        self.speechLocales = []
+        self.dictationLocales = []
+        self.supportedLocales = sfSpeechLocales
+        
+        if #available(iOS 26.0, *) {
+            self.speechLocales = await SpeechTranscriber.supportedLocales.map {
+                $0.identifier
+            }
+            self.dictationLocales = await DictationTranscriber.supportedLocales.map {
+                $0.identifier
+            }
+            Log.log("[Coordinator] sfSpeechLocales: \(self.sfSpeechLocales)")
+            Log.log("[Coordinator] speechLocales: \(self.speechLocales)")
+            Log.log("[Coordinator] dictationLocales: \(self.dictationLocales)")
+            self.supportedLocales = Array(
+                Set(sfSpeechLocales)
+                .union(Set(speechLocales))
+                .union(Set(dictationLocales))
+            )
+        }
+    }
+
+    func ensureLocale(localeString: String?) async {
+        let identifier = localeString ?? "en-US"
+        if self.equivalentsCountedFor == identifier {
+            // All locales has been counted already, might be nil, but use them
+            Log.log("[Coordinator] ensureLocale: \(identifier) -> Already counted ")
+            return
+        }
+        if #available(iOS 26.0, *) {
+            let speechEquivalent = await SpeechTranscriber.supportedLocale(
+                equivalentTo: Locale(identifier: identifier)
+            )?.identifier
+            if let speechEquivalent, speechLocales.contains(speechEquivalent) {
+                self.speechLocale = Locale(identifier: speechEquivalent)
+            } else {
+                self.speechLocale = nil
+            }
+            
+            let dictationEquivalent = await DictationTranscriber.supportedLocale(
+                equivalentTo: Locale(identifier: identifier)
+            )?.identifier
+            if let dictationEquivalent, self.dictationLocales.contains(dictationEquivalent) {
+                self.dictationLocale = Locale(identifier: dictationEquivalent)
+            } else {
+                self.dictationLocale = nil
+            }
+        }
+        if sfSpeechLocales.contains(identifier) {
+            self.SFLocale = Locale(identifier: identifier)
+        } else {
+            self.SFLocale = nil
+        }
+        self.equivalentsCountedFor = identifier
+        Log.log("[Coordinator] equivalents: speechLocale: \(self.speechLocale?.identifier), dictationLocale: \(self.dictationLocale?.identifier), SFLocale: \(self.SFLocale?.identifier)")
+        Log.log("[Coordinator] ensureLocale: \(identifier) -> New")
+    }
+}

package/ios/{AppStateObserver.swift → Shared/AppStateObserver.swift}

@@ -1,7 +1,7 @@
 import Foundation
 import UIKit
 
-class AppStateObserver {
+final class AppStateObserver {
     private var observer: NSObjectProtocol?
     private let onResignActive: () -> Void
     
@@ -28,4 +28,3 @@ class AppStateObserver {
         stop()
     }
 }
-

package/ios/Shared/AutoStopper.swift

@@ -0,0 +1,147 @@
+import Foundation
+
+final class AutoStopper {
+    private static let defaultSilenceThresholdMs = 8000.0
+    private static let defaultProgressIntervalMs = 1000.0
+    private static let minProgressIntervalMs = 50.0
+
+    private let lg = Lg(prefix: "AutoStopper", disable: false)
+    
+    private let queue = DispatchQueue(label: "com.margelo.nitrospeech.autostopper")
+
+    private var silenceThresholdMs: Double
+    private var progressIntervalMs: Double
+    private var timeLeftMs: Double
+    private var isStopped = false
+    private var didTimeout = false
+    private var timer: DispatchSourceTimer?
+
+    private let onProgress: (Double) -> Void
+    private let onTimeout: () -> Void
+
+    init(
+        silenceThresholdMs: Double?,
+        progressIntervalMs: Double?,
+        onProgress: @escaping (Double) -> Void,
+        onTimeout: @escaping () -> Void
+    ) {
+        let threshold = Self.clampMs(silenceThresholdMs ?? Self.defaultSilenceThresholdMs)
+        let interval = Self.clampMs(progressIntervalMs ?? Self.defaultProgressIntervalMs)
+        self.silenceThresholdMs = threshold
+        self.progressIntervalMs = interval
+        self.timeLeftMs = threshold
+        self.onProgress = onProgress
+        self.onTimeout = onTimeout
+    }
+
+    deinit {
+        queue.sync {
+            stopLocked()
+            timeLeftMs = 0
+        }
+    }
+
+    func resetTimer(from: String) {
+        queue.async { [weak self] in
+            guard let self, !self.isStopped else { return }
+            lg.log("[resetTimer] from:\(from)")
+            self.didTimeout = false
+            self.timeLeftMs = self.silenceThresholdMs
+            self.startOrRescheduleTimerLocked()
+            if self.timeLeftMs > 0 {
+                self.onProgress(self.timeLeftMs)
+            }
+        }
+    }
+
+    func updateThreshold(_ newThresholdMs: Double, from: String) {
+        queue.async { [weak self] in
+            guard let self, !self.isStopped else { return }
+            lg.log("[updateThreshold] from:\(from) newThresholdMs:\(newThresholdMs)")
+            self.silenceThresholdMs = Self.clampMs(newThresholdMs)
+        }
+    }
+
+    func addMsOnce(_ extraMs: Double, from: String) {
+        queue.async { [weak self] in
+            guard let self, !self.isStopped, extraMs.isFinite else { return }
+            lg.log("[addMsOnce] from:\(from) extraMs:\(extraMs)")
+            self.timeLeftMs += extraMs
+            self.didTimeout = false
+            if self.timeLeftMs > 0, self.timer != nil {
+                self.onProgress(self.timeLeftMs)
+            }
+        }
+    }
+
+    func updateProgressInterval(_ newIntervalMs: Double, from: String) {
+        queue.async { [weak self] in
+            guard let self, !self.isStopped else { return }
+            lg.log("[updateProgressInterval] from:\(from) newIntervalMs:\(newIntervalMs)")
+            self.progressIntervalMs = Self.clampMs(newIntervalMs)
+            if self.timer != nil {
+                self.startOrRescheduleTimerLocked()
+            }
+        }
+    }
+
+    func stop() {
+        queue.async { [weak self] in
+            guard let self else { return }
+            self.stopLocked()
+            self.timeLeftMs = 0
+        }
+    }
+
+    private func startOrRescheduleTimerLocked() {
+        timer?.cancel()
+        timer = nil
+
+        let source = DispatchSource.makeTimerSource(queue: queue)
+        let intervalNs = UInt64(progressIntervalMs * 1_000_000)
+        source.schedule(
+            deadline: .now() + .nanoseconds(Int(intervalNs)),
+            repeating: .nanoseconds(Int(intervalNs))
+        )
+        source.setEventHandler { [weak self] in
+            self?.tickLocked()
+        }
+        timer = source
+        source.resume()
+    }
+
+    private func tickLocked() {
+        guard !isStopped else { return }
+        guard !didTimeout else { return }
+
+        timeLeftMs -= progressIntervalMs
+        if timeLeftMs > 0 {
+            lg.log("[onProgress] timeLeftMs:\(timeLeftMs)")
+            onProgress(timeLeftMs)
+            return
+        }
+
+        timeLeftMs = 0
+        didTimeout = true
+        cancelTimerLocked()
+        lg.log("[onTimeout]")
+        onTimeout()
+    }
+
+    private func stopLocked() {
+        isStopped = true
+        cancelTimerLocked()
+    }
+
+    private func cancelTimerLocked() {
+        timer?.cancel()
+        timer = nil
+    }
+
+    private static func clampMs(_ value: Double) -> Double {
+        if !value.isFinite {
+            return minProgressIntervalMs
+        }
+        return max(minProgressIntervalMs, value)
+    }
+}

package/ios/Shared/HapticImpact.swift

@@ -0,0 +1,24 @@
+import Foundation
+import UIKit
+
+enum HapticImpact {
+    static func trigger(with: HapticFeedbackStyle?) {
+        // Default behavior - medium
+        let style = with ?? HapticFeedbackStyle.medium
+        let hapticStyle: UIImpactFeedbackGenerator.FeedbackStyle? = switch style {
+            case .light:
+                UIImpactFeedbackGenerator.FeedbackStyle.light
+            case .medium:
+                UIImpactFeedbackGenerator.FeedbackStyle.medium
+            case .heavy:
+                UIImpactFeedbackGenerator.FeedbackStyle.heavy
+            case .none:
+                nil
+        }
+        if let hapticStyle {
+            let impactGenerator = UIImpactFeedbackGenerator(style: hapticStyle)
+            impactGenerator.prepare()
+            impactGenerator.impactOccurred()
+        }
+    }
+}