@gmessier/nitro-speech 0.3.2 → 0.4.0

package/android/src/main/java/com/margelo/nitro/nitrospeech/recognizer/HybridRecognizer.kt

@@ -11,9 +11,14 @@ import android.util.Log
 import androidx.annotation.Keep
 import com.facebook.proguard.annotations.DoNotStrip
 import com.margelo.nitro.NitroModules
+import com.margelo.nitro.core.Promise
+import com.margelo.nitro.nitrospeech.MutableSpeechRecognitionConfig
 import com.margelo.nitro.nitrospeech.HybridRecognizerSpec
-import com.margelo.nitro.nitrospeech.SpeechToTextParams
+import com.margelo.nitro.nitrospeech.SpeechRecognitionConfig
+import com.margelo.nitro.nitrospeech.VolumeChangeEvent
 
+@DoNotStrip
+@Keep
 class HybridRecognizer: HybridRecognizerSpec() {
   companion object {
     private const val TAG = "HybridRecognizer"
@@ -21,7 +26,7 @@ class HybridRecognizer: HybridRecognizerSpec() {
   }
 
   private var isActive: Boolean = false
-  private var config: SpeechToTextParams? = null
+  private var config: SpeechRecognitionConfig? = null
   private var autoStopper: AutoStopper? = null
   private var speechRecognizer: SpeechRecognizer? = null
   private val mainHandler = Handler(Looper.getMainLooper())
@@ -33,15 +38,19 @@ class HybridRecognizer: HybridRecognizerSpec() {
   override var onAutoFinishProgress: ((timeLeftMs: Double) -> Unit)? = null
   override var onError: ((error: String) -> Unit)? = null
   override var onPermissionDenied: (() -> Unit)? = null
-  override var onVolumeChange: ((normVolume: Double) -> Unit)? = null
+  override var onVolumeChange: ((event: VolumeChangeEvent) -> Unit)? = null
 
-  override fun getIsActive(): Boolean {
-    return isActive
+  @DoNotStrip
+  @Keep
+  override fun prewarm(defaultParams: SpeechRecognitionConfig?): Promise<Unit> {
+    // no-op
+    // nothing to prewarm
+    return Promise()
   }
 
   @DoNotStrip
   @Keep
-  override fun startListening(params: SpeechToTextParams) {
+  override fun startListening(params: SpeechRecognitionConfig?) {
     Log.d(TAG, "startListening: $params")
     if (isActive) {
       onFinishRecognition(
@@ -98,25 +107,84 @@ class HybridRecognizer: HybridRecognizerSpec() {
     }, POST_RECOGNITION_DELAY)
   }
 
+  @DoNotStrip
+  @Keep
+  override fun resetAutoFinishTime() {
+    if (!isActive) return
+    autoStopper?.resetTimer()
+  }
+
   @DoNotStrip
   @Keep
   override fun addAutoFinishTime(additionalTimeMs: Double?) {
-    Log.d(TAG, "stopListening called")
+    Log.d(TAG, "addAutoFinishTime")
     if (!isActive) return
-    autoStopper?.indicateRecordingActivity()
+
+    if (additionalTimeMs != null) {
+      autoStopper?.addMsOnce(additionalTimeMs)
+    } else {
+      // Reset timer to original baseline.
+      autoStopper?.resetTimer()
+    }
   }
 
   @DoNotStrip
   @Keep
-  override fun updateAutoFinishTime(newTimeMs: Double, withRefresh: Boolean?) {
-    Log.d(TAG, "updateAutoFinishTime: $newTimeMs")
+  override fun updateConfig(
+    newConfig: MutableSpeechRecognitionConfig?,
+    resetAutoFinishTime: Boolean?
+  ) {
+    Log.d(TAG, "updateConfig $newConfig",)
     if (!isActive) return
-    autoStopper?.updateSilenceThreshold(newTimeMs.toLong())
-    if (withRefresh == true) {
-      autoStopper?.indicateRecordingActivity()
+
+    val newTimeMs = if (newConfig?.autoFinishRecognitionMs != null) newConfig.autoFinishRecognitionMs else config?.autoFinishRecognitionMs
+    if (newTimeMs != null && newTimeMs != config?.autoFinishRecognitionMs) {
+      autoStopper?.updateSilenceThreshold(newTimeMs)
     }
+    val newInterval = if (newConfig?.autoFinishProgressIntervalMs != null) newConfig.autoFinishProgressIntervalMs else config?.autoFinishProgressIntervalMs
+    if (newInterval != null && newInterval != config?.autoFinishProgressIntervalMs) {
+      autoStopper?.updateProgressInterval(newInterval)
+    }
+
+    if (resetAutoFinishTime == true) {
+      autoStopper?.resetTimer()
+    }
+
+    if (newConfig != null) {
+      config = SpeechRecognitionConfig(
+        locale = config?.locale,
+        contextualStrings = config?.contextualStrings,
+        maskOffensiveWords = config?.maskOffensiveWords,
+        autoFinishRecognitionMs = newConfig.autoFinishRecognitionMs ?: config?.autoFinishRecognitionMs,
+        autoFinishProgressIntervalMs = newConfig.autoFinishProgressIntervalMs ?: config?.autoFinishProgressIntervalMs,
+        resetAutoFinishVoiceSensitivity = newConfig.resetAutoFinishVoiceSensitivity ?: config?.resetAutoFinishVoiceSensitivity,
+        disableRepeatingFilter = newConfig.disableRepeatingFilter ?: config?.disableRepeatingFilter,
+        startHapticFeedbackStyle = newConfig.startHapticFeedbackStyle ?: config?.startHapticFeedbackStyle,
+        stopHapticFeedbackStyle = newConfig.stopHapticFeedbackStyle ?: config?.stopHapticFeedbackStyle,
+        androidFormattingPreferQuality = config?.androidFormattingPreferQuality,
+        androidUseWebSearchModel = config?.androidUseWebSearchModel,
+        androidDisableBatchHandling = config?.androidDisableBatchHandling,
+        iosAddPunctuation = config?.iosAddPunctuation,
+        iosPreset = config?.iosPreset,
+        iosAtypicalSpeech = config?.iosAtypicalSpeech
+      )
+    }
+  }
+
+  @DoNotStrip
+  @Keep
+  override fun getIsActive(): Boolean {
+    return isActive
   }
 
+  @DoNotStrip
+  @Keep
+  override fun getSupportedLocalesIOS(): Array<String> {
+    return emptyArray()
+  }
+
+  @DoNotStrip
+  @Keep
   override fun dispose() {
     stopListening()
   }
@@ -125,12 +193,16 @@ class HybridRecognizer: HybridRecognizerSpec() {
     mainHandler.post {
       try {
         speechRecognizer = SpeechRecognizer.createSpeechRecognizer(context)
-        val silenceThreshold = config?.autoFinishRecognitionMs?.toLong() ?: 8000
         autoStopper = AutoStopper(
-            silenceThreshold,
-        ) {
-            stopListening()
-        }
+            silenceThresholdMs = config?.autoFinishRecognitionMs,
+            progressIntervalMs = config?.autoFinishProgressIntervalMs,
+            onProgress = { timeLeftMs ->
+              onAutoFinishProgress?.invoke(timeLeftMs)
+            },
+            onTimeout = {
+              stopListening()
+            }
+        )
         val recognitionListenerSession = RecognitionListenerSession(
           autoStopper,
           config,
@@ -175,6 +247,7 @@ class HybridRecognizer: HybridRecognizerSpec() {
           if (isActive) {
             onReadyForSpeech?.invoke()
             onFinishRecognition(arrayListOf(), null, false)
+            autoStopper?.resetTimer()
           }
         }, 500)
       } catch (e: Exception) {
@@ -189,7 +262,7 @@ class HybridRecognizer: HybridRecognizerSpec() {
 
   private fun cleanup() {
     try {
-      Log.d(TAG, "stopListening called")
+      Log.d(TAG, "cleanup called")
       autoStopper?.stop()
       autoStopper = null
       speechRecognizer?.stopListening()
@@ -197,7 +270,7 @@ class HybridRecognizer: HybridRecognizerSpec() {
       speechRecognizer = null
       isActive = false
       // Reset voice meter in JS consumers after stop/error cleanup.
-      onVolumeChange?.invoke(0.0)
+      onVolumeChange?.invoke(VolumeChangeEvent(0.0,0.0,null))
     } catch (e: Exception) {
       onFinishRecognition(
         null,

package/android/src/main/java/com/margelo/nitro/nitrospeech/recognizer/RecognitionListenerSession.kt

@@ -4,19 +4,20 @@ import android.os.Bundle
 import android.speech.RecognitionListener
 import android.speech.SpeechRecognizer
 import android.util.Log
-import com.margelo.nitro.nitrospeech.SpeechToTextParams
+import com.margelo.nitro.nitrospeech.SpeechRecognitionConfig
+import com.margelo.nitro.nitrospeech.VolumeChangeEvent
 import kotlin.math.max
 import kotlin.math.roundToInt
 
 class RecognitionListenerSession (
     private val autoStopper: AutoStopper?,
-    private val config: SpeechToTextParams?,
-    private val onVolumeChange: ((normVolume: Double) -> Unit)?,
+    private val config: SpeechRecognitionConfig?,
+    private val onVolumeChange: ((event: VolumeChangeEvent) -> Unit)?,
     private val onFinishRecognition: (result: ArrayList<String>?, errorMessage: String?, recordingStopped: Boolean) -> Unit,
 ) {
     companion object {
         private const val TAG = "HybridRecognizer"
-        private const val SPEECH_LEVEL_THRESHOLD = 0.08f
+        private const val SPEECH_LEVEL_THRESHOLD = 0.35
         private const val FLOOR_RISE_ALPHA = 0.01f
         private const val FLOOR_FALL_ALPHA = 0.20f
         private const val PEAK_ATTACK_ALPHA = 0.25f
@@ -38,10 +39,14 @@ class RecognitionListenerSession (
             override fun onReadyForSpeech(params: Bundle?) {}
             override fun onBeginningOfSpeech() {}
             override fun onRmsChanged(rmsdB: Float) {
-                val normLevel = normalizeRmsDb(rmsdB)
-                onVolumeChange?.invoke(normLevel.toDouble())
-                if (normLevel > SPEECH_LEVEL_THRESHOLD) {
-                    autoStopper?.indicateRecordingActivity()
+                val volumeEvent = getVolume(rmsdB)
+                onVolumeChange?.invoke(volumeEvent)
+                val threshold =
+                    config?.resetAutoFinishVoiceSensitivity?.coerceIn(0.0, 1.0)
+                        ?: SPEECH_LEVEL_THRESHOLD.toDouble()
+                Log.d(TAG, "onRmsChanged: ${volumeEvent}")
+                if (volumeEvent.rawVolume > threshold) {
+                    autoStopper?.resetTimer()
                 }
             }
             override fun onBufferReceived(buffer: ByteArray?) {}
@@ -66,18 +71,17 @@ class RecognitionListenerSession (
                     true
                 )
                 autoStopper?.stop()
-                autoStopper?.forceStopRecording()
+                autoStopper?.onTimeout()
             }
 
             override fun onResults(results: Bundle?) {
                 Log.d(TAG, "onResults: $resultBatches")
                 onFinishRecognition(resultBatches, null, true)
                 autoStopper?.stop()
-                autoStopper?.forceStopRecording()
+                autoStopper?.onTimeout()
             }
 
             override fun onPartialResults(partialResults: Bundle?) {
-                autoStopper?.indicateRecordingActivity()
                 val matches = partialResults?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION)
 
                 if (matches.isNullOrEmpty() || matches[0] == "") {
@@ -85,6 +89,7 @@ class RecognitionListenerSession (
                     return
                 }
 
+                autoStopper?.resetTimer()
                 Log.d(TAG, "onPartialResults[0], add ${matches[0]}")
                 var currentBatches = resultBatches
                 if (currentBatches.isNullOrEmpty()) {
@@ -144,9 +149,9 @@ class RecognitionListenerSession (
         return joiner.toString()
     }
 
-    private fun normalizeRmsDb(rmsdB: Float): Double {
+    private fun getVolume(rmsdB: Float): VolumeChangeEvent {
         if (!rmsdB.isFinite()) {
-            return 0.0
+            return VolumeChangeEvent(0.0,0.0,null)
         }
 
         if (noiseFloorDb.isNaN()) {
@@ -166,7 +171,14 @@ class RecognitionListenerSession (
         val raw = ((rmsdB - noiseFloorDb) / span).coerceIn(0f, 1f)
         val smoothingCoeff = if (raw > levelSmoothed) METER_ATTACK else METER_RELEASE
         levelSmoothed += smoothingCoeff * (raw - levelSmoothed)
-
-        return ((levelSmoothed * PRECISION_SCALE).roundToInt() / PRECISION_SCALE).toDouble()
+        val roundedSmoothed = ((levelSmoothed * PRECISION_SCALE).roundToInt() / PRECISION_SCALE).toDouble()
+        val roundedRaw = ((raw * PRECISION_SCALE).roundToInt() / PRECISION_SCALE).toDouble()
+        val db = (rmsdB * 1000).roundToInt() / 1000.0
+
+        return VolumeChangeEvent(
+            smoothedVolume = roundedSmoothed,
+            rawVolume = roundedRaw,
+            db = db
+        )
     }
   }

package/ios/{BufferUtil.swift → Audio/AudioBufferConverter.swift}

@@ -1,6 +1,5 @@
 import Foundation
 import AVFoundation
-import Accelerate
 
 private final class SendablePCMBufferBox: @unchecked Sendable {
     let buffer: AVAudioPCMBuffer
@@ -10,42 +9,12 @@ private final class SendablePCMBufferBox: @unchecked Sendable {
     }
 }
 
-class BufferUtil {
-    private static let meterMinDb: Float = -70   // silence floor
-    private static let meterMaxDb: Float = -10   // loud speech ceiling
-    private static let meterAttack: Float = 0.35 // rise speed
-    private static let meterRelease: Float = 0.08 // fall speed
-    
-    func calcRmsVolume(
-        levelSmoothed: Float,
-        buffer: AVAudioPCMBuffer
-    ) -> (Float, Float)? {
-        guard let samples = buffer.floatChannelData?[0] else { return nil }
-        
-        let frameL = Int(buffer.frameLength)
-        var rms: Float = 0
-        
-        vDSP_rmsqv(samples, 1, &rms, vDSP_Length(frameL))
-        
-        // 2) RMS -> dBFS
-        let db = 20 * log10(rms + 0.00001)
-
-        // 3) Normalize dB to 0...1
-        let raw = (db - Self.meterMinDb) / (Self.meterMaxDb - Self.meterMinDb)
-        let normalized = max(0, min(1, raw))
-
-        // 4) Smooth (fast attack, slow release)
-        let coeff = normalized > levelSmoothed ? Self.meterAttack : Self.meterRelease
-        let nextLevelSmoothed = levelSmoothed + coeff * (normalized - levelSmoothed)
-        
-        return (rms, nextLevelSmoothed)
-    }
-
-    func convertBuffer(
+enum AudioBufferConverter {
+    static func convertBuffer(
         converter: AVAudioConverter,
         audioFormat: AVAudioFormat,
         pcmBuffer: AVAudioPCMBuffer
-    ) throws -> AVAudioPCMBuffer?  {
+    ) throws -> AVAudioPCMBuffer? {
         let resampledCapacity = AVAudioFrameCount(
             (Double(pcmBuffer.frameLength) * (audioFormat.sampleRate / pcmBuffer.format.sampleRate)).rounded(.up)
         )

package/ios/Audio/AudioLevelTracker.swift

@@ -0,0 +1,66 @@
+import Foundation
+import AVFoundation
+import Accelerate
+
+struct AudioLevelSample {
+    let smoothed: Double
+    let raw: Double
+    let db: Double
+    let resetTimer: Bool
+}
+
+final class AudioLevelTracker {
+    private static let meterMinDb: Float = -70
+    private static let meterMaxDb: Float = -10
+    private static let meterAttack: Float = 0.35
+    private static let meterRelease: Float = 0.08
+    private static let defaultAutoStopResetThreshold: Double = 0.4
+
+    private var autoStopResetThreshold: Double
+    private var smoothedLevel: Float = 0
+    
+    init(resetAutoFinishVoiceSensitivity: Double?) {
+        if let resetAutoFinishVoiceSensitivity {
+            // Clamp value between 0 and 1
+            self.autoStopResetThreshold = max(0, min(1, resetAutoFinishVoiceSensitivity))
+        } else {
+            self.autoStopResetThreshold = Self.defaultAutoStopResetThreshold
+        }
+    }
+    
+    func updateResetAutoFinishVoiceSensitivity(newValue: Double?) {
+        if let newValue {
+            // Clamp value between 0 and 1
+            self.autoStopResetThreshold = max(0, min(1, newValue))
+        } else {
+            self.autoStopResetThreshold = Self.defaultAutoStopResetThreshold
+        }
+    }
+
+    func reset() {
+        smoothedLevel = 0
+        self.autoStopResetThreshold = Self.defaultAutoStopResetThreshold
+    }
+
+    func process(_ buffer: AVAudioPCMBuffer) -> AudioLevelSample? {
+        guard let samples = buffer.floatChannelData?[0] else { return nil }
+
+        let frameCount = Int(buffer.frameLength)
+        var rms: Float = 0
+        vDSP_rmsqv(samples, 1, &rms, vDSP_Length(frameCount))
+
+        let db = 20 * log10(rms + 0.00001)
+        let raw = (db - Self.meterMinDb) / (Self.meterMaxDb - Self.meterMinDb)
+        let normalized = max(0, min(1, raw))
+
+        let coeff = normalized > smoothedLevel ? Self.meterAttack : Self.meterRelease
+        smoothedLevel += coeff * (normalized - smoothedLevel)
+        
+        return AudioLevelSample(
+            smoothed: Double(smoothedLevel * 1_000_000).rounded() / 1_000_000,
+            raw: Double(normalized * 1_000_000).rounded() / 1_000_000,
+            db: Double(db * 1_000).rounded() / 1_000,
+            resetTimer: Double(normalized) >= self.autoStopResetThreshold
+        )
+    }
+}

package/ios/Coordinator.swift

@@ -0,0 +1,105 @@
+import Foundation
+import NitroModules
+import Speech
+
+enum RecognizerBackend {
+    case speechTranscriber
+    case dictationTranscriber
+    case sfSpeech
+}
+
+final class Coordinator {
+    weak var recognizerDelegate: RecognizerDelegate?
+    private var localeManager: LocaleManager?
+    private var candidates: [RecognizerBackend] = []
+    private var localeTask: Task<Void, Never>?
+    
+    init() {
+        self.localeTask = Task {
+            self.localeManager = await LocaleManager()
+        }
+    }
+    
+    func initialize() async {
+        let params = self.recognizerDelegate?.config
+        Log.log("[Coordinator] LocaleManager - init (\(params?.locale))")
+        if self.localeManager == nil {
+            self.localeTask?.cancel()
+            self.localeTask = nil
+            self.localeManager = await LocaleManager()
+        }
+        guard let localeManager else { return }
+        await localeManager.ensureLocale(localeString: params?.locale)
+        self.candidates = []
+        guard #available(iOS 26.0, *) else {
+            if localeManager.SFLocale != nil {
+                self.candidates = [.sfSpeech]
+            }
+            return
+        }
+        
+        if params?.iosPreset == IosPreset.shortform
+        || params?.iosAddPunctuation == false
+        || params?.iosAtypicalSpeech == true {
+            // DictationTranscriber priority
+            if localeManager.dictationLocale != nil {
+                self.candidates.append(.dictationTranscriber)
+            }
+            if localeManager.speechLocale != nil {
+                self.candidates.append(.speechTranscriber)
+            }
+        } else {
+            // SpeechTranscriber priority
+            if localeManager.speechLocale != nil {
+                self.candidates.append(.speechTranscriber)
+            }
+            if localeManager.dictationLocale != nil {
+                self.candidates.append(.dictationTranscriber)
+            }
+        }
+        // Add SF Engine at the end
+        if localeManager.SFLocale != nil {
+            self.candidates.append(.sfSpeech)
+        }
+        Log.log("[Coordinator] candidates: \(self.candidates)")
+    }
+    
+    func getEngine() -> RecognizerEngine? {
+        Log.log("[Coordinator] getEngine")
+        guard let recognizerDelegate else { return nil }
+        guard let localeManager else { return nil }
+        guard let backend = candidates.first else { return nil }
+        Log.log("[Coordinator] backend: \(backend)")
+        if backend == .sfSpeech, let locale = localeManager.SFLocale {
+            Log.log("[Coordinator] SFSpeechEngine Activated")
+            return SFSpeechEngine(locale: locale, delegate: recognizerDelegate)
+        }
+        if #available(iOS 26.0, *) {
+            if backend == .speechTranscriber, let locale = localeManager.speechLocale {
+                Log.log("[Coordinator] SpeechTranscriber Activated")
+                return AnalyzerEngine(
+                    backend: .speechTranscriber,
+                    locale: locale,
+                    delegate: recognizerDelegate
+                )
+            }
+            if backend == .dictationTranscriber, let locale = localeManager.dictationLocale {
+                Log.log("[Coordinator] DictationTranscriber Activated")
+                return AnalyzerEngine(
+                    backend: .dictationTranscriber,
+                    locale: locale,
+                    delegate: recognizerDelegate
+                )
+            }
+        }
+        return nil
+    }
+    
+    func reportEngineFailure() {
+        self.candidates = Array(self.candidates.dropFirst())
+    }
+    
+    func getSupportedLocales() -> [String] {
+        return localeManager?.supportedLocales ?? []
+    }
+}