@gmessier/nitro-speech 0.0.1

package/android/src/main/java/com/margelo/nitro/nitrospeech/recognizer/RecognitionListenerSession.kt

@@ -0,0 +1,106 @@
+package com.margelo.nitro.nitrospeech.recognizer
+
+import android.os.Bundle
+import android.speech.RecognitionListener
+import android.speech.SpeechRecognizer
+import android.util.Log
+import com.margelo.nitro.nitrospeech.SpeechToTextParams
+
+class RecognitionListenerSession (
+    private val autoStopper: AutoStopper?,
+    private val config: SpeechToTextParams?,
+    private val onFinishRecognition: (result: ArrayList<String>?, errorMessage: String?, recordingStopped: Boolean) -> Unit,
+) {
+    companion object {
+        private const val TAG = "HybridRecognizer"
+    }
+
+    private var resultBatches: ArrayList<String>? = null
+
+    fun createRecognitionListener(): RecognitionListener {
+        resultBatches = null
+        return object : RecognitionListener {
+            override fun onReadyForSpeech(params: Bundle?) {}
+            override fun onBeginningOfSpeech() {}
+            override fun onRmsChanged(rmsdB: Float) {
+                autoStopper?.indicateRecordingActivity()
+            }
+            override fun onBufferReceived(buffer: ByteArray?) {}
+            override fun onEndOfSpeech() {}
+
+            override fun onError(error: Int) {
+                val message = when (error) {
+                    SpeechRecognizer.ERROR_AUDIO -> "Audio recording error"
+                    SpeechRecognizer.ERROR_CLIENT -> "Client side error"
+                    SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS -> "Insufficient permissions"
+                    SpeechRecognizer.ERROR_NETWORK -> "Network error"
+                    SpeechRecognizer.ERROR_NETWORK_TIMEOUT -> "Network timeout"
+                    SpeechRecognizer.ERROR_NO_MATCH -> "No match"
+                    SpeechRecognizer.ERROR_RECOGNIZER_BUSY -> "Recognizer busy"
+                    SpeechRecognizer.ERROR_SERVER -> "Server error"
+                    SpeechRecognizer.ERROR_SPEECH_TIMEOUT -> "No speech input"
+                    else -> "Unknown error"
+                }
+                onFinishRecognition(
+                    null,
+                    "Error at RecognitionListener: $message",
+                    true
+                )
+                autoStopper?.stop()
+                autoStopper?.forceStopRecording()
+            }
+
+            override fun onResults(results: Bundle?) {
+                Log.d(TAG, "onResults: $resultBatches")
+                onFinishRecognition(resultBatches, null, true)
+                autoStopper?.stop()
+                autoStopper?.forceStopRecording()
+            }
+
+            override fun onPartialResults(partialResults: Bundle?) {
+                autoStopper?.indicateRecordingActivity()
+                val matches = partialResults?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION)
+
+                if (matches.isNullOrEmpty() || matches[0] == "") {
+                    Log.d(TAG, "onPartialResults[0], skip, NO RECOGNIZE")
+                    return
+                }
+
+                Log.d(TAG, "onPartialResults[0], add ${matches[0]}")
+                var currentBatches = resultBatches
+                if (currentBatches.isNullOrEmpty()) {
+                    Log.d(TAG, "onPartialResults[1], NO BATCHES YET | add first")
+                    currentBatches = arrayListOf(matches[0])
+                } else {
+                    Log.d(TAG, "onPartialResults[1], current batches $currentBatches")
+                    val prevBatchLength = currentBatches[currentBatches.lastIndex].length
+                    val match = if (config?.disableRepeatingFilter == true) matches[0] else repeatingFilter(matches[0])
+                    val matchLength = match.length
+                    if (config?.androidDisableBatchHandling == true || matchLength + 3 < prevBatchLength) {
+                        Log.d(TAG, "onPartialResults[2], append new batch")
+                        currentBatches.add(match)
+                    } else {
+                        Log.d(TAG, "onPartialResults[2], update batch, replace #${currentBatches.lastIndex}")
+                        currentBatches[currentBatches.lastIndex] = match
+                    }
+                }
+                resultBatches = currentBatches
+                onFinishRecognition(currentBatches, null, false)
+            }
+
+            override fun onEvent(eventType: Int, params: Bundle?) {}     
+        }
+    }
+
+    // Filters out 2 or more repeating words in a row, like "and and"
+    private fun repeatingFilter(text: String): String {
+        val words = text.split(Regex("\\s+")).toMutableList()
+        var joiner = words[0]
+        for (i in words.indices) {
+            if (i == 0) continue
+            if (words[i] == words[i-1]) continue
+            joiner += " ${words[i]}"
+        }
+        return joiner
+    }
+  }

package/ios/AppStateObserver.swift

@@ -0,0 +1,31 @@
+import Foundation
+import UIKit
+
+class AppStateObserver {
+    private var observer: NSObjectProtocol?
+    private let onResignActive: () -> Void
+    
+    init(onResignActive: @escaping () -> Void) {
+        self.onResignActive = onResignActive
+        
+        observer = NotificationCenter.default.addObserver(
+            forName: UIApplication.willResignActiveNotification,
+            object: nil,
+            queue: .main
+        ) { [weak self] _ in
+            self?.onResignActive()
+        }
+    }
+    
+    func stop() {
+        if let observer = observer {
+            NotificationCenter.default.removeObserver(observer)
+            self.observer = nil
+        }
+    }
+    
+    deinit {
+        stop()
+    }
+}
+

package/ios/AutoStopper.swift

@@ -0,0 +1,57 @@
+import Foundation
+import os.log
+
+class AutoStopper {
+    private let silenceThresholdMs: Double
+    private let onTimeout: () -> Void
+    private let onProgress: (Double) -> Void
+    private var progressWorkItem: DispatchWorkItem?
+    private var elapsedSeconds: Int = 0
+    private var isStopped = false
+    private let logger = Logger(subsystem: "com.margelo.nitro.nitrospeech", category: "AutoStopper")
+    
+    init(silenceThresholdMs: Double, onProgress: @escaping (Double) -> Void, onTimeout: @escaping () -> Void) {
+        self.silenceThresholdMs = silenceThresholdMs
+        self.onProgress = onProgress
+        self.onTimeout = onTimeout
+    }
+    
+    func indicateRecordingActivity(from: String) {
+        logger.info("indicateRecordingActivity: \(from)")
+        self.onProgress(self.silenceThresholdMs)
+        progressWorkItem?.cancel()
+        elapsedSeconds = 0
+        if isStopped { return }
+        
+        scheduleNextTick()
+    }
+    
+    private func scheduleNextTick() {
+        let item = DispatchWorkItem { [weak self] in
+            guard let self = self, !self.isStopped else { return }
+            
+            self.elapsedSeconds += 1
+            let elapsedMs = Double(self.elapsedSeconds) * 1000
+            let timeLeftMs = self.silenceThresholdMs - elapsedMs
+            
+            if timeLeftMs <= 0 {
+                self.onTimeout()
+            } else {
+                self.onProgress(timeLeftMs)
+                self.scheduleNextTick()
+            }
+        }
+        progressWorkItem = item
+        DispatchQueue.main.asyncAfter(deadline: .now() + 1.0, execute: item)
+    }
+    
+    func stop() {
+        isStopped = true
+        progressWorkItem?.cancel()
+        progressWorkItem = nil
+    }
+    
+    deinit {
+        stop()
+    }
+}

package/ios/Bridge.h

@@ -0,0 +1,8 @@
+//
+//  Bridge.h
+//  NitroSpeech
+//
+//  Created by Marc Rousavy on 22.07.24.
+//
+
+#pragma once

package/ios/HybridNitroSpeech.swift

@@ -0,0 +1,6 @@
+import Foundation
+import NitroModules
+
+class HybridNitroSpeech : HybridNitroSpeechSpec {
+    var recognizer: HybridRecognizerSpec = HybridRecognizer()
+}

package/ios/HybridRecognizer.swift

@@ -0,0 +1,201 @@
+import Foundation
+import Speech
+import NitroModules
+
+class HybridRecognizer: HybridRecognizerSpec {
+    private static let defaultAutoFinishRecognitionMs = 8000.0
+    
+    var onReadyForSpeech: (() -> Void)?
+    var onRecordingStopped: (() -> Void)?
+    var onResult: (([String]) -> Void)?
+    var onAutoFinishProgress: ((Double) -> Void)?
+    var onError: ((String) -> Void)?
+    var onPermissionDenied: (() -> Void)?
+    
+    private var recognitionRequest: SFSpeechAudioBufferRecognitionRequest?
+    private var recognitionTask: SFSpeechRecognitionTask?
+    private var audioEngine: AVAudioEngine?
+    private var autoStopper: AutoStopper?
+    private var appStateObserver: AppStateObserver?
+    private var isActive: Bool = false
+    
+    func startListening(params: SpeechToTextParams) {
+        if isActive {
+            onError?("Previous recognition session is still active")
+            return
+        }
+        
+        SFSpeechRecognizer.requestAuthorization { [weak self] authStatus in
+            DispatchQueue.main.async {
+                guard let self = self else { return }
+                
+                switch authStatus {
+                case .authorized:
+                    self.requestMicrophonePermission(params: params)
+                case .denied, .restricted:
+                    self.onPermissionDenied?()
+                case .notDetermined:
+                    self.onError?("Speech recognition not determined")
+                @unknown default:
+                    self.onError?("Unknown authorization status")
+                }
+            }
+        }
+    }
+    
+    func stopListening() {
+        guard isActive else { return }
+        cleanup()
+        onRecordingStopped?()
+    }
+
+    private func requestMicrophonePermission(params: SpeechToTextParams) {
+        AVAudioSession.sharedInstance().requestRecordPermission { [weak self] granted in
+            DispatchQueue.main.async {
+                guard let self = self else { return }
+                
+                if granted {
+                    self.startRecognition(params: params)
+                } else {
+                    self.onPermissionDenied?()
+                }
+            }
+        }
+    }
+    
+    private func startRecognition(params: SpeechToTextParams) {
+        let locale = Locale(identifier: params.locale ?? "en-US")
+        guard let speechRecognizer = SFSpeechRecognizer(locale: locale), speechRecognizer.isAvailable else {
+            onError?("Speech recognizer not available")
+            return
+        }
+        
+        autoStopper = AutoStopper(
+            silenceThresholdMs: params.autoFinishRecognitionMs ?? Self.defaultAutoFinishRecognitionMs,
+            onProgress: { [weak self] timeLeftMs in
+                self?.onAutoFinishProgress?(timeLeftMs)
+            },
+            onTimeout: { [weak self] in
+                self?.stopListening()
+            }
+        )
+        
+        do {
+            let audioSession = AVAudioSession.sharedInstance()
+            try audioSession.setCategory(.record, mode: .measurement, options: .duckOthers)
+            try audioSession.setActive(true, options: .notifyOthersOnDeactivation)
+        } catch {
+            onError?("Failed to set up audio session: \(error.localizedDescription)")
+            return
+        }
+        
+        audioEngine = AVAudioEngine()
+        recognitionRequest = SFSpeechAudioBufferRecognitionRequest()
+        
+        guard let recognitionRequest = recognitionRequest, let audioEngine = audioEngine else {
+            onError?("Failed to create recognition request or audio engine")
+            return
+        }
+        
+        recognitionRequest.shouldReportPartialResults = true
+        
+        if let contextualStrings = params.contextualStrings, !contextualStrings.isEmpty {
+            recognitionRequest.contextualStrings = contextualStrings
+        }
+        
+        if #available(iOS 16, *) {
+            if let addPunctiation = params.iosAddPunctuation, addPunctiation == false {
+                recognitionRequest.addsPunctuation = false
+            } else {
+                recognitionRequest.addsPunctuation = true
+            }
+        }
+        
+        let disableRepeatingFilter = params.disableRepeatingFilter ?? false
+        
+        recognitionTask = speechRecognizer.recognitionTask(with: recognitionRequest) { [weak self] result, error in
+            guard let self = self else { return }
+            
+            if let result = result {
+                self.autoStopper?.indicateRecordingActivity(from: "partial results")
+                
+                var transcription = result.bestTranscription.formattedString
+                if !transcription.isEmpty {
+                    if !disableRepeatingFilter {
+                        transcription = self.repeatingFilter(text: transcription)
+                    }
+                    self.onResult?([transcription])
+                }
+                
+                if result.isFinal {
+                    self.stopListening()
+                }
+            }
+            
+            if let error = error {
+                self.onError?("Recognition error: \(error.localizedDescription)")
+                self.stopListening()
+            }
+        }
+        
+        let inputNode = audioEngine.inputNode
+        let recordingFormat = inputNode.outputFormat(forBus: 0)
+        
+        inputNode.installTap(onBus: 0, bufferSize: 1024, format: recordingFormat) { [weak self] buffer, _ in
+            self?.recognitionRequest?.append(buffer)
+        }
+        
+        // Observe app going to background
+        appStateObserver = AppStateObserver { [weak self] in
+            guard let self = self, self.isActive else { return }
+            self.stopListening()
+        }
+        
+        do {
+            audioEngine.prepare()
+            try audioEngine.start()
+            isActive = true
+            autoStopper?.indicateRecordingActivity(from: "startListening")
+            onReadyForSpeech?()
+            onResult?([])
+        } catch {
+            cleanup()
+            onError?("Failed to start audio engine: \(error.localizedDescription)")
+        }
+    }
+    
+    private func cleanup() {
+        autoStopper?.stop()
+        autoStopper = nil
+        
+        appStateObserver?.stop()
+        appStateObserver = nil
+        
+        recognitionRequest?.endAudio()
+        recognitionTask?.cancel()
+        
+        if let audioEngine = audioEngine {
+            if audioEngine.isRunning {
+                audioEngine.stop()
+            }
+            audioEngine.inputNode.removeTap(onBus: 0)
+        }
+        
+        try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation)
+        
+        recognitionRequest = nil
+        recognitionTask = nil
+        audioEngine = nil
+        isActive = false
+    }
+
+    private func repeatingFilter(text: String) -> String {
+        let words = text.split { $0.isWhitespace }.map { String($0) }
+        var joiner = words[0]
+        for i in words.indices {
+            if i == 0 || words[i] == words[i-1] {continue}
+            joiner += " \(words[i])"
+        }
+        return joiner
+    }
+}

package/lib/commonjs/index.js

@@ -0,0 +1,10 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.Recognizer = void 0;
+var _reactNativeNitroModules = require("react-native-nitro-modules");
+const NitroSpeech = _reactNativeNitroModules.NitroModules.createHybridObject('NitroSpeech');
+const Recognizer = exports.Recognizer = NitroSpeech.recognizer;
+//# sourceMappingURL=index.js.map

package/lib/commonjs/index.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["_reactNativeNitroModules","require","NitroSpeech","NitroModules","createHybridObject","Recognizer","exports","recognizer"],"sourceRoot":"../../src","sources":["index.ts"],"mappings":";;;;;;AAAA,IAAAA,wBAAA,GAAAC,OAAA;AAGA,MAAMC,WAAW,GACfC,qCAAY,CAACC,kBAAkB,CAAkB,aAAa,CAAC;AAE1D,MAAMC,UAAU,GAAAC,OAAA,CAAAD,UAAA,GAAGH,WAAW,CAACK,UAAU","ignoreList":[]}

package/lib/commonjs/package.json

@@ -0,0 +1 @@
+{"type":"commonjs"}

package/lib/commonjs/specs/NitroSpeech.nitro.js

@@ -0,0 +1,6 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+//# sourceMappingURL=NitroSpeech.nitro.js.map

package/lib/commonjs/specs/NitroSpeech.nitro.js.map

@@ -0,0 +1 @@
+{"version":3,"names":[],"sourceRoot":"../../../src","sources":["specs/NitroSpeech.nitro.ts"],"mappings":"","ignoreList":[]}

package/lib/module/index.js

@@ -0,0 +1,6 @@
+"use strict";
+
+import { NitroModules } from 'react-native-nitro-modules';
+const NitroSpeech = NitroModules.createHybridObject('NitroSpeech');
+export const Recognizer = NitroSpeech.recognizer;
+//# sourceMappingURL=index.js.map

package/lib/module/index.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["NitroModules","NitroSpeech","createHybridObject","Recognizer","recognizer"],"sourceRoot":"../../src","sources":["index.ts"],"mappings":";;AAAA,SAASA,YAAY,QAAQ,4BAA4B;AAGzD,MAAMC,WAAW,GACfD,YAAY,CAACE,kBAAkB,CAAkB,aAAa,CAAC;AAEjE,OAAO,MAAMC,UAAU,GAAGF,WAAW,CAACG,UAAU","ignoreList":[]}

package/lib/module/package.json

@@ -0,0 +1 @@
+{"type":"module"}

package/lib/module/specs/NitroSpeech.nitro.js

@@ -0,0 +1,4 @@
+"use strict";
+
+export {};
+//# sourceMappingURL=NitroSpeech.nitro.js.map

package/lib/module/specs/NitroSpeech.nitro.js.map

@@ -0,0 +1 @@
+{"version":3,"names":[],"sourceRoot":"../../../src","sources":["specs/NitroSpeech.nitro.ts"],"mappings":"","ignoreList":[]}