@gmessier/nitro-speech 0.0.1

package/NitroSpeech.podspec

@@ -0,0 +1,31 @@
+require "json"
+
+package = JSON.parse(File.read(File.join(__dir__, "package.json")))
+
+Pod::Spec.new do |s|
+  s.name         = "NitroSpeech"
+  s.version      = package["version"]
+  s.summary      = package["description"]
+  s.homepage     = package["homepage"]
+  s.license      = package["license"]
+  s.authors      = package["author"]
+
+  s.platforms    = { :ios => min_ios_version_supported, :visionos => 1.0 }
+  s.source       = { :git => "https://github.com/mrousavy/nitro.git", :tag => "#{s.version}" }
+
+  s.source_files = [
+    # Implementation (Swift)
+    "ios/**/*.{swift}",
+    # Autolinking/Registration (Objective-C++)
+    "ios/**/*.{m,mm}",
+    # Implementation (C++ objects)
+    "cpp/**/*.{hpp,cpp}",
+  ]
+
+  load 'nitrogen/generated/ios/NitroSpeech+autolinking.rb'
+  add_nitrogen_files(s)
+
+  s.dependency 'React-jsi'
+  s.dependency 'React-callinvoker'
+  install_modules_dependencies(s)
+end

package/README.md

@@ -0,0 +1,55 @@
+# nitro-speech
+
+> **⚠️ Work in Progress**
+> 
+> This library is under active development.
+
+Speech recognition for React Native, powered by [Nitro Modules](https://github.com/mrousavy/nitro).
+
+## Installation
+
+```bash
+npm install nitro-speech react-native-nitro-modules
+# or
+yarn add nitro-speech react-native-nitro-modules
+# or 
+bun add nitro-speech react-native-nitro-modules
+```
+
+### iOS
+
+```bash
+cd ios && pod install
+```
+
+### Android
+
+No additional setup required.
+
+## Permissions
+
+### Android
+
+The library declares the required permission in its `AndroidManifest.xml` (merged automatically):
+
+```xml
+<uses-permission android:name="android.permission.RECORD_AUDIO" />
+```
+
+### iOS
+
+Add the following keys to your app's `Info.plist`:
+
+```xml
+<key>NSMicrophoneUsageDescription</key>
+<string>This app needs microphone access for speech recognition</string>
+<key>NSSpeechRecognitionUsageDescription</key>
+<string>This app needs speech recognition to convert speech to text</string>
+```
+
+Both permissions are required for speech recognition to work on iOS.
+
+## TODO
+
+- [ ] (Android) Timer till the auto finish is called
+- [ ] (Android) Cleanup when app loses the focus

package/android/CMakeLists.txt

@@ -0,0 +1,29 @@
+project(NitroSpeech)
+cmake_minimum_required(VERSION 3.9.0)
+
+set (PACKAGE_NAME NitroSpeech)
+set (CMAKE_VERBOSE_MAKEFILE ON)
+set (CMAKE_CXX_STANDARD 20)
+
+# Define C++ library and add all sources
+add_library(${PACKAGE_NAME} SHARED
+        src/main/cpp/cpp-adapter.cpp
+)
+
+# Add Nitrogen specs :)
+include(${CMAKE_SOURCE_DIR}/../nitrogen/generated/android/NitroSpeech+autolinking.cmake)
+
+# Set up local includes
+include_directories(
+        "src/main/cpp"
+        "../cpp"
+)
+
+find_library(LOG_LIB log)
+
+# Link all libraries together
+target_link_libraries(
+        ${PACKAGE_NAME}
+        ${LOG_LIB}
+        android                                   # <-- Android core
+)

package/android/build.gradle

@@ -0,0 +1,148 @@
+buildscript {
+  repositories {
+    google()
+    mavenCentral()
+  }
+
+  dependencies {
+    classpath "com.android.tools.build:gradle:8.13.1"
+  }
+}
+
+def reactNativeArchitectures() {
+  def value = rootProject.getProperties().get("reactNativeArchitectures")
+  return value ? value.split(",") : ["armeabi-v7a", "x86", "x86_64", "arm64-v8a"]
+}
+
+def isNewArchitectureEnabled() {
+  return rootProject.hasProperty("newArchEnabled") && rootProject.getProperty("newArchEnabled") == "true"
+}
+
+apply plugin: "com.android.library"
+apply plugin: 'org.jetbrains.kotlin.android'
+apply from: '../nitrogen/generated/android/NitroSpeech+autolinking.gradle'
+apply from: "./fix-prefab.gradle"
+
+if (isNewArchitectureEnabled()) {
+  apply plugin: "com.facebook.react"
+}
+
+def getExtOrDefault(name) {
+  return rootProject.ext.has(name) ? rootProject.ext.get(name) : project.properties["NitroSpeech_" + name]
+}
+
+def getExtOrIntegerDefault(name) {
+  return rootProject.ext.has(name) ? rootProject.ext.get(name) : (project.properties["NitroSpeech_" + name]).toInteger()
+}
+
+android {
+  namespace "com.margelo.nitro.nitrospeech"
+
+  ndkVersion getExtOrDefault("ndkVersion")
+  compileSdkVersion getExtOrIntegerDefault("compileSdkVersion")
+
+  defaultConfig {
+    minSdkVersion getExtOrIntegerDefault("minSdkVersion")
+    targetSdkVersion getExtOrIntegerDefault("targetSdkVersion")
+    buildConfigField "boolean", "IS_NEW_ARCHITECTURE_ENABLED", isNewArchitectureEnabled().toString()
+
+    externalNativeBuild {
+      cmake {
+        cppFlags "-frtti -fexceptions -Wall -Wextra -fstack-protector-all"
+        arguments "-DANDROID_STL=c++_shared", "-DANDROID_SUPPORT_FLEXIBLE_PAGE_SIZES=ON"
+        abiFilters (*reactNativeArchitectures())
+
+        buildTypes {
+          debug {
+            cppFlags "-O1 -g"
+          }
+          release {
+            cppFlags "-O2"
+          }
+        }
+      }
+    }
+  }
+
+  externalNativeBuild {
+    cmake {
+      path "CMakeLists.txt"
+    }
+  }
+
+  packagingOptions {
+    excludes = [
+            "META-INF",
+            "META-INF/**",
+            "**/libc++_shared.so",
+            "**/libfbjni.so",
+            "**/libjsi.so",
+            "**/libfolly_json.so",
+            "**/libfolly_runtime.so",
+            "**/libglog.so",
+            "**/libhermes.so",
+            "**/libhermes-executor-debug.so",
+            "**/libhermes_executor.so",
+            "**/libreactnative.so",
+            "**/libreactnativejni.so",
+            "**/libturbomodulejsijni.so",
+            "**/libreact_nativemodule_core.so",
+            "**/libjscexecutor.so"
+    ]
+  }
+
+  buildFeatures {
+    buildConfig true
+    prefab true
+  }
+
+  buildTypes {
+    release {
+      minifyEnabled false
+    }
+  }
+
+  lintOptions {
+    disable "GradleCompatible"
+  }
+
+  compileOptions {
+    sourceCompatibility JavaVersion.VERSION_1_8
+    targetCompatibility JavaVersion.VERSION_1_8
+  }
+
+  sourceSets {
+    main {
+      if (isNewArchitectureEnabled()) {
+        java.srcDirs += [
+          // React Codegen files
+          "${project.buildDir}/generated/source/codegen/java"
+        ]
+      }
+    }
+  }
+}
+
+repositories {
+  mavenCentral()
+  google()
+}
+
+
+dependencies {
+  // For < 0.71, this will be from the local maven repo
+  // For > 0.71, this will be replaced by `com.facebook.react:react-android:$version` by react gradle plugin
+  //noinspection GradleDynamicVersion
+  implementation "com.facebook.react:react-native:+"
+
+  // Add a dependency on NitroModules
+  implementation project(":react-native-nitro-modules")
+}
+
+if (isNewArchitectureEnabled()) {
+  react {
+    jsRootDir = file("../src/")
+    libraryName = "NitroSpeech"
+    codegenJavaPackageName = "com.margelo.nitro.nitrospeech"
+  }
+}

package/android/fix-prefab.gradle

@@ -0,0 +1,51 @@
+tasks.configureEach { task ->
+  // Make sure that we generate our prefab publication file only after having built the native library
+  // so that not a header publication file, but a full configuration publication will be generated, which
+  // will include the .so file
+
+  def prefabConfigurePattern = ~/^prefab(.+)ConfigurePackage$/
+  def matcher = task.name =~ prefabConfigurePattern
+  if (matcher.matches()) {
+    def variantName = matcher[0][1]
+    task.outputs.upToDateWhen { false }
+    task.dependsOn("externalNativeBuild${variantName}")
+  }
+}
+
+afterEvaluate {
+  def abis = reactNativeArchitectures()
+  rootProject.allprojects.each { proj ->
+    if (proj === rootProject) return
+
+    def dependsOnThisLib = proj.configurations.findAll { it.canBeResolved }.any { config ->
+      config.dependencies.any { dep ->
+        dep.group == project.group && dep.name == project.name
+      }
+    }
+    if (!dependsOnThisLib && proj != project) return
+
+    if (!proj.plugins.hasPlugin('com.android.application') && !proj.plugins.hasPlugin('com.android.library')) {
+      return
+    }
+
+    def variants = proj.android.hasProperty('applicationVariants') ? proj.android.applicationVariants : proj.android.libraryVariants
+    // Touch the prefab_config.json files to ensure that in ExternalNativeJsonGenerator.kt we will re-trigger the prefab CLI to
+    // generate a libnameConfig.cmake file that will contain our native library (.so).
+    // See this condition: https://cs.android.com/android-studio/platform/tools/base/+/mirror-goog-studio-main:build-system/gradle-core/src/main/java/com/android/build/gradle/tasks/ExternalNativeJsonGenerator.kt;l=207-219?q=createPrefabBuildSystemGlue
+    variants.all { variant ->
+      def variantName = variant.name
+      abis.each { abi ->
+        def searchDir = new File(proj.projectDir, ".cxx/${variantName}")
+        if (!searchDir.exists()) return
+        def matches = []
+        searchDir.eachDir { randomDir ->
+          def prefabFile = new File(randomDir, "${abi}/prefab_config.json")
+          if (prefabFile.exists()) matches << prefabFile
+        }
+        matches.each { prefabConfig ->
+          prefabConfig.setLastModified(System.currentTimeMillis())
+        }
+      }
+    }
+  }
+}

package/android/gradle.properties

@@ -0,0 +1,5 @@
+NitroSpeech_kotlinVersion=2.1.20
+NitroSpeech_minSdkVersion=23
+NitroSpeech_targetSdkVersion=36
+NitroSpeech_compileSdkVersion=36
+NitroSpeech_ndkVersion=27.1.12297006

package/android/src/main/AndroidManifest.xml

@@ -0,0 +1,3 @@
+<manifest xmlns:android="http://schemas.android.com/apk/res/android">
+  <uses-permission android:name="android.permission.RECORD_AUDIO" />
+</manifest>

package/android/src/main/cpp/cpp-adapter.cpp

@@ -0,0 +1,6 @@
+#include <jni.h>
+#include "NitroSpeechOnLoad.hpp"
+
+JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void*) {
+  return margelo::nitro::nitrospeech::initialize(vm);
+}

package/android/src/main/java/com/margelo/nitro/nitrospeech/HybridNitroSpeech.kt

@@ -0,0 +1,12 @@
+package com.margelo.nitro.nitrospeech
+
+import androidx.annotation.Keep
+import com.facebook.proguard.annotations.DoNotStrip
+import com.margelo.nitro.nitrospeech.recognizer.HybridRecognizer
+
+class HybridNitroSpeech: HybridNitroSpeechSpec() {
+
+  @DoNotStrip
+  @Keep
+  override var recognizer: HybridRecognizerSpec = HybridRecognizer()
+}

package/android/src/main/java/com/margelo/nitro/nitrospeech/NitroSpeechPackage.kt

@@ -0,0 +1,20 @@
+package com.margelo.nitro.nitrospeech
+
+import com.facebook.react.bridge.NativeModule
+import com.facebook.react.bridge.ReactApplicationContext
+import com.facebook.react.module.model.ReactModuleInfoProvider
+import com.facebook.react.BaseReactPackage
+import com.margelo.nitro.nitrospeech.NitroSpeechOnLoad;
+
+
+class NitroSpeechPackage : BaseReactPackage() {
+    override fun getModule(name: String, reactContext: ReactApplicationContext): NativeModule? = null
+
+    override fun getReactModuleInfoProvider(): ReactModuleInfoProvider = ReactModuleInfoProvider { HashMap() }
+
+    companion object {
+        init {
+            NitroSpeechOnLoad.initializeNative()
+        }
+    }
+}

package/android/src/main/java/com/margelo/nitro/nitrospeech/recognizer/AudioPermissionRequester.kt

@@ -0,0 +1,39 @@
+package com.margelo.nitro.nitrospeech.recognizer
+
+import android.Manifest
+import android.app.Activity
+import android.content.pm.PackageManager
+import androidx.activity.ComponentActivity
+import androidx.activity.result.contract.ActivityResultContracts
+import androidx.core.content.ContextCompat
+
+class AudioPermissionRequester (
+  private val activity: Activity
+) {
+  private val recordAudioPermission = Manifest.permission.RECORD_AUDIO
+  private val componentActivity = activity as? ComponentActivity ?: error("Host activity must be a ComponentActivity")
+
+  private var callback: ((Boolean) -> Unit)? = null
+
+  private val launcher = componentActivity.activityResultRegistry.register(
+    "record_audio_key", ActivityResultContracts.RequestPermission()
+  ) { granted ->
+    callback?.invoke(granted)
+  }
+
+  fun checkAndRequest(onResult: (Boolean) -> Unit) {
+    val audioGranted =
+      ContextCompat.checkSelfPermission(
+        activity,
+        recordAudioPermission
+      ) == PackageManager.PERMISSION_GRANTED
+
+    if (audioGranted) {
+      onResult(true)
+      return
+    }
+
+    callback = onResult
+    launcher.launch(recordAudioPermission)
+  }
+}

package/android/src/main/java/com/margelo/nitro/nitrospeech/recognizer/AutoStopper.kt

@@ -0,0 +1,35 @@
+package com.margelo.nitro.nitrospeech.recognizer
+
+import android.os.Handler
+import android.os.Looper
+import android.util.Log
+
+class AutoStopper (
+    private val silenceThreshold: Long,
+    val forceStopRecording: () -> Unit,
+) {
+    companion object {
+        private const val TAG = "HybridRecognizer"
+    }
+
+    private var isStopped = false
+    private val handler = Handler(Looper.getMainLooper())
+
+    private val autoStopRecording = Runnable {
+        if (isStopped) return@Runnable
+        Log.d(TAG, "forceStopRecording, ms: ${System.currentTimeMillis()}")
+        forceStopRecording()
+    }
+
+    fun indicateRecordingActivity() {
+        Log.d(TAG, "indicateRecordingActivity | isStopped: $isStopped | ms: ${System.currentTimeMillis()}")
+        handler.removeCallbacks(autoStopRecording)
+        if (isStopped) return
+        handler.postDelayed(autoStopRecording, silenceThreshold)
+    }
+
+    fun stop() {
+        isStopped = true
+        handler.removeCallbacks(autoStopRecording)
+    }
+}

package/android/src/main/java/com/margelo/nitro/nitrospeech/recognizer/HybridRecognizer.kt

@@ -0,0 +1,181 @@
+package com.margelo.nitro.nitrospeech.recognizer
+
+import android.content.Context
+import android.content.Intent
+import android.os.Build
+import android.os.Handler
+import android.os.Looper
+import android.speech.RecognizerIntent
+import android.speech.SpeechRecognizer
+import android.util.Log
+import androidx.annotation.Keep
+import com.facebook.proguard.annotations.DoNotStrip
+import com.margelo.nitro.NitroModules
+import com.margelo.nitro.nitrospeech.HybridRecognizerSpec
+import com.margelo.nitro.nitrospeech.SpeechToTextParams
+
+class HybridRecognizer: HybridRecognizerSpec() {
+  companion object {
+    private const val TAG = "HybridRecognizer"
+    private const val POST_RECOGNITION_DELAY = 250L
+  }
+
+  private var isActive: Boolean = false
+  private var config: SpeechToTextParams? = null
+  private var autoStopper: AutoStopper? = null
+  private var speechRecognizer: SpeechRecognizer? = null
+  private val mainHandler = Handler(Looper.getMainLooper())
+
+  override var onReadyForSpeech: (() -> Unit)? = null
+  override var onRecordingStopped: (() -> Unit)? = null
+  override var onResult: ((resultBatches: Array<String>) -> Unit)? = null
+
+  override var onAutoFinishProgress: ((timeLeftMs: Double) -> Unit)? = null
+  override var onError: ((error: String) -> Unit)? = null
+  override var onPermissionDenied: (() -> Unit)? = null
+
+  @DoNotStrip
+  @Keep
+  override fun startListening(params: SpeechToTextParams) {
+    Log.d(TAG, "startListening: $params")
+    if (isActive) {
+      onFinishRecognition(
+        null,
+        "Error at startListening: Previous SpeechRecognizer is still active",
+        false
+      )
+      return
+    }
+
+    val context = NitroModules.applicationContext
+    if (context == null) {
+      onFinishRecognition(
+        null,
+        "Error at startListening: Context not available",
+        true
+      )
+      return
+    }
+    val activity = context.currentActivity
+    if (activity == null) {
+      onFinishRecognition(
+        null,
+        "Error at startListening: Activity not found",
+        true
+      )
+      return
+    }
+
+    val permissionRequester = AudioPermissionRequester(activity)
+    permissionRequester.checkAndRequest { granted ->
+      if (!granted) {
+        onPermissionDenied?.invoke()
+        return@checkAndRequest
+      }
+      config = params
+      start(context)
+    }
+  }
+
+  @DoNotStrip
+  @Keep
+  override fun stopListening() {
+    Log.d(TAG, "stopListening called")
+    if (!isActive) return
+    onFinishRecognition(null, null, true)
+    mainHandler.postDelayed({
+      cleanup()
+    }, POST_RECOGNITION_DELAY)
+  }
+
+  private fun start(context: Context) {
+    mainHandler.post {
+      try {
+        speechRecognizer = SpeechRecognizer.createSpeechRecognizer(context)
+        val silenceThreshold = config?.autoFinishRecognitionMs?.toLong() ?: 8000
+        autoStopper = AutoStopper(
+            silenceThreshold,
+        ) {
+            stopListening()
+        }
+        val recognitionListenerSession = RecognitionListenerSession(
+          autoStopper,
+          config,
+        ) { result: ArrayList<String>?, errorMessage: String?, recordingStopped: Boolean ->
+          onFinishRecognition(result, errorMessage, recordingStopped)
+        }
+        speechRecognizer?.setRecognitionListener(recognitionListenerSession.createRecognitionListener())
+
+        val languageModel = if (config?.androidUseWebSearchModel == true) RecognizerIntent.LANGUAGE_MODEL_WEB_SEARCH else RecognizerIntent.LANGUAGE_MODEL_FREE_FORM
+
+        val intent = Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH)
+        intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, languageModel)
+        intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE, config?.locale ?: "en-US")
+        intent.putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true)
+        // set many secs to avoid cutting early
+        intent.putExtra(RecognizerIntent.EXTRA_SPEECH_INPUT_COMPLETE_SILENCE_LENGTH_MILLIS, 300000)
+
+        if (config?.androidMaskOffensiveWords != true && Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
+          intent.putExtra(RecognizerIntent.EXTRA_MASK_OFFENSIVE_WORDS, false)
+        }
+
+        if (config?.androidFormattingPreferQuality == true && Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
+          intent.putExtra(RecognizerIntent.EXTRA_ENABLE_FORMATTING, RecognizerIntent.FORMATTING_OPTIMIZE_QUALITY)
+        }
+
+        val contextualStrings = config?.contextualStrings
+        if (!contextualStrings.isNullOrEmpty() && Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
+            intent.putExtra(
+                RecognizerIntent.EXTRA_BIASING_STRINGS,
+                ArrayList(contextualStrings.toList()),
+            )
+        }
+
+        speechRecognizer?.startListening(intent)
+        isActive = true
+        mainHandler.postDelayed({
+          if (isActive) {
+            onReadyForSpeech?.invoke()
+            onFinishRecognition(arrayListOf(), null, false)
+          }
+        }, 500)
+      } catch (e: Exception) {
+        onFinishRecognition(
+          null,
+          "Error at start.mainHandler.post: ${e.message ?: "Unknown error"}",
+          true
+        )
+      }
+    }
+  }
+
+  private fun cleanup() {
+    try {
+      Log.d(TAG, "stopListening called")
+      autoStopper?.stop()
+      autoStopper = null
+      speechRecognizer?.stopListening()
+      speechRecognizer?.destroy()
+      speechRecognizer = null
+      isActive = false
+    } catch (e: Exception) {
+      onFinishRecognition(
+        null,
+        "Error at stopListening.mainHandler.postDelayed: ${e.message ?: "Unknown error"}",
+        true
+      )
+    }
+  }
+
+  private fun onFinishRecognition(result: ArrayList<String>?, errorMessage: String?, recordingStopped: Boolean) {
+    if (recordingStopped) {
+      onRecordingStopped?.invoke()
+    }
+    if (!errorMessage.isNullOrEmpty()) {
+      onError?.invoke(errorMessage)
+    }
+    if (!result.isNullOrEmpty()) {
+      onResult?.invoke(result.toTypedArray())
+    }
+  }
+}