@siteed/sherpa-onnx.rn 1.0.0

package/README.md

@@ -0,0 +1,374 @@
+# @siteed/sherpa-onnx.rn
+
+React Native wrapper for [sherpa-onnx](https://github.com/k2-fsa/sherpa-onnx) providing Text-to-Speech (TTS) and Speech-to-Text (STT) capabilities for mobile applications.
+
+**Status: 🚧 In Development**
+
+✅ **Native Integration Testing**: Complete framework with 100% test success rate  
+✅ **Model Management**: Lightweight strategy for CI to production environments  
+✅ **Cross-Platform**: Android support validated, iOS structure in place  
+⏳ **Architecture Support**: Old/New React Native architecture compatibility planned
+
+## Features
+
+- Speech-to-text (STT) using Sherpa-ONNX
+- Streaming recognition support
+- Low-level API for direct access to Sherpa-ONNX capabilities
+- Support for both old and new React Native architectures
+- Pre-built native libraries for Android and iOS
+- **Web/WASM support** — all 10 features work in the browser via a combined WebAssembly build
+
+## Installation
+
+```sh
+npm install @siteed/sherpa-onnx.rn
+# or
+yarn add @siteed/sherpa-onnx.rn
+```
+
+### Linking
+
+This package is a native module and requires proper linking. For React Native 0.60 and above, linking should happen automatically through auto-linking.
+
+If you're using Expo, you'll need to use a custom development client or eject to a bare workflow:
+
+```sh
+npx expo prebuild
+```
+
+### iOS
+
+For iOS, run pod install:
+
+```bash
+cd ios && pod install
+```
+
+### Android
+
+Android integration is handled automatically by the package.
+
+## React Native Compatibility
+
+This module is compatible with both the old and new React Native architectures. See [COMPATIBILITY.md](./COMPATIBILITY.md) for details on how this is achieved and considerations when using this module.
+
+## Usage
+
+### Text-to-Speech (TTS)
+
+This example demonstrates how to use the TTS functionality:
+
+```typescript
+import SherpaOnnx, { TtsModelConfig } from '@siteed/sherpa-onnx.rn';
+import { Audio } from 'expo-av';
+import { Platform } from 'react-native';
+
+// Initialize TTS
+const initTts = async () => {
+  try {
+    // First validate the library is loaded
+    const validateResult = await SherpaOnnx.validateLibraryLoaded();
+    if (!validateResult.loaded) {
+      throw new Error(`Library validation failed: ${validateResult.status}`);
+    }
+    
+    // Configure TTS with your model
+    const modelConfig: TtsModelConfig = {
+      modelDir: 'assets/tts/kokoro-en-v0_19',
+      modelName: 'model.onnx',
+      voices: 'voices.bin',
+      dataDir: 'assets/tts/kokoro-en-v0_19/espeak-ng-data',
+    };
+    
+    // Initialize TTS
+    const initResult = await SherpaOnnx.initTts(modelConfig);
+    console.log(`TTS initialized with ${initResult.numSpeakers} speakers at ${initResult.sampleRate}Hz`);
+    return initResult;
+  } catch (error) {
+    console.error('Failed to initialize TTS:', error);
+    throw error;
+  }
+};
+
+// Generate speech
+const generateSpeech = async (text: string) => {
+  try {
+    // Generate TTS without playing (we'll play with Expo AV)
+    const result = await SherpaOnnx.generateTts(text, {
+      speakerId: 0,
+      speakingRate: 1.0,
+      playAudio: false
+    });
+    
+    // Play with Expo AV
+    const { sound } = await Audio.Sound.createAsync({ 
+      uri: Platform.OS === 'ios' ? result.filePath : `file://${result.filePath}` 
+    });
+    await sound.playAsync();
+    
+    return { sound, result };
+  } catch (error) {
+    console.error('Failed to generate speech:', error);
+    throw error;
+  }
+};
+
+// Clean up resources
+const cleanup = async () => {
+  await SherpaOnnx.releaseTts();
+};
+```
+
+### Basic Speech-to-Text
+
+```typescript
+import { SherpaOnnxAPI } from '@siteed/sherpa-onnx.rn';
+
+// Initialize the API
+const sherpaOnnx = new SherpaOnnxAPI({
+  modelPath: '/path/to/models',
+  sampleRate: 16000,
+  channels: 1,
+  language: 'en',
+});
+
+// Initialize the recognizer
+await sherpaOnnx.initialize();
+
+// Convert speech to text from audio buffer
+const audioData = [...]; // Array of audio samples
+const result = await sherpaOnnx.speechToText(audioData);
+console.log('Recognized text:', result.text);
+
+// Or from an audio file
+const fileResult = await sherpaOnnx.speechToText('/path/to/audio.wav');
+console.log('Recognized text from file:', fileResult.text);
+
+// Clean up when done
+await sherpaOnnx.release();
+```
+
+### Streaming Recognition
+
+```typescript
+import { SherpaOnnxAPI } from '@siteed/sherpa-onnx.rn';
+
+const sherpaOnnx = new SherpaOnnxAPI({
+  modelPath: '/path/to/models',
+  sampleRate: 16000,
+});
+
+// Initialize
+await sherpaOnnx.initialize();
+
+// Start streaming
+await sherpaOnnx.startStreaming();
+
+// Feed audio chunks
+const audioChunk = [...]; // Array of audio samples from microphone
+const partialResult = await sherpaOnnx.feedAudioContent(audioChunk);
+console.log('Partial result:', partialResult.text);
+
+// Stop streaming when done
+const finalResult = await sherpaOnnx.stopStreaming();
+console.log('Final result:', finalResult.text);
+
+// Clean up
+await sherpaOnnx.release();
+```
+
+### Low-level API
+
+For advanced usage, you can access the low-level API that maps directly to Sherpa-ONNX functions:
+
+```typescript
+import { NativeModules } from 'react-native';
+const { SherpaOnnx } = NativeModules;
+
+// Create a recognizer
+const config = {
+  sampleRate: 16000,
+  featureDim: 80,
+  modelType: 'transducer',
+  transducer: {
+    encoder: '/path/to/encoder.onnx',
+    decoder: '/path/to/decoder.onnx',
+    joiner: '/path/to/joiner.onnx',
+  },
+  tokens: '/path/to/tokens.txt',
+  enableEndpoint: true,
+};
+
+const recognizerId = await SherpaOnnx.createRecognizer(config);
+
+// Create a stream
+const streamId = await SherpaOnnx.createStream(recognizerId, '');
+
+// Process audio
+const samples = [...]; // Float array of audio samples
+await SherpaOnnx.acceptWaveform(streamId, samples, 16000);
+await SherpaOnnx.decode(recognizerId, streamId);
+
+// Get results
+const result = await SherpaOnnx.getResult(recognizerId, streamId);
+console.log('Text:', result.text);
+
+// Clean up
+await SherpaOnnx.releaseStream(streamId);
+await SherpaOnnx.releaseRecognizer(recognizerId);
+```
+
+## Models
+
+You need to provide Sherpa-ONNX model files to use this library. You can download pre-trained models from the [Sherpa-ONNX](https://github.com/k2-fsa/sherpa-onnx) repository or train your own.
+
+The most common model types are:
+- Transducer models (encoder, decoder, joiner)
+- CTC models (single model file)
+- Paraformer models (encoder, decoder)
+
+## Building from Source
+
+If you need to build the native libraries yourself:
+
+```bash
+# Clone the repository
+git clone https://github.com/yourusername/yourrepo.git
+cd yourrepo/packages/sherpa-onnx.rn
+
+# Set up dependencies
+./setup.sh
+
+# Build for all platforms
+./build-all.sh
+```
+
+## API Reference
+
+### Library Validation
+
+#### `validateLibraryLoaded()`
+
+Validates if the Sherpa ONNX library is properly loaded.
+
+**Returns:** Promise<ValidateResult>
+
+### Text-to-Speech
+
+#### `initTts(config: TtsModelConfig)`
+
+Initializes the TTS engine with the provided model configuration.
+
+**Parameters:**
+- `config`: TTS model configuration
+
+**Returns:** Promise<TtsInitResult>
+
+#### `generateTts(text: string, options?: TtsOptions)`
+
+Generates speech from the given text.
+
+**Parameters:**
+- `text`: Text to synthesize
+- `options`: TTS generation options (optional)
+
+**Returns:** Promise<TtsGenerateResult>
+
+#### `stopTts()`
+
+Stops any ongoing TTS generation.
+
+**Returns:** Promise<{ stopped: boolean; message?: string }>
+
+#### `releaseTts()`
+
+Releases TTS resources.
+
+**Returns:** Promise<{ released: boolean }>
+
+## Model Configuration
+
+### TtsModelConfig
+
+```typescript
+interface TtsModelConfig {
+  // Directory containing model files
+  modelDir: string;
+  
+  // Model file name for VITS models
+  modelName?: string;
+  
+  // Acoustic model file name for Matcha models
+  acousticModelName?: string;
+  
+  // Vocoder file name for Matcha models
+  vocoder?: string;
+  
+  // Voices file name for Kokoro models
+  voices?: string;
+  
+  // Lexicon file name
+  lexicon?: string;
+  
+  // Data directory path
+  dataDir?: string;
+  
+  // Dictionary directory path
+  dictDir?: string;
+  
+  // Rule FSTs file paths (comma-separated)
+  ruleFsts?: string;
+  
+  // Rule FARs file paths (comma-separated)
+  ruleFars?: string;
+  
+  // Number of threads to use for processing
+  numThreads?: number;
+}
+```
+
+## Troubleshooting
+
+### Android
+
+If you're having issues with the Android build, check that:
+
+1. The Kotlin standard library is properly included in your app's `build.gradle`
+2. JNI libraries are properly included in the build
+
+### iOS
+
+For iOS, ensure that:
+
+1. The necessary static libraries are included in your Xcode project
+2. The pod installation completed successfully
+
+## Documentation
+
+- **[Web/WASM Guide](docs/WEB.md)** - How to deploy on web: setup, model customization, feature selection
+- **[WASM Build Guide](docs/WASM_BUILD.md)** - Building the WASM binary from source
+- **[Testing Framework](docs/testing/)** - Native integration testing documentation
+- **[Architecture](docs/architecture/)** - React Native architecture support and planning
+- **[Integration](docs/integration/)** - Platform integration guides
+- **[Compatibility](COMPATIBILITY.md)** - React Native version and platform compatibility
+
+## Development Status
+
+### ✅ Completed
+- Native integration testing framework (12 tests, 100% success rate)
+- Model management strategy for testing environments
+- Android native library integration validation
+- Cross-platform documentation structure
+
+### ⏳ In Progress / Planned
+- React Native architecture-specific validation (Old vs New Architecture)
+- Real ONNX model functionality testing
+- iOS integration validation
+
+## Contributing
+
+See the testing framework in [`docs/testing/`](docs/testing/) for validation methodology and development workflow.
+
+## License
+
+MIT 

package/android/CMakeLists.txt

@@ -0,0 +1,46 @@
+cmake_minimum_required(VERSION 3.4.1)
+
+# Set path to prebuilt libraries
+set(PREBUILT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src/main/jniLibs)
+
+# Include C++ header files
+include_directories(${PREBUILT_DIR}/include)
+
+# Add sherpa-onnx-jni library
+add_library(sherpa-onnx-jni SHARED IMPORTED)
+set_target_properties(sherpa-onnx-jni PROPERTIES IMPORTED_LOCATION
+        ${PREBUILT_DIR}/${ANDROID_ABI}/libsherpa-onnx-jni.so)
+
+# Add onnxruntime library
+add_library(onnxruntime SHARED IMPORTED)
+set_target_properties(onnxruntime PROPERTIES IMPORTED_LOCATION
+        ${PREBUILT_DIR}/${ANDROID_ABI}/libonnxruntime.so)
+
+# Add dummy implementation to help with IDE resolving
+add_library(sherpa-onnx-dummy SHARED
+  ${CMAKE_CURRENT_SOURCE_DIR}/../cpp/dummy/dummy.cpp
+)
+
+# 16KB page size alignment for Android 15+ (required for Play Store)
+target_link_options(sherpa-onnx-dummy PRIVATE "-Wl,-z,max-page-size=16384")
+
+# Check if we should include TurboModule support
+if(${NEW_ARCHITECTURE_ENABLED})
+    # These paths would be needed for TurboModule implementation
+    include_directories(
+        "${NODE_MODULES_DIR}/react-native/ReactCommon"
+        "${NODE_MODULES_DIR}/react-native/ReactCommon/callinvoker"
+        "${NODE_MODULES_DIR}/react-native/ReactCommon/jsi"
+        "${NODE_MODULES_DIR}/react-native/ReactCommon/turbomodule/core"
+        "${NODE_MODULES_DIR}/react-native/ReactCommon/react/nativemodule/core"
+    )
+endif()
+
+# If we need to add our own native code, uncomment and customize:
+# add_library(sherpa-onnx-extra SHARED
+#   src/main/cpp/sherpa-onnx-extra.cpp
+# )
+
+# Create an interface library for the JNI libraries
+add_library(sherpa-onnx-rn INTERFACE)
+target_link_libraries(sherpa-onnx-rn INTERFACE sherpa-onnx-jni onnxruntime sherpa-onnx-dummy) 

package/android/README.md

@@ -0,0 +1,77 @@
+# Android Implementation for @siteed/sherpa-onnx.rn
+
+This directory contains the Android native implementation for the Sherpa ONNX React Native module.
+
+## Architecture
+
+The implementation consists of:
+
+1. **Kotlin Native Module** (`SherpaOnnxModule.kt`): A simple React Native module implementation that loads the Sherpa ONNX JNI library and provides placeholder methods for the JavaScript API.
+
+2. **JNI Libraries** (in `src/main/jniLibs/`): Native `.so` libraries for different architectures that implement the actual speech recognition functionality.
+
+3. **New Architecture Compatibility** (in `build/generated/source/codegen/jni/`): Placeholder C++ files to ensure compatibility with React Native's New Architecture.
+
+## Building the Module
+
+The module has been set up with automated scripts to handle the build process:
+
+1. **setup.sh**: Clones the Sherpa ONNX repository into `third_party/`.
+
+2. **build-sherpa-android.sh**: Builds the Sherpa ONNX native libraries for Android using the Sherpa ONNX build scripts.
+
+3. **copy-libs.sh**: Copies the built native libraries to the correct location in the Android project.
+
+## Build Process
+
+The standard build process is:
+
+1. Run `npm run setup` to clone the Sherpa ONNX repository.
+2. Run `npm run build:android` to build the Sherpa ONNX native libraries.
+3. Run `npm run copy:libs` to copy the libraries to the correct locations.
+
+These steps are automated as part of the module's `postinstall` script.
+
+## Compatibility with React Native
+
+The module supports both the Old and New React Native architectures:
+
+- **Old Architecture**: Uses Kotlin implementation via `ReactContextBaseJavaModule`.
+- **New Architecture**: Uses placeholder C++ files to satisfy build requirements.
+
+## Implementing Speech Recognition
+
+The current implementation provides a minimal set of placeholder methods. To implement actual speech recognition functionality, you'll need to:
+
+1. Call the Sherpa ONNX JNI methods directly from the Kotlin code
+2. Process audio data and return results
+3. Update the TypeScript interfaces to match your implementation
+
+## JVM Compatibility
+
+The module is configured to use Java 17, which is required for modern React Native applications. This is set in the `build.gradle` file:
+
+```gradle
+compileOptions {
+    sourceCompatibility JavaVersion.VERSION_17
+    targetCompatibility JavaVersion.VERSION_17
+}
+
+kotlinOptions {
+    jvmTarget = '17'
+}
+```
+
+## Troubleshooting
+
+### Missing JNI Libraries
+If you encounter errors about missing native libraries, make sure you've run:
+```
+npm run build:android
+npm run copy:libs
+```
+
+### New Architecture Compatibility
+If you have build errors in a New Architecture app, check that:
+1. The placeholder C++ files exist in `build/generated/source/codegen/jni/`
+2. Your app's gradle version is compatible with the module's configuration 

package/android/build.gradle

@@ -0,0 +1,140 @@
+buildscript {
+    ext.safeExtGet = { prop, fallback ->
+        rootProject.ext.has(prop) ? rootProject.ext.get(prop) : fallback
+    }
+
+    repositories {
+        google()
+        mavenCentral()
+    }
+
+    dependencies {
+        classpath("com.android.tools.build:gradle:7.2.1")
+        classpath("org.jetbrains.kotlin:kotlin-gradle-plugin:1.8.0")
+    }
+}
+
+// For backwards compatibility with older React Native versions
+def REACT_NATIVE_VERSION = safeExtGet('reactNativeVersion', '+')
+
+apply plugin: 'com.android.library'
+apply plugin: 'com.facebook.react'
+apply plugin: 'kotlin-android'
+apply plugin: 'kotlin-parcelize'
+
+android {
+    namespace "net.siteed.sherpaonnx"
+    compileSdkVersion safeExtGet('compileSdkVersion', 34)
+    buildToolsVersion safeExtGet('buildToolsVersion', '34.0.0')
+
+    defaultConfig {
+        minSdkVersion safeExtGet('minSdkVersion', 21)
+        targetSdkVersion safeExtGet('targetSdkVersion', 34)
+        versionCode 1
+        versionName "1.0"
+
+        // Test instrumentation runner for integration tests
+        testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
+
+        ndk {
+            abiFilters 'arm64-v8a', 'armeabi-v7a', 'x86_64'
+        }
+    }
+
+    sourceSets {
+        main {
+            // Include prebuilt libraries in the correct location
+            jniLibs.srcDirs = ['src/main/jniLibs']
+
+            // Add Java source directory for compatibility
+            java.srcDirs = ['src/main/java']
+        }
+    }
+
+    // Configure Kotlin source directories
+    kotlin {
+        sourceSets {
+            main {
+                kotlin.srcDirs = ['src/main/kotlin']
+            }
+        }
+    }
+
+    buildTypes {
+        release {
+            minifyEnabled false
+            proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
+        }
+    }
+
+    // IMPORTANT: These Java compatibility settings must match the React Native app
+    compileOptions {
+        sourceCompatibility JavaVersion.VERSION_17
+        targetCompatibility JavaVersion.VERSION_17
+    }
+
+    kotlinOptions {
+        jvmTarget = '17'
+    }
+
+    // This ensures the library is compiled as an AAR with the JNI libraries included
+    packagingOptions {
+        pickFirst '**/*.so'
+    }
+    
+    // Add local project sources for Sherpa ONNX Kotlin classes
+    externalNativeBuild {
+        cmake {
+            path "CMakeLists.txt"
+        }
+    }
+}
+
+
+repositories {
+    google()
+    mavenCentral()
+    mavenLocal()
+}
+
+dependencies {
+    // Kotlin standard library must come first - this is critical
+    implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.8.0"
+    implementation "org.jetbrains.kotlin:kotlin-stdlib:1.8.0"
+    implementation "org.jetbrains.kotlin:kotlin-stdlib-common:1.8.0"
+    implementation "org.jetbrains.kotlin:kotlin-reflect:1.8.0"
+    
+    // React Native dependencies
+    implementation "com.facebook.react:react-native:${REACT_NATIVE_VERSION}"
+    
+    // Android support libraries
+    implementation "androidx.annotation:annotation:1.5.0"
+    implementation "androidx.core:core-ktx:1.9.0"
+    implementation "androidx.media:media:1.6.0"  // For AudioTrack and related classes
+
+    // Apache Commons Compress for tar.bz2 extraction
+    implementation "org.apache.commons:commons-compress:1.23.0"
+    
+    // OkHttp for model downloads
+    implementation "com.squareup.okhttp3:okhttp:4.11.0"
+    
+    // Kotlin coroutines for async operations
+    implementation "org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.3"
+    implementation "org.jetbrains.kotlinx:kotlinx-coroutines-android:1.7.3"
+    
+    // Make sure we have the actual sherpa-onnx library dependencies
+    // First check if there are prebuilt libraries in the jniLibs folder
+    if (file("src/main/jniLibs").exists()) {
+        println("Using prebuilt sherpa-onnx libraries from jniLibs directory")
+    } else {
+        println("WARNING: No prebuilt libraries found in jniLibs directory!")
+    }
+
+    // Test dependencies
+    testImplementation "junit:junit:4.13.2"
+    androidTestImplementation "androidx.test.ext:junit:1.1.5"
+    androidTestImplementation "androidx.test.espresso:espresso-core:3.5.1"
+    androidTestImplementation "androidx.test:runner:1.5.2"
+    androidTestImplementation "androidx.test:rules:1.5.0"
+
+} 

package/android/proguard-rules.pro

@@ -0,0 +1,32 @@
+# Keep sherpa-onnx JNI classes
+-keep class com.k2fsa.sherpa.onnx.** { *; }
+
+# Keep React Native related code
+-keep,allowobfuscation @interface com.facebook.proguard.annotations.DoNotStrip
+-keep,allowobfuscation @interface com.facebook.proguard.annotations.KeepGettersAndSetters
+-keep,allowobfuscation @interface com.facebook.common.internal.DoNotStrip
+
+-keep @com.facebook.proguard.annotations.DoNotStrip class *
+-keep @com.facebook.common.internal.DoNotStrip class *
+-keepclassmembers class * {
+    @com.facebook.proguard.annotations.DoNotStrip *;
+    @com.facebook.common.internal.DoNotStrip *;
+}
+
+-keepclassmembers @com.facebook.proguard.annotations.KeepGettersAndSetters class * {
+  void set*(***);
+  *** get*();
+}
+
+# Keep JNI methods
+-keepclasseswithmembernames class * {
+    native <methods>;
+}
+
+# Keep native libraries
+-keepclasseswithmembernames class * {
+    native <methods>;
+}
+
+# Keep all classes in our namespace
+-keep class net.siteed.sherpaonnx.** { *; }