@mentra/bluetooth-sdk 0.1.8 → 0.1.10

package/android/src/main/java/com/mentra/bluetoothsdk/stt/VadGateSpeechPolicy.kt

@@ -0,0 +1,229 @@
+package com.mentra.bluetoothsdk.stt
+
+import android.content.Context
+import android.util.Log
+import com.konovalov.vad.silero.Vad
+import com.konovalov.vad.silero.VadSilero
+import com.konovalov.vad.silero.config.FrameSize
+import com.konovalov.vad.silero.config.Mode
+import com.konovalov.vad.silero.config.SampleRate
+import java.nio.ByteBuffer
+import java.nio.ByteOrder
+
+/**
+ * VadGateSpeechPolicy handles Voice Activity Detection using Silero VAD.
+ *
+ * Features:
+ * - Dynamic VAD check throttling (adjusts frequency based on silence duration)
+ * - 8-second silence timeout before declaring "not speaking"
+ * - Bypass modes for debugging and PCM streaming
+ * - 512-sample frame size (matches Sherpa-ONNX requirements)
+ *
+ * Ported from android_core VadGateSpeechPolicy.java
+ */
+class VadGateSpeechPolicy(private val context: Context) {
+    companion object {
+        private const val TAG = "VadGateSpeechPolicy"
+
+        // Total required silence duration
+        private const val REQUIRED_SILENCE_DURATION_MS = 8000L
+
+        // Dynamic VAD check intervals
+        private const val INITIAL_SILENCE_VAD_INTERVAL_MS = 50L // Check frequently at first
+        private const val MEDIUM_SILENCE_VAD_INTERVAL_MS = 100L // Medium frequency after some time
+        private const val LONG_SILENCE_VAD_INTERVAL_MS =
+                200L // Less frequent after extended silence
+
+        // Thresholds for switching intervals
+        private const val MEDIUM_SILENCE_THRESHOLD_MS =
+                20000L // Switch to medium interval after 20s
+        private const val LONG_SILENCE_THRESHOLD_MS = 60000L // Switch to long interval after 60s
+
+        private const val FRAME_SIZE = 512
+    }
+
+    private var vad: VadSilero? = null
+    private var isCurrentlySpeech = false
+    private var bypassVadForDebugging = false
+    private var bypassVadForPCM = false
+
+    /** Notified whenever the speech/silence state actually flips. */
+    var onSpeechStateChanged: ((Boolean) -> Unit)? = null
+
+    // Timestamp of the last detected speech
+    private var lastSpeechDetectedTime = 0L
+    // Throttle timer for silence VAD checks
+    private var lastVadCheckTime = 0L
+
+    /** Initialize the Silero VAD model */
+    fun init(blockSizeSamples: Int) {
+        startVad()
+    }
+
+    private fun startVad() {
+        try {
+            // Set internal silence duration very low; external logic manages the full required
+            // silence duration
+            vad =
+                    Vad.builder()
+                            .setContext(context)
+                            .setSampleRate(SampleRate.SAMPLE_RATE_16K)
+                            .setFrameSize(FrameSize.FRAME_SIZE_512)
+                            .setMode(Mode.NORMAL)
+                            .setSilenceDurationMs(50)
+                            .setSpeechDurationMs(50)
+                            .build()
+
+            Log.d(TAG, "VAD initialized successfully")
+        } catch (e: Exception) {
+            Log.e(TAG, "Failed to initialize VAD", e)
+            vad = null
+        }
+    }
+
+    /** Check if audio should be passed to the recognizer based on VAD state */
+    fun shouldPassAudioToRecognizer(): Boolean {
+        // CRITICAL: Handle VAD null case
+        if (vad == null) {
+            return bypassVadForDebugging || bypassVadForPCM || true
+        }
+
+        return bypassVadForDebugging || bypassVadForPCM || isCurrentlySpeech
+    }
+
+    /** Process audio bytes through VAD to detect speech */
+    fun processAudioBytes(bytes: ByteArray, offset: Int, length: Int) {
+        val now = System.currentTimeMillis()
+
+        // If in speech state and it hasn't been 8 seconds since the last speech was detected, skip
+        // processing
+        if (isCurrentlySpeech && (now - lastSpeechDetectedTime < REQUIRED_SILENCE_DURATION_MS)) {
+            return
+        }
+
+        // Calculate silence duration
+        val silenceDuration = now - lastSpeechDetectedTime
+
+        // Determine which VAD interval to use based on silence duration
+        val currentVadInterval =
+                when {
+                    silenceDuration < MEDIUM_SILENCE_THRESHOLD_MS -> INITIAL_SILENCE_VAD_INTERVAL_MS
+                    silenceDuration < LONG_SILENCE_THRESHOLD_MS -> MEDIUM_SILENCE_VAD_INTERVAL_MS
+                    else -> LONG_SILENCE_VAD_INTERVAL_MS
+                }
+
+        // During silence, throttle VAD checks based on the dynamic interval
+        if (!isCurrentlySpeech && (now - lastVadCheckTime < currentVadInterval)) {
+            return
+        }
+        lastVadCheckTime = now
+
+        val audioBytesFull = bytesToShort(bytes)
+        val totalSamples = audioBytesFull.size
+
+        if (totalSamples % FRAME_SIZE != 0) {
+            Log.e(
+                    TAG,
+                    "Invalid audio frame size: $totalSamples samples. Needs to be multiple of $FRAME_SIZE."
+            )
+            return
+        }
+
+        val currentVad =
+                vad
+                        ?: run {
+                            Log.w(TAG, "VAD not initialized, skipping audio processing")
+                            return
+                        }
+
+        var previousSpeechState = isCurrentlySpeech
+
+        // Process each 512-sample frame
+        val numFrames = totalSamples / FRAME_SIZE
+        for (i in 0 until numFrames) {
+            val currentTime = System.currentTimeMillis()
+            val startIdx = i * FRAME_SIZE
+            val audioFrame = audioBytesFull.copyOfRange(startIdx, startIdx + FRAME_SIZE)
+            val detectedSpeech = currentVad.isSpeech(audioFrame)
+
+            if (detectedSpeech) {
+                isCurrentlySpeech = true
+                // Update the last speech detection timestamp
+                lastSpeechDetectedTime = currentTime
+            } else {
+                // If no speech detected, and 8 seconds have elapsed since the last speech, mark as
+                // silence
+                if (currentTime - lastSpeechDetectedTime >= REQUIRED_SILENCE_DURATION_MS) {
+                    isCurrentlySpeech = false
+                }
+            }
+
+            if (isCurrentlySpeech != previousSpeechState) {
+                Log.d(
+                        TAG,
+                        "Speech detection changed to: ${if (isCurrentlySpeech) "SPEECH" else "SILENCE"}"
+                )
+                previousSpeechState = isCurrentlySpeech
+                onSpeechStateChanged?.invoke(isCurrentlySpeech)
+            }
+        }
+    }
+
+    /** Convert byte array (PCM16LE) to short array */
+    private fun bytesToShort(bytes: ByteArray): ShortArray {
+        val shorts = ShortArray(bytes.size / 2)
+        ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN).asShortBuffer().get(shorts)
+        return shorts
+    }
+
+    /** Stop VAD and release resources */
+    fun stop() {
+        try {
+            vad?.close()
+            vad = null
+            Log.d(TAG, "VAD stopped and resources released")
+        } catch (e: Exception) {
+            Log.e(TAG, "Error stopping VAD", e)
+        }
+    }
+
+    /** Reset VAD state */
+    fun reset() {
+        val wasSpeaking = isCurrentlySpeech
+        isCurrentlySpeech = false
+        lastSpeechDetectedTime = 0
+        lastVadCheckTime = 0
+        if (wasSpeaking) onSpeechStateChanged?.invoke(false)
+    }
+
+    /** Change bypass VAD for debugging state */
+    fun changeBypassVadForDebugging(bypass: Boolean) {
+        bypassVadForDebugging = bypass
+        Log.d(TAG, "Bypass VAD for debugging: $bypass")
+    }
+
+    /** Change bypass VAD for PCM streaming state */
+    fun changeBypassVadForPCM(bypass: Boolean) {
+        Log.d(TAG, "VAD PCM Bypass State Change: $bypassVadForPCM -> $bypass")
+        bypassVadForPCM = bypass
+    }
+
+    /** Handle microphone state changes */
+    fun microphoneStateChanged(state: Boolean) {
+        if (!state) {
+            // Microphone turned off: force immediate silence
+            val wasSpeaking = isCurrentlySpeech
+            isCurrentlySpeech = false
+            lastSpeechDetectedTime = 0
+            lastVadCheckTime = 0
+
+            // Optionally flush the VAD's internal state by processing a silent frame
+            vad?.let { vadInstance ->
+                val silentFrame = ShortArray(FRAME_SIZE)
+                vadInstance.isSpeech(silentFrame)
+            }
+            Log.d(TAG, "Microphone turned off; forced state to SILENCE.")
+            if (wasSpeaking) onSpeechStateChanged?.invoke(false)
+        }
+    }
+}

package/android/src/main/java/com/mentra/bluetoothsdk/tts/TTSTools.kt

@@ -0,0 +1,180 @@
+package com.mentra.core.tts
+
+import android.content.Context
+import android.content.SharedPreferences
+import com.k2fsa.sherpa.onnx.GenerationConfig
+import com.k2fsa.sherpa.onnx.OfflineTts
+import com.k2fsa.sherpa.onnx.OfflineTtsConfig
+import com.k2fsa.sherpa.onnx.OfflineTtsModelConfig
+import com.k2fsa.sherpa.onnx.OfflineTtsSupertonicModelConfig
+import com.mentra.bluetoothsdk.Bridge
+import java.io.File
+
+/**
+ * Utilities for offline Sherpa-ONNX Supertonic 3 TTS model management.
+ *
+ * Supertonic 3 is a single multilingual model bundle (31 languages) with 10
+ * preset voice styles packed into a single voice.bin. Speaker IDs are indexed
+ * alphabetically by JSON filename, so the order is F1, F2, F3, F4, F5, M1, M2,
+ * M3, M4, M5 (sids 0..9). We default to F1 (sid 0) for a female voice.
+ */
+object TTSTools {
+    private const val PREFS_NAME = "MentraPrefs"
+    private const val KEY_TTS_MODEL_PATH = "TTSModelPath"
+    private const val KEY_TTS_MODEL_LANGUAGE = "TTSModelLanguageCode"
+
+    private const val FILE_DURATION_PREDICTOR = "duration_predictor.int8.onnx"
+    private const val FILE_TEXT_ENCODER = "text_encoder.int8.onnx"
+    private const val FILE_VECTOR_ESTIMATOR = "vector_estimator.int8.onnx"
+    private const val FILE_VOCODER = "vocoder.int8.onnx"
+    private const val FILE_TTS_JSON = "tts.json"
+    private const val FILE_UNICODE_INDEXER = "unicode_indexer.bin"
+    private const val FILE_VOICE_STYLE = "voice.bin"
+
+    private val REQUIRED_FILES = listOf(
+            FILE_DURATION_PREDICTOR,
+            FILE_TEXT_ENCODER,
+            FILE_VECTOR_ESTIMATOR,
+            FILE_VOCODER,
+            FILE_TTS_JSON,
+            FILE_UNICODE_INDEXER,
+            FILE_VOICE_STYLE,
+    )
+
+    fun setTtsModelDetails(context: Context, path: String, languageCode: String) {
+        val prefs = getPrefs(context)
+        prefs.edit().apply {
+            putString(KEY_TTS_MODEL_PATH, path)
+            putString(KEY_TTS_MODEL_LANGUAGE, languageCode)
+            apply()
+        }
+        Bridge.log("TTS model details saved: path=$path, language=$languageCode")
+    }
+
+    fun getTtsModelPath(context: Context): String {
+        val prefs = getPrefs(context)
+        return prefs.getString(KEY_TTS_MODEL_PATH, "") ?: ""
+    }
+
+    fun getTtsModelLanguage(context: Context): String {
+        val prefs = getPrefs(context)
+        return prefs.getString(KEY_TTS_MODEL_LANGUAGE, "en-US") ?: "en-US"
+    }
+
+    fun checkTTSModelAvailable(context: Context): Boolean {
+        val modelPath = getTtsModelPath(context)
+        if (modelPath.isEmpty()) {
+            return false
+        }
+        return validateTTSModel(modelPath)
+    }
+
+    fun validateTTSModel(path: String): Boolean {
+        val modelDir = File(path)
+        if (!modelDir.exists() || !modelDir.isDirectory) {
+            Bridge.log("TTS model path does not exist or is not a directory: $path")
+            return false
+        }
+
+        for (name in REQUIRED_FILES) {
+            val file = File(modelDir, name)
+            if (!file.exists() || !file.canRead() || file.length() == 0L) {
+                Bridge.log("TTS model missing required file '$name' at: $path")
+                return false
+            }
+        }
+
+        return true
+    }
+
+    private val SUPPORTED_LANGS = setOf(
+            "en", "ko", "ja", "ar", "bg", "cs", "da", "de", "el", "es", "et",
+            "fi", "fr", "hi", "hr", "hu", "id", "it", "lt", "lv", "nl", "pl",
+            "pt", "ro", "ru", "sk", "sl", "sv", "tr", "uk", "vi",
+    )
+
+    /** Convert a BCP-47 tag like "en-US" or "fr-FR" to a Supertonic 2-letter code. */
+    private fun languageTagToSupertonicLang(tag: String?): String {
+        if (tag.isNullOrBlank()) return "en"
+        val primary = tag.substringBefore('-').lowercase()
+        return if (primary in SUPPORTED_LANGS) primary else "en"
+    }
+
+    fun generateTtsAudio(
+            context: Context,
+            text: String,
+            modelPath: String,
+            outputPath: String,
+            speakerId: Int,
+            speed: Float,
+    ): Boolean {
+        val languageTag = getTtsModelLanguage(context)
+        if (text.isBlank()) {
+            Bridge.log("TTS_ERROR: text is empty")
+            return false
+        }
+        if (!validateTTSModel(modelPath)) {
+            Bridge.log("TTS_ERROR: model is invalid: $modelPath")
+            return false
+        }
+
+        val modelDir = File(modelPath)
+        val outputFile = File(outputPath)
+        outputFile.parentFile?.mkdirs()
+
+        var tts: OfflineTts? = null
+        return try {
+            val supertonic = OfflineTtsSupertonicModelConfig(
+                    durationPredictor = File(modelDir, FILE_DURATION_PREDICTOR).absolutePath,
+                    textEncoder = File(modelDir, FILE_TEXT_ENCODER).absolutePath,
+                    vectorEstimator = File(modelDir, FILE_VECTOR_ESTIMATOR).absolutePath,
+                    vocoder = File(modelDir, FILE_VOCODER).absolutePath,
+                    ttsJson = File(modelDir, FILE_TTS_JSON).absolutePath,
+                    unicodeIndexer = File(modelDir, FILE_UNICODE_INDEXER).absolutePath,
+                    voiceStyle = File(modelDir, FILE_VOICE_STYLE).absolutePath,
+            )
+
+            val modelConfig = OfflineTtsModelConfig(
+                    supertonic = supertonic,
+                    numThreads = 2,
+                    provider = "cpu",
+            )
+
+            val config = OfflineTtsConfig(
+                    model = modelConfig,
+                    maxNumSentences = 1,
+                    silenceScale = 0.2f,
+            )
+
+            tts = OfflineTts(null, config)
+
+            // sid out of range falls back to F1 (sid 0); clamp 0..9.
+            val sid = speakerId.coerceIn(0, tts.numSpeakers() - 1)
+            val clampedSpeed = speed.coerceIn(0.5f, 2.0f)
+            val lang = languageTagToSupertonicLang(languageTag)
+            val genConfig = GenerationConfig(
+                    sid = sid,
+                    speed = clampedSpeed,
+                    extra = mapOf("lang" to lang),
+            )
+            val audio = tts.generateWithConfig(text = text, config = genConfig)
+            val saved = audio.save(outputFile.absolutePath)
+            Bridge.log("TTS generated ${outputFile.absolutePath}: saved=$saved sid=$sid lang=$lang")
+            saved
+        } catch (e: Exception) {
+            Bridge.log("TTS_ERROR: ${e.javaClass.simpleName}: ${e.message}")
+            e.printStackTrace()
+            false
+        } finally {
+            try {
+                tts?.release()
+            } catch (e: Exception) {
+                Bridge.log("TTS release failed: ${e.message}")
+            }
+        }
+    }
+
+    private fun getPrefs(context: Context): SharedPreferences {
+        return context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE)
+    }
+}