@mentra/bluetooth-sdk 0.1.5 → 0.1.7

package/android/src/main/java/com/mentra/bluetoothsdk/stt/VadGateSpeechPolicy.kt

@@ -1,221 +0,0 @@
-package com.mentra.bluetoothsdk.stt
-
-import android.content.Context
-import android.util.Log
-import com.konovalov.vad.silero.Vad
-import com.konovalov.vad.silero.VadSilero
-import com.konovalov.vad.silero.config.FrameSize
-import com.konovalov.vad.silero.config.Mode
-import com.konovalov.vad.silero.config.SampleRate
-import java.nio.ByteBuffer
-import java.nio.ByteOrder
-
-/**
- * VadGateSpeechPolicy handles Voice Activity Detection using Silero VAD.
- *
- * Features:
- * - Dynamic VAD check throttling (adjusts frequency based on silence duration)
- * - 8-second silence timeout before declaring "not speaking"
- * - Bypass modes for debugging and PCM streaming
- * - 512-sample frame size (matches Sherpa-ONNX requirements)
- *
- * Ported from android_core VadGateSpeechPolicy.java
- */
-class VadGateSpeechPolicy(private val context: Context) {
-    companion object {
-        private const val TAG = "VadGateSpeechPolicy"
-
-        // Total required silence duration
-        private const val REQUIRED_SILENCE_DURATION_MS = 8000L
-
-        // Dynamic VAD check intervals
-        private const val INITIAL_SILENCE_VAD_INTERVAL_MS = 50L // Check frequently at first
-        private const val MEDIUM_SILENCE_VAD_INTERVAL_MS = 100L // Medium frequency after some time
-        private const val LONG_SILENCE_VAD_INTERVAL_MS =
-                200L // Less frequent after extended silence
-
-        // Thresholds for switching intervals
-        private const val MEDIUM_SILENCE_THRESHOLD_MS =
-                20000L // Switch to medium interval after 20s
-        private const val LONG_SILENCE_THRESHOLD_MS = 60000L // Switch to long interval after 60s
-
-        private const val FRAME_SIZE = 512
-    }
-
-    private var vad: VadSilero? = null
-    private var isCurrentlySpeech = false
-    private var bypassVadForDebugging = false
-    private var bypassVadForPCM = false
-
-    // Timestamp of the last detected speech
-    private var lastSpeechDetectedTime = 0L
-    // Throttle timer for silence VAD checks
-    private var lastVadCheckTime = 0L
-
-    /** Initialize the Silero VAD model */
-    fun init(blockSizeSamples: Int) {
-        startVad()
-    }
-
-    private fun startVad() {
-        try {
-            // Set internal silence duration very low; external logic manages the full required
-            // silence duration
-            vad =
-                    Vad.builder()
-                            .setContext(context)
-                            .setSampleRate(SampleRate.SAMPLE_RATE_16K)
-                            .setFrameSize(FrameSize.FRAME_SIZE_512)
-                            .setMode(Mode.NORMAL)
-                            .setSilenceDurationMs(50)
-                            .setSpeechDurationMs(50)
-                            .build()
-
-            Log.d(TAG, "VAD initialized successfully")
-        } catch (e: Exception) {
-            Log.e(TAG, "Failed to initialize VAD", e)
-            vad = null
-        }
-    }
-
-    /** Check if audio should be passed to the recognizer based on VAD state */
-    fun shouldPassAudioToRecognizer(): Boolean {
-        // CRITICAL: Handle VAD null case
-        if (vad == null) {
-            return bypassVadForDebugging || bypassVadForPCM || true
-        }
-
-        return bypassVadForDebugging || bypassVadForPCM || isCurrentlySpeech
-    }
-
-    /** Process audio bytes through VAD to detect speech */
-    fun processAudioBytes(bytes: ByteArray, offset: Int, length: Int) {
-        val now = System.currentTimeMillis()
-
-        // If in speech state and it hasn't been 8 seconds since the last speech was detected, skip
-        // processing
-        if (isCurrentlySpeech && (now - lastSpeechDetectedTime < REQUIRED_SILENCE_DURATION_MS)) {
-            return
-        }
-
-        // Calculate silence duration
-        val silenceDuration = now - lastSpeechDetectedTime
-
-        // Determine which VAD interval to use based on silence duration
-        val currentVadInterval =
-                when {
-                    silenceDuration < MEDIUM_SILENCE_THRESHOLD_MS -> INITIAL_SILENCE_VAD_INTERVAL_MS
-                    silenceDuration < LONG_SILENCE_THRESHOLD_MS -> MEDIUM_SILENCE_VAD_INTERVAL_MS
-                    else -> LONG_SILENCE_VAD_INTERVAL_MS
-                }
-
-        // During silence, throttle VAD checks based on the dynamic interval
-        if (!isCurrentlySpeech && (now - lastVadCheckTime < currentVadInterval)) {
-            return
-        }
-        lastVadCheckTime = now
-
-        val audioBytesFull = bytesToShort(bytes)
-        val totalSamples = audioBytesFull.size
-
-        if (totalSamples % FRAME_SIZE != 0) {
-            Log.e(
-                    TAG,
-                    "Invalid audio frame size: $totalSamples samples. Needs to be multiple of $FRAME_SIZE."
-            )
-            return
-        }
-
-        val currentVad =
-                vad
-                        ?: run {
-                            Log.w(TAG, "VAD not initialized, skipping audio processing")
-                            return
-                        }
-
-        var previousSpeechState = isCurrentlySpeech
-
-        // Process each 512-sample frame
-        val numFrames = totalSamples / FRAME_SIZE
-        for (i in 0 until numFrames) {
-            val currentTime = System.currentTimeMillis()
-            val startIdx = i * FRAME_SIZE
-            val audioFrame = audioBytesFull.copyOfRange(startIdx, startIdx + FRAME_SIZE)
-            val detectedSpeech = currentVad.isSpeech(audioFrame)
-
-            if (detectedSpeech) {
-                isCurrentlySpeech = true
-                // Update the last speech detection timestamp
-                lastSpeechDetectedTime = currentTime
-            } else {
-                // If no speech detected, and 8 seconds have elapsed since the last speech, mark as
-                // silence
-                if (currentTime - lastSpeechDetectedTime >= REQUIRED_SILENCE_DURATION_MS) {
-                    isCurrentlySpeech = false
-                }
-            }
-
-            if (isCurrentlySpeech != previousSpeechState) {
-                Log.d(
-                        TAG,
-                        "Speech detection changed to: ${if (isCurrentlySpeech) "SPEECH" else "SILENCE"}"
-                )
-                previousSpeechState = isCurrentlySpeech
-            }
-        }
-    }
-
-    /** Convert byte array (PCM16LE) to short array */
-    private fun bytesToShort(bytes: ByteArray): ShortArray {
-        val shorts = ShortArray(bytes.size / 2)
-        ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN).asShortBuffer().get(shorts)
-        return shorts
-    }
-
-    /** Stop VAD and release resources */
-    fun stop() {
-        try {
-            vad?.close()
-            vad = null
-            Log.d(TAG, "VAD stopped and resources released")
-        } catch (e: Exception) {
-            Log.e(TAG, "Error stopping VAD", e)
-        }
-    }
-
-    /** Reset VAD state */
-    fun reset() {
-        isCurrentlySpeech = false
-        lastSpeechDetectedTime = 0
-        lastVadCheckTime = 0
-    }
-
-    /** Change bypass VAD for debugging state */
-    fun changeBypassVadForDebugging(bypass: Boolean) {
-        bypassVadForDebugging = bypass
-        Log.d(TAG, "Bypass VAD for debugging: $bypass")
-    }
-
-    /** Change bypass VAD for PCM streaming state */
-    fun changeBypassVadForPCM(bypass: Boolean) {
-        Log.d(TAG, "VAD PCM Bypass State Change: $bypassVadForPCM -> $bypass")
-        bypassVadForPCM = bypass
-    }
-
-    /** Handle microphone state changes */
-    fun microphoneStateChanged(state: Boolean) {
-        if (!state) {
-            // Microphone turned off: force immediate silence
-            isCurrentlySpeech = false
-            lastSpeechDetectedTime = 0
-            lastVadCheckTime = 0
-
-            // Optionally flush the VAD's internal state by processing a silent frame
-            vad?.let { vadInstance ->
-                val silentFrame = ShortArray(FRAME_SIZE)
-                vadInstance.isSpeech(silentFrame)
-            }
-            Log.d(TAG, "Microphone turned off; forced state to SILENCE.")
-        }
-    }
-}

package/ios/Packages/VAD/Common/VAD/VADQuality.swift

@@ -1,41 +0,0 @@
-//
-//  VADQuality.swift
-//  ios-vad
-//
-//  Created by baochuquan on 2024/11/9.
-//
-
-import Foundation
-
-enum VADQuality: String, CaseIterable {
-    case normal = "NORMAL"
-    case low_bitrate = "LOW_BITERATE"
-    case aggressive = "AGGRESSIVE"
-    case very_aggressive = "VERY_AGGRESSIVE"
-
-    var desc: String {
-        rawValue
-    }
-
-    var threshold: Float {
-        switch self {
-        case .normal: return 0.3
-        case .low_bitrate: return 0.7
-        case .aggressive: return 0.8
-        case .very_aggressive: return 0.90
-        }
-    }
-
-    var webrtcMode: Int {
-        switch self {
-        case .normal: return 0
-        case .low_bitrate: return 1
-        case .aggressive: return 2
-        case .very_aggressive: return 3
-        }
-    }
-
-    static let webrtc: [VADQuality] = [.normal, .low_bitrate, .aggressive, .very_aggressive]
-    static let silero: [VADQuality] = [.normal, .aggressive, .very_aggressive]
-    static let yamnet: [VADQuality] = [.normal]
-}

package/ios/Packages/VAD/Common/VAD/VADState.swift

@@ -1,26 +0,0 @@
-//
-//  VADState.swift
-//  ios-vad
-//
-//  Created by baochuquan on 2024/11/9.
-//
-
-import Foundation
-
-enum VADState {
-    case start // 开始说话
-    case speeching // 说话中
-    case end // 结束说话
-    case silence // 静默中
-    case error // 出错了
-
-    var desc: String {
-        switch self {
-        case .start: return "VAD Start"
-        case .speeching: return "VAD Speeching"
-        case .end: return "VAD End"
-        case .silence: return "VAD Silence"
-        case .error: return "Error"
-        }
-    }
-}

package/ios/Packages/VAD/Common/VAD/VADStrategy.swift

@@ -1,29 +0,0 @@
-//
-//  VADStrategy.swift
-//  ios-vad
-//
-//  Created by baochuquan on 2024/11/9.
-//
-
-import Foundation
-
-protocol VADStrategy {
-    /// vad setup method
-    /// - Parameters:
-    ///   - sampleRate: sample rate
-    ///   - frameSize: frame size
-    ///   - quality: vad quality level
-    ///   - silenceTriggerDurationMs: the minimum duration required to switch from speech to silence. Unit: millisecond.
-    ///   - speechTriggerDurationMs: the minimum duration required to switch from silence to speech. Unit: millisecond.
-    func setup(sampleRate: SampleRate, frameSize: FrameSize, quality: VADQuality, silenceTriggerDurationMs: Int64, speechTriggerDurationMs: Int64)
-
-    /// vad check result callback
-    /// - Parameters:
-    ///   - pcm: pcm Data
-    ///   - handler: callback.
-    func checkVAD(pcm: [Int16], handler: @escaping (VADState) -> Void)
-
-    /// current vad state
-    /// - Returns: vad state
-    func currentState() -> VADState
-}

package/ios/Packages/VAD/Common/VAD/VADType.swift

@@ -1,14 +0,0 @@
-//
-//  VADType.swift
-//  ios-vad
-//
-//  Created by baochuquan on 2024/11/9.
-//
-
-import Foundation
-
-enum VADType: String {
-    case webrtc = "WebRTC VAD"
-    case silero = "Silero VAD"
-    case yamnet = "Yamnet VAD"
-}

package/ios/Packages/VAD/Data/Configuration.swift

@@ -1,68 +0,0 @@
-//
-//  Configuration.swift
-//  ios-vad
-//
-//  Created by baochuquan on 2024/11/9.
-//
-
-import Foundation
-
-// MARK: - Sample Rate Config
-
-struct SampleRateConfiguration: Hashable {
-    var selectedOption: SampleRate
-    var options: [SampleRate]
-
-    static let webrtc = SampleRateConfiguration(selectedOption: SampleRate.webrtc[0], options: SampleRate.webrtc)
-
-    static let silero = SampleRateConfiguration(selectedOption: SampleRate.silero[0], options: SampleRate.silero)
-
-    static let yamnet = SampleRateConfiguration(selectedOption: SampleRate.yamnet[0], options: SampleRate.yamnet)
-
-    func frameSizeConfiguration(type: VADType) -> FrameSizeConfiguration {
-        let options = selectedOption.frameSizeOptions(type: type)
-        return FrameSizeConfiguration(selectedOption: options[0], options: options)
-    }
-}
-
-// MARK: - Frame Size Config
-
-struct FrameSizeConfiguration: Hashable {
-    var selectedOption: FrameSize
-    var options: [FrameSize]
-}
-
-// MARK: - Mode Config
-
-struct QualityConfiguration: Hashable {
-    var selectedOption: VADQuality
-    var options: [VADQuality]
-
-    static let webrtc = QualityConfiguration(
-        selectedOption: .very_aggressive,
-        options: [
-            .normal,
-            .low_bitrate,
-            .aggressive,
-            .very_aggressive,
-        ]
-    )
-
-    static let silero = QualityConfiguration(
-        selectedOption: .normal,
-        options: [
-            .normal,
-            .aggressive,
-            .very_aggressive,
-        ]
-    )
-
-    static let yamnet = QualityConfiguration(
-        selectedOption: .normal,
-        options: [
-            .normal,
-            .aggressive,
-            .very_aggressive,
-        ]
-    )
-}

package/ios/Packages/VAD/Data/FrameSize.swift

@@ -1,39 +0,0 @@
-//
-//  FrameSize.swift
-//  ios-vad
-//
-//  Created by baochuquan on 2024/11/9.
-//
-
-import Foundation
-
-enum FrameSize: Int, CaseIterable {
-    case size_80 = 80
-    case size_160 = 160
-    case size_240 = 240
-    case size_320 = 320
-    case size_256 = 256
-    case size_480 = 480
-    case size_512 = 512
-    case size_640 = 640
-    case size_768 = 768
-    case size_960 = 960
-    case size_1024 = 1024
-    case size_1440 = 1440
-    case size_1536 = 1536
-    case size_15600 = 15600
-
-    var desc: String {
-        "FRAME_SIZE_\(rawValue)"
-    }
-
-    static let webrtc_8k: [FrameSize] = [.size_80, .size_160, .size_240]
-    static let webrtc_16k: [FrameSize] = [.size_160, .size_320, .size_480]
-    static let webrtc_32k: [FrameSize] = [.size_320, .size_640, .size_960]
-    static let webrtc_48k: [FrameSize] = [.size_480, .size_960, .size_1440]
-
-    static let silero_8k: [FrameSize] = [.size_256, .size_512, .size_768]
-    static let silero_16k: [FrameSize] = [.size_512, .size_1024, .size_1536]
-
-    static let yamnet_15k: [FrameSize] = [.size_15600]
-}

package/ios/Packages/VAD/Data/Record.swift

@@ -1,13 +0,0 @@
-//
-//  Record.swift
-//  ios-vad
-//
-//  Created by baochuquan on 2024/11/9.
-//
-
-import Foundation
-
-enum VADRecord: Hashable {
-    case idle
-    case work
-}

package/ios/Packages/VAD/Data/Result.swift

@@ -1,22 +0,0 @@
-//
-//  Result.swift
-//  ios-vad
-//
-//  Created by baochuquan on 2024/11/9.
-//
-
-import Foundation
-
-enum VADResult {
-    case idle
-    case speech
-    case silence
-
-    var desc: String {
-        switch self {
-        case .idle: return "Press button to start VAD!"
-        case .speech: return "Speeching"
-        case .silence: return "Silence"
-        }
-    }
-}

package/ios/Packages/VAD/Data/SampleRate.swift

@@ -1,48 +0,0 @@
-//
-//  SampleRate.swift
-//  ios-vad
-//
-//  Created by baochuquan on 2024/11/9.
-//
-
-import Foundation
-
-enum SampleRate: Int, CaseIterable {
-    case rate_8k = 8000
-    case rate_15K = 15600
-    case rate_16k = 16000
-    case rate_32k = 32000
-    case rate_48k = 48000
-
-    var desc: String {
-        "SAMPLE_RATE_\(rawValue)"
-    }
-
-    static let webrtc: [SampleRate] = [.rate_8k, .rate_16k, .rate_32k, .rate_48k]
-    static let silero: [SampleRate] = [.rate_8k, .rate_16k]
-    static let yamnet: [SampleRate] = [.rate_15K]
-
-    func frameSizeOptions(type: VADType) -> [FrameSize] {
-        switch type {
-        case .webrtc:
-            switch self {
-            case .rate_8k: return FrameSize.webrtc_8k
-            case .rate_16k: return FrameSize.webrtc_16k
-            case .rate_32k: return FrameSize.webrtc_32k
-            case .rate_48k: return FrameSize.webrtc_48k
-            default: return []
-            }
-        case .silero:
-            switch self {
-            case .rate_8k: return FrameSize.silero_8k
-            case .rate_16k: return FrameSize.silero_16k
-            default: return []
-            }
-        case .yamnet:
-            switch self {
-            case .rate_15K: return FrameSize.yamnet_15k
-            default: return []
-            }
-        }
-    }
-}