@mentra/bluetooth-sdk 0.1.9 → 0.1.11

package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt

@@ -27,6 +27,9 @@ import com.mentra.bluetoothsdk.utils.MicMap
 import com.mentra.bluetoothsdk.utils.MicTypes
 import com.mentra.lc3Lib.Lc3Cpp
 import com.mentra.bluetoothsdk.stt.SherpaOnnxTranscriber
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.launch
 import java.text.SimpleDateFormat
 import java.util.*
 import java.util.concurrent.CountDownLatch
@@ -132,6 +135,13 @@ class DeviceManager {
         get() = DeviceStore.store.get("bluetooth", "power_saving_mode") as? Boolean ?: false
         set(value) = DeviceStore.apply("bluetooth", "power_saving_mode", value)
 
+    // Phone-side VAD gating switch. Default is OFF (VAD runs) so the
+    // coordinator can drive per-utterance offline/online STT switching from
+    // `vad_status` events. Set to `true` only as an emergency kill-switch.
+    private var bypassVad: Boolean
+        get() = DeviceStore.store.get("bluetooth", "bypass_vad") as? Boolean ?: false
+        set(value) = DeviceStore.apply("bluetooth", "bypass_vad", value)
+
     private var offlineCaptionsRunning: Boolean
         get() = DeviceStore.store.get("bluetooth", "offline_captions_running") as? Boolean ?: false
         set(value) = DeviceStore.apply("bluetooth", "offline_captions_running", value)
@@ -192,6 +202,7 @@ class DeviceManager {
     // Guard against duplicate ready callbacks firing back-to-back.
     private var lastReadyHandledAtMs: Long = 0L
     private var lastReadyHandledKey: String = ""
+    private var lastSystemTimeSyncConnectionKey: String = ""
 
     private var systemMicUnavailable: Boolean
         get() = DeviceStore.store.get("bluetooth", "systemMicUnavailable") as? Boolean ?: false
@@ -237,6 +248,11 @@ class DeviceManager {
     private var lastLc3Event: Long? = null
     private var micReinitRunnable: Runnable? = null
 
+    // VAD
+    private val vadBuffer = mutableListOf<ByteArray>()
+    private var isSpeaking = false
+    private var vadPolicy: com.mentra.bluetoothsdk.stt.VadGateSpeechPolicy? = null
+
     // STT
     private var transcriber: SherpaOnnxTranscriber? = null
 
@@ -272,6 +288,9 @@ class DeviceManager {
         } catch (e: Exception) {
             Bridge.log("Failed to initialize SherpaOnnxTranscriber: ${e.message}")
             transcriber = null
+        } catch (e: LinkageError) {
+            Bridge.log("Failed to initialize SherpaOnnxTranscriber: ${e.message}")
+            transcriber = null
         }
 
         // Initialize LC3 encoder/decoder for unified audio encoding
@@ -286,6 +305,30 @@ class DeviceManager {
             lc3DecoderPtr = 0
         }
 
+        // Initialize phone-side Silero VAD. Used by handlePcm to gate audio
+        // fan-out and to drive per-utterance offline/online STT switching
+        // (see LocalSttFallbackCoordinator on the JS side). If a previous
+        // policy somehow exists (re-init on hot reload), stop it first to
+        // release the ONNX session.
+        try {
+            vadPolicy?.stop()
+            val ctx = Bridge.getContext()
+            val policy = com.mentra.bluetoothsdk.stt.VadGateSpeechPolicy(ctx)
+            policy.init(blockSizeSamples = 512)
+            policy.onSpeechStateChanged = { speaking ->
+                isSpeaking = speaking
+                Bridge.sendVoiceActivityDetectionStatus(speaking)
+            }
+            vadPolicy = policy
+            Bridge.log("VadGateSpeechPolicy initialized")
+        } catch (e: Exception) {
+            Bridge.log("Failed to initialize VadGateSpeechPolicy: ${e.message}")
+            vadPolicy = null
+        } catch (e: LinkageError) {
+            Bridge.log("Failed to initialize VadGateSpeechPolicy: ${e.message}")
+            vadPolicy = null
+        }
+
         // Mic reinit check every 10 seconds
         val micReinitR =
                 object : Runnable {
@@ -590,7 +633,12 @@ class DeviceManager {
             var layoutType: String,
             var text: String,
             var data: String?,
-            var animationData: Map<String, Any>?
+            var animationData: Map<String, Any>?,
+            // Optional bitmap_view container position/size (used by G2; ignored by others)
+            var bmpX: Int? = null,
+            var bmpY: Int? = null,
+            var bmpWidth: Int? = null,
+            var bmpHeight: Int? = null
     )
     // MARK: - End Unique
 
@@ -652,9 +700,17 @@ class DeviceManager {
     }
 
     fun handlePcm(pcmData: ByteArray) {
+        // Audio always flows. The previous phone-side Silero VAD gate was a
+        // bandwidth-saver that ate transcripts when the mic delivered frames
+        // not aligned to 512 samples (the case for Android AudioRecord on the
+        // phone internal mic) and was never wired up correctly anyway —
+        // `bypass_vad_for_debugging` was dead, cloud-side `bypass_vad` was
+        // the only knob, and the policy double-VAD'd what the cloud already
+        // VADs server-side. VadGateSpeechPolicy is kept around because
+        // hardware-side VAD events from the glasses route through the same
+        // class to fire `vad_status` (a separate signal the cloud SDK
+        // surfaces as `session.audio.isSpeaking`).
         handleSendingPcm(pcmData)
-
-        // Send PCM to local transcriber (always needs raw PCM)
         if (shouldSendTranscript || offlineCaptionsRunning || localSttFallbackActive) {
             transcriber?.acceptAudio(pcmData)
         }
@@ -831,7 +887,15 @@ class DeviceManager {
                 sgc?.sendTextWall("${currentViewState.title}\n\n${currentViewState.text}")
             }
             "bitmap_view" -> {
-                currentViewState.data?.let { data -> sgc?.displayBitmap(data) }
+                currentViewState.data?.let { data ->
+                    sgc?.displayBitmap(
+                            data,
+                            currentViewState.bmpX,
+                            currentViewState.bmpY,
+                            currentViewState.bmpWidth,
+                            currentViewState.bmpHeight
+                    )
+                }
             }
             "clear_view" -> sgc?.clearDisplay()
             else -> Bridge.log("MAN: UNHANDLED LAYOUT_TYPE ${currentViewState.layoutType}")
@@ -974,6 +1038,7 @@ class DeviceManager {
             Bridge.log("MAN: Cleaning up previous sgc type: ${sgc?.type}")
             sgc?.cleanup()
             sgc = null
+            lastSystemTimeSyncConnectionKey = ""
         }
 
         if (sgc != null) {
@@ -992,10 +1057,9 @@ class DeviceManager {
         } else if (wearable.contains(DeviceTypes.NEX)) {
             sgc = MentraNex()
         } else if (wearable.contains(DeviceTypes.MACH1)) {
-            sgc = Mach1()
+            sgc = createOptionalMach1Sgc(DeviceTypes.MACH1)
         } else if (wearable.contains(DeviceTypes.Z100)) {
-            sgc = Mach1() // Z100 uses same hardware/SDK as Mach1
-            sgc?.type = DeviceTypes.Z100 // Override type to Z100
+            sgc = createOptionalMach1Sgc(DeviceTypes.Z100)
         } else if (wearable.contains(DeviceTypes.FRAME)) {
             // sgc = FrameManager()
         }
@@ -1003,6 +1067,19 @@ class DeviceManager {
         DeviceStore.apply("glasses", "deviceModel", sgc?.type ?: "")
     }
 
+    private fun createOptionalMach1Sgc(deviceType: String): SGCManager? {
+        return try {
+            Mach1().also { sgc ->
+                if (deviceType == DeviceTypes.Z100) {
+                    sgc.type = DeviceTypes.Z100
+                }
+            }
+        } catch (e: LinkageError) {
+            Bridge.log("Failed to initialize $deviceType support: ${e.message}")
+            null
+        }
+    }
+
     fun initController(controllerType: String) {
         Bridge.log("MAN: Initializing controller: $controllerType")
         if (controller != null && controller?.type != controllerType) {
@@ -1048,8 +1125,24 @@ class DeviceManager {
         defaultWearable = sgc?.type ?: ""
         searching = false
 
-        // Apply dashboard position before any boot text so content doesn't jump.
-        sgc?.setDashboardPosition(dashboardHeight, dashboardDepth)
+        syncSystemTimeOnceForConnection(readyKey)
+
+        // re-apply display height/depth after reconnection
+        mainHandler.postDelayed(
+                {
+                    val h =
+                            (DeviceStore.store.get("bluetooth", "dashboard_height") as? Number)
+                                    ?.toInt()
+                                    ?: 4
+                    val rawDepth =
+                            (DeviceStore.store.get("bluetooth", "dashboard_depth") as? Number)
+                                    ?.toInt()
+                                    ?: dashboardDepth // canonical default (2), not 1
+                    val d = rawDepth.coerceIn(1, 4)
+                    sgc?.setDashboardPosition(h, d)
+                },
+                2000
+        )
 
         // Show welcome message on first connect for all display glasses
         if (shouldSendBootingMessage) {
@@ -1084,6 +1177,21 @@ class DeviceManager {
         Bridge.saveSetting("device_address", deviceAddress)
     }
 
+    private fun syncSystemTimeOnceForConnection(connectionKey: String) {
+        val activeSgc = sgc ?: return
+        if (activeSgc.type.contains(DeviceTypes.SIMULATED)) {
+            return
+        }
+        if (connectionKey == lastSystemTimeSyncConnectionKey) {
+            return
+        }
+
+        lastSystemTimeSyncConnectionKey = connectionKey
+        val timestampMs = System.currentTimeMillis()
+        Bridge.log("MAN: Syncing glasses system time once for connection: $timestampMs")
+        activeSgc.sendSetSystemTime(timestampMs)
+    }
+
     private fun handleG1Ready() {
         // G1-specific setup (if any needed in the future)
         // Note: G1-specific settings like silent mode, battery status,
@@ -1096,8 +1204,9 @@ class DeviceManager {
 
     fun handleDeviceDisconnected() {
         Bridge.log("MAN: Device disconnected")
+        lastSystemTimeSyncConnectionKey = ""
         DeviceStore.apply("glasses", "headUp", false)
-        DeviceStore.apply("glasses", "voiceActivityDetectionEnabled", true)
+        DeviceStore.apply("glasses", "voiceActivityDetectionEnabled", BluetoothSdkDefaults.VOICE_ACTIVITY_DETECTION_ENABLED)
     }
 
     fun handleControllerReady() {
@@ -1154,7 +1263,26 @@ class DeviceManager {
         val title = parsePlaceholders(layout.getString("title", " "))
         val data = layout["data"] as? String
 
-        var newViewState = ViewState(topText, bottomText, title, layoutType ?: "", text, data, null)
+        // Optional bitmap_view container position/size (forwarded to the SGC; used by G2).
+        val bmpX = (layout["x"] as? Number)?.toInt()
+        val bmpY = (layout["y"] as? Number)?.toInt()
+        val bmpWidth = (layout["width"] as? Number)?.toInt()
+        val bmpHeight = (layout["height"] as? Number)?.toInt()
+
+        var newViewState =
+                ViewState(
+                        topText,
+                        bottomText,
+                        title,
+                        layoutType ?: "",
+                        text,
+                        data,
+                        null,
+                        bmpX,
+                        bmpY,
+                        bmpWidth,
+                        bmpHeight
+                )
 
         val currentState = viewStates[stateIndex]
 
@@ -1176,6 +1304,12 @@ class DeviceManager {
         sgc?.showDashboard()
     }
 
+    fun showNotificationsPanel() {
+        CoroutineScope(Dispatchers.Main).launch {
+            sgc?.showNotificationsPanel()
+        }
+    }
+
     fun ping() {
         sgc?.ping()
     }
@@ -1232,6 +1366,11 @@ class DeviceManager {
         sgc?.sendHotspotState(enabled)
     }
 
+    fun setSystemTime(timestampMs: Long) {
+        Bridge.log("MAN: Setting glasses system time: $timestampMs")
+        sgc?.sendSetSystemTime(timestampMs)
+    }
+
     fun queryGalleryStatus() {
         Bridge.log("MAN: Querying gallery status from glasses")
         sgc?.queryGalleryStatus()
@@ -1251,6 +1390,41 @@ class DeviceManager {
         (sgc as? MentraLive)?.sendOtaQueryStatus()
     }
 
+    fun sendGalleryMode(requestId: String, enabled: Boolean) {
+        val live = sgc as? MentraLive ?: throw IllegalStateException("unsupported_device")
+        live.sendGalleryMode(requestId, enabled)
+    }
+
+    fun sendButtonPhotoSettings(requestId: String, size: String) {
+        val live = sgc as? MentraLive ?: throw IllegalStateException("unsupported_device")
+        live.sendButtonPhotoSettings(requestId, size)
+    }
+
+    fun sendButtonVideoRecordingSettings(requestId: String, width: Int, height: Int, fps: Int) {
+        val live = sgc as? MentraLive ?: throw IllegalStateException("unsupported_device")
+        live.sendButtonVideoRecordingSettings(requestId, width, height, fps)
+    }
+
+    fun sendButtonCameraLedSetting(requestId: String, enabled: Boolean) {
+        val live = sgc as? MentraLive ?: throw IllegalStateException("unsupported_device")
+        live.sendButtonCameraLedSetting(requestId, enabled)
+    }
+
+    fun sendButtonMaxRecordingTime(requestId: String, minutes: Int) {
+        val live = sgc as? MentraLive ?: throw IllegalStateException("unsupported_device")
+        live.sendButtonMaxRecordingTime(requestId, minutes)
+    }
+
+    fun sendCameraFovSetting(requestId: String, fov: Int, roiPosition: Int) {
+        val live = sgc as? MentraLive ?: throw IllegalStateException("unsupported_device")
+        live.sendCameraFovSetting(requestId, fov, roiPosition)
+    }
+
+    fun retryOtaVersionCheck() {
+        Bridge.log("MAN: ⏰ Retrying glasses OTA version check after clock sync")
+        (sgc as? MentraLive)?.sendOtaRetryVersionCheck()
+    }
+
     /**
      * Read glasses media step volume (0–15) via K900 on Mentra Live only. Blocks until response,
      * error, or timeout (used from JS AsyncFunction on a worker thread).
@@ -1324,11 +1498,19 @@ class DeviceManager {
         sgc?.sendReboot()
     }
 
-    fun startVideoRecording(requestId: String, save: Boolean, sound: Boolean) {
+    fun startVideoRecording(
+            requestId: String,
+            save: Boolean,
+            sound: Boolean,
+            width: Int = 0,
+            height: Int = 0,
+            fps: Int = 0,
+    ) {
         Bridge.log(
-                "MAN: onStartVideoRecording: requestId=$requestId, save=$save, flash=true, sound=$sound"
+                "MAN: onStartVideoRecording: requestId=$requestId, save=$save, flash=true, sound=$sound, " +
+                        "resolution=${width}x${height}@${fps}fps"
         )
-        sgc?.startVideoRecording(requestId, save, true, sound)
+        sgc?.startVideoRecording(requestId, save, true, sound, width, height, fps)
     }
 
     fun stopVideoRecording(requestId: String) {
@@ -1339,7 +1521,15 @@ class DeviceManager {
     fun setMicState() {
         val willSendPcm = shouldSendPcm || shouldSendLc3
         val willSendTranscript = shouldSendTranscript || offlineCaptionsRunning || localSttFallbackActive
-        micEnabled = willSendPcm || willSendTranscript
+        val nextEnabled = willSendPcm || willSendTranscript
+        // Tell VAD when the mic is shutting down so it doesn't get stuck in
+        // a stale "speaking" state and keep emitting vad_status=true after
+        // audio stops flowing.
+        if (micEnabled && !nextEnabled) {
+            vadPolicy?.microphoneStateChanged(false)
+        }
+        micEnabled = nextEnabled
+        vadBuffer.clear()
         updateMicState()
     }
 
@@ -1351,6 +1541,7 @@ class DeviceManager {
             authToken: String?,
             compress: String,
             flash: Boolean,
+            save: Boolean,
             sound: Boolean,
             exposureTimeNs: Double? = null,
             iso: Int? = null,
@@ -1364,7 +1555,7 @@ class DeviceManager {
                 }
         val manualIso = if (exposureNs != null) iso?.takeIf { it > 0 } else null
         Bridge.log(
-                "MAN: PHOTO PIPELINE [4/6] DeviceManager.requestPhoto requestId=$requestId appId=$appId size=$size compress=$compress flash=$flash sound=$sound exposureTimeNs=$exposureNs iso=${manualIso ?: "auto"} sgc=${sgc?.javaClass?.simpleName ?: "null"}"
+                "MAN: PHOTO PIPELINE [4/6] DeviceManager.requestPhoto requestId=$requestId appId=$appId size=$size compress=$compress flash=$flash save=$save sound=$sound exposureTimeNs=$exposureNs iso=${manualIso ?: "auto"} sgc=${sgc?.javaClass?.simpleName ?: "null"}"
         )
         val activeSgc = sgc
         if (activeSgc == null) {
@@ -1373,7 +1564,7 @@ class DeviceManager {
             )
             return
         }
-        activeSgc.requestPhoto(requestId, appId, size, webhookUrl, authToken, compress, flash, sound, exposureNs, manualIso)
+        activeSgc.requestPhoto(requestId, appId, size, webhookUrl, authToken, compress, flash, save, sound, exposureNs, manualIso)
     }
 
     fun rgbLedControl(
@@ -1495,6 +1686,7 @@ class DeviceManager {
         sgc?.clearDisplay()
         sgc?.disconnect()
         sgc = null // Clear the SGC reference after disconnect
+        lastSystemTimeSyncConnectionKey = ""
         searching = false
         micEnabled = false
         updateMicState()
@@ -1503,7 +1695,7 @@ class DeviceManager {
         DeviceStore.apply("glasses", "deviceModel", "")
         DeviceStore.apply("glasses", "fullyBooted", false)
         DeviceStore.apply("glasses", "connected", false)
-        DeviceStore.apply("glasses", "voiceActivityDetectionEnabled", true)
+        DeviceStore.apply("glasses", "voiceActivityDetectionEnabled", BluetoothSdkDefaults.VOICE_ACTIVITY_DETECTION_ENABLED)
         // disconnect the controller as well:
         searchingController = false
         DeviceStore.apply("glasses", "controllerConnected", false)

package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt

@@ -66,7 +66,7 @@ object DeviceStore {
         store.set("glasses", "deviceModel", "")
         store.set("glasses", "firmwareVersion", "")
         store.set("glasses", "micEnabled", false)
-        store.set("glasses", "voiceActivityDetectionEnabled", true)
+        store.set("glasses", "voiceActivityDetectionEnabled", BluetoothSdkDefaults.VOICE_ACTIVITY_DETECTION_ENABLED)
         store.set("glasses", "bluetoothClassicConnected", false)
         store.set("glasses", "caseRemoved", true)
         store.set("glasses", "caseOpen", true)
@@ -121,9 +121,10 @@ object DeviceStore {
         store.set("bluetooth", "dashboard_height", 4)
         store.set("bluetooth", "dashboard_depth", 2)
         store.set("bluetooth", "head_up_angle", 30)
+        store.set("bluetooth", "imu_enabled", false)
         store.set("bluetooth", "contextual_dashboard", true)
         store.set("bluetooth", "gallery_mode", true)
-        store.set("bluetooth", "voice_activity_detection_enabled", true)
+        store.set("bluetooth", "voice_activity_detection_enabled", BluetoothSdkDefaults.VOICE_ACTIVITY_DETECTION_ENABLED)
         store.set("bluetooth", "screen_disabled", false)
         store.set("bluetooth", "button_photo_size", "medium")
         store.set("bluetooth", "button_camera_led", true)
@@ -140,6 +141,9 @@ object DeviceStore {
         store.set("bluetooth", "should_send_lc3", false)
         store.set("bluetooth", "should_send_transcript", false)
         store.set("bluetooth", "use_native_dashboard", false)
+        // Mentra Nex feature flags (off by default; toggled from Nex Developer Settings):
+        store.set("bluetooth", "nex_chinese_captions", false)
+        store.set("bluetooth", "nex_audio_playback", false)
     }
 
     fun get(category: String, key: String): Any? {
@@ -235,18 +239,42 @@ object DeviceStore {
                     DeviceManager.getInstance().sgc?.setHeadUpAngle(angle)
                 }
             }
+            "bluetooth" to "imu_enabled" -> {
+                (value as? Boolean)?.let { enabled ->
+                    CoroutineScope(Dispatchers.Main).launch {
+                        DeviceManager.getInstance().sgc?.setImuEnabled(enabled)
+                    }
+                }
+            }
             "bluetooth" to "menu_apps" -> {
                 @Suppress("UNCHECKED_CAST")
                 (value as? List<Map<String, Any>>)?.let { items ->
                     DeviceManager.getInstance().sgc?.setDashboardMenu(items)
                 }
             }
+            // `core` category is normalized to `bluetooth` before reaching this switch.
+            "bluetooth" to "calendar_events" -> {
+                @Suppress("UNCHECKED_CAST")
+                (value as? List<Map<String, Any>>)?.let { events ->
+                    DeviceManager.getInstance().sgc?.sendCalendarEvents(events)
+                }
+            }
+            "bluetooth" to "metric_system",
+            "bluetooth" to "twelve_hour_time" -> {
+                DeviceManager.getInstance().sgc?.sendDashboardDisplaySettings()
+            }
             "bluetooth" to "gallery_mode" -> {
                 DeviceManager.getInstance().sgc?.sendGalleryMode()
             }
             "bluetooth" to "voice_activity_detection_enabled" -> {
                 DeviceManager.getInstance().sgc?.sendVoiceActivityDetectionSetting()
             }
+            "bluetooth" to "nex_audio_playback" -> {
+                (value as? Boolean)?.let { enabled ->
+                    Bridge.log("DeviceStore: nex_audio_playback changed to $enabled")
+                    DeviceManager.getInstance().sgc?.applyNexAudioPlaybackSetting()
+                }
+            }
             "bluetooth" to "screen_disabled" -> {
                 (value as? Boolean)?.let { disabled ->
                     if (disabled) {