@mentra/bluetooth-sdk 0.1.19 → 0.1.21-beta.0

package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraNex.kt

@@ -334,7 +334,7 @@ class MentraNex : SGCManager() {
     }
 
     // Network Management
-    override fun requestWifiScan () { Bridge.log("Nex: requestWifiScan operation not supported") }
+    override fun requestWifiScan (scanId: String?) { Bridge.log("Nex: requestWifiScan operation not supported") }
     override fun sendWifiCredentials(ssid: String, password: String) { Bridge.log("Nex: sendWifiCredentials operation not supported") }
     override fun forgetWifiNetwork(ssid: String) { }
     override fun sendHotspotState(enabled: Boolean) { Bridge.log("Nex: sendHotspotState operation not supported") }

package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Nimo.kt

@@ -1427,7 +1427,7 @@ class Nimo : SGCManager() {
 
     // ---------- SGCManager: Network (no WiFi) ----------
 
-    override fun requestWifiScan() {}
+    override fun requestWifiScan(scanId: String?) {}
     override fun sendWifiCredentials(ssid: String, password: String) {}
     override fun forgetWifiNetwork(ssid: String) {}
     override fun sendHotspotState(enabled: Boolean) {}

package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt

@@ -46,6 +46,8 @@ abstract class SGCManager {
 
     // Audio Control
     abstract fun setMicEnabled(enabled: Boolean)
+    open val isMicSuspendedForAudio: Boolean
+        get() = false
     abstract fun sortMicRanking(list: MutableList<String>): MutableList<String>
 
     // Camera & Media
@@ -296,12 +298,12 @@ abstract class SGCManager {
     abstract fun dbg2()
 
     // Network Management
-    abstract fun requestWifiScan()
+    abstract fun requestWifiScan(scanId: String?)
     abstract fun sendWifiCredentials(ssid: String, password: String)
     abstract fun forgetWifiNetwork(ssid: String)
     abstract fun sendHotspotState(enabled: Boolean)
 
-    /** Set glasses system clock (Mentra Live only; no-op on other devices). */
+    /** Set glasses system clock (Mentra Live and G2; no-op on other devices). */
     open fun sendSetSystemTime(timestampMs: Long) {
         Bridge.log("SGC: sendSetSystemTime not supported on $type")
     }

package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Simulated.kt

@@ -188,7 +188,7 @@ class Simulated : SGCManager() {
     override fun dbg2() {}
 
     // Network Management
-    override fun requestWifiScan() {
+    override fun requestWifiScan(scanId: String?) {
         Bridge.log("requestWifiScan")
     }
 

package/android/src/main/java/com/mentra/bluetoothsdk/status/WifiHotspotStatus.kt

@@ -66,15 +66,31 @@ sealed interface WifiStatus {
     }
 }
 
-data class WifiStatusEvent(
+data class WifiStatusEvent @JvmOverloads constructor(
     val status: WifiStatus,
+    /**
+     * Glasses-reported provisioning failure reason when THIS event is the verdict of a
+     * failed connect attempt; null for routine link-state updates. An attempt property,
+     * not a link property — which is why it lives on the event and not on [WifiStatus]:
+     * "connect_timeout" arrives with a disconnected status (never associated), while
+     * "connected_to_other_network" arrives with a *connected* status — the attempt
+     * failed and Android's auto-join left the glasses on (or returned them to) a
+     * different SSID than requested, so the link is genuinely up while the request
+     * genuinely failed. Sent by ASG client builds that include the WiFi error
+     * surfacing (v40+); older glasses never set it.
+     */
+    val error: String? = null,
 ) {
-    internal constructor(values: Map<String, Any>) : this(WifiStatus.fromMap(values) ?: WifiStatus.Disconnected)
+    internal constructor(values: Map<String, Any>) : this(
+        WifiStatus.fromMap(values) ?: WifiStatus.Disconnected,
+        stringValue(values, "error")?.takeIf { it.isNotEmpty() },
+    )
     internal constructor(connected: Boolean, ssid: String?, localIp: String?) : this(
         WifiStatus.fromStoreFields(connected, ssid, localIp) ?: WifiStatus.Disconnected
     )
 
-    val values: Map<String, Any> get() = status.toEventMap()
+    val values: Map<String, Any>
+        get() = status.toEventMap() + (error?.let { mapOf("error" to it) } ?: emptyMap())
 }
 
 sealed interface HotspotStatus {

package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt

@@ -436,12 +436,25 @@ sealed interface StreamStatus {
 data class StreamStatusEvent(
     val status: StreamStatus,
 ) {
-    constructor(values: Map<String, Any>) : this(StreamStatus.fromMap(values))
+    // True when the glasses will retry the failed publisher themselves
+    // (emitting side lands in PR #3488); absent on older firmware and on
+    // events not parsed from a glasses status map. Carried here instead of
+    // on StreamStatus.Error so the public Error shape stays unchanged.
+    var willRetry: Boolean? = null
+        private set
+
+    constructor(values: Map<String, Any>) : this(StreamStatus.fromMap(values)) {
+        willRetry = boolValue(values, "willRetry")
+    }
 
     val state: StreamState get() = status.state
     val streamId: String? get() = status.streamId
     val resolvedConfig: StreamResolvedConfig? get() = status.resolvedConfig
-    val values: Map<String, Any> get() = status.toEventMap()
+    val values: Map<String, Any>
+        get() = buildMap {
+            putAll(status.toEventMap())
+            willRetry?.let { put("willRetry", it) }
+        }
 }
 
 data class KeepAliveAckEvent(

package/android/src/main/java/com/mentra/bluetoothsdk/tts/TTSTools.kt

@@ -9,6 +9,9 @@ import com.k2fsa.sherpa.onnx.OfflineTtsModelConfig
 import com.k2fsa.sherpa.onnx.OfflineTtsSupertonicModelConfig
 import com.mentra.bluetoothsdk.Bridge
 import java.io.File
+import java.util.concurrent.Executors
+import java.util.concurrent.ScheduledFuture
+import java.util.concurrent.TimeUnit
 
 /**
  * Utilities for offline Sherpa-ONNX Supertonic 3 TTS model management.
@@ -31,6 +34,21 @@ object TTSTools {
     private const val FILE_UNICODE_INDEXER = "unicode_indexer.bin"
     private const val FILE_VOICE_STYLE = "voice.bin"
 
+    /**
+     * Keep the native model warm briefly after use to reduce repeated TTS latency while
+     * still reclaiming its memory when speech has been idle for a while.
+     */
+    private const val MODEL_IDLE_TIMEOUT_MINUTES = 3L
+    private val modelLock = Any()
+    private val teardownExecutor =
+            Executors.newSingleThreadScheduledExecutor { runnable ->
+                Thread(runnable, "MentraTtsIdleTeardown").apply { isDaemon = true }
+            }
+    private var cachedTts: OfflineTts? = null
+    private var cachedModelPath: String? = null
+    private var teardownFuture: ScheduledFuture<*>? = null
+    private var teardownGeneration = 0L
+
     private val REQUIRED_FILES = listOf(
             FILE_DURATION_PREDICTOR,
             FILE_TEXT_ENCODER,
@@ -42,6 +60,11 @@ object TTSTools {
     )
 
     fun setTtsModelDetails(context: Context, path: String, languageCode: String) {
+        synchronized(modelLock) {
+            if (cachedModelPath != null && cachedModelPath != path) {
+                releaseCachedModelLocked("model path changed")
+            }
+        }
         val prefs = getPrefs(context)
         prefs.edit().apply {
             putString(KEY_TTS_MODEL_PATH, path)
@@ -118,59 +141,104 @@ object TTSTools {
             return false
         }
 
-        val modelDir = File(modelPath)
         val outputFile = File(outputPath)
         outputFile.parentFile?.mkdirs()
 
-        var tts: OfflineTts? = null
-        return try {
-            val supertonic = OfflineTtsSupertonicModelConfig(
-                    durationPredictor = File(modelDir, FILE_DURATION_PREDICTOR).absolutePath,
-                    textEncoder = File(modelDir, FILE_TEXT_ENCODER).absolutePath,
-                    vectorEstimator = File(modelDir, FILE_VECTOR_ESTIMATOR).absolutePath,
-                    vocoder = File(modelDir, FILE_VOCODER).absolutePath,
-                    ttsJson = File(modelDir, FILE_TTS_JSON).absolutePath,
-                    unicodeIndexer = File(modelDir, FILE_UNICODE_INDEXER).absolutePath,
-                    voiceStyle = File(modelDir, FILE_VOICE_STYLE).absolutePath,
-            )
-
-            val modelConfig = OfflineTtsModelConfig(
-                    supertonic = supertonic,
-                    numThreads = 2,
-                    provider = "cpu",
-            )
-
-            val config = OfflineTtsConfig(
-                    model = modelConfig,
-                    maxNumSentences = 1,
-                    silenceScale = 0.2f,
-            )
-
-            tts = OfflineTts(null, config)
-
-            // sid out of range falls back to F1 (sid 0); clamp 0..9.
-            val sid = speakerId.coerceIn(0, tts.numSpeakers() - 1)
-            val clampedSpeed = speed.coerceIn(0.5f, 2.0f)
-            val lang = languageTagToSupertonicLang(languageTag)
-            val genConfig = GenerationConfig(
-                    sid = sid,
-                    speed = clampedSpeed,
-                    extra = mapOf("lang" to lang),
-            )
-            val audio = tts.generateWithConfig(text = text, config = genConfig)
-            val saved = audio.save(outputFile.absolutePath)
-            Bridge.log("TTS generated ${outputFile.absolutePath}: saved=$saved sid=$sid lang=$lang")
-            saved
-        } catch (e: Exception) {
-            Bridge.log("TTS_ERROR: ${e.javaClass.simpleName}: ${e.message}")
-            e.printStackTrace()
-            false
-        } finally {
+        return synchronized(modelLock) {
             try {
-                tts?.release()
+                val tts = getOrCreateModelLocked(modelPath)
+
+                // sid out of range falls back to F1 (sid 0); clamp 0..9.
+                val sid = speakerId.coerceIn(0, tts.numSpeakers() - 1)
+                val clampedSpeed = speed.coerceIn(0.5f, 2.0f)
+                val lang = languageTagToSupertonicLang(languageTag)
+                val genConfig = GenerationConfig(
+                        sid = sid,
+                        speed = clampedSpeed,
+                        numSteps = 5,
+                        extra = mapOf("lang" to lang),
+                )
+                val audio = tts.generateWithConfig(text = text, config = genConfig)
+                val saved = audio.save(outputFile.absolutePath)
+                Bridge.log("TTS generated ${outputFile.absolutePath}: saved=$saved sid=$sid lang=$lang")
+                saved
             } catch (e: Exception) {
-                Bridge.log("TTS release failed: ${e.message}")
+                Bridge.log("TTS_ERROR: ${e.javaClass.simpleName}: ${e.message}")
+                e.printStackTrace()
+                false
+            } finally {
+                scheduleIdleTeardownLocked()
+            }
+        }
+    }
+
+    private fun getOrCreateModelLocked(modelPath: String): OfflineTts {
+        cachedTts?.let { cached ->
+            if (cachedModelPath == modelPath) {
+                Bridge.log("TTS reusing cached model")
+                return cached
             }
+            releaseCachedModelLocked("model path changed")
+        }
+
+        val modelDir = File(modelPath)
+        val supertonic = OfflineTtsSupertonicModelConfig(
+                durationPredictor = File(modelDir, FILE_DURATION_PREDICTOR).absolutePath,
+                textEncoder = File(modelDir, FILE_TEXT_ENCODER).absolutePath,
+                vectorEstimator = File(modelDir, FILE_VECTOR_ESTIMATOR).absolutePath,
+                vocoder = File(modelDir, FILE_VOCODER).absolutePath,
+                ttsJson = File(modelDir, FILE_TTS_JSON).absolutePath,
+                unicodeIndexer = File(modelDir, FILE_UNICODE_INDEXER).absolutePath,
+                voiceStyle = File(modelDir, FILE_VOICE_STYLE).absolutePath,
+        )
+        val modelConfig = OfflineTtsModelConfig(
+                supertonic = supertonic,
+                numThreads = 8,
+                provider = "cpu",
+        )
+        val config = OfflineTtsConfig(
+                model = modelConfig,
+                maxNumSentences = 1,
+                silenceScale = 0.2f,
+        )
+        return OfflineTts(null, config).also {
+            cachedTts = it
+            cachedModelPath = modelPath
+            Bridge.log("TTS loaded model; keeping it warm for $MODEL_IDLE_TIMEOUT_MINUTES minutes after use")
+        }
+    }
+
+    private fun scheduleIdleTeardownLocked() {
+        if (cachedTts == null) return
+        teardownFuture?.cancel(false)
+        val generation = ++teardownGeneration
+        teardownFuture =
+                teardownExecutor.schedule(
+                        {
+                            synchronized(modelLock) {
+                                if (generation == teardownGeneration) {
+                                    releaseCachedModelLocked("idle for $MODEL_IDLE_TIMEOUT_MINUTES minutes")
+                                }
+                            }
+                        },
+                        MODEL_IDLE_TIMEOUT_MINUTES,
+                        TimeUnit.MINUTES,
+                )
+    }
+
+    private fun releaseCachedModelLocked(reason: String) {
+        teardownFuture?.cancel(false)
+        teardownFuture = null
+        teardownGeneration++
+        val tts = cachedTts
+        cachedTts = null
+        cachedModelPath = null
+        if (tts == null) return
+        try {
+            tts.release()
+            Bridge.log("TTS released cached model: $reason")
+        } catch (e: Exception) {
+            Bridge.log("TTS release failed: ${e.message}")
         }
     }
 

package/android/src/main/java/com/mentra/bluetoothsdk/types/DeviceModels.kt

@@ -2,7 +2,6 @@ package com.mentra.bluetoothsdk
 
 import com.mentra.bluetoothsdk.utils.ControllerTypes
 import com.mentra.bluetoothsdk.utils.DeviceTypes
-import expo.modules.kotlin.exception.CodedException
 
 data class MentraBluetoothSdkConfig @JvmOverloads constructor(
     val deliverCallbacksOnMainThread: Boolean = true,
@@ -10,10 +9,10 @@ data class MentraBluetoothSdkConfig @JvmOverloads constructor(
 )
 
 class BluetoothSdkException(
-    code: String,
+    val code: String,
     message: String,
     cause: Throwable? = null,
-) : CodedException(code, message, cause)
+) : IllegalStateException(message, cause)
 
 enum class DeviceModel(val deviceType: String) {
     G1(DeviceTypes.G1),

package/android/src/main/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadService.java

@@ -37,10 +37,10 @@ import org.json.JSONObject;
  */
 public class BlePhotoUploadService {
     private static final String TAG = "BlePhotoUploadService";
-    // 100: the source already went through a lossy AVIF pass on the glasses, so
-    // this re-encode must not compound the loss. Phone CPU and upload bandwidth
-    // are cheap relative to what was paid to get the bytes over BLE.
-    private static final int JPEG_QUALITY = 100;
+    // AVIF sources already went through a lossy pass on the glasses; re-encode at
+    // high-but-not-max quality to avoid inflating upload size. JPEG fast-path
+    // payloads from text mode are uploaded as-is.
+    private static final int AVIF_TO_JPEG_QUALITY = 90;
 
     public interface UploadCallback {
         void onSuccess(String requestId, String responseBody);
@@ -84,6 +84,18 @@ public class BlePhotoUploadService {
      */
     @VisibleForTesting
     static byte[] convertToJpegPreservingExif(byte[] imageData) throws Exception {
+        long conversionStartMs = System.currentTimeMillis();
+        if (isJpeg(imageData)) {
+            Log.d(
+                    TAG,
+                    "BLE relay pass-through: input already JPEG ("
+                            + imageData.length
+                            + " bytes), skipping decode/re-encode in "
+                            + (System.currentTimeMillis() - conversionStartMs)
+                            + "ms");
+            return imageData;
+        }
+
         File inputFile = File.createTempFile("ble_photo_in_", guessExtension(imageData));
         File outputFile = File.createTempFile("ble_photo_out_", ".jpg");
         try {
@@ -95,19 +107,39 @@ public class BlePhotoUploadService {
 
             String imuJson = readImuJsonFromBleImage(imageData, inputFile.getAbsolutePath());
 
+            long decodeStartMs = System.currentTimeMillis();
             Bitmap bitmap = decodeImage(imageData);
+            long decodeDurationMs = System.currentTimeMillis() - decodeStartMs;
             if (bitmap == null) {
                 throw new Exception("Failed to decode image data");
             }
 
-            Log.d(TAG, "Decoded image to bitmap: " + bitmap.getWidth() + "x" + bitmap.getHeight());
-
+            Log.d(
+                    TAG,
+                    "AVIF decode complete: "
+                            + bitmap.getWidth()
+                            + "x"
+                            + bitmap.getHeight()
+                            + " in "
+                            + decodeDurationMs
+                            + "ms");
+
+            long encodeStartMs = System.currentTimeMillis();
             try (FileOutputStream fos = new FileOutputStream(outputFile)) {
-                bitmap.compress(Bitmap.CompressFormat.JPEG, JPEG_QUALITY, fos);
+                bitmap.compress(Bitmap.CompressFormat.JPEG, AVIF_TO_JPEG_QUALITY, fos);
             } finally {
                 bitmap.recycle();
             }
+            long encodeDurationMs = System.currentTimeMillis() - encodeStartMs;
+            Log.d(
+                    TAG,
+                    "AVIF->JPEG encode complete: quality="
+                            + AVIF_TO_JPEG_QUALITY
+                            + " in "
+                            + encodeDurationMs
+                            + "ms");
 
+            long metadataStartMs = System.currentTimeMillis();
             if (imuJson != null && !imuJson.isEmpty()) {
                 logImuData(imuJson);
                 // IMU EXIF is enrichment, not the payload: a write failure must not drop the
@@ -131,8 +163,27 @@ public class BlePhotoUploadService {
                                 + "). If rawHasExifMarker=true, EXIF may be present but unreadable via"
                                 + " ExifInterface on this container.");
             }
+            Log.d(
+                    TAG,
+                    "AVIF metadata handling complete in "
+                            + (System.currentTimeMillis() - metadataStartMs)
+                            + "ms");
 
-            return java.nio.file.Files.readAllBytes(outputFile.toPath());
+            byte[] jpegData = java.nio.file.Files.readAllBytes(outputFile.toPath());
+            Log.d(
+                    TAG,
+                    "Phone image conversion complete: AVIF "
+                            + imageData.length
+                            + " bytes -> JPEG "
+                            + jpegData.length
+                            + " bytes in "
+                            + (System.currentTimeMillis() - conversionStartMs)
+                            + "ms (decode="
+                            + decodeDurationMs
+                            + "ms, encode="
+                            + encodeDurationMs
+                            + "ms)");
+            return jpegData;
         } finally {
             if (!inputFile.delete()) {
                 inputFile.deleteOnExit();
@@ -254,6 +305,12 @@ public class BlePhotoUploadService {
         exif.saveAttributes();
     }
 
+    private static boolean isJpeg(byte[] imageData) {
+        return imageData.length >= 2
+                && (imageData[0] & 0xFF) == 0xFF
+                && (imageData[1] & 0xFF) == 0xD8;
+    }
+
     private static String guessExtension(byte[] imageData) {
         if (imageData.length > 12
                 && imageData[4] == 'f'

package/android/src/main/java/com/mentra/bluetoothsdk/utils/K900ProtocolUtils.java

@@ -30,7 +30,9 @@ public class K900ProtocolUtils {
     public static final byte CMD_TYPE_BINARY_MSG = BleWireProtocol.CMD_TYPE_BINARY_MSG;
 
     // File transfer constants
-    public static final int FILE_PACK_SIZE = 400; // Max data size per packet
+    // Negotiated file protocol ceiling. Legacy/GATT transfers remain smaller; 800-byte frames are
+    // accepted only after file_payload_v2 negotiation and an open CoC channel.
+    public static final int FILE_PACK_SIZE = 800;
     public static final int LENGTH_FILE_START = 2;
     public static final int LENGTH_FILE_TYPE = 1;
     public static final int LENGTH_FILE_PACKSIZE = 2;
@@ -274,9 +276,6 @@ public class K900ProtocolUtils {
             return false;
         }
 
-        Log.d("K900ProtocolUtils", "isK900ProtocolFormat: " + data[0] + " " + data[1]);
-        Log.d("K900ProtocolUtils", "CMD_START_CODE: " + CMD_START_CODE[0] + " " + CMD_START_CODE[1]);
-        
         return data[0] == CMD_START_CODE[0] && 
                data[1] == CMD_START_CODE[1];
     }
@@ -792,7 +791,9 @@ public class K900ProtocolUtils {
             Log.e("K900ProtocolUtils", "File packet checksum failed. Expected: " +
                   String.format("%02X", info.verifyCode) + ", Calculated: " +
                   String.format("%02X", calculatedVerify));
-        } else {
+        } else if (info.packIndex == 0
+                || info.packIndex % 32 == 0
+                || info.packIndex == (info.fileSize + FILE_PACK_SIZE - 1) / FILE_PACK_SIZE - 1) {
             Log.d("K900ProtocolUtils", "File packet extracted successfully: index=" + info.packIndex +
                   ", size=" + info.packSize + ", fileName=" + info.fileName);
         }