@mentra/bluetooth-sdk 0.1.20 → 0.1.21-beta.1

package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/ar99/ota/OtaGattTransport.java

@@ -0,0 +1,11 @@
+package com.mentra.bluetoothsdk.sgcs.ar99.ota;
+
+public interface OtaGattTransport {
+  boolean enableOtaNotification();
+
+  void sendOtaData(byte[] data);
+
+  void requestMtu(int mtu);
+
+  boolean isBleConnected();
+}

package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/ar99/ota/OtaProtocol.java

@@ -0,0 +1,92 @@
+package com.mentra.bluetoothsdk.sgcs.ar99.ota;
+
+import java.io.ByteArrayOutputStream;
+
+public final class OtaProtocol {
+  private OtaProtocol() {}
+
+  public static byte[] buildFrame(byte cmd, byte paramType, byte[] payload) {
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+    out.write(OtaCommandConstants.OTA.SVC_ID & 0xFF);
+    out.write(cmd & 0xFF);
+    out.write(paramType & 0xFF);
+    int len = payload != null ? payload.length : 0;
+    out.write(len & 0xFF);
+    out.write((len >> 8) & 0xFF);
+    if (payload != null && payload.length > 0) {
+      try {
+        out.write(payload);
+      } catch (Exception ignored) {
+      }
+    }
+    return out.toByteArray();
+  }
+
+  public static byte[] buildRequestUpgrade(int packageSize, int blockSize, long crc32) {
+    // Wire layout matches the vendor bring-up frame: type marker, then TLV
+    // 0x0A/len4 = package size (LE). CRC is validated per-block in SEND_IMAGE_DATA;
+    // blockSize is negotiated separately in CONNECT_NEGOTIATION.
+    ByteArrayOutputStream payload = new ByteArrayOutputStream();
+    payload.write(0x09);
+    payload.write(0x01);
+    payload.write(0x00);
+    payload.write(0x01);
+    payload.write(0x0A);
+    payload.write(0x04);
+    payload.write(packageSize & 0xFF);
+    payload.write((packageSize >> 8) & 0xFF);
+    payload.write((packageSize >> 16) & 0xFF);
+    payload.write((packageSize >> 24) & 0xFF);
+    payload.write(0x00);
+    return buildFrame(OtaCommandConstants.OTA.REQUEST_UPGRADE, (byte) 0x80, payload.toByteArray());
+  }
+
+  public static byte[] buildConnectNegotiation(int blockSize) {
+    return buildFrame(OtaCommandConstants.OTA.CONNECT_NEGOTIATION, (byte) 0x80, null);
+  }
+
+  public static byte[] buildNegotiationResult() {
+    ByteArrayOutputStream payload = new ByteArrayOutputStream();
+    payload.write(0x01);
+    payload.write(0x01);
+    payload.write(0x00);
+    payload.write(0x01);
+    return buildFrame(OtaCommandConstants.OTA.NEGOTIATION_RESULT, (byte) 0x80, payload.toByteArray());
+  }
+
+  public static byte[] buildSendImageData(int psn, byte[] data) {
+    ByteArrayOutputStream payload = new ByteArrayOutputStream();
+    payload.write(psn & 0xFF);
+    long crc32 = OtaCrc32Util.calculate(data);
+    payload.write((byte) (crc32 & 0xFF));
+    payload.write((byte) ((crc32 >> 8) & 0xFF));
+    payload.write((byte) ((crc32 >> 16) & 0xFF));
+    payload.write((byte) ((crc32 >> 24) & 0xFF));
+    if (data != null && data.length > 0) {
+      try {
+        payload.write(data);
+      } catch (Exception ignored) {
+      }
+    }
+    return buildFrame(OtaCommandConstants.OTA.SEND_IMAGE_DATA, (byte) 0x80, payload.toByteArray());
+  }
+
+  public static FrameHeader parseFrameHeader(byte[] frame) {
+    if (frame == null || frame.length < 5) return null;
+    FrameHeader header = new FrameHeader();
+    header.svcId = frame[0];
+    header.cmd = frame[1];
+    header.paramType = frame[2];
+    header.paramLen = OtaByteUtils.readShortLE(frame, 3) & 0xFFFF;
+    header.payloadOffset = 5;
+    return header;
+  }
+
+  public static class FrameHeader {
+    public byte svcId;
+    public byte cmd;
+    public byte paramType;
+    public int paramLen;
+    public int payloadOffset;
+  }
+}

package/android/src/main/java/com/mentra/bluetoothsdk/status/WifiHotspotStatus.kt

@@ -66,15 +66,31 @@ sealed interface WifiStatus {
     }
 }
 
-data class WifiStatusEvent(
+data class WifiStatusEvent @JvmOverloads constructor(
     val status: WifiStatus,
+    /**
+     * Glasses-reported provisioning failure reason when THIS event is the verdict of a
+     * failed connect attempt; null for routine link-state updates. An attempt property,
+     * not a link property — which is why it lives on the event and not on [WifiStatus]:
+     * "connect_timeout" arrives with a disconnected status (never associated), while
+     * "connected_to_other_network" arrives with a *connected* status — the attempt
+     * failed and Android's auto-join left the glasses on (or returned them to) a
+     * different SSID than requested, so the link is genuinely up while the request
+     * genuinely failed. Sent by ASG client builds that include the WiFi error
+     * surfacing (v40+); older glasses never set it.
+     */
+    val error: String? = null,
 ) {
-    internal constructor(values: Map<String, Any>) : this(WifiStatus.fromMap(values) ?: WifiStatus.Disconnected)
+    internal constructor(values: Map<String, Any>) : this(
+        WifiStatus.fromMap(values) ?: WifiStatus.Disconnected,
+        stringValue(values, "error")?.takeIf { it.isNotEmpty() },
+    )
     internal constructor(connected: Boolean, ssid: String?, localIp: String?) : this(
         WifiStatus.fromStoreFields(connected, ssid, localIp) ?: WifiStatus.Disconnected
     )
 
-    val values: Map<String, Any> get() = status.toEventMap()
+    val values: Map<String, Any>
+        get() = status.toEventMap() + (error?.let { mapOf("error" to it) } ?: emptyMap())
 }
 
 sealed interface HotspotStatus {

package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt

@@ -168,6 +168,38 @@ data class StreamResolvedConfig @JvmOverloads constructor(
     }
 }
 
+/** Live encoder and device telemetry reported by the glasses while streaming. */
+data class StreamLiveStats @JvmOverloads constructor(
+    val bitrate: Long? = null,
+    val fps: Double? = null,
+    val droppedFrames: Long? = null,
+    val duration: Long? = null,
+    val temperatureC: Double? = null,
+) {
+    fun toMap(): Map<String, Any> =
+        buildMap {
+            bitrate?.let { put("bitrate", it) }
+            fps?.let { put("fps", it) }
+            droppedFrames?.let { put("droppedFrames", it) }
+            duration?.let { put("duration", it) }
+            temperatureC?.let { put("temperatureC", it) }
+        }
+
+    companion object {
+        @JvmStatic
+        fun fromMap(values: Map<String, Any>?): StreamLiveStats? {
+            values ?: return null
+            return StreamLiveStats(
+                bitrate = longValue(values, "bitrate"),
+                fps = doubleValue(values, "fps"),
+                droppedFrames = longValue(values, "droppedFrames"),
+                duration = longValue(values, "duration"),
+                temperatureC = doubleValue(values, "temperatureC"),
+            )
+        }
+    }
+}
+
 data class StreamRequest @JvmOverloads constructor(
     val streamUrl: String,
     val streamId: String = "",
@@ -435,13 +467,31 @@ sealed interface StreamStatus {
 
 data class StreamStatusEvent(
     val status: StreamStatus,
+    val stats: StreamLiveStats? = null,
 ) {
-    constructor(values: Map<String, Any>) : this(StreamStatus.fromMap(values))
+    // True when the glasses will retry the failed publisher themselves
+    // (emitting side lands in PR #3488); absent on older firmware and on
+    // events not parsed from a glasses status map. Carried here instead of
+    // on StreamStatus.Error so the public Error shape stays unchanged.
+    var willRetry: Boolean? = null
+        private set
+
+    constructor(values: Map<String, Any>) : this(
+        status = StreamStatus.fromMap(values),
+        stats = StreamLiveStats.fromMap(stringMapValue(values["stats"])),
+    ) {
+        willRetry = boolValue(values, "willRetry")
+    }
 
     val state: StreamState get() = status.state
     val streamId: String? get() = status.streamId
     val resolvedConfig: StreamResolvedConfig? get() = status.resolvedConfig
-    val values: Map<String, Any> get() = status.toEventMap()
+    val values: Map<String, Any>
+        get() = buildMap {
+            putAll(status.toEventMap())
+            stats?.let { put("stats", it.toMap()) }
+            willRetry?.let { put("willRetry", it) }
+        }
 }
 
 data class KeepAliveAckEvent(

package/android/src/main/java/com/mentra/bluetoothsdk/tts/TTSTools.kt

@@ -9,6 +9,9 @@ import com.k2fsa.sherpa.onnx.OfflineTtsModelConfig
 import com.k2fsa.sherpa.onnx.OfflineTtsSupertonicModelConfig
 import com.mentra.bluetoothsdk.Bridge
 import java.io.File
+import java.util.concurrent.Executors
+import java.util.concurrent.ScheduledFuture
+import java.util.concurrent.TimeUnit
 
 /**
  * Utilities for offline Sherpa-ONNX Supertonic 3 TTS model management.
@@ -31,6 +34,21 @@ object TTSTools {
     private const val FILE_UNICODE_INDEXER = "unicode_indexer.bin"
     private const val FILE_VOICE_STYLE = "voice.bin"
 
+    /**
+     * Keep the native model warm briefly after use to reduce repeated TTS latency while
+     * still reclaiming its memory when speech has been idle for a while.
+     */
+    private const val MODEL_IDLE_TIMEOUT_MINUTES = 3L
+    private val modelLock = Any()
+    private val teardownExecutor =
+            Executors.newSingleThreadScheduledExecutor { runnable ->
+                Thread(runnable, "MentraTtsIdleTeardown").apply { isDaemon = true }
+            }
+    private var cachedTts: OfflineTts? = null
+    private var cachedModelPath: String? = null
+    private var teardownFuture: ScheduledFuture<*>? = null
+    private var teardownGeneration = 0L
+
     private val REQUIRED_FILES = listOf(
             FILE_DURATION_PREDICTOR,
             FILE_TEXT_ENCODER,
@@ -42,6 +60,11 @@ object TTSTools {
     )
 
     fun setTtsModelDetails(context: Context, path: String, languageCode: String) {
+        synchronized(modelLock) {
+            if (cachedModelPath != null && cachedModelPath != path) {
+                releaseCachedModelLocked("model path changed")
+            }
+        }
         val prefs = getPrefs(context)
         prefs.edit().apply {
             putString(KEY_TTS_MODEL_PATH, path)
@@ -118,59 +141,104 @@ object TTSTools {
             return false
         }
 
-        val modelDir = File(modelPath)
         val outputFile = File(outputPath)
         outputFile.parentFile?.mkdirs()
 
-        var tts: OfflineTts? = null
-        return try {
-            val supertonic = OfflineTtsSupertonicModelConfig(
-                    durationPredictor = File(modelDir, FILE_DURATION_PREDICTOR).absolutePath,
-                    textEncoder = File(modelDir, FILE_TEXT_ENCODER).absolutePath,
-                    vectorEstimator = File(modelDir, FILE_VECTOR_ESTIMATOR).absolutePath,
-                    vocoder = File(modelDir, FILE_VOCODER).absolutePath,
-                    ttsJson = File(modelDir, FILE_TTS_JSON).absolutePath,
-                    unicodeIndexer = File(modelDir, FILE_UNICODE_INDEXER).absolutePath,
-                    voiceStyle = File(modelDir, FILE_VOICE_STYLE).absolutePath,
-            )
-
-            val modelConfig = OfflineTtsModelConfig(
-                    supertonic = supertonic,
-                    numThreads = 2,
-                    provider = "cpu",
-            )
-
-            val config = OfflineTtsConfig(
-                    model = modelConfig,
-                    maxNumSentences = 1,
-                    silenceScale = 0.2f,
-            )
-
-            tts = OfflineTts(null, config)
-
-            // sid out of range falls back to F1 (sid 0); clamp 0..9.
-            val sid = speakerId.coerceIn(0, tts.numSpeakers() - 1)
-            val clampedSpeed = speed.coerceIn(0.5f, 2.0f)
-            val lang = languageTagToSupertonicLang(languageTag)
-            val genConfig = GenerationConfig(
-                    sid = sid,
-                    speed = clampedSpeed,
-                    extra = mapOf("lang" to lang),
-            )
-            val audio = tts.generateWithConfig(text = text, config = genConfig)
-            val saved = audio.save(outputFile.absolutePath)
-            Bridge.log("TTS generated ${outputFile.absolutePath}: saved=$saved sid=$sid lang=$lang")
-            saved
-        } catch (e: Exception) {
-            Bridge.log("TTS_ERROR: ${e.javaClass.simpleName}: ${e.message}")
-            e.printStackTrace()
-            false
-        } finally {
+        return synchronized(modelLock) {
             try {
-                tts?.release()
+                val tts = getOrCreateModelLocked(modelPath)
+
+                // sid out of range falls back to F1 (sid 0); clamp 0..9.
+                val sid = speakerId.coerceIn(0, tts.numSpeakers() - 1)
+                val clampedSpeed = speed.coerceIn(0.5f, 2.0f)
+                val lang = languageTagToSupertonicLang(languageTag)
+                val genConfig = GenerationConfig(
+                        sid = sid,
+                        speed = clampedSpeed,
+                        numSteps = 5,
+                        extra = mapOf("lang" to lang),
+                )
+                val audio = tts.generateWithConfig(text = text, config = genConfig)
+                val saved = audio.save(outputFile.absolutePath)
+                Bridge.log("TTS generated ${outputFile.absolutePath}: saved=$saved sid=$sid lang=$lang")
+                saved
             } catch (e: Exception) {
-                Bridge.log("TTS release failed: ${e.message}")
+                Bridge.log("TTS_ERROR: ${e.javaClass.simpleName}: ${e.message}")
+                e.printStackTrace()
+                false
+            } finally {
+                scheduleIdleTeardownLocked()
+            }
+        }
+    }
+
+    private fun getOrCreateModelLocked(modelPath: String): OfflineTts {
+        cachedTts?.let { cached ->
+            if (cachedModelPath == modelPath) {
+                Bridge.log("TTS reusing cached model")
+                return cached
             }
+            releaseCachedModelLocked("model path changed")
+        }
+
+        val modelDir = File(modelPath)
+        val supertonic = OfflineTtsSupertonicModelConfig(
+                durationPredictor = File(modelDir, FILE_DURATION_PREDICTOR).absolutePath,
+                textEncoder = File(modelDir, FILE_TEXT_ENCODER).absolutePath,
+                vectorEstimator = File(modelDir, FILE_VECTOR_ESTIMATOR).absolutePath,
+                vocoder = File(modelDir, FILE_VOCODER).absolutePath,
+                ttsJson = File(modelDir, FILE_TTS_JSON).absolutePath,
+                unicodeIndexer = File(modelDir, FILE_UNICODE_INDEXER).absolutePath,
+                voiceStyle = File(modelDir, FILE_VOICE_STYLE).absolutePath,
+        )
+        val modelConfig = OfflineTtsModelConfig(
+                supertonic = supertonic,
+                numThreads = 8,
+                provider = "cpu",
+        )
+        val config = OfflineTtsConfig(
+                model = modelConfig,
+                maxNumSentences = 1,
+                silenceScale = 0.2f,
+        )
+        return OfflineTts(null, config).also {
+            cachedTts = it
+            cachedModelPath = modelPath
+            Bridge.log("TTS loaded model; keeping it warm for $MODEL_IDLE_TIMEOUT_MINUTES minutes after use")
+        }
+    }
+
+    private fun scheduleIdleTeardownLocked() {
+        if (cachedTts == null) return
+        teardownFuture?.cancel(false)
+        val generation = ++teardownGeneration
+        teardownFuture =
+                teardownExecutor.schedule(
+                        {
+                            synchronized(modelLock) {
+                                if (generation == teardownGeneration) {
+                                    releaseCachedModelLocked("idle for $MODEL_IDLE_TIMEOUT_MINUTES minutes")
+                                }
+                            }
+                        },
+                        MODEL_IDLE_TIMEOUT_MINUTES,
+                        TimeUnit.MINUTES,
+                )
+    }
+
+    private fun releaseCachedModelLocked(reason: String) {
+        teardownFuture?.cancel(false)
+        teardownFuture = null
+        teardownGeneration++
+        val tts = cachedTts
+        cachedTts = null
+        cachedModelPath = null
+        if (tts == null) return
+        try {
+            tts.release()
+            Bridge.log("TTS released cached model: $reason")
+        } catch (e: Exception) {
+            Bridge.log("TTS release failed: ${e.message}")
         }
     }
 

package/android/src/main/java/com/mentra/bluetoothsdk/types/DeviceModels.kt

@@ -23,6 +23,7 @@ enum class DeviceModel(val deviceType: String) {
     Z100(DeviceTypes.Z100),
     FRAME(DeviceTypes.FRAME),
     NIMO(DeviceTypes.NIMO),
+    AR99(DeviceTypes.AR99),
     SIMULATED(DeviceTypes.SIMULATED),
     R1(ControllerTypes.R1);
 
@@ -38,6 +39,7 @@ data class Device(
     val name: String,
     /** Android Bluetooth address when available. */
     val address: String? = null,
+    val projectName: String? = null,
     val rssi: Int? = null,
     /** Stable app-facing scan-result key. Do not parse; use typed fields instead. */
     val id: String = address?.takeIf { it.isNotBlank() } ?: "${model.deviceType}:$name",
@@ -50,6 +52,9 @@ data class Device(
             address?.takeIf { it.isNotBlank() }?.let {
                 put("address", it)
             }
+            projectName?.takeIf { it.isNotBlank() }?.let {
+                put("projectName", it)
+            }
             rssi?.let { put("rssi", it) }
         }
 
@@ -58,12 +63,14 @@ data class Device(
             val model = stringValue(values, "model") ?: return null
             val name = stringValue(values, "name") ?: return null
             val address = stringValue(values, "address")?.takeIf { it.isNotBlank() }
+            val projectName = stringValue(values, "projectName")?.takeIf { it.isNotBlank() }
             val rssi = numberValue(values, "rssi")
             val id = stringValue(values, "id")?.takeIf { it.isNotBlank() } ?: address ?: "${model}:$name"
             return Device(
                 model = DeviceModel.fromDeviceType(model),
                 name = name,
                 address = address,
+                projectName = projectName,
                 rssi = rssi,
                 id = id,
             )

package/android/src/main/java/com/mentra/bluetoothsdk/utils/BleJsonCompact.java

@@ -66,6 +66,14 @@ public final class BleJsonCompact {
         putKey("transferMethod", "tm");
         putKey("manual", "m");
         putKey("reconnecting", "rc");
+        putKey("stats", "st");
+        putKey("width", "w");
+        putKey("height", "h");
+        putKey("bitrate", "br");
+        putKey("fps", "f");
+        putKey("droppedFrames", "df");
+        putKey("duration", "du");
+        putKey("temperatureC", "tc");
 
         putEnum(PHOTO_STATUS_TO_INT, PHOTO_STATUS_FROM_INT, "accepted", 0);
         putEnum(PHOTO_STATUS_TO_INT, PHOTO_STATUS_FROM_INT, "queued", 1);

package/android/src/main/java/com/mentra/bluetoothsdk/utils/Constants.kt

@@ -10,7 +10,8 @@ object DeviceTypes {
     const val FRAME = "Brilliant Frame"
     const val G2 = "Even Realities G2"
     const val NIMO = "NIMO"
-    val ALL = arrayOf(SIMULATED, G1, G2, MACH1, LIVE, Z100, FRAME, NEX, NIMO)
+    const val AR99 = "AR99"
+    val ALL = arrayOf(SIMULATED, G1, G2, MACH1, LIVE, Z100, FRAME, NEX, NIMO, AR99)
 }
 
 object ControllerTypes {

package/android/src/main/java/com/mentra/bluetoothsdk/utils/PhoneAudioMonitor.kt

@@ -26,6 +26,14 @@ class PhoneAudioMonitor private constructor(private val context: Context) {
         @Volatile
         private var instance: PhoneAudioMonitor? = null
 
+        internal fun shouldKeepOwnAppAudioSignal(
+            signalledPlaying: Boolean,
+            systemAudioPlaying: Boolean,
+            nowMs: Long,
+            holdoffUntilMs: Long,
+        ): Boolean =
+            signalledPlaying && (systemAudioPlaying || nowMs < holdoffUntilMs)
+
         @JvmStatic
         fun getInstance(context: Context): PhoneAudioMonitor {
             return instance ?: synchronized(this) {
@@ -48,6 +56,10 @@ class PhoneAudioMonitor private constructor(private val context: Context) {
     private var listener: Listener? = null
     private var isMonitoring = false
     private var lastKnownState = false
+    // Set explicitly by AudioPlaybackService. Unlike isPlaying(), this is
+    // limited to Mentra's own playback and is safe for mic-watchdog gating.
+    @Volatile
+    private var ownAppAudioPlaying = false
 
     // Rate limiting: max 1 state change notification per 500ms
     // If state changes rapidly (true/false/true), we wait and send the final state
@@ -78,6 +90,37 @@ class PhoneAudioMonitor private constructor(private val context: Context) {
         return audioManager.isMusicActive
     }
 
+    /**
+     * Return the explicit React Native playback hint, reconciling it with
+     * Android's actual media state.
+     *
+     * Expo can miss a completion callback while its Activity is backgrounded,
+     * leaving the explicit hint stuck at PLAYING. The hint only protects the
+     * short A2DP codec transition; after that hold-off, Android's
+     * isMusicActive is authoritative. Clearing a stale hint lets the glasses
+     * mic watchdog recover instead of suppressing recovery forever.
+     */
+    @Synchronized
+    fun isOwnAppAudioPlaying(): Boolean {
+        val systemAudioPlaying = isPlaying()
+        if (
+            shouldKeepOwnAppAudioSignal(
+                ownAppAudioPlaying,
+                systemAudioPlaying,
+                System.currentTimeMillis(),
+                audioStartHoldoffUntil,
+            )
+        ) {
+            return true
+        }
+
+        if (ownAppAudioPlaying) {
+            ownAppAudioPlaying = false
+            Bridge.log("$TAG: Cleared stale own-app audio flag (system audio is inactive)")
+        }
+        return false
+    }
+
     /**
      * Notify the monitor that our own app started/stopped playing audio
      * Called from RN AudioPlaybackService via Bridge
@@ -87,7 +130,9 @@ class PhoneAudioMonitor private constructor(private val context: Context) {
      * there's a brief gap where the system reports 0 active configs (codec is switching).
      * The hold-off prevents us from reporting STOPPED during this transition.
      */
+    @Synchronized
     fun setOwnAppAudioPlaying(playing: Boolean) {
+        ownAppAudioPlaying = playing
         Bridge.log("$TAG: Own app audio -> ${if (playing) "PLAYING" else "STOPPED"}")
 
         if (playing) {