@mentra/bluetooth-sdk 0.1.18 → 0.1.19

package/README.md

@@ -475,7 +475,7 @@ For bare native iOS apps, use the public SwiftPM repository:
 https://github.com/Mentra-Community/mentra-bluetooth-sdk-ios.git
 ```
 
-Select version `0.1.18`, then add the `MentraBluetoothSDK` product to your app target.
+Select version `0.1.19`, then add the `MentraBluetoothSDK` product to your app target.
 
 For local SDK development, add this package folder directly in Xcode:
 

package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt

@@ -16,6 +16,8 @@ import com.mentra.bluetoothsdk.services.ForegroundService
 import com.mentra.bluetoothsdk.services.PhoneMic
 import com.mentra.bluetoothsdk.sgcs.G1
 import com.mentra.bluetoothsdk.sgcs.G2
+import com.mentra.bluetoothsdk.sgcs.SceneElement
+import com.mentra.bluetoothsdk.sgcs.SceneFrame
 import com.mentra.bluetoothsdk.sgcs.Mach1
 import com.mentra.bluetoothsdk.sgcs.MentraLive
 import com.mentra.bluetoothsdk.sgcs.MentraNex
@@ -144,10 +146,6 @@ class DeviceManager {
         get() = DeviceStore.store.get("bluetooth", "bypass_vad") as? Boolean ?: false
         set(value) = DeviceStore.apply("bluetooth", "bypass_vad", value)
 
-    private var offlineCaptionsRunning: Boolean
-        get() = DeviceStore.store.get("bluetooth", "offline_captions_running") as? Boolean ?: false
-        set(value) = DeviceStore.apply("bluetooth", "offline_captions_running", value)
-
     private var localSttFallbackActive: Boolean
         get() = DeviceStore.store.get("bluetooth", "local_stt_fallback_active") as? Boolean ?: false
         set(value) = DeviceStore.apply("bluetooth", "local_stt_fallback_active", value)
@@ -660,6 +658,13 @@ class DeviceManager {
         var borderWidth: Int? = null,
         var borderRadius: Int? = null
     )
+
+    // Scene slots — one whole SceneFrame per view (main/dashboard), parallel to
+    // viewStates. When a slot holds a scene, viewStates carries a "scene"
+    // sentinel so sendCurrentState routes here. Holding the WHOLE frame keeps
+    // native re-dispatch coherent (dashboard exit re-applies a complete scene,
+    // not whatever element happened to arrive last).
+    private val sceneStates = arrayOfNulls<SceneFrame>(2)
     // MARK: - End Unique
 
     // MARK: - Voice Data Handling
@@ -740,7 +745,7 @@ class DeviceManager {
         // class to fire `vad_status` (a separate signal the cloud SDK
         // surfaces as `session.audio.isSpeaking`).
         handleSendingPcm(pcmData)
-        if (shouldSendTranscript || offlineCaptionsRunning || localSttFallbackActive) {
+        if (shouldSendTranscript || localSttFallbackActive) {
             if (ensureTranscriberInitialized()) {
                 transcriber?.acceptAudio(pcmData)
             }
@@ -877,16 +882,8 @@ class DeviceManager {
         }
 
         // executor.execute {
-        var currentViewState: ViewState
-        if (hUp) {
-            currentViewState = viewStates[1]
-        } else {
-            currentViewState = viewStates[0]
-        }
-
-        if (hUp && !contextualDashboard) {
-            currentViewState = viewStates[0]
-        }
+        val currentStateIndex = if (hUp && contextualDashboard) 1 else 0
+        val currentViewState: ViewState = viewStates[currentStateIndex]
 
         if (sgc?.type?.contains(DeviceTypes.SIMULATED) == true) {
             // dont send the event to glasses that aren't there:
@@ -944,6 +941,10 @@ class DeviceManager {
                 )
             }
 
+            "scene" -> {
+                sceneStates[currentStateIndex]?.let { sgc?.applySceneFrame(it) }
+            }
+
             "clear_view" -> sgc?.clearDisplay()
             else -> Bridge.log("MAN: UNHANDLED LAYOUT_TYPE ${currentViewState.layoutType}")
         }
@@ -1362,9 +1363,31 @@ class DeviceManager {
         val isDashboard = view == "dashboard"
         val stateIndex = if (isDashboard) 1 else 0
 
+        // Scene frames (display.render() pipeline) take their own path: the
+        // whole frame is the unit, not a layout, and the host's per-element
+        // annotations make redundant frames self-deduping (all-"unchanged"
+        // frames no-op in the SGC base handler).
+        @Suppress("UNCHECKED_CAST") val sceneMap = event["scene"] as? Map<String, Any>
+        if (sceneMap != null) {
+            handleSceneEvent(stateIndex, sceneMap)
+            return
+        }
+
         @Suppress("UNCHECKED_CAST") val layout = event["layout"] as? Map<String, Any> ?: return
 
         val layoutType = layout["layoutType"] as? String
+
+        // Scene→legacy handoff: a legacy layout is about to draw over a scene
+        // (e.g. a cloud app taking the view from a miniapp). Sweep the scene's
+        // elements first so they don't linger under the new content; clear_view
+        // wipes everything anyway.
+        sceneStates[stateIndex]?.let { prevFrame ->
+            sceneStates[stateIndex] = null
+            if (layoutType != "clear_view") {
+                sgc?.clearSceneElements(prevFrame.elements.map { it.id })
+            }
+        }
+
         val text = parsePlaceholders(layout.getString("text", " "))
         val topText = parsePlaceholders(layout.getString("topText", " "))
         val bottomText = parsePlaceholders(layout.getString("bottomText", " "))
@@ -1414,6 +1437,86 @@ class DeviceManager {
         }
     }
 
+    /** Parse + store a scene frame, then dispatch it if its view is visible. */
+    private fun handleSceneEvent(stateIndex: Int, sceneMap: Map<String, Any>) {
+        var frame = parseSceneFrame(sceneMap) ?: return
+        val prevFrame = sceneStates[stateIndex]
+
+        if (prevFrame == null) {
+            // Legacy→scene handoff: stale legacy content (e.g. a cloud app's
+            // text wall) must not linger under the scene's elements.
+            // clearDisplay is the per-device "wipe what's there" (blank-in-place
+            // on G2 — no page rebuild).
+            val prevLegacyType = viewStates[stateIndex].layoutType
+            if (prevLegacyType.isNotEmpty() && prevLegacyType != "clear_view" && prevLegacyType != "scene") {
+                sgc?.clearDisplay()
+            }
+        } else if (prevFrame.appId != frame.appId) {
+            // Cross-app switch: the host's diff baseline is per-app, so the new
+            // app's annotations don't know the old app's elements are on the
+            // glasses. Sweep the old app's elements (SGC registries still map
+            // them), then paint the new frame from scratch. In practice the
+            // boot message interposes between apps, so this isn't visible as a
+            // blank.
+            sgc?.clearSceneElements(prevFrame.elements.map { it.id })
+            frame = frame.copy(replay = true, elements = frame.elements.map { it.copy(change = "created") })
+        }
+
+        // Store the REDISPATCH form: any later sendCurrentState (dashboard
+        // exit, head-up return) must repaint the whole frame — the original
+        // annotations are only valid for the first dispatch right now.
+        sceneStates[stateIndex] =
+            frame.copy(replay = true, elements = frame.elements.map { it.copy(change = "created") })
+        viewStates[stateIndex] = ViewState(" ", " ", " ", "scene", " ", null, null)
+
+        val hUp = headUp && contextualDashboard
+        if ((stateIndex == 0 && !hUp) || (stateIndex == 1 && hUp)) {
+            dispatchSceneFrame(frame)
+        }
+    }
+
+    /** Guarded scene dispatch — mirrors sendCurrentState's send conditions. */
+    private fun dispatchSceneFrame(frame: SceneFrame) {
+        if (screenDisabled) return
+        if (sgc?.type?.contains(DeviceTypes.SIMULATED) == true) return
+        if (sgc?.fullyBooted != true) {
+            Bridge.log("MAN: dispatchSceneFrame(): sgc not ready")
+            return
+        }
+        sgc?.applySceneFrame(frame)
+    }
+
+    @Suppress("UNCHECKED_CAST")
+    private fun parseSceneFrame(sceneMap: Map<String, Any>): SceneFrame? {
+        val elementsRaw = sceneMap["elements"] as? List<Map<String, Any>> ?: return null
+        val elements =
+            elementsRaw.mapNotNull { el ->
+                val box = el["box"] as? Map<String, Any> ?: return@mapNotNull null
+                val style = el["style"] as? Map<String, Any>
+                SceneElement(
+                    id = el["id"] as? String ?: return@mapNotNull null,
+                    type = el["type"] as? String ?: return@mapNotNull null,
+                    x = (box["x"] as? Number)?.toInt() ?: 0,
+                    y = (box["y"] as? Number)?.toInt() ?: 0,
+                    w = (box["w"] as? Number)?.toInt() ?: 0,
+                    h = (box["h"] as? Number)?.toInt() ?: 0,
+                    text = (el["text"] as? String)?.let { parsePlaceholders(it) },
+                    data = el["data"] as? String,
+                    border = (style?.get("border") as? Number)?.toInt() ?: 0,
+                    radius = (style?.get("radius") as? Number)?.toInt() ?: 0,
+                    change = el["change"] as? String ?: "created",
+                    contentHash = el["contentHash"] as? String ?: ""
+                )
+            }
+        return SceneFrame(
+            appId = sceneMap["appId"] as? String ?: "",
+            epoch = (sceneMap["sceneEpoch"] as? Number)?.toInt() ?: 0,
+            replay = sceneMap["replay"] as? Boolean ?: false,
+            elements = elements,
+            removed = (sceneMap["removed"] as? List<*>)?.filterIsInstance<String>() ?: emptyList()
+        )
+    }
+
     fun showDashboard() {
         sgc?.showDashboard()
     }
@@ -1669,7 +1772,7 @@ class DeviceManager {
 
     fun setMicState() {
         val willSendPcm = shouldSendPcm || shouldSendLc3
-        val willSendTranscript = shouldSendTranscript || offlineCaptionsRunning || localSttFallbackActive
+        val willSendTranscript = shouldSendTranscript || localSttFallbackActive
         val nextEnabled = willSendPcm || willSendTranscript
         // Tell VAD when the mic is shutting down so it doesn't get stuck in
         // a stale "speaking" state and keep emitting vad_status=true after

package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt

@@ -303,12 +303,6 @@ object DeviceStore {
                     DeviceManager.getInstance().setMicState()
                 }
             }
-            "bluetooth" to "offline_captions_running" -> {
-                (value as? Boolean)?.let { running ->
-                    Bridge.log("DeviceStore: offline_captions_running changed to $running")
-                    DeviceManager.getInstance().setMicState()
-                }
-            }
             "bluetooth" to "local_stt_fallback_active" -> {
                 (value as? Boolean)?.let { active ->
                     Bridge.log("DeviceStore: local_stt_fallback_active changed to $active")

package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt

@@ -67,6 +67,7 @@ class MentraBluetoothSdk private constructor(
         private val SCAN_STATE_KEYS = setOf("searching", "searchingController", "searchResults")
         private const val DEFAULT_SCAN_TIMEOUT_MS = 15_000L
         private const val DEFAULT_REQUEST_TIMEOUT_MS = 15_000L
+        private const val WIFI_SCAN_TIMEOUT_MS = 20_000L
         private const val VIDEO_UPLOAD_STOP_TIMEOUT_MS = 10 * 60 * 1000L
         private const val STREAM_START_TIMEOUT_MS = 30_000L
         private const val STREAM_STOP_TIMEOUT_MS = 15_000L
@@ -633,35 +634,35 @@ class MentraBluetoothSdk private constructor(
     fun requestWifiScan(): List<WifiScanResult> {
         val pending = PendingResponse<List<WifiScanResult>>("WiFi scan request")
         val request = PendingWifiScan(pending)
-        synchronized(oneShotLock) {
-            if (pendingWifiScan != null) {
-                throw BluetoothSdkException(
-                    "request_in_flight",
-                    "A WiFi scan is already waiting for a glasses response.",
-                )
+        val existing =
+            synchronized(oneShotLock) {
+                pendingWifiScan ?: run {
+                    pendingWifiScan = request
+                    null
+                }
             }
-            pendingWifiScan = request
+        if (existing != null) {
+            // Join the in-flight scan instead of failing with request_in_flight;
+            // the scan screen auto-starts a scan on mount and can be pushed twice.
+            return existing.pending.await(WIFI_SCAN_TIMEOUT_MS)
         }
         try {
             deviceManager.requestWifiScan()
-            return pending.await()
-        } catch (error: BluetoothSdkException) {
-            if (error.code == "request_timeout") {
-                val fallbackResults =
-                    synchronized(oneShotLock) {
-                        if (request.latestResults.isNotEmpty()) {
-                            if (pendingWifiScan === request) {
-                                pendingWifiScan = null
-                            }
-                            request.latestResults
-                        } else {
-                            emptyList()
-                        }
-                    }
+            // The glasses wait up to 15s for scan-results broadcasts before sending
+            // scan_complete, so give them longer than that before falling back.
+            return pending.await(WIFI_SCAN_TIMEOUT_MS)
+        } catch (error: Throwable) {
+            if (error is BluetoothSdkException && error.code == "request_timeout") {
+                val fallbackResults = synchronized(oneShotLock) { request.latestResults }
                 if (fallbackResults.isNotEmpty()) {
+                    // Resolve so callers joined on the same scan get the fallback too.
+                    pending.resolve(fallbackResults)
                     return fallbackResults
                 }
             }
+            // Fail joined callers immediately instead of letting them run out their
+            // own timeout.
+            pending.reject(error)
             throw error
         } finally {
             synchronized(oneShotLock) {

package/android/src/main/java/com/mentra/bluetoothsdk/OtaManifest.kt

@@ -57,7 +57,7 @@ internal object OtaManifestChecker {
         return try {
             val status = connection.responseCode
             if (status !in 200..299) {
-                throw BluetoothSdkException("ota_manifest_request_failed", "OTA manifest request failed with HTTP $status.")
+                throw BluetoothSdkException("ota_manifest_request_failed", "OTA manifest request failed with HTTP $status for $otaVersionUrl.")
             }
             JSONObject(connection.inputStream.bufferedReader().use { it.readText() })
         } finally {