@mentra/bluetooth-sdk 0.1.18 → 0.1.20

package/ios/Source/BluetoothSdkDefaults.swift

@@ -7,7 +7,7 @@ enum BluetoothSdkDefaults {
     }
 
     static let voiceActivityDetectionEnabled = false
-    private static let swiftPackageSdkVersion = "0.1.18"
+    private static let swiftPackageSdkVersion = "0.1.20"
     private static let swiftPackageSdkVersionPlaceholder = "__MENTRA" + "_BLUETOOTH_SDK_VERSION__"
 
     private static func normalizedSdkVersion(_ value: String?) -> String? {

package/ios/Source/DeviceManager.swift

@@ -186,11 +186,6 @@ struct ViewState {
         set { DeviceStore.shared.apply("bluetooth", "bypass_vad", newValue) }
     }
 
-    private var offlineCaptionsRunning: Bool {
-        get { DeviceStore.shared.get("bluetooth", "offline_captions_running") as? Bool ?? false }
-        set { DeviceStore.shared.apply("bluetooth", "offline_captions_running", newValue) }
-    }
-
     private var localSttFallbackActive: Bool {
         get { DeviceStore.shared.get("bluetooth", "local_stt_fallback_active") as? Bool ?? false }
         set { DeviceStore.shared.apply("bluetooth", "local_stt_fallback_active", newValue) }
@@ -325,6 +320,12 @@ struct ViewState {
         ),
     ]
 
+    // Scene slots — one whole SceneFrame per view (main/dashboard), parallel to
+    // viewStates. When a slot holds a scene, viewStates carries a "scene"
+    // sentinel so sendCurrentState routes here. Holding the WHOLE frame keeps
+    // native re-dispatch coherent (dashboard exit re-applies a complete scene).
+    var sceneStates: [SceneFrame?] = [nil, nil]
+
     override init() {
         Bridge.log("MAN: init()")
         super.init()
@@ -421,7 +422,7 @@ struct ViewState {
 
         // Send PCM to local transcriber.
 #if !SWIFT_PACKAGE || MENTRA_FEATURE_LOCAL_STT
-        if shouldSendTranscript || offlineCaptionsRunning || localSttFallbackActive {
+        if shouldSendTranscript || localSttFallbackActive {
             transcriber?.acceptAudio(pcm16le: pcmData)
         }
 #endif
@@ -732,6 +733,11 @@ struct ViewState {
                     borderWidth: currentViewState.borderWidth ?? 0,
                     borderRadius: currentViewState.borderRadius ?? 0
                 )
+            case "scene":
+                let sceneIndex = (headUp && self.contextualDashboard) ? 1 : 0
+                if let frame = self.sceneStates[sceneIndex] {
+                    await sgc?.applySceneFrame(frame)
+                }
             case "clear_view":
                 sgc?.clearDisplay()
             default:
@@ -1044,8 +1050,35 @@ struct ViewState {
             stateIndex = 0
         }
 
-        let layout = event["layout"] as! [String: Any]
-        let layoutType = layout["layoutType"] as! String
+        // Scene frames (display.render() pipeline) take their own path: the
+        // whole frame is the unit, not a layout, and the host's per-element
+        // annotations make redundant frames self-deduping (all-"unchanged"
+        // frames no-op in the SGC base handler).
+        if let sceneMap = event["scene"] as? [String: Any] {
+            handleSceneEvent(stateIndex, sceneMap)
+            return
+        }
+
+        guard let layout = event["layout"] as? [String: Any],
+              let layoutType = layout["layoutType"] as? String
+        else {
+            Bridge.log("MAN: displayEvent missing layout")
+            return
+        }
+
+        // Scene→legacy handoff: a legacy layout is about to draw over a scene
+        // (e.g. a cloud app taking the view from a miniapp). Sweep the scene's
+        // elements first so they don't linger under the new content; clear_view
+        // wipes everything anyway.
+        if let prevFrame = sceneStates[stateIndex] {
+            sceneStates[stateIndex] = nil
+            if layoutType != "clear_view" {
+                let ids = prevFrame.elements.map(\.id)
+                Task { [weak self] in
+                    await self?.sgc?.clearSceneElements(ids)
+                }
+            }
+        }
         var text = layout["text"] as? String ?? " "
         var topText = layout["topText"] as? String ?? " "
         var bottomText = layout["bottomText"] as? String ?? " "
@@ -1090,26 +1123,11 @@ struct ViewState {
             }
         }
 
-        // positioned_text is a sticky overlay container (e.g. the nav trip-stats
-        // label) that lives ALONGSIDE the main view. It must NOT flow through the
-        // single replaceable viewState[stateIndex] — otherwise the constantly-
-        // refreshing main text_wall (maneuver text) and the minimap bitmap_view
-        // overwrite it every frame and it never renders. Route it straight to the
-        // SGC, which keeps its own persistent text container for that rect.
-        if layoutType == "positioned_text" {
-            Bridge.log(
-                "MAN: positioned_text (sticky) → text='\(text)' rect=\(bmpX ?? 0),\(bmpY ?? 0) \(bmpWidth ?? 576)x\(bmpHeight ?? 288)"
-            )
-            Task { [weak self] in
-                await self?.sgc?.sendPositionedText(
-                    text,
-                    x: bmpX ?? 0, y: bmpY ?? 0,
-                    width: bmpWidth ?? 576, height: bmpHeight ?? 288,
-                    borderWidth: borderWidth ?? 0, borderRadius: borderRadius ?? 0
-                )
-            }
-            return
-        }
+        // NOTE: positioned_text used to bypass the viewState slot here (the
+        // "sticky overlay" hack for the old nav HUD). Scenes made that
+        // obsolete: multi-element frames arrive as ONE scene event, so nothing
+        // clobbers anything. Legacy positioned_text now flows through the slot
+        // like every other layout (matching Android).
 
         let cS = viewStates[stateIndex]
         let nS = newViewState
@@ -1136,6 +1154,93 @@ struct ViewState {
         }
     }
 
+    /// Parse + store a scene frame, then dispatch it if its view is visible.
+    private func handleSceneEvent(_ stateIndex: Int, _ sceneMap: [String: Any]) {
+        guard var frame = parseSceneFrame(sceneMap) else { return }
+        let prevFrame = sceneStates[stateIndex]
+
+        if prevFrame == nil {
+            // Legacy→scene handoff: stale legacy content (e.g. a cloud app's
+            // text wall) must not linger under the scene's elements.
+            // clearDisplay is the per-device "wipe what's there" (blank-in-place
+            // on G2 — no page rebuild).
+            let prevLegacyType = viewStates[stateIndex].layoutType
+            if !prevLegacyType.isEmpty, prevLegacyType != "clear_view", prevLegacyType != "scene" {
+                sgc?.clearDisplay()
+            }
+        } else if let prevFrame, prevFrame.appId != frame.appId {
+            // Cross-app switch: the host's diff baseline is per-app, so the new
+            // app's annotations don't know the old app's elements are on the
+            // glasses. Sweep the old app's elements (SGC registries still map
+            // them), then paint the new frame from scratch. The boot message
+            // interposes between apps in practice, so this isn't visible.
+            let ids = prevFrame.elements.map(\.id)
+            Task { [weak self] in
+                await self?.sgc?.clearSceneElements(ids)
+            }
+            frame = frame.asReplay()
+        }
+
+        // Store the REDISPATCH form: any later sendCurrentState (dashboard
+        // exit, head-up return) must repaint the whole frame — the original
+        // annotations are only valid for the first dispatch right now.
+        sceneStates[stateIndex] = frame.asReplay()
+        viewStates[stateIndex] = ViewState(
+            topText: " ", bottomText: " ", title: " ", layoutType: "scene",
+            text: " ", data: nil, animationData: nil
+        )
+
+        let hUp = headUp && contextualDashboard
+        if (stateIndex == 0 && !hUp) || (stateIndex == 1 && hUp) {
+            dispatchSceneFrame(frame)
+        }
+    }
+
+    /// Guarded scene dispatch — mirrors sendCurrentState's send conditions.
+    private func dispatchSceneFrame(_ frame: SceneFrame) {
+        if screenDisabled { return }
+        if sgc?.type.contains(DeviceTypes.SIMULATED) ?? true { return }
+        guard sgc?.fullyBooted == true else {
+            Bridge.log("MAN: dispatchSceneFrame(): sgc not ready")
+            return
+        }
+        Task { [weak self] in
+            await self?.sgc?.applySceneFrame(frame)
+        }
+    }
+
+    private func parseSceneFrame(_ sceneMap: [String: Any]) -> SceneFrame? {
+        guard let elementsRaw = sceneMap["elements"] as? [[String: Any]] else { return nil }
+        let elements: [SceneElement] = elementsRaw.compactMap { el in
+            guard let id = el["id"] as? String,
+                  let type = el["type"] as? String,
+                  let box = el["box"] as? [String: Any]
+            else { return nil }
+            let style = el["style"] as? [String: Any]
+            return SceneElement(
+                id: id,
+                type: type,
+                x: (box["x"] as? NSNumber)?.int32Value ?? 0,
+                y: (box["y"] as? NSNumber)?.int32Value ?? 0,
+                w: (box["w"] as? NSNumber)?.int32Value ?? 0,
+                h: (box["h"] as? NSNumber)?.int32Value ?? 0,
+                text: (el["text"] as? String).map { parsePlaceholders($0) },
+                data: el["data"] as? String,
+                border: ((style?["border"]) as? NSNumber)?.int32Value ?? 0,
+                radius: ((style?["radius"]) as? NSNumber)?.int32Value ?? 0,
+                change: el["change"] as? String ?? "created",
+                contentHash: el["contentHash"] as? String ?? ""
+            )
+        }
+        return SceneFrame(
+            appId: sceneMap["appId"] as? String ?? "",
+            epoch: (sceneMap["sceneEpoch"] as? NSNumber)?.intValue ?? 0,
+            replay: sceneMap["replay"] as? Bool ?? false,
+            elements: elements,
+            removed: (sceneMap["removed"] as? [String]) ?? []
+        )
+    }
+
     func showDashboard() {
         sgc?.showDashboard()
     }
@@ -1321,7 +1426,7 @@ struct ViewState {
 
     func setMicState() {
         let willSendPcm = shouldSendPcm || shouldSendLc3
-        let willSendTranscript = shouldSendTranscript || offlineCaptionsRunning || localSttFallbackActive
+        let willSendTranscript = shouldSendTranscript || localSttFallbackActive
         micEnabled = willSendPcm || willSendTranscript
         updateMicState()
     }
@@ -1400,7 +1505,8 @@ struct ViewState {
             mfnr: request.mfnr,
             zsl: request.zsl,
             ispDigitalGain: request.ispDigitalGain,
-            ispAnalogGain: request.ispAnalogGain
+            ispAnalogGain: request.ispAnalogGain,
+            mode: request.mode
         )
         Bridge.log(
             "MAN: PHOTO PIPELINE [4/6] DeviceManager.requestPhoto requestId=\(routed.requestId) webhookUrl=\(routed.webhookUrl ?? "nil") size=\(routed.size.rawValue) compress=\(routed.compress?.rawValue ?? "none") save=\(routed.save) sound=\(routed.sound) exposureTimeNs=\(manualExposureNs.map { String($0) } ?? "nil") iso=\(manualIso.map { String($0) } ?? "auto") aeDivisor=\(routed.aeExposureDivisor.map { String($0) } ?? "nil") isoCap=\(routed.isoCap.map { String($0) } ?? "nil") sgc=\(sgc != nil ? String(describing: type(of: sgc!)) : "null")"

package/ios/Source/DeviceStore.swift

@@ -263,6 +263,11 @@ class DeviceStore {
         case ("bluetooth", "camera_fov"):
             DeviceManager.shared.sgc?.sendCameraFovSetting()
 
+        case ("bluetooth", "button_video_settings"):
+            DeviceManager.shared.sgc?.sendButtonVideoRecordingSettings()
+
+        // Legacy scalar keys remain supported for older hosts. New code should write the
+        // canonical button_video_settings object so width/height/fps update atomically.
         case ("bluetooth", "button_video_width"), ("bluetooth", "button_video_height"),
              ("bluetooth", "button_video_fps"):
             DeviceManager.shared.sgc?.sendButtonVideoRecordingSettings()
@@ -273,11 +278,6 @@ class DeviceStore {
                 DeviceManager.shared.setMicState()
             }
 
-        case ("bluetooth", "offline_captions_running"):
-            if let running = value as? Bool {
-                DeviceManager.shared.setMicState()
-            }
-
         case ("bluetooth", "local_stt_fallback_active"):
             if let active = value as? Bool {
                 DeviceManager.shared.setMicState()

package/ios/Source/MentraBluetoothSDK.swift

@@ -148,6 +148,7 @@ private final class PendingResponse<T> {
 
 @MainActor
 public final class MentraBluetoothSDK {
+    private static let wifiScanTimeoutMs = 20_000
     private static let otaBesVersionWaitMs = 5_000
     private static let otaMtkVersionWaitMs = 2_000
     private static let otaVersionPollMs = 100
@@ -180,6 +181,7 @@ public final class MentraBluetoothSDK {
     private var pendingOtaQuery: PendingResponse<OtaQueryResult>?
     private var pendingOtaStart: PendingResponse<OtaStartAckEvent>?
     private var pendingWifiScan: PendingWifiScan?
+    private var wifiScanTask: Task<[WifiScanResult], Error>?
     private var pendingWifiStatus: PendingWifiStatusRequest?
     private var pendingHotspotStatus: PendingHotspotStatusRequest?
     private var pendingVersionInfo: PendingResponse<VersionInfoResult>?
@@ -485,6 +487,7 @@ public final class MentraBluetoothSDK {
         DeviceStore.shared.apply(ObservableStore.bluetoothCategory, "voice_activity_detection_enabled", enabled)
     }
 
+    @available(*, deprecated, message: "Sticky action-button photo presets are deprecated. Prefer per-request requestPhoto(...) options (e.g. mode .text for AE ÷3, or explicit per-shot fields). This method still works but will be removed in a future release.")
     public func setPhotoCaptureDefaults(_ settings: PhotoCaptureDefaults) async throws -> SettingsAckEvent {
         try await performSettingsCommand(
             setting: "button_photo",
@@ -548,6 +551,12 @@ public final class MentraBluetoothSDK {
         try await performSettingsCommand(
             setting: "button_video_recording",
             updateStore: { _ in
+                DeviceStore.shared.set(
+                    ObservableStore.bluetoothCategory,
+                    "button_video_settings",
+                    ["width": defaults.width, "height": defaults.height, "fps": defaults.fps]
+                )
+                // Keep legacy cache keys readable for older internal callers during migration.
                 DeviceStore.shared.set(ObservableStore.bluetoothCategory, "button_video_width", defaults.width)
                 DeviceStore.shared.set(ObservableStore.bluetoothCategory, "button_video_height", defaults.height)
                 DeviceStore.shared.set(ObservableStore.bluetoothCategory, "button_video_fps", defaults.fps)
@@ -664,37 +673,38 @@ public final class MentraBluetoothSDK {
     }
 
     public func requestWifiScan() async throws -> [WifiScanResult] {
-        guard pendingWifiScan == nil else {
-            throw BluetoothSdkError(
-                code: "request_in_flight",
-                message: "A WiFi scan is already waiting for a glasses response."
-            )
+        if let existing = wifiScanTask {
+            // Join the in-flight scan instead of failing with request_in_flight;
+            // the scan screen auto-starts a scan on mount and can be pushed twice.
+            return try await existing.value
         }
         let pending = PendingResponse<[WifiScanResult]>(operation: "WiFi scan request")
         let request = PendingWifiScan(pending: pending)
         pendingWifiScan = request
         DeviceManager.shared.requestWifiScan()
-        do {
-            let results = try await pending.wait()
-            if pendingWifiScan?.pending === pending {
-                pendingWifiScan = nil
-            }
-            return results
-        } catch {
-            let fallbackResults: [WifiScanResult]
-            if (error as? BluetoothSdkError)?.code == "request_timeout" {
-                fallbackResults = request.latestResults
-            } else {
-                fallbackResults = []
-            }
-            if pendingWifiScan?.pending === pending {
-                pendingWifiScan = nil
+        let task = Task { @MainActor [weak self] () async throws -> [WifiScanResult] in
+            defer {
+                if let self {
+                    if self.pendingWifiScan === request {
+                        self.pendingWifiScan = nil
+                    }
+                    self.wifiScanTask = nil
+                }
             }
-            if !fallbackResults.isEmpty {
-                return fallbackResults
+            do {
+                // The glasses wait up to 15s for scan-results broadcasts before sending
+                // scan_complete, so give them longer than that before falling back.
+                return try await pending.wait(timeoutMs: MentraBluetoothSDK.wifiScanTimeoutMs)
+            } catch {
+                if (error as? BluetoothSdkError)?.code == "request_timeout",
+                   !request.latestResults.isEmpty {
+                    return request.latestResults
+                }
+                throw error
             }
-            throw error
         }
+        wifiScanTask = task
+        return try await task.value
     }
 
     public func sendWifiCredentials(ssid: String, password: String) async throws -> WifiStatusEvent {
@@ -1406,9 +1416,15 @@ public final class MentraBluetoothSDK {
             case .streaming:
                 pendingStreamStarts.removeValue(forKey: streamId)
                 pending.resolve(event)
-            case .error, .reconnectFailed, .stopped:
+            case .reconnectFailed, .stopped:
                 pendingStreamStarts.removeValue(forKey: streamId)
                 pending.reject(streamStatusError(event, code: "stream_start_failed"))
+            case .error:
+                // The glasses publisher automatically retries transient transport
+                // errors. Keep the start pending so a subsequent `streaming`
+                // status can resolve it; `reconnectFailed` or the request timeout
+                // still terminates a publisher that cannot recover.
+                break
             default:
                 break
             }

package/ios/Source/OtaManifest.swift

@@ -78,7 +78,7 @@ enum OtaManifestChecker {
         guard (200 ... 299).contains(httpResponse.statusCode) else {
             throw BluetoothSdkError(
                 code: "ota_manifest_request_failed",
-                message: "OTA manifest request failed with HTTP \(httpResponse.statusCode)."
+                message: "OTA manifest request failed with HTTP \(httpResponse.statusCode) for \(otaVersionUrl)."
             )
         }
         return try JSONDecoder().decode(OtaManifest.self, from: data)

package/ios/Source/camera/CameraModels.swift

@@ -35,6 +35,15 @@ public enum PhotoSize: String {
     }
 }
 
+public enum PhotoMode: String {
+    case photo
+    case text
+
+    public init(normalizedRawValue value: String?) {
+        self = PhotoMode(rawValue: value ?? "") ?? .photo
+    }
+}
+
 public enum ButtonPhotoSize: String {
     case low
     case medium
@@ -234,6 +243,7 @@ public struct CameraFovResult: CustomStringConvertible {
 public struct PhotoRequest {
     public let requestId: String
     public let size: PhotoSize
+    public let mode: PhotoMode
     public let webhookUrl: String?
     public let authToken: String?
     public let compress: PhotoCompression?
@@ -269,7 +279,8 @@ public struct PhotoRequest {
         mfnr: Bool? = nil,
         zsl: Bool? = nil,
         ispDigitalGain: Int? = nil,
-        ispAnalogGain: String? = nil
+        ispAnalogGain: String? = nil,
+        mode: PhotoMode = .photo
     ) {
         self.requestId = nonBlankRequestId(requestId) ?? generatedCameraRequestId("photo")
         self.size = size
@@ -288,6 +299,7 @@ public struct PhotoRequest {
         self.zsl = zsl
         self.ispDigitalGain = ispDigitalGain
         self.ispAnalogGain = ispAnalogGain
+        self.mode = mode
     }
 
     public static func from(params: [String: Any]) -> PhotoRequest {
@@ -354,7 +366,8 @@ public struct PhotoRequest {
             mfnr: optionalBool("mfnr"),
             zsl: optionalBool("zsl"),
             ispDigitalGain: optionalInt("ispDigitalGain"),
-            ispAnalogGain: params["ispAnalogGain"] as? String
+            ispAnalogGain: params["ispAnalogGain"] as? String,
+            mode: PhotoMode(normalizedRawValue: params["mode"] as? String)
         )
     }
 
@@ -403,7 +416,8 @@ public struct PhotoRequest {
             mfnr: mfnr,
             zsl: zsl,
             ispDigitalGain: ispDigitalGain,
-            ispAnalogGain: ispAnalogGain
+            ispAnalogGain: ispAnalogGain,
+            mode: mode
         )
     }
 }

package/ios/Source/sgcs/G1.swift

@@ -739,7 +739,7 @@ class G1: NSObject, SGCManager {
 
     @objc func RN_sendText(_ text: String) {
         Task {
-            let displayText = "\(text)"
+            let displayText = G1Text.sanitizeForDisplay(text)
             guard let textData = displayText.data(using: .utf8) else { return }
 
             var command: [UInt8] = [