@mentra/bluetooth-sdk 0.1.17 → 0.1.19

package/ios/LocalPhotoUploadServer.swift

@@ -104,6 +104,7 @@ final class LocalPhotoUploadServer {
   }
 
   private func handle(_ connection: NWConnection) {
+    traceWifiInput("photo_receiver_connection", connection: connection, state: nil)
     connection.start(queue: queue)
     receive(connection, state: RequestReadState())
   }
@@ -171,6 +172,14 @@ final class LocalPhotoUploadServer {
       let request = HttpRequest.parse(headerText)
       state.request = request
       onLog("\(request.method) \(request.path)")
+      var requestTraceValues: [String: Any] = [
+        "method": request.method,
+        "path": request.path,
+      ]
+      if let contentLength = request.headers["content-length"].flatMap(Int.init) {
+        requestTraceValues["contentLength"] = contentLength
+      }
+      traceWifiInput("photo_receiver_request", connection: connection, state: state, values: requestTraceValues)
 
       if request.method == "GET", request.path == "/" || request.path == "/health" {
         writeJson(connection, status: 200, body: #"{"ok":true,"service":"mentra-photo-upload-receiver"}"#)
@@ -263,6 +272,29 @@ final class LocalPhotoUploadServer {
     )
 
     onLog("upload fields=\(parsed.fields.keys.joined(separator: ",")) requestId=\(requestId ?? "") bytes=\(photoPart.byteCount) saved=\(photoFile.path)")
+    var uploadReceivedTraceValues: [String: Any] = [
+      "durationMs": Int(Date().timeIntervalSince1970 * 1000 - state.startMs),
+      "photoBytes": photoPart.byteCount,
+      "contentLength": state.contentLength,
+      "method": request.method,
+      "path": request.path,
+    ]
+    if let requestId {
+      uploadReceivedTraceValues["requestId"] = requestId
+    }
+    traceWifiInput("photo_receiver_upload_received", connection: connection, state: state, values: uploadReceivedTraceValues)
+    var uploadResponseTraceValues: [String: Any] = [
+      "statusCode": 200,
+      "success": true,
+      "durationMs": Int(Date().timeIntervalSince1970 * 1000 - state.startMs),
+      "contentLength": state.contentLength,
+      "method": request.method,
+      "path": request.path,
+    ]
+    if let requestId {
+      uploadResponseTraceValues["requestId"] = requestId
+    }
+    traceWifiOutput("photo_receiver_response", connection: connection, state: state, values: uploadResponseTraceValues)
     writeJson(
       connection,
       status: 200,
@@ -370,6 +402,51 @@ final class LocalPhotoUploadServer {
     }
   }
 
+  private func traceWifiInput(
+    _ type: String,
+    connection: NWConnection,
+    state: RequestReadState?,
+    values: [String: Any] = [:]
+  ) {
+    traceWifi(direction: "wifi_to_phone", layer: "wifi_http_input", type: type, connection: connection, state: state, values: values)
+  }
+
+  private func traceWifiOutput(
+    _ type: String,
+    connection: NWConnection,
+    state: RequestReadState?,
+    values: [String: Any] = [:]
+  ) {
+    traceWifi(direction: "phone_to_wifi", layer: "wifi_http_output", type: type, connection: connection, state: state, values: values)
+  }
+
+  private func traceWifi(
+    direction: String,
+    layer: String,
+    type: String,
+    connection: NWConnection,
+    state: RequestReadState?,
+    values: [String: Any]
+  ) {
+    var payload = endpointPayload(type: type, connection: connection)
+    if let state {
+      payload["startMs"] = state.startMs
+    }
+    for (key, value) in values {
+      payload[key] = value
+    }
+    BleTraceLogger.logMap(direction: direction, layer: layer, type: type, payload: payload)
+  }
+
+  private func endpointPayload(type: String, connection: NWConnection) -> [String: Any] {
+    var payload: [String: Any] = ["type": type]
+    if case let .hostPort(host, port) = connection.endpoint {
+      payload["remoteHost"] = String(describing: host)
+      payload["remotePort"] = Int(port.rawValue)
+    }
+    return payload
+  }
+
   private struct HttpRequest {
     let method: String
     let path: String
@@ -613,6 +690,7 @@ final class LocalPhotoUploadServer {
   }
 
   private final class RequestReadState {
+    let startMs = Date().timeIntervalSince1970 * 1000
     var buffer = Data()
     var request: HttpRequest?
     var contentLength = 0

package/ios/MentraPhotoReceiverModule.swift

@@ -79,6 +79,16 @@ public class MentraPhotoReceiverModule: Module {
   }
 
   private func handlePhotoUpload(_ upload: PhotoUpload) {
+    BleTraceLogger.logMap(
+      direction: "phone_to_app",
+      layer: "photo_receiver_event",
+      type: "photo_upload",
+      payload: [
+        "requestId": upload.requestId ?? "",
+        "fileName": upload.photoFile.lastPathComponent,
+        "byteCount": upload.byteCount,
+      ]
+    )
     sendEvent("photoUpload", [
       "requestId": upload.requestId as Any,
       "fileUri": upload.photoFile.absoluteString,

package/ios/Source/BluetoothSdkDefaults.swift

@@ -7,7 +7,7 @@ enum BluetoothSdkDefaults {
     }
 
     static let voiceActivityDetectionEnabled = false
-    private static let swiftPackageSdkVersion = "0.1.17"
+    private static let swiftPackageSdkVersion = "0.1.19"
     private static let swiftPackageSdkVersionPlaceholder = "__MENTRA" + "_BLUETOOTH_SDK_VERSION__"
 
     private static func normalizedSdkVersion(_ value: String?) -> String? {

package/ios/Source/Bridge.swift

@@ -15,6 +15,17 @@ class Bridge {
     private static let micChannels = 1
     private static let lc3FrameDurationMs = 10
     private static let defaultLc3FrameSizeBytes = 60
+    private static let audioTraceMetadataKeys = [
+        "sampleRate",
+        "bitsPerSample",
+        "channels",
+        "encoding",
+        "frameDurationMs",
+        "frameSizeBytes",
+        "bitrate",
+        "packetizedFromGlasses",
+        "voiceActivityDetectionEnabled",
+    ]
     private static let eventSinkLock = NSLock()
     private static let defaultEventSinkId = "default"
     private static var eventSinks: [String: (String, [String: Any]) -> Void] = [:]
@@ -486,7 +497,59 @@ class Bridge {
     static func sendTypedMessage(_ type: String, body: [String: Any]) {
         var body = body
         body["type"] = type
+        if let tracePayload = tracePayloadForTypedMessage(type, body: body) {
+            BleTraceLogger.logMap(
+                direction: "phone_to_app",
+                layer: "sdk_event_dispatch",
+                type: type,
+                payload: tracePayload
+            )
+        }
         // Send directly using type as event name - no JSON serialization
         dispatchEvent(type, body)
     }
+
+    private static func tracePayloadForTypedMessage(_ type: String, body: [String: Any]) -> [String: Any]? {
+        if type == "log" {
+            return nil
+        }
+        if isAudioPayloadEvent(type) {
+            return audioTracePayload(type, body: body)
+        }
+        return body
+    }
+
+    private static func isAudioPayloadEvent(_ type: String) -> Bool {
+        type == "mic_pcm" || type == "mic_lc3"
+    }
+
+    private static func audioTracePayload(_ type: String, body: [String: Any]) -> [String: Any] {
+        var payload: [String: Any] = [
+            "type": type,
+            "timestamp": Int(Date().timeIntervalSince1970 * 1000),
+            "payloadOmitted": true,
+            "payloadOmittedReason": "audio",
+        ]
+
+        switch type {
+        case "mic_pcm":
+            if let data = body["pcm"] as? Data {
+                payload["audioBytes"] = data.count
+            }
+        case "mic_lc3":
+            if let data = body["lc3"] as? Data {
+                payload["audioBytes"] = data.count
+            }
+        default:
+            break
+        }
+
+        for key in audioTraceMetadataKeys {
+            if let value = body[key] {
+                payload[key] = value
+            }
+        }
+
+        return payload
+    }
 }

package/ios/Source/DeviceManager.swift

@@ -186,11 +186,6 @@ struct ViewState {
         set { DeviceStore.shared.apply("bluetooth", "bypass_vad", newValue) }
     }
 
-    private var offlineCaptionsRunning: Bool {
-        get { DeviceStore.shared.get("bluetooth", "offline_captions_running") as? Bool ?? false }
-        set { DeviceStore.shared.apply("bluetooth", "offline_captions_running", newValue) }
-    }
-
     private var localSttFallbackActive: Bool {
         get { DeviceStore.shared.get("bluetooth", "local_stt_fallback_active") as? Bool ?? false }
         set { DeviceStore.shared.apply("bluetooth", "local_stt_fallback_active", newValue) }
@@ -325,6 +320,12 @@ struct ViewState {
         ),
     ]
 
+    // Scene slots — one whole SceneFrame per view (main/dashboard), parallel to
+    // viewStates. When a slot holds a scene, viewStates carries a "scene"
+    // sentinel so sendCurrentState routes here. Holding the WHOLE frame keeps
+    // native re-dispatch coherent (dashboard exit re-applies a complete scene).
+    var sceneStates: [SceneFrame?] = [nil, nil]
+
     override init() {
         Bridge.log("MAN: init()")
         super.init()
@@ -421,7 +422,7 @@ struct ViewState {
 
         // Send PCM to local transcriber.
 #if !SWIFT_PACKAGE || MENTRA_FEATURE_LOCAL_STT
-        if shouldSendTranscript || offlineCaptionsRunning || localSttFallbackActive {
+        if shouldSendTranscript || localSttFallbackActive {
             transcriber?.acceptAudio(pcm16le: pcmData)
         }
 #endif
@@ -732,6 +733,11 @@ struct ViewState {
                     borderWidth: currentViewState.borderWidth ?? 0,
                     borderRadius: currentViewState.borderRadius ?? 0
                 )
+            case "scene":
+                let sceneIndex = (headUp && self.contextualDashboard) ? 1 : 0
+                if let frame = self.sceneStates[sceneIndex] {
+                    await sgc?.applySceneFrame(frame)
+                }
             case "clear_view":
                 sgc?.clearDisplay()
             default:
@@ -1044,8 +1050,35 @@ struct ViewState {
             stateIndex = 0
         }
 
-        let layout = event["layout"] as! [String: Any]
-        let layoutType = layout["layoutType"] as! String
+        // Scene frames (display.render() pipeline) take their own path: the
+        // whole frame is the unit, not a layout, and the host's per-element
+        // annotations make redundant frames self-deduping (all-"unchanged"
+        // frames no-op in the SGC base handler).
+        if let sceneMap = event["scene"] as? [String: Any] {
+            handleSceneEvent(stateIndex, sceneMap)
+            return
+        }
+
+        guard let layout = event["layout"] as? [String: Any],
+              let layoutType = layout["layoutType"] as? String
+        else {
+            Bridge.log("MAN: displayEvent missing layout")
+            return
+        }
+
+        // Scene→legacy handoff: a legacy layout is about to draw over a scene
+        // (e.g. a cloud app taking the view from a miniapp). Sweep the scene's
+        // elements first so they don't linger under the new content; clear_view
+        // wipes everything anyway.
+        if let prevFrame = sceneStates[stateIndex] {
+            sceneStates[stateIndex] = nil
+            if layoutType != "clear_view" {
+                let ids = prevFrame.elements.map(\.id)
+                Task { [weak self] in
+                    await self?.sgc?.clearSceneElements(ids)
+                }
+            }
+        }
         var text = layout["text"] as? String ?? " "
         var topText = layout["topText"] as? String ?? " "
         var bottomText = layout["bottomText"] as? String ?? " "
@@ -1090,26 +1123,11 @@ struct ViewState {
             }
         }
 
-        // positioned_text is a sticky overlay container (e.g. the nav trip-stats
-        // label) that lives ALONGSIDE the main view. It must NOT flow through the
-        // single replaceable viewState[stateIndex] — otherwise the constantly-
-        // refreshing main text_wall (maneuver text) and the minimap bitmap_view
-        // overwrite it every frame and it never renders. Route it straight to the
-        // SGC, which keeps its own persistent text container for that rect.
-        if layoutType == "positioned_text" {
-            Bridge.log(
-                "MAN: positioned_text (sticky) → text='\(text)' rect=\(bmpX ?? 0),\(bmpY ?? 0) \(bmpWidth ?? 576)x\(bmpHeight ?? 288)"
-            )
-            Task { [weak self] in
-                await self?.sgc?.sendPositionedText(
-                    text,
-                    x: bmpX ?? 0, y: bmpY ?? 0,
-                    width: bmpWidth ?? 576, height: bmpHeight ?? 288,
-                    borderWidth: borderWidth ?? 0, borderRadius: borderRadius ?? 0
-                )
-            }
-            return
-        }
+        // NOTE: positioned_text used to bypass the viewState slot here (the
+        // "sticky overlay" hack for the old nav HUD). Scenes made that
+        // obsolete: multi-element frames arrive as ONE scene event, so nothing
+        // clobbers anything. Legacy positioned_text now flows through the slot
+        // like every other layout (matching Android).
 
         let cS = viewStates[stateIndex]
         let nS = newViewState
@@ -1136,6 +1154,93 @@ struct ViewState {
         }
     }
 
+    /// Parse + store a scene frame, then dispatch it if its view is visible.
+    private func handleSceneEvent(_ stateIndex: Int, _ sceneMap: [String: Any]) {
+        guard var frame = parseSceneFrame(sceneMap) else { return }
+        let prevFrame = sceneStates[stateIndex]
+
+        if prevFrame == nil {
+            // Legacy→scene handoff: stale legacy content (e.g. a cloud app's
+            // text wall) must not linger under the scene's elements.
+            // clearDisplay is the per-device "wipe what's there" (blank-in-place
+            // on G2 — no page rebuild).
+            let prevLegacyType = viewStates[stateIndex].layoutType
+            if !prevLegacyType.isEmpty, prevLegacyType != "clear_view", prevLegacyType != "scene" {
+                sgc?.clearDisplay()
+            }
+        } else if let prevFrame, prevFrame.appId != frame.appId {
+            // Cross-app switch: the host's diff baseline is per-app, so the new
+            // app's annotations don't know the old app's elements are on the
+            // glasses. Sweep the old app's elements (SGC registries still map
+            // them), then paint the new frame from scratch. The boot message
+            // interposes between apps in practice, so this isn't visible.
+            let ids = prevFrame.elements.map(\.id)
+            Task { [weak self] in
+                await self?.sgc?.clearSceneElements(ids)
+            }
+            frame = frame.asReplay()
+        }
+
+        // Store the REDISPATCH form: any later sendCurrentState (dashboard
+        // exit, head-up return) must repaint the whole frame — the original
+        // annotations are only valid for the first dispatch right now.
+        sceneStates[stateIndex] = frame.asReplay()
+        viewStates[stateIndex] = ViewState(
+            topText: " ", bottomText: " ", title: " ", layoutType: "scene",
+            text: " ", data: nil, animationData: nil
+        )
+
+        let hUp = headUp && contextualDashboard
+        if (stateIndex == 0 && !hUp) || (stateIndex == 1 && hUp) {
+            dispatchSceneFrame(frame)
+        }
+    }
+
+    /// Guarded scene dispatch — mirrors sendCurrentState's send conditions.
+    private func dispatchSceneFrame(_ frame: SceneFrame) {
+        if screenDisabled { return }
+        if sgc?.type.contains(DeviceTypes.SIMULATED) ?? true { return }
+        guard sgc?.fullyBooted == true else {
+            Bridge.log("MAN: dispatchSceneFrame(): sgc not ready")
+            return
+        }
+        Task { [weak self] in
+            await self?.sgc?.applySceneFrame(frame)
+        }
+    }
+
+    private func parseSceneFrame(_ sceneMap: [String: Any]) -> SceneFrame? {
+        guard let elementsRaw = sceneMap["elements"] as? [[String: Any]] else { return nil }
+        let elements: [SceneElement] = elementsRaw.compactMap { el in
+            guard let id = el["id"] as? String,
+                  let type = el["type"] as? String,
+                  let box = el["box"] as? [String: Any]
+            else { return nil }
+            let style = el["style"] as? [String: Any]
+            return SceneElement(
+                id: id,
+                type: type,
+                x: (box["x"] as? NSNumber)?.int32Value ?? 0,
+                y: (box["y"] as? NSNumber)?.int32Value ?? 0,
+                w: (box["w"] as? NSNumber)?.int32Value ?? 0,
+                h: (box["h"] as? NSNumber)?.int32Value ?? 0,
+                text: (el["text"] as? String).map { parsePlaceholders($0) },
+                data: el["data"] as? String,
+                border: ((style?["border"]) as? NSNumber)?.int32Value ?? 0,
+                radius: ((style?["radius"]) as? NSNumber)?.int32Value ?? 0,
+                change: el["change"] as? String ?? "created",
+                contentHash: el["contentHash"] as? String ?? ""
+            )
+        }
+        return SceneFrame(
+            appId: sceneMap["appId"] as? String ?? "",
+            epoch: (sceneMap["sceneEpoch"] as? NSNumber)?.intValue ?? 0,
+            replay: sceneMap["replay"] as? Bool ?? false,
+            elements: elements,
+            removed: (sceneMap["removed"] as? [String]) ?? []
+        )
+    }
+
     func showDashboard() {
         sgc?.showDashboard()
     }
@@ -1321,7 +1426,7 @@ struct ViewState {
 
     func setMicState() {
         let willSendPcm = shouldSendPcm || shouldSendLc3
-        let willSendTranscript = shouldSendTranscript || offlineCaptionsRunning || localSttFallbackActive
+        let willSendTranscript = shouldSendTranscript || localSttFallbackActive
         micEnabled = willSendPcm || willSendTranscript
         updateMicState()
     }

package/ios/Source/DeviceStore.swift

@@ -273,11 +273,6 @@ class DeviceStore {
                 DeviceManager.shared.setMicState()
             }
 
-        case ("bluetooth", "offline_captions_running"):
-            if let running = value as? Bool {
-                DeviceManager.shared.setMicState()
-            }
-
         case ("bluetooth", "local_stt_fallback_active"):
             if let active = value as? Bool {
                 DeviceManager.shared.setMicState()

package/ios/Source/MentraBluetoothSDK.swift

@@ -148,6 +148,7 @@ private final class PendingResponse<T> {
 
 @MainActor
 public final class MentraBluetoothSDK {
+    private static let wifiScanTimeoutMs = 20_000
     private static let otaBesVersionWaitMs = 5_000
     private static let otaMtkVersionWaitMs = 2_000
     private static let otaVersionPollMs = 100
@@ -180,6 +181,7 @@ public final class MentraBluetoothSDK {
     private var pendingOtaQuery: PendingResponse<OtaQueryResult>?
     private var pendingOtaStart: PendingResponse<OtaStartAckEvent>?
     private var pendingWifiScan: PendingWifiScan?
+    private var wifiScanTask: Task<[WifiScanResult], Error>?
     private var pendingWifiStatus: PendingWifiStatusRequest?
     private var pendingHotspotStatus: PendingHotspotStatusRequest?
     private var pendingVersionInfo: PendingResponse<VersionInfoResult>?
@@ -664,37 +666,38 @@ public final class MentraBluetoothSDK {
     }
 
     public func requestWifiScan() async throws -> [WifiScanResult] {
-        guard pendingWifiScan == nil else {
-            throw BluetoothSdkError(
-                code: "request_in_flight",
-                message: "A WiFi scan is already waiting for a glasses response."
-            )
+        if let existing = wifiScanTask {
+            // Join the in-flight scan instead of failing with request_in_flight;
+            // the scan screen auto-starts a scan on mount and can be pushed twice.
+            return try await existing.value
         }
         let pending = PendingResponse<[WifiScanResult]>(operation: "WiFi scan request")
         let request = PendingWifiScan(pending: pending)
         pendingWifiScan = request
         DeviceManager.shared.requestWifiScan()
-        do {
-            let results = try await pending.wait()
-            if pendingWifiScan?.pending === pending {
-                pendingWifiScan = nil
-            }
-            return results
-        } catch {
-            let fallbackResults: [WifiScanResult]
-            if (error as? BluetoothSdkError)?.code == "request_timeout" {
-                fallbackResults = request.latestResults
-            } else {
-                fallbackResults = []
-            }
-            if pendingWifiScan?.pending === pending {
-                pendingWifiScan = nil
+        let task = Task { @MainActor [weak self] () async throws -> [WifiScanResult] in
+            defer {
+                if let self {
+                    if self.pendingWifiScan === request {
+                        self.pendingWifiScan = nil
+                    }
+                    self.wifiScanTask = nil
+                }
             }
-            if !fallbackResults.isEmpty {
-                return fallbackResults
+            do {
+                // The glasses wait up to 15s for scan-results broadcasts before sending
+                // scan_complete, so give them longer than that before falling back.
+                return try await pending.wait(timeoutMs: MentraBluetoothSDK.wifiScanTimeoutMs)
+            } catch {
+                if (error as? BluetoothSdkError)?.code == "request_timeout",
+                   !request.latestResults.isEmpty {
+                    return request.latestResults
+                }
+                throw error
             }
-            throw error
         }
+        wifiScanTask = task
+        return try await task.value
     }
 
     public func sendWifiCredentials(ssid: String, password: String) async throws -> WifiStatusEvent {

package/ios/Source/OtaManifest.swift

@@ -78,7 +78,7 @@ enum OtaManifestChecker {
         guard (200 ... 299).contains(httpResponse.statusCode) else {
             throw BluetoothSdkError(
                 code: "ota_manifest_request_failed",
-                message: "OTA manifest request failed with HTTP \(httpResponse.statusCode)."
+                message: "OTA manifest request failed with HTTP \(httpResponse.statusCode) for \(otaVersionUrl)."
             )
         }
         return try JSONDecoder().decode(OtaManifest.self, from: data)