@mentra/bluetooth-sdk 0.1.6 → 0.1.8

package/ios/Source/MentraBluetoothSDK.swift

@@ -1,6 +1,20 @@
 import CoreBluetooth
 import Foundation
 
+@MainActor
+private final class ActiveStreamKeepAlive {
+    let streamId: String
+    let intervalSeconds: Int
+    var pendingAckId: String?
+    var missedAckCount = 0
+    var task: Task<Void, Never>?
+
+    init(streamId: String, intervalSeconds: Int) {
+        self.streamId = streamId
+        self.intervalSeconds = intervalSeconds
+    }
+}
+
 @MainActor
 private final class ActiveScanSession {
     let model: DeviceModel
@@ -33,6 +47,7 @@ public final class MentraBluetoothSDK {
     private var suppressDefaultDeviceEvents = false
     private var defaultDeviceApplyGeneration = 0
     private var activeScanSessions: [UUID: ActiveScanSession] = [:]
+    private var activeStreamKeepAlive: ActiveStreamKeepAlive?
 
     public init(configuration: MentraBluetoothSDKConfiguration = .default) {
         self.configuration = configuration
@@ -268,8 +283,12 @@ public final class MentraBluetoothSDK {
         DeviceStore.shared.apply(ObservableStore.bluetoothCategory, "screen_disabled", disabled)
     }
 
-    public func setGalleryMode(_ mode: GalleryMode) async throws {
-        DeviceStore.shared.apply(ObservableStore.bluetoothCategory, "galleryModeAuto", mode == .auto)
+    public func setGalleryModeEnabled(_ enabled: Bool) async throws {
+        DeviceStore.shared.apply(ObservableStore.bluetoothCategory, "gallery_mode", enabled)
+    }
+
+    public func setVoiceActivityDetectionEnabled(_ enabled: Bool) async throws {
+        DeviceStore.shared.apply(ObservableStore.bluetoothCategory, "voice_activity_detection_enabled", enabled)
     }
 
     public func setButtonPhotoSettings(size: ButtonPhotoSize) async throws {
@@ -280,14 +299,14 @@ public final class MentraBluetoothSDK {
         try await setButtonPhotoSettings(size: settings.size)
     }
 
-    public func setButtonVideoRecordingSettings(width: Int, height: Int, frameRate: Int) async throws {
+    public func setButtonVideoRecordingSettings(width: Int, height: Int, fps: Int) async throws {
         DeviceStore.shared.apply(ObservableStore.bluetoothCategory, "button_video_width", width)
         DeviceStore.shared.apply(ObservableStore.bluetoothCategory, "button_video_height", height)
-        DeviceStore.shared.apply(ObservableStore.bluetoothCategory, "button_video_fps", frameRate)
+        DeviceStore.shared.apply(ObservableStore.bluetoothCategory, "button_video_fps", fps)
     }
 
     public func setButtonVideoRecordingSettings(_ settings: ButtonVideoRecordingSettings) async throws {
-        try await setButtonVideoRecordingSettings(width: settings.width, height: settings.height, frameRate: settings.frameRate)
+        try await setButtonVideoRecordingSettings(width: settings.width, height: settings.height, fps: settings.fps)
     }
 
     public func setButtonCameraLed(enabled: Bool) async throws {
@@ -305,7 +324,6 @@ public final class MentraBluetoothSDK {
     public func setMicState(
         enabled: Bool,
         useGlassesMic: Bool = true,
-        bypassVad: Bool = true,
         sendTranscript: Bool = false,
         sendLc3Data: Bool = false
     ) {
@@ -319,7 +337,6 @@ public final class MentraBluetoothSDK {
         applyMicState(
             sendPcmData: enabled,
             sendTranscript: enabled && sendTranscript,
-            bypassVad: bypassVad,
             sendLc3Data: enabled && sendLc3Data
         )
     }
@@ -327,13 +344,11 @@ public final class MentraBluetoothSDK {
     private func applyMicState(
         sendPcmData: Bool,
         sendTranscript: Bool,
-        bypassVad: Bool,
         sendLc3Data: Bool
     ) {
         DeviceStore.shared.apply(ObservableStore.bluetoothCategory, "should_send_pcm", sendPcmData)
         DeviceStore.shared.apply(ObservableStore.bluetoothCategory, "should_send_lc3", sendLc3Data)
         DeviceStore.shared.apply(ObservableStore.bluetoothCategory, "should_send_transcript", sendTranscript)
-        DeviceStore.shared.apply(ObservableStore.bluetoothCategory, "bypass_vad", bypassVad)
         DeviceManager.shared.setMicState()
     }
 
@@ -376,6 +391,9 @@ public final class MentraBluetoothSDK {
     }
 
     public func requestPhoto(_ request: PhotoRequest) {
+        Bridge.log(
+            "NATIVE: PHOTO PIPELINE [3b/6] MentraBluetoothSdk.requestPhoto requestId=\(request.requestId) appId=\(request.appId)"
+        )
         DeviceManager.shared.requestPhoto(
             request.requestId,
             request.appId,
@@ -383,7 +401,10 @@ public final class MentraBluetoothSDK {
             request.webhookUrl,
             request.authToken,
             request.compress?.rawValue,
-            request.sound
+            request.flash,
+            request.sound,
+            exposureTimeNs: request.exposureTimeNs,
+            iso: request.iso
         )
     }
 
@@ -392,10 +413,17 @@ public final class MentraBluetoothSDK {
     }
 
     public func startStream(_ request: StreamRequest) {
-        DeviceManager.shared.startStream(request.values)
+        var values = request.values
+        let streamId = stringValue(values, "streamId").flatMap { $0.isEmpty ? nil : $0 } ?? "sdk-\(UUID().uuidString)"
+        values["streamId"] = streamId
+        stopStreamKeepAliveMonitor()
+        DeviceManager.shared.startStream(values)
+        if request.keepAlive, !request.isExternallyManagedKeepAlive {
+            startStreamKeepAliveMonitor(streamId: streamId, intervalSeconds: request.keepAliveIntervalSeconds)
+        }
     }
 
-    public func keepStreamAlive(_ request: StreamKeepAliveRequest) {
+    func sendCloudStreamKeepAlive(_ request: StreamKeepAliveRequest) {
         DeviceManager.shared.keepStreamAlive(request.values)
     }
 
@@ -412,6 +440,7 @@ public final class MentraBluetoothSDK {
     }
 
     public func stopStream() {
+        stopStreamKeepAliveMonitor()
         DeviceManager.shared.stopStream()
     }
 
@@ -452,6 +481,7 @@ public final class MentraBluetoothSDK {
     }
 
     public func invalidate() {
+        stopStreamKeepAliveMonitor()
         if let bridgeEventSinkId {
             Bridge.removeEventSink(bridgeEventSinkId)
             self.bridgeEventSinkId = nil
@@ -463,6 +493,80 @@ public final class MentraBluetoothSDK {
         delegate = nil
     }
 
+    private func startStreamKeepAliveMonitor(streamId: String, intervalSeconds requestedIntervalSeconds: Int) {
+        let intervalSeconds = requestedIntervalSeconds > 0 ? requestedIntervalSeconds : 5
+        let tracker = ActiveStreamKeepAlive(streamId: streamId, intervalSeconds: intervalSeconds)
+        activeStreamKeepAlive = tracker
+        sendNextStreamKeepAlive(for: tracker)
+    }
+
+    private func stopStreamKeepAliveMonitor() {
+        activeStreamKeepAlive?.task?.cancel()
+        activeStreamKeepAlive = nil
+    }
+
+    private func sendNextStreamKeepAlive(for tracker: ActiveStreamKeepAlive) {
+        guard activeStreamKeepAlive === tracker else { return }
+
+        if tracker.pendingAckId != nil {
+            tracker.missedAckCount += 1
+            if tracker.missedAckCount >= 3 {
+                activeStreamKeepAlive = nil
+                tracker.task?.cancel()
+                let event = StreamStatusEvent(
+                    status: .error(
+                        streamId: tracker.streamId,
+                        errorDetails: "Stream keep-alive timed out after \(tracker.missedAckCount) missed ACKs",
+                        timestamp: Int(Date().timeIntervalSince1970 * 1000),
+                        resolvedConfig: nil
+                    )
+                )
+                delegate?.mentraBluetoothSDK(self, didReceive: .streamStatus(event))
+                return
+            }
+        }
+
+        let ackId = "ack-\(Int(Date().timeIntervalSince1970 * 1000))"
+        tracker.pendingAckId = ackId
+        DeviceManager.shared.keepStreamAlive(
+            StreamKeepAliveRequest(streamId: tracker.streamId, ackId: ackId).values
+        )
+
+        tracker.task?.cancel()
+        tracker.task = Task { @MainActor [weak self, weak tracker] in
+            guard let tracker else { return }
+            try? await Task.sleep(nanoseconds: UInt64(tracker.intervalSeconds) * 1_000_000_000)
+            self?.sendNextStreamKeepAlive(for: tracker)
+        }
+    }
+
+    private func handleStreamKeepAliveAck(_ event: KeepAliveAckEvent) -> Bool {
+        guard let tracker = activeStreamKeepAlive,
+              event.streamId == tracker.streamId,
+              event.ackId == tracker.pendingAckId
+        else {
+            return false
+        }
+        tracker.pendingAckId = nil
+        tracker.missedAckCount = 0
+        return true
+    }
+
+    private func handleStreamStatusForKeepAlive(_ status: StreamStatus) {
+        if let streamId = status.streamId,
+           activeStreamKeepAlive?.streamId != streamId
+        {
+            return
+        }
+
+        switch status.state {
+        case .stopped, .stopping, .error, .reconnectFailed:
+            stopStreamKeepAliveMonitor()
+        default:
+            break
+        }
+    }
+
     private func dispatchStoreUpdate(_ category: String, _ changes: [String: Any]) {
         switch ObservableStore.normalizeCategory(category) {
         case "glasses":
@@ -573,6 +677,16 @@ public final class MentraBluetoothSDK {
                 values: data
             )
             delegate?.mentraBluetoothSDK(self, didReceive: .localTranscription(event))
+        case "voice_activity_detection_status":
+            delegate?.mentraBluetoothSDK(
+                self,
+                didReceive: .voiceActivityDetectionStatus(VoiceActivityDetectionStatusEvent(values: data))
+            )
+        case "speaking_status":
+            delegate?.mentraBluetoothSDK(
+                self,
+                didReceive: .speakingStatus(SpeakingStatusEvent(values: data))
+            )
         case "hotspot_status_change":
             delegate?.mentraBluetoothSDK(self, didReceive: .hotspotStatus(HotspotStatusEvent(values: data)))
         case "wifi_status_change":
@@ -581,10 +695,17 @@ public final class MentraBluetoothSDK {
             delegate?.mentraBluetoothSDK(self, didReceive: .hotspotError(HotspotErrorEvent(values: data)))
         case "photo_response":
             delegate?.mentraBluetoothSDK(self, didReceive: .photoResponse(PhotoResponseEvent(values: data)))
+        case "photo_status":
+            delegate?.mentraBluetoothSDK(self, didReceive: .photoStatus(PhotoStatusEvent(values: data)))
         case "stream_status":
-            delegate?.mentraBluetoothSDK(self, didReceive: .streamStatus(StreamStatusEvent(values: data)))
+            let event = StreamStatusEvent(values: data)
+            handleStreamStatusForKeepAlive(event.status)
+            delegate?.mentraBluetoothSDK(self, didReceive: .streamStatus(event))
         case "keep_alive_ack":
-            delegate?.mentraBluetoothSDK(self, didReceive: .keepAliveAck(KeepAliveAckEvent(values: data)))
+            let event = KeepAliveAckEvent(values: data)
+            if !handleStreamKeepAliveAck(event) {
+                delegate?.mentraBluetoothSDK(self, didReceive: .keepAliveAck(event))
+            }
         case "compatible_glasses_search_stop":
             delegate?.mentraBluetoothSDK(self, didStopScan: .completed)
         case "pair_failure":

package/ios/Source/Status/DeviceStatus.swift

@@ -133,6 +133,7 @@ struct GlassesStatus: CustomStringConvertible {
     var fullyBooted: Bool { boolValue(values, "fullyBooted") ?? false }
     var connected: Bool { boolValue(values, "connected") ?? false }
     var micEnabled: Bool { boolValue(values, "micEnabled") ?? false }
+    var voiceActivityDetectionEnabled: Bool { boolValue(values, "voiceActivityDetectionEnabled") ?? true }
     var connectionState: GlassesConnectionState { GlassesConnectionState(stringValue(values, "connectionState")) }
     var bluetoothClassicConnected: Bool { boolValue(values, "bluetoothClassicConnected") ?? false }
     var signalStrength: Int { intValue(values["signalStrength"]) ?? -1 }
@@ -246,6 +247,10 @@ struct BluetoothStatus: CustomStringConvertible {
         if let searchResults = values["searchResults"] as? [[String: Any]] {
             normalizedValues["searchResults"] = searchResults.compactMap { Device(values: $0)?.dictionary }
         }
+        if let galleryMode = boolValue(values, "gallery_mode") {
+            normalizedValues["galleryModeEnabled"] = galleryMode
+            normalizedValues.removeValue(forKey: "gallery_mode")
+        }
         return normalizedValues
     }
 
@@ -294,7 +299,7 @@ struct BluetoothStatus: CustomStringConvertible {
     var dashboardDepth: Int { intValue(values["dashboard_depth"]) ?? 2 }
     var headUpAngle: Int { intValue(values["head_up_angle"]) ?? 30 }
     var contextualDashboard: Bool { boolValue(values, "contextual_dashboard") ?? true }
-    var galleryModeAuto: Bool { boolValue(values, "galleryModeAuto") ?? true }
+    var galleryModeEnabled: Bool { boolValue(values, "gallery_mode") ?? boolValue(values, "galleryModeEnabled") ?? true }
     var buttonPhotoSize: ButtonPhotoSize {
         ButtonPhotoSize(rawValue: stringValue(values, "button_photo_size") ?? "") ?? .medium
     }
@@ -306,7 +311,6 @@ struct BluetoothStatus: CustomStringConvertible {
     var shouldSendPcm: Bool { boolValue(values, "should_send_pcm") ?? false }
     var shouldSendLc3: Bool { boolValue(values, "should_send_lc3") ?? false }
     var shouldSendTranscript: Bool { boolValue(values, "should_send_transcript") ?? false }
-    var bypassVad: Bool { boolValue(values, "bypass_vad") ?? true }
     var offlineCaptionsRunning: Bool { boolValue(values, "offline_captions_running") ?? false }
     var localSttFallbackActive: Bool { boolValue(values, "local_stt_fallback_active") ?? false }
     var shouldSendBootingMessage: Bool { boolValue(values, "shouldSendBootingMessage") ?? true }
@@ -335,6 +339,7 @@ struct GlassesStatusUpdate: CustomStringConvertible {
     var fullyBooted: Bool? { optionalBoolValue(values, "fullyBooted") }
     var connected: Bool? { optionalBoolValue(values, "connected") }
     var micEnabled: Bool? { optionalBoolValue(values, "micEnabled") }
+    var voiceActivityDetectionEnabled: Bool? { optionalBoolValue(values, "voiceActivityDetectionEnabled") }
     var connectionState: GlassesConnectionState? {
         GlassesConnectionState.fromValue(optionalStringValue(values, "connectionState"))
     }
@@ -403,6 +408,10 @@ struct BluetoothStatusUpdate: CustomStringConvertible {
         if let searchResults = values["searchResults"] as? [[String: Any]] {
             normalizedValues["searchResults"] = searchResults.compactMap { Device(values: $0)?.dictionary }
         }
+        if let galleryMode = optionalBoolValue(values, "gallery_mode") {
+            normalizedValues["galleryModeEnabled"] = galleryMode
+            normalizedValues.removeValue(forKey: "gallery_mode")
+        }
         self.values = normalizedValues
     }
 
@@ -440,7 +449,7 @@ struct BluetoothStatusUpdate: CustomStringConvertible {
     var dashboardDepth: Int? { optionalIntValue(values, "dashboard_depth") }
     var headUpAngle: Int? { optionalIntValue(values, "head_up_angle") }
     var contextualDashboard: Bool? { optionalBoolValue(values, "contextual_dashboard") }
-    var galleryModeAuto: Bool? { optionalBoolValue(values, "galleryModeAuto") }
+    var galleryModeEnabled: Bool? { optionalBoolValue(values, "gallery_mode") ?? optionalBoolValue(values, "galleryModeEnabled") }
     var buttonPhotoSize: ButtonPhotoSize? {
         optionalStringValue(values, "button_photo_size").flatMap(ButtonPhotoSize.init(rawValue:))
     }
@@ -452,7 +461,6 @@ struct BluetoothStatusUpdate: CustomStringConvertible {
     var shouldSendPcm: Bool? { optionalBoolValue(values, "should_send_pcm") }
     var shouldSendLc3: Bool? { optionalBoolValue(values, "should_send_lc3") }
     var shouldSendTranscript: Bool? { optionalBoolValue(values, "should_send_transcript") }
-    var bypassVad: Bool? { optionalBoolValue(values, "bypass_vad") }
     var offlineCaptionsRunning: Bool? { optionalBoolValue(values, "offline_captions_running") }
     var localSttFallbackActive: Bool? { optionalBoolValue(values, "local_stt_fallback_active") }
     var shouldSendBootingMessage: Bool? { optionalBoolValue(values, "shouldSendBootingMessage") }

package/ios/Source/Status/RuntimeState.swift

@@ -121,6 +121,7 @@ public enum GlassesRuntimeState: CustomStringConvertible {
         hotspot: HotspotStatus,
         ready: Bool,
         signal: SignalState,
+        voiceActivityDetectionEnabled: Bool,
         wifi: WifiStatus
     )
 
@@ -153,6 +154,7 @@ public enum GlassesRuntimeState: CustomStringConvertible {
                 strengthDbm: status.signalStrength == -1 ? nil : status.signalStrength,
                 updatedAt: status.signalStrengthUpdatedAt <= 0 ? nil : status.signalStrengthUpdatedAt
             ),
+            voiceActivityDetectionEnabled: status.voiceActivityDetectionEnabled,
             wifi: status.wifi
         )
     }
@@ -168,7 +170,7 @@ public enum GlassesRuntimeState: CustomStringConvertible {
         switch self {
         case let .disconnected(connection):
             connection
-        case let .connected(_, connection, _, _, _, _, _, _):
+        case let .connected(_, connection, _, _, _, _, _, _, _):
             connection
         }
     }
@@ -177,48 +179,55 @@ public enum GlassesRuntimeState: CustomStringConvertible {
         switch self {
         case .disconnected:
             false
-        case let .connected(_, _, _, _, _, ready, _, _):
+        case let .connected(_, _, _, _, _, ready, _, _, _):
             ready
         }
     }
 
     public var battery: GlassesBatteryState? {
-        guard case let .connected(battery, _, _, _, _, _, _, _) = self else {
+        guard case let .connected(battery, _, _, _, _, _, _, _, _) = self else {
             return nil
         }
         return battery
     }
 
     public var device: ConnectedGlassesInfo? {
-        guard case let .connected(_, _, device, _, _, _, _, _) = self else {
+        guard case let .connected(_, _, device, _, _, _, _, _, _) = self else {
             return nil
         }
         return device
     }
 
     public var firmware: FirmwareInfo? {
-        guard case let .connected(_, _, _, firmware, _, _, _, _) = self else {
+        guard case let .connected(_, _, _, firmware, _, _, _, _, _) = self else {
             return nil
         }
         return firmware
     }
 
     public var hotspot: HotspotStatus? {
-        guard case let .connected(_, _, _, _, hotspot, _, _, _) = self else {
+        guard case let .connected(_, _, _, _, hotspot, _, _, _, _) = self else {
             return nil
         }
         return hotspot
     }
 
     public var signal: SignalState? {
-        guard case let .connected(_, _, _, _, _, _, signal, _) = self else {
+        guard case let .connected(_, _, _, _, _, _, signal, _, _) = self else {
             return nil
         }
         return signal
     }
 
+    public var voiceActivityDetectionEnabled: Bool {
+        guard case let .connected(_, _, _, _, _, _, _, voiceActivityDetectionEnabled, _) = self else {
+            return false
+        }
+        return voiceActivityDetectionEnabled
+    }
+
     public var wifi: WifiStatus? {
-        guard case let .connected(_, _, _, _, _, _, _, wifi) = self else {
+        guard case let .connected(_, _, _, _, _, _, _, _, wifi) = self else {
             return nil
         }
         return wifi
@@ -228,17 +237,17 @@ public enum GlassesRuntimeState: CustomStringConvertible {
         switch self {
         case let .disconnected(connection):
             "GlassesRuntimeState(\(connection.rawValue))"
-        case let .connected(_, _, device, _, _, ready, _, _):
+        case let .connected(_, _, device, _, _, ready, _, _, _):
             "GlassesRuntimeState(connected: \(device), ready: \(ready))"
         }
     }
 }
 
 public struct GalleryModeState: Equatable {
-    public let desired: GalleryMode
+    public let enabled: Bool
 
-    public init(desired: GalleryMode) {
-        self.desired = desired
+    public init(enabled: Bool) {
+        self.enabled = enabled
     }
 }
 
@@ -288,7 +297,7 @@ public struct PhoneSdkRuntimeState: CustomStringConvertible {
     init(status: BluetoothStatus) {
         currentMic = MicMode(rawValue: status.currentMic)
         defaultDevice = status.defaultDevice
-        galleryMode = GalleryModeState(desired: status.galleryModeAuto ? .auto : .manual)
+        galleryMode = GalleryModeState(enabled: status.galleryModeEnabled)
         lastLog = status.lastLog
         micRanking = status.micRanking.compactMap(MicMode.init(rawValue:))
         otherBluetoothConnected = status.otherBtConnected