@mentra/bluetooth-sdk 0.1.6 → 0.1.7

package/ios/Source/DeviceStore.swift

@@ -28,6 +28,7 @@ class DeviceStore {
         store.set("glasses", "deviceModel", "")
         store.set("glasses", "firmwareVersion", "")
         store.set("glasses", "micEnabled", false)
+        store.set("glasses", "voiceActivityDetectionEnabled", true)
         store.set("glasses", "bluetoothClassicConnected", false)
         store.set("glasses", "caseRemoved", true)
         store.set("glasses", "caseOpen", true)
@@ -71,21 +72,26 @@ class DeviceStore {
         store.set("bluetooth", "pending_wearable", "")
         store.set("bluetooth", "device_name", "")
         store.set("bluetooth", "device_address", "")
+        store.set("bluetooth", "default_controller", "")
+        store.set("bluetooth", "pending_controller", "")
+        store.set("bluetooth", "controller_device_name", "")
         store.set("bluetooth", "screen_disabled", false)
         store.set("bluetooth", "preferred_mic", "auto")
         store.set("bluetooth", "sensing_enabled", true)
+        store.set("bluetooth", "power_saving_mode", false)
         store.set("bluetooth", "brightness", 50)
         store.set("bluetooth", "auto_brightness", true)
         store.set("bluetooth", "dashboard_height", 4)
         store.set("bluetooth", "dashboard_depth", 2)
         store.set("bluetooth", "head_up_angle", 30)
         store.set("bluetooth", "contextual_dashboard", true)
-        store.set("bluetooth", "galleryModeAuto", true)
+        store.set("bluetooth", "gallery_mode", true)
+        store.set("bluetooth", "voice_activity_detection_enabled", true)
         store.set("bluetooth", "screen_disabled", false)
         store.set("bluetooth", "button_photo_size", "medium")
         store.set("bluetooth", "button_camera_led", true)
         store.set("bluetooth", "button_max_recording_time", 10)
-        store.set("bluetooth", "camera_fov", ["fov": 118, "roiPosition": 0])
+        store.set("bluetooth", "camera_fov", ["fov": 118, "roi_position": 0])
         store.set("bluetooth", "button_video_width", 1280)
         store.set("bluetooth", "button_video_height", 720)
         store.set("bluetooth", "button_video_fps", 30)
@@ -96,7 +102,6 @@ class DeviceStore {
         store.set("bluetooth", "should_send_pcm", false)
         store.set("bluetooth", "should_send_lc3", false)
         store.set("bluetooth", "should_send_transcript", false)
-        store.set("bluetooth", "bypass_vad", true)
         store.set("bluetooth", "use_native_dashboard", false)
     }
 
@@ -215,9 +220,12 @@ class DeviceStore {
                 DeviceManager.shared.sgc?.setDashboardMenu(items)
             }
 
-        case ("bluetooth", "galleryModeAuto"):
+        case ("bluetooth", "gallery_mode"):
             DeviceManager.shared.sgc?.sendGalleryMode()
 
+        case ("bluetooth", "voice_activity_detection_enabled"):
+            DeviceManager.shared.sgc?.sendVoiceActivityDetectionSetting()
+
         case ("bluetooth", "screen_disabled"):
             if let disabled = value as? Bool {
                 if disabled {

package/ios/Source/Events/BluetoothEvents.swift

@@ -44,9 +44,39 @@ public struct TouchEvent: CustomStringConvertible {
     }
 }
 
+public struct VoiceActivityDetectionStatusEvent: CustomStringConvertible {
+    public let voiceActivityDetectionEnabled: Bool
+    public let values: [String: Any]
+
+    public init(values: [String: Any]) {
+        voiceActivityDetectionEnabled = boolValue(values, "voiceActivityDetectionEnabled") ?? true
+        self.values = values
+    }
+
+    public var description: String {
+        "VoiceActivityDetectionStatusEvent(voiceActivityDetectionEnabled: \(voiceActivityDetectionEnabled))"
+    }
+}
+
+public struct SpeakingStatusEvent: CustomStringConvertible {
+    public let speaking: Bool
+    public let values: [String: Any]
+
+    public init(values: [String: Any]) {
+        speaking = boolValue(values, "speaking") ?? false
+        self.values = values
+    }
+
+    public var description: String {
+        "SpeakingStatusEvent(speaking: \(speaking))"
+    }
+}
+
 public enum BluetoothEvent: CustomStringConvertible {
     case buttonPress(ButtonPressEvent)
     case touch(TouchEvent)
+    case voiceActivityDetectionStatus(VoiceActivityDetectionStatusEvent)
+    case speakingStatus(SpeakingStatusEvent)
     case wifiStatus(WifiStatusEvent)
     case hotspotStatus(HotspotStatusEvent)
     case hotspotError(HotspotErrorEvent)
@@ -62,6 +92,10 @@ public enum BluetoothEvent: CustomStringConvertible {
             event.description
         case let .touch(event):
             event.description
+        case let .voiceActivityDetectionStatus(event):
+            event.description
+        case let .speakingStatus(event):
+            event.description
         case let .wifiStatus(event):
             event.description
         case let .hotspotStatus(event):

package/ios/Source/MentraBluetoothSDK.swift

@@ -268,8 +268,12 @@ public final class MentraBluetoothSDK {
         DeviceStore.shared.apply(ObservableStore.bluetoothCategory, "screen_disabled", disabled)
     }
 
-    public func setGalleryMode(_ mode: GalleryMode) async throws {
-        DeviceStore.shared.apply(ObservableStore.bluetoothCategory, "galleryModeAuto", mode == .auto)
+    public func setGalleryModeEnabled(_ enabled: Bool) async throws {
+        DeviceStore.shared.apply(ObservableStore.bluetoothCategory, "gallery_mode", enabled)
+    }
+
+    public func setVoiceActivityDetectionEnabled(_ enabled: Bool) async throws {
+        DeviceStore.shared.apply(ObservableStore.bluetoothCategory, "voice_activity_detection_enabled", enabled)
     }
 
     public func setButtonPhotoSettings(size: ButtonPhotoSize) async throws {
@@ -280,14 +284,14 @@ public final class MentraBluetoothSDK {
         try await setButtonPhotoSettings(size: settings.size)
     }
 
-    public func setButtonVideoRecordingSettings(width: Int, height: Int, frameRate: Int) async throws {
+    public func setButtonVideoRecordingSettings(width: Int, height: Int, fps: Int) async throws {
         DeviceStore.shared.apply(ObservableStore.bluetoothCategory, "button_video_width", width)
         DeviceStore.shared.apply(ObservableStore.bluetoothCategory, "button_video_height", height)
-        DeviceStore.shared.apply(ObservableStore.bluetoothCategory, "button_video_fps", frameRate)
+        DeviceStore.shared.apply(ObservableStore.bluetoothCategory, "button_video_fps", fps)
     }
 
     public func setButtonVideoRecordingSettings(_ settings: ButtonVideoRecordingSettings) async throws {
-        try await setButtonVideoRecordingSettings(width: settings.width, height: settings.height, frameRate: settings.frameRate)
+        try await setButtonVideoRecordingSettings(width: settings.width, height: settings.height, fps: settings.fps)
     }
 
     public func setButtonCameraLed(enabled: Bool) async throws {
@@ -305,7 +309,6 @@ public final class MentraBluetoothSDK {
     public func setMicState(
         enabled: Bool,
         useGlassesMic: Bool = true,
-        bypassVad: Bool = true,
         sendTranscript: Bool = false,
         sendLc3Data: Bool = false
     ) {
@@ -319,7 +322,6 @@ public final class MentraBluetoothSDK {
         applyMicState(
             sendPcmData: enabled,
             sendTranscript: enabled && sendTranscript,
-            bypassVad: bypassVad,
             sendLc3Data: enabled && sendLc3Data
         )
     }
@@ -327,13 +329,11 @@ public final class MentraBluetoothSDK {
     private func applyMicState(
         sendPcmData: Bool,
         sendTranscript: Bool,
-        bypassVad: Bool,
         sendLc3Data: Bool
     ) {
         DeviceStore.shared.apply(ObservableStore.bluetoothCategory, "should_send_pcm", sendPcmData)
         DeviceStore.shared.apply(ObservableStore.bluetoothCategory, "should_send_lc3", sendLc3Data)
         DeviceStore.shared.apply(ObservableStore.bluetoothCategory, "should_send_transcript", sendTranscript)
-        DeviceStore.shared.apply(ObservableStore.bluetoothCategory, "bypass_vad", bypassVad)
         DeviceManager.shared.setMicState()
     }
 
@@ -376,6 +376,9 @@ public final class MentraBluetoothSDK {
     }
 
     public func requestPhoto(_ request: PhotoRequest) {
+        Bridge.log(
+            "NATIVE: PHOTO PIPELINE [3b/6] MentraBluetoothSdk.requestPhoto requestId=\(request.requestId) appId=\(request.appId)"
+        )
         DeviceManager.shared.requestPhoto(
             request.requestId,
             request.appId,
@@ -383,7 +386,9 @@ public final class MentraBluetoothSDK {
             request.webhookUrl,
             request.authToken,
             request.compress?.rawValue,
-            request.sound
+            request.flash,
+            request.sound,
+            exposureTimeNs: request.exposureTimeNs
         )
     }
 
@@ -573,6 +578,16 @@ public final class MentraBluetoothSDK {
                 values: data
             )
             delegate?.mentraBluetoothSDK(self, didReceive: .localTranscription(event))
+        case "voice_activity_detection_status":
+            delegate?.mentraBluetoothSDK(
+                self,
+                didReceive: .voiceActivityDetectionStatus(VoiceActivityDetectionStatusEvent(values: data))
+            )
+        case "speaking_status":
+            delegate?.mentraBluetoothSDK(
+                self,
+                didReceive: .speakingStatus(SpeakingStatusEvent(values: data))
+            )
         case "hotspot_status_change":
             delegate?.mentraBluetoothSDK(self, didReceive: .hotspotStatus(HotspotStatusEvent(values: data)))
         case "wifi_status_change":

package/ios/Source/Status/DeviceStatus.swift

@@ -133,6 +133,7 @@ struct GlassesStatus: CustomStringConvertible {
     var fullyBooted: Bool { boolValue(values, "fullyBooted") ?? false }
     var connected: Bool { boolValue(values, "connected") ?? false }
     var micEnabled: Bool { boolValue(values, "micEnabled") ?? false }
+    var voiceActivityDetectionEnabled: Bool { boolValue(values, "voiceActivityDetectionEnabled") ?? true }
     var connectionState: GlassesConnectionState { GlassesConnectionState(stringValue(values, "connectionState")) }
     var bluetoothClassicConnected: Bool { boolValue(values, "bluetoothClassicConnected") ?? false }
     var signalStrength: Int { intValue(values["signalStrength"]) ?? -1 }
@@ -246,6 +247,10 @@ struct BluetoothStatus: CustomStringConvertible {
         if let searchResults = values["searchResults"] as? [[String: Any]] {
             normalizedValues["searchResults"] = searchResults.compactMap { Device(values: $0)?.dictionary }
         }
+        if let galleryMode = boolValue(values, "gallery_mode") {
+            normalizedValues["galleryModeEnabled"] = galleryMode
+            normalizedValues.removeValue(forKey: "gallery_mode")
+        }
         return normalizedValues
     }
 
@@ -294,7 +299,7 @@ struct BluetoothStatus: CustomStringConvertible {
     var dashboardDepth: Int { intValue(values["dashboard_depth"]) ?? 2 }
     var headUpAngle: Int { intValue(values["head_up_angle"]) ?? 30 }
     var contextualDashboard: Bool { boolValue(values, "contextual_dashboard") ?? true }
-    var galleryModeAuto: Bool { boolValue(values, "galleryModeAuto") ?? true }
+    var galleryModeEnabled: Bool { boolValue(values, "gallery_mode") ?? boolValue(values, "galleryModeEnabled") ?? true }
     var buttonPhotoSize: ButtonPhotoSize {
         ButtonPhotoSize(rawValue: stringValue(values, "button_photo_size") ?? "") ?? .medium
     }
@@ -306,7 +311,6 @@ struct BluetoothStatus: CustomStringConvertible {
     var shouldSendPcm: Bool { boolValue(values, "should_send_pcm") ?? false }
     var shouldSendLc3: Bool { boolValue(values, "should_send_lc3") ?? false }
     var shouldSendTranscript: Bool { boolValue(values, "should_send_transcript") ?? false }
-    var bypassVad: Bool { boolValue(values, "bypass_vad") ?? true }
     var offlineCaptionsRunning: Bool { boolValue(values, "offline_captions_running") ?? false }
     var localSttFallbackActive: Bool { boolValue(values, "local_stt_fallback_active") ?? false }
     var shouldSendBootingMessage: Bool { boolValue(values, "shouldSendBootingMessage") ?? true }
@@ -335,6 +339,7 @@ struct GlassesStatusUpdate: CustomStringConvertible {
     var fullyBooted: Bool? { optionalBoolValue(values, "fullyBooted") }
     var connected: Bool? { optionalBoolValue(values, "connected") }
     var micEnabled: Bool? { optionalBoolValue(values, "micEnabled") }
+    var voiceActivityDetectionEnabled: Bool? { optionalBoolValue(values, "voiceActivityDetectionEnabled") }
     var connectionState: GlassesConnectionState? {
         GlassesConnectionState.fromValue(optionalStringValue(values, "connectionState"))
     }
@@ -403,6 +408,10 @@ struct BluetoothStatusUpdate: CustomStringConvertible {
         if let searchResults = values["searchResults"] as? [[String: Any]] {
             normalizedValues["searchResults"] = searchResults.compactMap { Device(values: $0)?.dictionary }
         }
+        if let galleryMode = optionalBoolValue(values, "gallery_mode") {
+            normalizedValues["galleryModeEnabled"] = galleryMode
+            normalizedValues.removeValue(forKey: "gallery_mode")
+        }
         self.values = normalizedValues
     }
 
@@ -440,7 +449,7 @@ struct BluetoothStatusUpdate: CustomStringConvertible {
     var dashboardDepth: Int? { optionalIntValue(values, "dashboard_depth") }
     var headUpAngle: Int? { optionalIntValue(values, "head_up_angle") }
     var contextualDashboard: Bool? { optionalBoolValue(values, "contextual_dashboard") }
-    var galleryModeAuto: Bool? { optionalBoolValue(values, "galleryModeAuto") }
+    var galleryModeEnabled: Bool? { optionalBoolValue(values, "gallery_mode") ?? optionalBoolValue(values, "galleryModeEnabled") }
     var buttonPhotoSize: ButtonPhotoSize? {
         optionalStringValue(values, "button_photo_size").flatMap(ButtonPhotoSize.init(rawValue:))
     }
@@ -452,7 +461,6 @@ struct BluetoothStatusUpdate: CustomStringConvertible {
     var shouldSendPcm: Bool? { optionalBoolValue(values, "should_send_pcm") }
     var shouldSendLc3: Bool? { optionalBoolValue(values, "should_send_lc3") }
     var shouldSendTranscript: Bool? { optionalBoolValue(values, "should_send_transcript") }
-    var bypassVad: Bool? { optionalBoolValue(values, "bypass_vad") }
     var offlineCaptionsRunning: Bool? { optionalBoolValue(values, "offline_captions_running") }
     var localSttFallbackActive: Bool? { optionalBoolValue(values, "local_stt_fallback_active") }
     var shouldSendBootingMessage: Bool? { optionalBoolValue(values, "shouldSendBootingMessage") }

package/ios/Source/Status/RuntimeState.swift

@@ -121,6 +121,7 @@ public enum GlassesRuntimeState: CustomStringConvertible {
         hotspot: HotspotStatus,
         ready: Bool,
         signal: SignalState,
+        voiceActivityDetectionEnabled: Bool,
         wifi: WifiStatus
     )
 
@@ -153,6 +154,7 @@ public enum GlassesRuntimeState: CustomStringConvertible {
                 strengthDbm: status.signalStrength == -1 ? nil : status.signalStrength,
                 updatedAt: status.signalStrengthUpdatedAt <= 0 ? nil : status.signalStrengthUpdatedAt
             ),
+            voiceActivityDetectionEnabled: status.voiceActivityDetectionEnabled,
             wifi: status.wifi
         )
     }
@@ -168,7 +170,7 @@ public enum GlassesRuntimeState: CustomStringConvertible {
         switch self {
         case let .disconnected(connection):
             connection
-        case let .connected(_, connection, _, _, _, _, _, _):
+        case let .connected(_, connection, _, _, _, _, _, _, _):
             connection
         }
     }
@@ -177,48 +179,55 @@ public enum GlassesRuntimeState: CustomStringConvertible {
         switch self {
         case .disconnected:
             false
-        case let .connected(_, _, _, _, _, ready, _, _):
+        case let .connected(_, _, _, _, _, ready, _, _, _):
             ready
         }
     }
 
     public var battery: GlassesBatteryState? {
-        guard case let .connected(battery, _, _, _, _, _, _, _) = self else {
+        guard case let .connected(battery, _, _, _, _, _, _, _, _) = self else {
             return nil
         }
         return battery
     }
 
     public var device: ConnectedGlassesInfo? {
-        guard case let .connected(_, _, device, _, _, _, _, _) = self else {
+        guard case let .connected(_, _, device, _, _, _, _, _, _) = self else {
             return nil
         }
         return device
     }
 
     public var firmware: FirmwareInfo? {
-        guard case let .connected(_, _, _, firmware, _, _, _, _) = self else {
+        guard case let .connected(_, _, _, firmware, _, _, _, _, _) = self else {
             return nil
         }
         return firmware
     }
 
     public var hotspot: HotspotStatus? {
-        guard case let .connected(_, _, _, _, hotspot, _, _, _) = self else {
+        guard case let .connected(_, _, _, _, hotspot, _, _, _, _) = self else {
             return nil
         }
         return hotspot
     }
 
     public var signal: SignalState? {
-        guard case let .connected(_, _, _, _, _, _, signal, _) = self else {
+        guard case let .connected(_, _, _, _, _, _, signal, _, _) = self else {
             return nil
         }
         return signal
     }
 
+    public var voiceActivityDetectionEnabled: Bool {
+        guard case let .connected(_, _, _, _, _, _, _, voiceActivityDetectionEnabled, _) = self else {
+            return false
+        }
+        return voiceActivityDetectionEnabled
+    }
+
     public var wifi: WifiStatus? {
-        guard case let .connected(_, _, _, _, _, _, _, wifi) = self else {
+        guard case let .connected(_, _, _, _, _, _, _, _, wifi) = self else {
             return nil
         }
         return wifi
@@ -228,17 +237,17 @@ public enum GlassesRuntimeState: CustomStringConvertible {
         switch self {
         case let .disconnected(connection):
             "GlassesRuntimeState(\(connection.rawValue))"
-        case let .connected(_, _, device, _, _, ready, _, _):
+        case let .connected(_, _, device, _, _, ready, _, _, _):
             "GlassesRuntimeState(connected: \(device), ready: \(ready))"
         }
     }
 }
 
 public struct GalleryModeState: Equatable {
-    public let desired: GalleryMode
+    public let enabled: Bool
 
-    public init(desired: GalleryMode) {
-        self.desired = desired
+    public init(enabled: Bool) {
+        self.enabled = enabled
     }
 }
 
@@ -288,7 +297,7 @@ public struct PhoneSdkRuntimeState: CustomStringConvertible {
     init(status: BluetoothStatus) {
         currentMic = MicMode(rawValue: status.currentMic)
         defaultDevice = status.defaultDevice
-        galleryMode = GalleryModeState(desired: status.galleryModeAuto ? .auto : .manual)
+        galleryMode = GalleryModeState(enabled: status.galleryModeEnabled)
         lastLog = status.lastLog
         micRanking = status.micRanking.compactMap(MicMode.init(rawValue:))
         otherBluetoothConnected = status.otherBtConnected

package/ios/Source/Streaming/StreamModels.swift

@@ -4,18 +4,18 @@ public struct StreamVideoConfig {
     public let width: Int?
     public let height: Int?
     public let bitrate: Int?
-    public let frameRate: Int?
+    public let fps: Int?
 
     public init(
         width: Int? = nil,
         height: Int? = nil,
         bitrate: Int? = nil,
-        frameRate: Int? = nil
+        fps: Int? = nil
     ) {
         self.width = width
         self.height = height
         self.bitrate = bitrate
-        self.frameRate = frameRate
+        self.fps = fps
     }
 
     var dictionary: [String: Any] {
@@ -23,7 +23,7 @@ public struct StreamVideoConfig {
         if let width { values["width"] = width }
         if let height { values["height"] = height }
         if let bitrate { values["bitrate"] = bitrate }
-        if let frameRate { values["frameRate"] = frameRate }
+        if let fps { values["frameRate"] = fps }
         return values
     }
 
@@ -33,7 +33,7 @@ public struct StreamVideoConfig {
             width: intValue(values["width"]),
             height: intValue(values["height"]),
             bitrate: intValue(values["bitrate"]),
-            frameRate: intValue(values["frameRate"])
+            fps: intValue(values["fps"])
         )
     }
 }

package/ios/Source/controllers/ControllerManager.swift

@@ -20,7 +20,7 @@ protocol ControllerManager {
 
     func requestPhoto(
         _ requestId: String, appId: String, size: String?, webhookUrl: String?, authToken: String?,
-        compress: String?, flash: Bool, sound: Bool
+        compress: String?, flash: Bool, sound: Bool, exposureTimeNs: Double?
     )
     func startStream(_ message: [String: Any])
     func stopStream()
@@ -153,8 +153,8 @@ extension ControllerManager {
         DeviceStore.shared.get("glasses", "micEnabled") as? Bool ?? false
     }
 
-    var vadEnabled: Bool {
-        DeviceStore.shared.get("glasses", "vadEnabled") as? Bool ?? false
+    var voiceActivityDetectionEnabled: Bool {
+        DeviceStore.shared.get("glasses", "voiceActivityDetectionEnabled") as? Bool ?? true
     }
 
     var batteryLevel: Int {

package/ios/Source/controllers/R1.swift

@@ -503,7 +503,7 @@ class R1: NSObject, ControllerManager {
     func sendJson(_: [String: Any], wakeUp _: Bool, requireAck _: Bool) {}
     func requestPhoto(
         _: String, appId _: String, size _: String?, webhookUrl _: String?, authToken _: String?,
-        compress _: String?, flash _: Bool, sound _: Bool
+        compress _: String?, flash _: Bool, sound _: Bool, exposureTimeNs _: Double?
     ) {}
     func startVideoRecording(requestId _: String, save _: Bool, flash _: Bool, sound _: Bool) {}
     func stopVideoRecording(requestId _: String) {}

package/ios/Source/sgcs/G1.swift

@@ -278,7 +278,7 @@ class G1: NSObject, SGCManager {
 
     func requestPhoto(
         _: String, appId _: String, size _: String?, webhookUrl _: String?, authToken _: String?,
-        compress _: String?, flash _: Bool, sound _: Bool
+        compress _: String?, flash _: Bool, sound _: Bool, exposureTimeNs _: Double?
     ) {}
 
     func startStream(_: [String: Any]) {}

package/ios/Source/sgcs/G2.swift

@@ -1454,8 +1454,7 @@ class G2: NSObject, SGCManager {
                     DeviceStore.shared.apply("glasses", "bluetoothName", btName)
                     DeviceStore.shared.apply("glasses", "deviceModel", DeviceTypes.G2)
 
-                    DeviceStore.shared.apply("glasses", "connected", true)
-                    DeviceStore.shared.apply("glasses", "fullyBooted", true)
+                    self.setFullyConnected()
 
                     // connnect a controller if we have one:
                     self.connectController()
@@ -2648,7 +2647,7 @@ class G2: NSObject, SGCManager {
 
     func requestPhoto(
         _: String, appId _: String, size _: String?, webhookUrl _: String?, authToken _: String?,
-        compress _: String?, flash _: Bool, sound _: Bool
+        compress _: String?, flash _: Bool, sound _: Bool, exposureTimeNs _: Double?
     ) {}
     func startVideoRecording(requestId _: String, save _: Bool, flash _: Bool, sound _: Bool) {}
     func startStream(_: [String: Any]) {}

package/ios/Source/sgcs/Mach1.swift

@@ -17,7 +17,7 @@ class Mach1: UltraliteBaseViewController, SGCManager {
 
     func requestPhoto(
         _: String, appId _: String, size _: String?, webhookUrl _: String?, authToken _: String?,
-        compress _: String?, flash _: Bool, sound _: Bool
+        compress _: String?, flash _: Bool, sound _: Bool, exposureTimeNs _: Double?
     ) {}
 
     func sendGalleryMode() {}