@mentra/bluetooth-sdk 0.1.5 → 0.1.7

package/ios/Source/Camera/CameraModels.swift

@@ -0,0 +1,248 @@
+import Foundation
+
+public enum PhotoSize: String {
+    case small
+    case medium
+    case large
+    case full
+}
+
+public enum ButtonPhotoSize: String {
+    case small
+    case medium
+    case large
+}
+
+public enum PhotoCompression: String {
+    case none
+    case medium
+    case heavy
+}
+
+public struct ButtonPhotoSettings {
+    public let size: ButtonPhotoSize
+
+    public init(size: ButtonPhotoSize) {
+        self.size = size
+    }
+}
+
+public struct ButtonVideoRecordingSettings {
+    public let width: Int
+    public let height: Int
+    public let fps: Int
+
+    public init(width: Int, height: Int, fps: Int) {
+        self.width = width
+        self.height = height
+        self.fps = fps
+    }
+}
+
+public enum CameraFov {
+    case standard
+    case wide
+
+    var value: [String: Int] {
+        switch self {
+        case .standard:
+            ["fov": 118, "roi_position": 0]
+        case .wide:
+            ["fov": 118, "roi_position": 0]
+        }
+    }
+}
+
+public struct PhotoRequest {
+    public let requestId: String
+    public let appId: String
+    public let size: PhotoSize
+    public let webhookUrl: String?
+    public let authToken: String?
+    public let compress: PhotoCompression?
+    public let flash: Bool
+    public let sound: Bool
+    /// Sensor exposure time for this capture only (ns), or nil for auto exposure
+    public let exposureTimeNs: Double?
+
+    public init(
+        requestId: String,
+        appId: String,
+        size: PhotoSize,
+        webhookUrl: String? = nil,
+        authToken: String? = nil,
+        compress: PhotoCompression? = nil,
+        flash: Bool = true,
+        sound: Bool,
+        exposureTimeNs: Double? = nil
+    ) {
+        self.requestId = requestId
+        self.appId = appId
+        self.size = size
+        self.webhookUrl = webhookUrl
+        self.authToken = authToken
+        self.compress = compress
+        self.flash = flash
+        self.sound = sound
+        self.exposureTimeNs = exposureTimeNs
+    }
+}
+
+public enum RgbLedAction: String {
+    case on
+    case off
+}
+
+public enum RgbLedColor: String {
+    case red
+    case green
+    case blue
+    case orange
+    case white
+}
+
+public struct RgbLedRequest {
+    public let requestId: String
+    public let packageName: String?
+    public let action: RgbLedAction
+    public let color: RgbLedColor?
+    public let onDurationMs: Int
+    public let offDurationMs: Int
+    public let count: Int
+
+    public init(
+        requestId: String,
+        packageName: String?,
+        action: RgbLedAction,
+        color: RgbLedColor?,
+        onDurationMs: Int,
+        offDurationMs: Int,
+        count: Int
+    ) {
+        self.requestId = requestId
+        self.packageName = packageName
+        self.action = action
+        self.color = color
+        self.onDurationMs = onDurationMs
+        self.offDurationMs = offDurationMs
+        self.count = count
+    }
+}
+
+public struct VideoRecordingRequest {
+    public let requestId: String
+    public let save: Bool
+    public let sound: Bool
+
+    public init(requestId: String, save: Bool, sound: Bool) {
+        self.requestId = requestId
+        self.save = save
+        self.sound = sound
+    }
+}
+
+public enum PhotoResponse: CustomStringConvertible, Equatable {
+    public enum State: String {
+        case success
+        case error
+    }
+
+    case success(requestId: String, uploadUrl: String, timestamp: Int)
+    case error(requestId: String, errorCode: String?, errorMessage: String, timestamp: Int)
+
+    public init(values: [String: Any]) {
+        let requestId = stringValue(values, "requestId") ?? ""
+        let timestamp = intValue(values["timestamp"]) ?? Int(Date().timeIntervalSince1970 * 1000)
+        let state = stringValue(values, "state")?.lowercased()
+        if state == State.success.rawValue {
+            self = .success(
+                requestId: requestId,
+                uploadUrl: stringValue(values, "uploadUrl") ?? "",
+                timestamp: timestamp
+            )
+        } else {
+            self = .error(
+                requestId: requestId,
+                errorCode: stringValue(values, "errorCode"),
+                errorMessage: stringValue(values, "errorMessage") ?? "Unknown photo error",
+                timestamp: timestamp
+            )
+        }
+    }
+
+    public var state: State {
+        switch self {
+        case .success:
+            .success
+        case .error:
+            .error
+        }
+    }
+
+    public var requestId: String {
+        switch self {
+        case let .success(requestId, _, _), let .error(requestId, _, _, _):
+            requestId
+        }
+    }
+
+    public var timestamp: Int {
+        switch self {
+        case let .success(_, _, timestamp), let .error(_, _, _, timestamp):
+            timestamp
+        }
+    }
+
+    public var values: [String: Any] {
+        switch self {
+        case let .success(requestId, uploadUrl, timestamp):
+            return [
+                "state": State.success.rawValue,
+                "requestId": requestId,
+                "uploadUrl": uploadUrl,
+                "timestamp": timestamp,
+            ]
+        case let .error(requestId, errorCode, errorMessage, timestamp):
+            var values: [String: Any] = [
+                "state": State.error.rawValue,
+                "requestId": requestId,
+                "errorMessage": errorMessage,
+                "timestamp": timestamp,
+            ]
+            if let errorCode, !errorCode.isEmpty {
+                values["errorCode"] = errorCode
+            }
+            return values
+        }
+    }
+
+    public var description: String {
+        "PhotoResponse(requestId: \(requestId), state: \(state.rawValue))"
+    }
+}
+
+public struct PhotoResponseEvent: CustomStringConvertible {
+    public let response: PhotoResponse
+
+    public init(response: PhotoResponse) {
+        self.response = response
+    }
+
+    public init(values: [String: Any]) {
+        self.response = PhotoResponse(values: values)
+    }
+
+    public var requestId: String {
+        response.requestId
+    }
+
+    public var values: [String: Any] {
+        var values = response.values
+        values["type"] = "photo_response"
+        return values
+    }
+
+    public var description: String {
+        "PhotoResponseEvent(requestId: \(requestId), state: \(response.state.rawValue))"
+    }
+}

package/ios/Source/Connection/ScanSession.swift

@@ -0,0 +1,27 @@
+import Foundation
+
+public enum ScanStopReason {
+    case completed
+    case cancelled
+    case error
+}
+
+@MainActor
+public final class ScanSession {
+    private let stopAction: () -> Void
+    private var stopped = false
+
+    init(stopAction: @escaping () -> Void) {
+        self.stopAction = stopAction
+    }
+
+    public func stop() {
+        guard !stopped else { return }
+        stopped = true
+        stopAction()
+    }
+
+    func markStopped() {
+        stopped = true
+    }
+}

package/ios/Source/DeviceManager.swift

@@ -167,11 +167,6 @@ struct ViewState {
         set { DeviceStore.shared.apply("bluetooth", "sensing_enabled", newValue) }
     }
 
-    private var bypassVad: Bool {
-        get { DeviceStore.shared.get("bluetooth", "bypass_vad") as? Bool ?? true }
-        set { DeviceStore.shared.apply("bluetooth", "bypass_vad", newValue) }
-    }
-
     private var offlineCaptionsRunning: Bool {
         get { DeviceStore.shared.get("bluetooth", "offline_captions_running") as? Bool ?? false }
         set { DeviceStore.shared.apply("bluetooth", "offline_captions_running", newValue) }
@@ -285,11 +280,6 @@ struct ViewState {
     private var lastLc3Event: Date?
     private var micReinitTimer: Timer?
 
-    // VAD:
-    private var vad: SileroVADStrategy?
-    private var vadBuffer = [Data]()
-    private var isSpeaking = false
-
     /// STT:
     private var transcriber: SherpaOnnxTranscriber?
 
@@ -314,7 +304,6 @@ struct ViewState {
 
     override init() {
         Bridge.log("MAN: init()")
-        vad = SileroVADStrategy()
         super.init()
 
         // Start memory monitoring (logs every 30s to help detect leaks)
@@ -336,16 +325,6 @@ struct ViewState {
             Bridge.log("SherpaOnnxTranscriber fully initialized")
         }
 
-        Task {
-            self.vad?.setup(
-                sampleRate: .rate_16k,
-                frameSize: .size_1024,
-                quality: .normal,
-                silenceTriggerDurationMs: 4000,
-                speechTriggerDurationMs: 50
-            )
-        }
-
         // Initialize persistent LC3 converter for unified audio encoding
         lc3Converter = PcmConverter()
         Bridge.log("LC3 converter initialized for unified audio encoding")
@@ -362,13 +341,6 @@ struct ViewState {
 
     // MARK: - AUX Voice Data Handling
 
-    private func checkSetVadStatus(speaking: Bool) {
-        if speaking != isSpeaking {
-            isSpeaking = speaking
-            Bridge.sendVadEvent(isSpeaking)
-        }
-    }
-
     private func convertAndSendMicLc3(_ pcmData: Data) {
         guard let lc3Converter = lc3Converter else {
             Bridge.log("MAN: ERROR - LC3 converter not initialized but format is LC3")
@@ -393,23 +365,6 @@ struct ViewState {
         }
     }
 
-    private func emptyVadBuffer() {
-        // go through the buffer, popping from the first element in the array (FIFO):
-        while !vadBuffer.isEmpty {
-            let chunk = vadBuffer.removeFirst()
-            handleSendingPcm(chunk)
-        }
-    }
-
-    private func addToVadBuffer(_ chunk: Data) {
-        let MAX_BUFFER_SIZE = 20
-        vadBuffer.append(chunk)
-        while vadBuffer.count > MAX_BUFFER_SIZE {
-            // pop from the front of the array:
-            vadBuffer.removeFirst()
-        }
-    }
-
     /**
      * Handle raw LC3 audio data from glasses.
      * Decodes the glasses LC3 to PCM, then forwards to handlePcm for processing.
@@ -432,59 +387,16 @@ struct ViewState {
             Bridge.log("MAN: Failed to decode glasses LC3 audio")
             return
         }
-        // Forward to handlePcm which handles VAD and encoding
+        // Forward to handlePcm which handles SDK audio events and encoding.
         handlePcm(pcmData)
     }
 
     func handlePcm(_ pcmData: Data) {
-        // handle incoming PCM data from the microphone manager and feed to the VAD:
-        if bypassVad {
-            handleSendingPcm(pcmData)
+        handleSendingPcm(pcmData)
 
-            // Send PCM to local transcriber (always needs raw PCM)
-            if shouldSendTranscript || offlineCaptionsRunning || localSttFallbackActive {
-                transcriber?.acceptAudio(pcm16le: pcmData)
-            }
-            return
-        }
-
-        // feed PCM to the VAD:
-        guard let vad = vad else {
-            Bridge.log("VAD not initialized")
-            return
-        }
-
-        // convert audioData to Int16 array for VAD:
-        let pcmDataArray = pcmData.withUnsafeBytes { pointer -> [Int16] in
-            Array(
-                UnsafeBufferPointer(
-                    start: pointer.bindMemory(to: Int16.self).baseAddress,
-                    count: pointer.count / MemoryLayout<Int16>.stride
-                )
-            )
-        }
-
-        vad.checkVAD(pcm: pcmDataArray) { [weak self] state in
-            guard let self = self else { return }
-            Bridge.log("VAD State: \(state)")
-        }
-
-        let vadState = vad.currentState()
-        if vadState == .speeching {
-            checkSetVadStatus(speaking: true)
-            // first send out whatever's in the vadBuffer (if there is anything):
-            emptyVadBuffer()
-
-            handleSendingPcm(pcmData)
-
-            // Send PCM to local transcriber (always needs raw PCM)
-            if shouldSendTranscript || offlineCaptionsRunning || localSttFallbackActive {
-                transcriber?.acceptAudio(pcm16le: pcmData)
-            }
-        } else {
-            checkSetVadStatus(speaking: false)
-            // add to the vadBuffer (stores PCM for potential later sending):
-            addToVadBuffer(pcmData)
+        // Send PCM to local transcriber.
+        if shouldSendTranscript || offlineCaptionsRunning || localSttFallbackActive {
+            transcriber?.acceptAudio(pcm16le: pcmData)
         }
     }
 
@@ -1013,6 +925,8 @@ struct ViewState {
 
     func handleDeviceDisconnected() {
         Bridge.log("MAN: Device disconnected")
+        DeviceStore.shared.apply("glasses", "headUp", false)
+        DeviceStore.shared.apply("glasses", "voiceActivityDetectionEnabled", true)
         // shouldSendBootingMessage = true  // Reset for next first connect
     }
 
@@ -1280,14 +1194,22 @@ struct ViewState {
         _ webhookUrl: String?,
         _ authToken: String?,
         _ compress: String?,
-        _ sound: Bool
+        _ flash: Bool,
+        _ sound: Bool,
+        exposureTimeNs: Double? = nil
     ) {
         Bridge.log(
-            "MAN: requestPhoto: \(requestId), \(appId), \(webhookUrl), size=\(size), compress=\(compress ?? "none"), flash=true, sound=\(sound)"
+            "MAN: PHOTO PIPELINE [4/6] DeviceManager.requestPhoto requestId=\(requestId) appId=\(appId) webhookUrl=\(webhookUrl ?? "nil") size=\(size) compress=\(compress ?? "none") flash=\(flash) sound=\(sound) exposureTimeNs=\(exposureTimeNs.map { String($0) } ?? "nil") sgc=\(sgc != nil ? String(describing: type(of: sgc!)) : "null")"
         )
-        sgc?.requestPhoto(
+        guard let sgc else {
+            Bridge.log(
+                "MAN: PHOTO PIPELINE — sgc is null (glasses not connected); dropping requestId=\(requestId)"
+            )
+            return
+        }
+        sgc.requestPhoto(
             requestId, appId: appId, size: size, webhookUrl: webhookUrl, authToken: authToken,
-            compress: compress, flash: true, sound: sound
+            compress: compress, flash: flash, sound: sound, exposureTimeNs: exposureTimeNs
         )
     }
 
@@ -1393,6 +1315,7 @@ struct ViewState {
         DeviceStore.shared.apply("glasses", "deviceModel", "")
         DeviceStore.shared.apply("glasses", "fullyBooted", false)
         DeviceStore.shared.apply("glasses", "connected", false)
+        DeviceStore.shared.apply("glasses", "voiceActivityDetectionEnabled", true)
         // disconnect the controller as well:
         searchingController = false
         DeviceStore.shared.apply("glasses", "controllerConnected", false)

package/ios/Source/DeviceStore.swift

@@ -28,6 +28,7 @@ class DeviceStore {
         store.set("glasses", "deviceModel", "")
         store.set("glasses", "firmwareVersion", "")
         store.set("glasses", "micEnabled", false)
+        store.set("glasses", "voiceActivityDetectionEnabled", true)
         store.set("glasses", "bluetoothClassicConnected", false)
         store.set("glasses", "caseRemoved", true)
         store.set("glasses", "caseOpen", true)
@@ -71,21 +72,26 @@ class DeviceStore {
         store.set("bluetooth", "pending_wearable", "")
         store.set("bluetooth", "device_name", "")
         store.set("bluetooth", "device_address", "")
+        store.set("bluetooth", "default_controller", "")
+        store.set("bluetooth", "pending_controller", "")
+        store.set("bluetooth", "controller_device_name", "")
         store.set("bluetooth", "screen_disabled", false)
         store.set("bluetooth", "preferred_mic", "auto")
         store.set("bluetooth", "sensing_enabled", true)
+        store.set("bluetooth", "power_saving_mode", false)
         store.set("bluetooth", "brightness", 50)
         store.set("bluetooth", "auto_brightness", true)
         store.set("bluetooth", "dashboard_height", 4)
         store.set("bluetooth", "dashboard_depth", 2)
         store.set("bluetooth", "head_up_angle", 30)
         store.set("bluetooth", "contextual_dashboard", true)
-        store.set("bluetooth", "galleryModeAuto", true)
+        store.set("bluetooth", "gallery_mode", true)
+        store.set("bluetooth", "voice_activity_detection_enabled", true)
         store.set("bluetooth", "screen_disabled", false)
         store.set("bluetooth", "button_photo_size", "medium")
         store.set("bluetooth", "button_camera_led", true)
         store.set("bluetooth", "button_max_recording_time", 10)
-        store.set("bluetooth", "camera_fov", ["fov": 118, "roiPosition": 0])
+        store.set("bluetooth", "camera_fov", ["fov": 118, "roi_position": 0])
         store.set("bluetooth", "button_video_width", 1280)
         store.set("bluetooth", "button_video_height", 720)
         store.set("bluetooth", "button_video_fps", 30)
@@ -96,7 +102,7 @@ class DeviceStore {
         store.set("bluetooth", "should_send_pcm", false)
         store.set("bluetooth", "should_send_lc3", false)
         store.set("bluetooth", "should_send_transcript", false)
-        store.set("bluetooth", "bypass_vad", true)
+        store.set("bluetooth", "use_native_dashboard", false)
     }
 
     func get(_ category: String, _ key: String) -> Any? {
@@ -214,9 +220,12 @@ class DeviceStore {
                 DeviceManager.shared.sgc?.setDashboardMenu(items)
             }
 
-        case ("bluetooth", "galleryModeAuto"):
+        case ("bluetooth", "gallery_mode"):
             DeviceManager.shared.sgc?.sendGalleryMode()
 
+        case ("bluetooth", "voice_activity_detection_enabled"):
+            DeviceManager.shared.sgc?.sendVoiceActivityDetectionSetting()
+
         case ("bluetooth", "screen_disabled"):
             if let disabled = value as? Bool {
                 if disabled {

package/ios/Source/Errors/BluetoothError.swift

@@ -0,0 +1,19 @@
+import Foundation
+
+public struct BluetoothError: Error, LocalizedError, CustomStringConvertible {
+    public let code: String
+    public let message: String
+
+    public init(code: String, message: String) {
+        self.code = code
+        self.message = message
+    }
+
+    public var errorDescription: String? {
+        message
+    }
+
+    public var description: String {
+        "\(code): \(message)"
+    }
+}

package/ios/Source/Events/BluetoothEvents.swift

@@ -0,0 +1,149 @@
+import Foundation
+
+public struct ButtonPressEvent: CustomStringConvertible {
+    public let buttonId: String
+    public let pressType: String
+    public let timestamp: Int?
+
+    public init(buttonId: String, pressType: String, timestamp: Int? = nil) {
+        self.buttonId = buttonId
+        self.pressType = pressType
+        self.timestamp = timestamp
+    }
+
+    public var description: String {
+        "ButtonPressEvent(buttonId: \(buttonId), pressType: \(pressType))"
+    }
+}
+
+public struct TouchEvent: CustomStringConvertible {
+    public let values: [String: Any]
+
+    public init(values: [String: Any]) {
+        self.values = values
+    }
+
+    public var deviceModel: String? {
+        stringValue(values, "deviceModel")
+    }
+
+    public var gestureName: String? {
+        stringValue(values, "gestureName")
+    }
+
+    public var timestamp: Int? {
+        intValue(values["timestamp"])
+    }
+
+    public var isSwipe: Bool {
+        gestureName?.localizedCaseInsensitiveContains("swipe") == true
+    }
+
+    public var description: String {
+        "TouchEvent(gestureName: \(gestureName ?? "unknown"))"
+    }
+}
+
+public struct VoiceActivityDetectionStatusEvent: CustomStringConvertible {
+    public let voiceActivityDetectionEnabled: Bool
+    public let values: [String: Any]
+
+    public init(values: [String: Any]) {
+        voiceActivityDetectionEnabled = boolValue(values, "voiceActivityDetectionEnabled") ?? true
+        self.values = values
+    }
+
+    public var description: String {
+        "VoiceActivityDetectionStatusEvent(voiceActivityDetectionEnabled: \(voiceActivityDetectionEnabled))"
+    }
+}
+
+public struct SpeakingStatusEvent: CustomStringConvertible {
+    public let speaking: Bool
+    public let values: [String: Any]
+
+    public init(values: [String: Any]) {
+        speaking = boolValue(values, "speaking") ?? false
+        self.values = values
+    }
+
+    public var description: String {
+        "SpeakingStatusEvent(speaking: \(speaking))"
+    }
+}
+
+public enum BluetoothEvent: CustomStringConvertible {
+    case buttonPress(ButtonPressEvent)
+    case touch(TouchEvent)
+    case voiceActivityDetectionStatus(VoiceActivityDetectionStatusEvent)
+    case speakingStatus(SpeakingStatusEvent)
+    case wifiStatus(WifiStatusEvent)
+    case hotspotStatus(HotspotStatusEvent)
+    case hotspotError(HotspotErrorEvent)
+    case photoResponse(PhotoResponseEvent)
+    case streamStatus(StreamStatusEvent)
+    case keepAliveAck(KeepAliveAckEvent)
+    case localTranscription(LocalTranscriptionEvent)
+    case raw(name: String, values: [String: Any])
+
+    public var description: String {
+        switch self {
+        case let .buttonPress(event):
+            event.description
+        case let .touch(event):
+            event.description
+        case let .voiceActivityDetectionStatus(event):
+            event.description
+        case let .speakingStatus(event):
+            event.description
+        case let .wifiStatus(event):
+            event.description
+        case let .hotspotStatus(event):
+            event.description
+        case let .hotspotError(event):
+            event.description
+        case let .photoResponse(event):
+            event.description
+        case let .streamStatus(event):
+            event.description
+        case let .keepAliveAck(event):
+            event.description
+        case let .localTranscription(event):
+            event.description
+        case let .raw(name, values):
+            "\(name): \(values)"
+        }
+    }
+}
+
+@MainActor
+public protocol MentraBluetoothSDKDelegate: AnyObject {
+    func mentraBluetoothSDK(_ sdk: MentraBluetoothSDK, didUpdate state: MentraBluetoothState)
+    func mentraBluetoothSDK(_ sdk: MentraBluetoothSDK, didUpdateGlasses glasses: GlassesRuntimeState)
+    func mentraBluetoothSDK(_ sdk: MentraBluetoothSDK, didUpdateSdkState sdkState: PhoneSdkRuntimeState)
+    func mentraBluetoothSDK(_ sdk: MentraBluetoothSDK, didUpdateScan scan: BluetoothScanState)
+    func mentraBluetoothSDK(_ sdk: MentraBluetoothSDK, didDiscover device: Device)
+    func mentraBluetoothSDK(_ sdk: MentraBluetoothSDK, didStopScan reason: ScanStopReason)
+    func mentraBluetoothSDK(_ sdk: MentraBluetoothSDK, didReceive event: BluetoothEvent)
+    func mentraBluetoothSDK(_ sdk: MentraBluetoothSDK, didReceiveMicPcm event: MicPcmEvent)
+    func mentraBluetoothSDK(_ sdk: MentraBluetoothSDK, didReceiveMicLc3 event: MicLc3Event)
+    func mentraBluetoothSDK(_ sdk: MentraBluetoothSDK, didChangeDefaultDevice device: Device?)
+    func mentraBluetoothSDK(_ sdk: MentraBluetoothSDK, didLog message: String)
+    func mentraBluetoothSDK(_ sdk: MentraBluetoothSDK, didFail error: BluetoothError)
+}
+
+@MainActor
+public extension MentraBluetoothSDKDelegate {
+    func mentraBluetoothSDK(_: MentraBluetoothSDK, didUpdate _: MentraBluetoothState) {}
+    func mentraBluetoothSDK(_: MentraBluetoothSDK, didUpdateGlasses _: GlassesRuntimeState) {}
+    func mentraBluetoothSDK(_: MentraBluetoothSDK, didUpdateSdkState _: PhoneSdkRuntimeState) {}
+    func mentraBluetoothSDK(_: MentraBluetoothSDK, didUpdateScan _: BluetoothScanState) {}
+    func mentraBluetoothSDK(_: MentraBluetoothSDK, didDiscover _: Device) {}
+    func mentraBluetoothSDK(_: MentraBluetoothSDK, didStopScan _: ScanStopReason) {}
+    func mentraBluetoothSDK(_: MentraBluetoothSDK, didReceive _: BluetoothEvent) {}
+    func mentraBluetoothSDK(_: MentraBluetoothSDK, didReceiveMicPcm _: MicPcmEvent) {}
+    func mentraBluetoothSDK(_: MentraBluetoothSDK, didReceiveMicLc3 _: MicLc3Event) {}
+    func mentraBluetoothSDK(_: MentraBluetoothSDK, didChangeDefaultDevice _: Device?) {}
+    func mentraBluetoothSDK(_: MentraBluetoothSDK, didLog _: String) {}
+    func mentraBluetoothSDK(_: MentraBluetoothSDK, didFail _: BluetoothError) {}
+}