@mentra/bluetooth-sdk 0.1.6 → 0.1.8

package/ios/Source/Bridge.swift

@@ -89,18 +89,22 @@ class Bridge {
 
     @MainActor
     private static func micPcmEventBody(_ data: Data) -> [String: Any] {
-        [
+        let voiceActivityDetectionEnabled =
+            DeviceStore.shared.get("glasses", "voiceActivityDetectionEnabled") as? Bool ?? true
+        return [
             "pcm": data,
             "sampleRate": micSampleRate,
             "bitsPerSample": pcmBitsPerSample,
             "channels": micChannels,
             "encoding": "pcm_s16le",
-            "vadGated": isVadGated(),
+            "voiceActivityDetectionEnabled": voiceActivityDetectionEnabled,
         ]
     }
 
     @MainActor
     private static func micLc3EventBody(_ data: Data) -> [String: Any] {
+        let voiceActivityDetectionEnabled =
+            DeviceStore.shared.get("glasses", "voiceActivityDetectionEnabled") as? Bool ?? true
         let frameSizeBytes = DeviceStore.shared.get("bluetooth", "lc3_frame_size") as? Int ?? defaultLc3FrameSizeBytes
         return [
             "lc3": data,
@@ -111,23 +115,30 @@ class Bridge {
             "frameSizeBytes": frameSizeBytes,
             "bitrate": frameSizeBytes * 8 * (1000 / lc3FrameDurationMs),
             "packetizedFromGlasses": false,
-            "vadGated": isVadGated(),
+            "voiceActivityDetectionEnabled": voiceActivityDetectionEnabled,
         ]
     }
 
-    @MainActor
-    private static func isVadGated() -> Bool {
-        !(DeviceStore.shared.get("bluetooth", "bypass_vad") as? Bool ?? true)
-    }
-
     static func saveSetting(_ key: String, _ value: Any) {
         let body = ["key": key, "value": value]
         Bridge.sendTypedMessage("save_setting", body: body)
     }
 
-    static func sendVadEvent(_ isSpeaking: Bool) {
-        let body: [String: Any] = ["status": isSpeaking]
-        Bridge.sendTypedMessage("vad_status", body: body)
+    @MainActor
+    static func sendVoiceActivityDetectionStatus(_ enabled: Bool) {
+        DeviceStore.shared.set("glasses", "voiceActivityDetectionEnabled", enabled)
+        let body: [String: Any] = [
+            "voiceActivityDetectionEnabled": enabled,
+        ]
+        Bridge.sendTypedMessage("voice_activity_detection_status", body: body)
+    }
+
+    static func sendSpeakingStatus(_ speaking: Bool) {
+        let body: [String: Any] = [
+            "speaking": speaking,
+            "timestamp": Int(Date().timeIntervalSince1970 * 1000),
+        ]
+        Bridge.sendTypedMessage("speaking_status", body: body)
     }
 
     static func sendBatteryStatus(level: Int, charging: Bool) {
@@ -259,11 +270,12 @@ class Bridge {
     }
 
     static func sendPhotoError(requestId: String, errorCode: String, errorMessage: String) {
+        let timestamp = Int(Date().timeIntervalSince1970 * 1000)
         var event: [String: Any] = [
             "type": "photo_response",
             "state": "error",
             "requestId": requestId,
-            "timestamp": Int(Date().timeIntervalSince1970 * 1000),
+            "timestamp": timestamp,
         ]
         if !errorCode.isEmpty {
             event["errorCode"] = errorCode
@@ -272,6 +284,24 @@ class Bridge {
             event["errorMessage"] = errorMessage
         }
         Bridge.sendTypedMessage("photo_response", body: event)
+
+        var status: [String: Any] = [
+            "type": "photo_status",
+            "status": "failed",
+            "requestId": requestId,
+            "timestamp": timestamp,
+        ]
+        if !errorCode.isEmpty {
+            status["errorCode"] = errorCode
+        }
+        if !errorMessage.isEmpty {
+            status["errorMessage"] = errorMessage
+        }
+        Bridge.sendTypedMessage("photo_status", body: status)
+    }
+
+    static func sendPhotoStatus(_ status: [String: Any]) {
+        Bridge.sendTypedMessage("photo_status", body: status)
     }
 
     static func sendMiniappSelected(packageName: String) {

package/ios/Source/Camera/CameraModels.swift

@@ -1,10 +1,5 @@
 import Foundation
 
-public enum GalleryMode {
-    case auto
-    case manual
-}
-
 public enum PhotoSize: String {
     case small
     case medium
@@ -35,26 +30,33 @@ public struct ButtonPhotoSettings {
 public struct ButtonVideoRecordingSettings {
     public let width: Int
     public let height: Int
-    public let frameRate: Int
+    public let fps: Int
 
-    public init(width: Int, height: Int, frameRate: Int) {
+    public init(width: Int, height: Int, fps: Int) {
         self.width = width
         self.height = height
-        self.frameRate = frameRate
+        self.fps = fps
     }
 }
 
-public enum CameraFov {
-    case standard
-    case wide
+public struct CameraFov {
+    public static let minFov = 82
+    public static let maxFov = 118
+    public static let defaultFov = 102
+    public static let minRoiPosition = 0
+    public static let maxRoiPosition = 2
+    public static let defaultRoiPosition = 0
+
+    public let fov: Int
+    public let roiPosition: Int
+
+    public init(fov: Int = CameraFov.defaultFov, roiPosition: Int = CameraFov.defaultRoiPosition) {
+        self.fov = min(max(fov, CameraFov.minFov), CameraFov.maxFov)
+        self.roiPosition = min(max(roiPosition, CameraFov.minRoiPosition), CameraFov.maxRoiPosition)
+    }
 
     var value: [String: Int] {
-        switch self {
-        case .standard:
-            ["fov": 118, "roiPosition": 0]
-        case .wide:
-            ["fov": 118, "roiPosition": 0]
-        }
+        ["fov": fov, "roi_position": roiPosition]
     }
 }
 
@@ -65,7 +67,12 @@ public struct PhotoRequest {
     public let webhookUrl: String?
     public let authToken: String?
     public let compress: PhotoCompression?
+    public let flash: Bool
     public let sound: Bool
+    /// Sensor exposure time for this capture only (ns), or nil for auto exposure
+    public let exposureTimeNs: Double?
+    /// Sensor ISO for this capture only. Only used when exposureTimeNs enables manual exposure.
+    public let iso: Int?
 
     public init(
         requestId: String,
@@ -74,7 +81,10 @@ public struct PhotoRequest {
         webhookUrl: String? = nil,
         authToken: String? = nil,
         compress: PhotoCompression? = nil,
-        sound: Bool
+        flash: Bool = true,
+        sound: Bool,
+        exposureTimeNs: Double? = nil,
+        iso: Int? = nil
     ) {
         self.requestId = requestId
         self.appId = appId
@@ -82,7 +92,10 @@ public struct PhotoRequest {
         self.webhookUrl = webhookUrl
         self.authToken = authToken
         self.compress = compress
+        self.flash = flash
         self.sound = sound
+        self.exposureTimeNs = exposureTimeNs
+        self.iso = iso
     }
 }
 
@@ -227,7 +240,7 @@ public struct PhotoResponseEvent: CustomStringConvertible {
     }
 
     public init(values: [String: Any]) {
-        self.response = PhotoResponse(values: values)
+        response = PhotoResponse(values: values)
     }
 
     public var requestId: String {
@@ -244,3 +257,45 @@ public struct PhotoResponseEvent: CustomStringConvertible {
         "PhotoResponseEvent(requestId: \(requestId), state: \(response.state.rawValue))"
     }
 }
+
+public struct PhotoStatusEvent: CustomStringConvertible {
+    public let values: [String: Any]
+
+    public init(values: [String: Any]) {
+        var values = values
+        values["type"] = "photo_status"
+        self.values = values
+    }
+
+    public var requestId: String {
+        stringValue(values, "requestId") ?? ""
+    }
+
+    public var status: String {
+        stringValue(values, "status") ?? ""
+    }
+
+    public var timestamp: Int64 {
+        if let value = values["timestamp"] as? Int64 { return value }
+        if let value = values["timestamp"] as? Int { return Int64(value) }
+        if let value = values["timestamp"] as? Double { return Int64(value) }
+        if let value = values["timestamp"] as? NSNumber { return value.int64Value }
+        return Int64(Date().timeIntervalSince1970 * 1000)
+    }
+
+    public var resolvedConfig: [String: Any]? {
+        values["resolvedConfig"] as? [String: Any]
+    }
+
+    public var errorCode: String? {
+        stringValue(values, "errorCode")
+    }
+
+    public var errorMessage: String? {
+        stringValue(values, "errorMessage")
+    }
+
+    public var description: String {
+        "PhotoStatusEvent(requestId: \(requestId), status: \(status))"
+    }
+}

package/ios/Source/DeviceManager.swift

@@ -167,11 +167,6 @@ struct ViewState {
         set { DeviceStore.shared.apply("bluetooth", "sensing_enabled", newValue) }
     }
 
-    private var bypassVad: Bool {
-        get { DeviceStore.shared.get("bluetooth", "bypass_vad") as? Bool ?? true }
-        set { DeviceStore.shared.apply("bluetooth", "bypass_vad", newValue) }
-    }
-
     private var offlineCaptionsRunning: Bool {
         get { DeviceStore.shared.get("bluetooth", "offline_captions_running") as? Bool ?? false }
         set { DeviceStore.shared.apply("bluetooth", "offline_captions_running", newValue) }
@@ -285,11 +280,6 @@ struct ViewState {
     private var lastLc3Event: Date?
     private var micReinitTimer: Timer?
 
-    // VAD:
-    private var vad: SileroVADStrategy?
-    private var vadBuffer = [Data]()
-    private var isSpeaking = false
-
     /// STT:
     private var transcriber: SherpaOnnxTranscriber?
 
@@ -314,7 +304,6 @@ struct ViewState {
 
     override init() {
         Bridge.log("MAN: init()")
-        vad = SileroVADStrategy()
         super.init()
 
         // Start memory monitoring (logs every 30s to help detect leaks)
@@ -336,16 +325,6 @@ struct ViewState {
             Bridge.log("SherpaOnnxTranscriber fully initialized")
         }
 
-        Task {
-            self.vad?.setup(
-                sampleRate: .rate_16k,
-                frameSize: .size_1024,
-                quality: .normal,
-                silenceTriggerDurationMs: 4000,
-                speechTriggerDurationMs: 50
-            )
-        }
-
         // Initialize persistent LC3 converter for unified audio encoding
         lc3Converter = PcmConverter()
         Bridge.log("LC3 converter initialized for unified audio encoding")
@@ -362,13 +341,6 @@ struct ViewState {
 
     // MARK: - AUX Voice Data Handling
 
-    private func checkSetVadStatus(speaking: Bool) {
-        if speaking != isSpeaking {
-            isSpeaking = speaking
-            Bridge.sendVadEvent(isSpeaking)
-        }
-    }
-
     private func convertAndSendMicLc3(_ pcmData: Data) {
         guard let lc3Converter = lc3Converter else {
             Bridge.log("MAN: ERROR - LC3 converter not initialized but format is LC3")
@@ -393,23 +365,6 @@ struct ViewState {
         }
     }
 
-    private func emptyVadBuffer() {
-        // go through the buffer, popping from the first element in the array (FIFO):
-        while !vadBuffer.isEmpty {
-            let chunk = vadBuffer.removeFirst()
-            handleSendingPcm(chunk)
-        }
-    }
-
-    private func addToVadBuffer(_ chunk: Data) {
-        let MAX_BUFFER_SIZE = 20
-        vadBuffer.append(chunk)
-        while vadBuffer.count > MAX_BUFFER_SIZE {
-            // pop from the front of the array:
-            vadBuffer.removeFirst()
-        }
-    }
-
     /**
      * Handle raw LC3 audio data from glasses.
      * Decodes the glasses LC3 to PCM, then forwards to handlePcm for processing.
@@ -432,59 +387,16 @@ struct ViewState {
             Bridge.log("MAN: Failed to decode glasses LC3 audio")
             return
         }
-        // Forward to handlePcm which handles VAD and encoding
+        // Forward to handlePcm which handles SDK audio events and encoding.
         handlePcm(pcmData)
     }
 
     func handlePcm(_ pcmData: Data) {
-        // handle incoming PCM data from the microphone manager and feed to the VAD:
-        if bypassVad {
-            handleSendingPcm(pcmData)
-
-            // Send PCM to local transcriber (always needs raw PCM)
-            if shouldSendTranscript || offlineCaptionsRunning || localSttFallbackActive {
-                transcriber?.acceptAudio(pcm16le: pcmData)
-            }
-            return
-        }
+        handleSendingPcm(pcmData)
 
-        // feed PCM to the VAD:
-        guard let vad = vad else {
-            Bridge.log("VAD not initialized")
-            return
-        }
-
-        // convert audioData to Int16 array for VAD:
-        let pcmDataArray = pcmData.withUnsafeBytes { pointer -> [Int16] in
-            Array(
-                UnsafeBufferPointer(
-                    start: pointer.bindMemory(to: Int16.self).baseAddress,
-                    count: pointer.count / MemoryLayout<Int16>.stride
-                )
-            )
-        }
-
-        vad.checkVAD(pcm: pcmDataArray) { [weak self] state in
-            guard let self = self else { return }
-            Bridge.log("VAD State: \(state)")
-        }
-
-        let vadState = vad.currentState()
-        if vadState == .speeching {
-            checkSetVadStatus(speaking: true)
-            // first send out whatever's in the vadBuffer (if there is anything):
-            emptyVadBuffer()
-
-            handleSendingPcm(pcmData)
-
-            // Send PCM to local transcriber (always needs raw PCM)
-            if shouldSendTranscript || offlineCaptionsRunning || localSttFallbackActive {
-                transcriber?.acceptAudio(pcm16le: pcmData)
-            }
-        } else {
-            checkSetVadStatus(speaking: false)
-            // add to the vadBuffer (stores PCM for potential later sending):
-            addToVadBuffer(pcmData)
+        // Send PCM to local transcriber.
+        if shouldSendTranscript || offlineCaptionsRunning || localSttFallbackActive {
+            transcriber?.acceptAudio(pcm16le: pcmData)
         }
     }
 
@@ -821,7 +733,7 @@ struct ViewState {
         if !glassesMicEnabled || !glassesConnected {
             return
         }
-        
+
         let timeSinceLastLc3Event = Date().timeIntervalSince(lastLc3Event ?? Date())
         if timeSinceLastLc3Event > 5 {
             Bridge.log("MAN: No audio activity in the last 5 seconds from glasses, reinitializing glasses mic")
@@ -1013,6 +925,8 @@ struct ViewState {
 
     func handleDeviceDisconnected() {
         Bridge.log("MAN: Device disconnected")
+        DeviceStore.shared.apply("glasses", "headUp", false)
+        DeviceStore.shared.apply("glasses", "voiceActivityDetectionEnabled", true)
         // shouldSendBootingMessage = true  // Reset for next first connect
     }
 
@@ -1280,14 +1194,24 @@ struct ViewState {
         _ webhookUrl: String?,
         _ authToken: String?,
         _ compress: String?,
-        _ sound: Bool
+        _ flash: Bool,
+        _ sound: Bool,
+        exposureTimeNs: Double? = nil,
+        iso: Int? = nil
     ) {
+        let manualIso = exposureTimeNs != nil ? iso.flatMap { $0 > 0 ? $0 : nil } : nil
         Bridge.log(
-            "MAN: requestPhoto: \(requestId), \(appId), \(webhookUrl), size=\(size), compress=\(compress ?? "none"), flash=true, sound=\(sound)"
+            "MAN: PHOTO PIPELINE [4/6] DeviceManager.requestPhoto requestId=\(requestId) appId=\(appId) webhookUrl=\(webhookUrl ?? "nil") size=\(size) compress=\(compress ?? "none") flash=\(flash) sound=\(sound) exposureTimeNs=\(exposureTimeNs.map { String($0) } ?? "nil") iso=\(manualIso.map { String($0) } ?? "auto") sgc=\(sgc != nil ? String(describing: type(of: sgc!)) : "null")"
         )
-        sgc?.requestPhoto(
+        guard let sgc else {
+            Bridge.log(
+                "MAN: PHOTO PIPELINE — sgc is null (glasses not connected); dropping requestId=\(requestId)"
+            )
+            return
+        }
+        sgc.requestPhoto(
             requestId, appId: appId, size: size, webhookUrl: webhookUrl, authToken: authToken,
-            compress: compress, flash: true, sound: sound
+            compress: compress, flash: flash, sound: sound, exposureTimeNs: exposureTimeNs, iso: manualIso
         )
     }
 
@@ -1393,6 +1317,7 @@ struct ViewState {
         DeviceStore.shared.apply("glasses", "deviceModel", "")
         DeviceStore.shared.apply("glasses", "fullyBooted", false)
         DeviceStore.shared.apply("glasses", "connected", false)
+        DeviceStore.shared.apply("glasses", "voiceActivityDetectionEnabled", true)
         // disconnect the controller as well:
         searchingController = false
         DeviceStore.shared.apply("glasses", "controllerConnected", false)

package/ios/Source/DeviceStore.swift

@@ -28,6 +28,7 @@ class DeviceStore {
         store.set("glasses", "deviceModel", "")
         store.set("glasses", "firmwareVersion", "")
         store.set("glasses", "micEnabled", false)
+        store.set("glasses", "voiceActivityDetectionEnabled", true)
         store.set("glasses", "bluetoothClassicConnected", false)
         store.set("glasses", "caseRemoved", true)
         store.set("glasses", "caseOpen", true)
@@ -71,21 +72,26 @@ class DeviceStore {
         store.set("bluetooth", "pending_wearable", "")
         store.set("bluetooth", "device_name", "")
         store.set("bluetooth", "device_address", "")
+        store.set("bluetooth", "default_controller", "")
+        store.set("bluetooth", "pending_controller", "")
+        store.set("bluetooth", "controller_device_name", "")
         store.set("bluetooth", "screen_disabled", false)
         store.set("bluetooth", "preferred_mic", "auto")
         store.set("bluetooth", "sensing_enabled", true)
+        store.set("bluetooth", "power_saving_mode", false)
         store.set("bluetooth", "brightness", 50)
         store.set("bluetooth", "auto_brightness", true)
         store.set("bluetooth", "dashboard_height", 4)
         store.set("bluetooth", "dashboard_depth", 2)
         store.set("bluetooth", "head_up_angle", 30)
         store.set("bluetooth", "contextual_dashboard", true)
-        store.set("bluetooth", "galleryModeAuto", true)
+        store.set("bluetooth", "gallery_mode", true)
+        store.set("bluetooth", "voice_activity_detection_enabled", true)
         store.set("bluetooth", "screen_disabled", false)
         store.set("bluetooth", "button_photo_size", "medium")
         store.set("bluetooth", "button_camera_led", true)
         store.set("bluetooth", "button_max_recording_time", 10)
-        store.set("bluetooth", "camera_fov", ["fov": 118, "roiPosition": 0])
+        store.set("bluetooth", "camera_fov", ["fov": 118, "roi_position": 0])
         store.set("bluetooth", "button_video_width", 1280)
         store.set("bluetooth", "button_video_height", 720)
         store.set("bluetooth", "button_video_fps", 30)
@@ -96,7 +102,6 @@ class DeviceStore {
         store.set("bluetooth", "should_send_pcm", false)
         store.set("bluetooth", "should_send_lc3", false)
         store.set("bluetooth", "should_send_transcript", false)
-        store.set("bluetooth", "bypass_vad", true)
         store.set("bluetooth", "use_native_dashboard", false)
     }
 
@@ -215,9 +220,12 @@ class DeviceStore {
                 DeviceManager.shared.sgc?.setDashboardMenu(items)
             }
 
-        case ("bluetooth", "galleryModeAuto"):
+        case ("bluetooth", "gallery_mode"):
             DeviceManager.shared.sgc?.sendGalleryMode()
 
+        case ("bluetooth", "voice_activity_detection_enabled"):
+            DeviceManager.shared.sgc?.sendVoiceActivityDetectionSetting()
+
         case ("bluetooth", "screen_disabled"):
             if let disabled = value as? Bool {
                 if disabled {

package/ios/Source/Events/BluetoothEvents.swift

@@ -44,13 +44,44 @@ public struct TouchEvent: CustomStringConvertible {
     }
 }
 
+public struct VoiceActivityDetectionStatusEvent: CustomStringConvertible {
+    public let voiceActivityDetectionEnabled: Bool
+    public let values: [String: Any]
+
+    public init(values: [String: Any]) {
+        voiceActivityDetectionEnabled = boolValue(values, "voiceActivityDetectionEnabled") ?? true
+        self.values = values
+    }
+
+    public var description: String {
+        "VoiceActivityDetectionStatusEvent(voiceActivityDetectionEnabled: \(voiceActivityDetectionEnabled))"
+    }
+}
+
+public struct SpeakingStatusEvent: CustomStringConvertible {
+    public let speaking: Bool
+    public let values: [String: Any]
+
+    public init(values: [String: Any]) {
+        speaking = boolValue(values, "speaking") ?? false
+        self.values = values
+    }
+
+    public var description: String {
+        "SpeakingStatusEvent(speaking: \(speaking))"
+    }
+}
+
 public enum BluetoothEvent: CustomStringConvertible {
     case buttonPress(ButtonPressEvent)
     case touch(TouchEvent)
+    case voiceActivityDetectionStatus(VoiceActivityDetectionStatusEvent)
+    case speakingStatus(SpeakingStatusEvent)
     case wifiStatus(WifiStatusEvent)
     case hotspotStatus(HotspotStatusEvent)
     case hotspotError(HotspotErrorEvent)
     case photoResponse(PhotoResponseEvent)
+    case photoStatus(PhotoStatusEvent)
     case streamStatus(StreamStatusEvent)
     case keepAliveAck(KeepAliveAckEvent)
     case localTranscription(LocalTranscriptionEvent)
@@ -62,6 +93,10 @@ public enum BluetoothEvent: CustomStringConvertible {
             event.description
         case let .touch(event):
             event.description
+        case let .voiceActivityDetectionStatus(event):
+            event.description
+        case let .speakingStatus(event):
+            event.description
         case let .wifiStatus(event):
             event.description
         case let .hotspotStatus(event):
@@ -70,6 +105,8 @@ public enum BluetoothEvent: CustomStringConvertible {
             event.description
         case let .photoResponse(event):
             event.description
+        case let .photoStatus(event):
+            event.description
         case let .streamStatus(event):
             event.description
         case let .keepAliveAck(event):