@mentra/bluetooth-sdk 0.1.7 → 0.1.8

package/ios/Source/Camera/CameraModels.swift

@@ -39,17 +39,24 @@ public struct ButtonVideoRecordingSettings {
     }
 }
 
-public enum CameraFov {
-    case standard
-    case wide
+public struct CameraFov {
+    public static let minFov = 82
+    public static let maxFov = 118
+    public static let defaultFov = 102
+    public static let minRoiPosition = 0
+    public static let maxRoiPosition = 2
+    public static let defaultRoiPosition = 0
+
+    public let fov: Int
+    public let roiPosition: Int
+
+    public init(fov: Int = CameraFov.defaultFov, roiPosition: Int = CameraFov.defaultRoiPosition) {
+        self.fov = min(max(fov, CameraFov.minFov), CameraFov.maxFov)
+        self.roiPosition = min(max(roiPosition, CameraFov.minRoiPosition), CameraFov.maxRoiPosition)
+    }
 
     var value: [String: Int] {
-        switch self {
-        case .standard:
-            ["fov": 118, "roi_position": 0]
-        case .wide:
-            ["fov": 118, "roi_position": 0]
-        }
+        ["fov": fov, "roi_position": roiPosition]
     }
 }
 
@@ -64,6 +71,8 @@ public struct PhotoRequest {
     public let sound: Bool
     /// Sensor exposure time for this capture only (ns), or nil for auto exposure
     public let exposureTimeNs: Double?
+    /// Sensor ISO for this capture only. Only used when exposureTimeNs enables manual exposure.
+    public let iso: Int?
 
     public init(
         requestId: String,
@@ -74,7 +83,8 @@ public struct PhotoRequest {
         compress: PhotoCompression? = nil,
         flash: Bool = true,
         sound: Bool,
-        exposureTimeNs: Double? = nil
+        exposureTimeNs: Double? = nil,
+        iso: Int? = nil
     ) {
         self.requestId = requestId
         self.appId = appId
@@ -85,6 +95,7 @@ public struct PhotoRequest {
         self.flash = flash
         self.sound = sound
         self.exposureTimeNs = exposureTimeNs
+        self.iso = iso
     }
 }
 
@@ -229,7 +240,7 @@ public struct PhotoResponseEvent: CustomStringConvertible {
     }
 
     public init(values: [String: Any]) {
-        self.response = PhotoResponse(values: values)
+        response = PhotoResponse(values: values)
     }
 
     public var requestId: String {
@@ -246,3 +257,45 @@ public struct PhotoResponseEvent: CustomStringConvertible {
         "PhotoResponseEvent(requestId: \(requestId), state: \(response.state.rawValue))"
     }
 }
+
+public struct PhotoStatusEvent: CustomStringConvertible {
+    public let values: [String: Any]
+
+    public init(values: [String: Any]) {
+        var values = values
+        values["type"] = "photo_status"
+        self.values = values
+    }
+
+    public var requestId: String {
+        stringValue(values, "requestId") ?? ""
+    }
+
+    public var status: String {
+        stringValue(values, "status") ?? ""
+    }
+
+    public var timestamp: Int64 {
+        if let value = values["timestamp"] as? Int64 { return value }
+        if let value = values["timestamp"] as? Int { return Int64(value) }
+        if let value = values["timestamp"] as? Double { return Int64(value) }
+        if let value = values["timestamp"] as? NSNumber { return value.int64Value }
+        return Int64(Date().timeIntervalSince1970 * 1000)
+    }
+
+    public var resolvedConfig: [String: Any]? {
+        values["resolvedConfig"] as? [String: Any]
+    }
+
+    public var errorCode: String? {
+        stringValue(values, "errorCode")
+    }
+
+    public var errorMessage: String? {
+        stringValue(values, "errorMessage")
+    }
+
+    public var description: String {
+        "PhotoStatusEvent(requestId: \(requestId), status: \(status))"
+    }
+}

package/ios/Source/DeviceManager.swift

@@ -733,7 +733,7 @@ struct ViewState {
         if !glassesMicEnabled || !glassesConnected {
             return
         }
-        
+
         let timeSinceLastLc3Event = Date().timeIntervalSince(lastLc3Event ?? Date())
         if timeSinceLastLc3Event > 5 {
             Bridge.log("MAN: No audio activity in the last 5 seconds from glasses, reinitializing glasses mic")
@@ -1196,10 +1196,12 @@ struct ViewState {
         _ compress: String?,
         _ flash: Bool,
         _ sound: Bool,
-        exposureTimeNs: Double? = nil
+        exposureTimeNs: Double? = nil,
+        iso: Int? = nil
     ) {
+        let manualIso = exposureTimeNs != nil ? iso.flatMap { $0 > 0 ? $0 : nil } : nil
         Bridge.log(
-            "MAN: PHOTO PIPELINE [4/6] DeviceManager.requestPhoto requestId=\(requestId) appId=\(appId) webhookUrl=\(webhookUrl ?? "nil") size=\(size) compress=\(compress ?? "none") flash=\(flash) sound=\(sound) exposureTimeNs=\(exposureTimeNs.map { String($0) } ?? "nil") sgc=\(sgc != nil ? String(describing: type(of: sgc!)) : "null")"
+            "MAN: PHOTO PIPELINE [4/6] DeviceManager.requestPhoto requestId=\(requestId) appId=\(appId) webhookUrl=\(webhookUrl ?? "nil") size=\(size) compress=\(compress ?? "none") flash=\(flash) sound=\(sound) exposureTimeNs=\(exposureTimeNs.map { String($0) } ?? "nil") iso=\(manualIso.map { String($0) } ?? "auto") sgc=\(sgc != nil ? String(describing: type(of: sgc!)) : "null")"
         )
         guard let sgc else {
             Bridge.log(
@@ -1209,7 +1211,7 @@ struct ViewState {
         }
         sgc.requestPhoto(
             requestId, appId: appId, size: size, webhookUrl: webhookUrl, authToken: authToken,
-            compress: compress, flash: flash, sound: sound, exposureTimeNs: exposureTimeNs
+            compress: compress, flash: flash, sound: sound, exposureTimeNs: exposureTimeNs, iso: manualIso
         )
     }
 

package/ios/Source/Events/BluetoothEvents.swift

@@ -81,6 +81,7 @@ public enum BluetoothEvent: CustomStringConvertible {
     case hotspotStatus(HotspotStatusEvent)
     case hotspotError(HotspotErrorEvent)
     case photoResponse(PhotoResponseEvent)
+    case photoStatus(PhotoStatusEvent)
     case streamStatus(StreamStatusEvent)
     case keepAliveAck(KeepAliveAckEvent)
     case localTranscription(LocalTranscriptionEvent)
@@ -104,6 +105,8 @@ public enum BluetoothEvent: CustomStringConvertible {
             event.description
         case let .photoResponse(event):
             event.description
+        case let .photoStatus(event):
+            event.description
         case let .streamStatus(event):
             event.description
         case let .keepAliveAck(event):

package/ios/Source/MentraBluetoothSDK.swift

@@ -1,6 +1,20 @@
 import CoreBluetooth
 import Foundation
 
+@MainActor
+private final class ActiveStreamKeepAlive {
+    let streamId: String
+    let intervalSeconds: Int
+    var pendingAckId: String?
+    var missedAckCount = 0
+    var task: Task<Void, Never>?
+
+    init(streamId: String, intervalSeconds: Int) {
+        self.streamId = streamId
+        self.intervalSeconds = intervalSeconds
+    }
+}
+
 @MainActor
 private final class ActiveScanSession {
     let model: DeviceModel
@@ -33,6 +47,7 @@ public final class MentraBluetoothSDK {
     private var suppressDefaultDeviceEvents = false
     private var defaultDeviceApplyGeneration = 0
     private var activeScanSessions: [UUID: ActiveScanSession] = [:]
+    private var activeStreamKeepAlive: ActiveStreamKeepAlive?
 
     public init(configuration: MentraBluetoothSDKConfiguration = .default) {
         self.configuration = configuration
@@ -388,7 +403,8 @@ public final class MentraBluetoothSDK {
             request.compress?.rawValue,
             request.flash,
             request.sound,
-            exposureTimeNs: request.exposureTimeNs
+            exposureTimeNs: request.exposureTimeNs,
+            iso: request.iso
         )
     }
 
@@ -397,10 +413,17 @@ public final class MentraBluetoothSDK {
     }
 
     public func startStream(_ request: StreamRequest) {
-        DeviceManager.shared.startStream(request.values)
+        var values = request.values
+        let streamId = stringValue(values, "streamId").flatMap { $0.isEmpty ? nil : $0 } ?? "sdk-\(UUID().uuidString)"
+        values["streamId"] = streamId
+        stopStreamKeepAliveMonitor()
+        DeviceManager.shared.startStream(values)
+        if request.keepAlive, !request.isExternallyManagedKeepAlive {
+            startStreamKeepAliveMonitor(streamId: streamId, intervalSeconds: request.keepAliveIntervalSeconds)
+        }
     }
 
-    public func keepStreamAlive(_ request: StreamKeepAliveRequest) {
+    func sendCloudStreamKeepAlive(_ request: StreamKeepAliveRequest) {
         DeviceManager.shared.keepStreamAlive(request.values)
     }
 
@@ -417,6 +440,7 @@ public final class MentraBluetoothSDK {
     }
 
     public func stopStream() {
+        stopStreamKeepAliveMonitor()
         DeviceManager.shared.stopStream()
     }
 
@@ -457,6 +481,7 @@ public final class MentraBluetoothSDK {
     }
 
     public func invalidate() {
+        stopStreamKeepAliveMonitor()
         if let bridgeEventSinkId {
             Bridge.removeEventSink(bridgeEventSinkId)
             self.bridgeEventSinkId = nil
@@ -468,6 +493,80 @@ public final class MentraBluetoothSDK {
         delegate = nil
     }
 
+    private func startStreamKeepAliveMonitor(streamId: String, intervalSeconds requestedIntervalSeconds: Int) {
+        let intervalSeconds = requestedIntervalSeconds > 0 ? requestedIntervalSeconds : 5
+        let tracker = ActiveStreamKeepAlive(streamId: streamId, intervalSeconds: intervalSeconds)
+        activeStreamKeepAlive = tracker
+        sendNextStreamKeepAlive(for: tracker)
+    }
+
+    private func stopStreamKeepAliveMonitor() {
+        activeStreamKeepAlive?.task?.cancel()
+        activeStreamKeepAlive = nil
+    }
+
+    private func sendNextStreamKeepAlive(for tracker: ActiveStreamKeepAlive) {
+        guard activeStreamKeepAlive === tracker else { return }
+
+        if tracker.pendingAckId != nil {
+            tracker.missedAckCount += 1
+            if tracker.missedAckCount >= 3 {
+                activeStreamKeepAlive = nil
+                tracker.task?.cancel()
+                let event = StreamStatusEvent(
+                    status: .error(
+                        streamId: tracker.streamId,
+                        errorDetails: "Stream keep-alive timed out after \(tracker.missedAckCount) missed ACKs",
+                        timestamp: Int(Date().timeIntervalSince1970 * 1000),
+                        resolvedConfig: nil
+                    )
+                )
+                delegate?.mentraBluetoothSDK(self, didReceive: .streamStatus(event))
+                return
+            }
+        }
+
+        let ackId = "ack-\(Int(Date().timeIntervalSince1970 * 1000))"
+        tracker.pendingAckId = ackId
+        DeviceManager.shared.keepStreamAlive(
+            StreamKeepAliveRequest(streamId: tracker.streamId, ackId: ackId).values
+        )
+
+        tracker.task?.cancel()
+        tracker.task = Task { @MainActor [weak self, weak tracker] in
+            guard let tracker else { return }
+            try? await Task.sleep(nanoseconds: UInt64(tracker.intervalSeconds) * 1_000_000_000)
+            self?.sendNextStreamKeepAlive(for: tracker)
+        }
+    }
+
+    private func handleStreamKeepAliveAck(_ event: KeepAliveAckEvent) -> Bool {
+        guard let tracker = activeStreamKeepAlive,
+              event.streamId == tracker.streamId,
+              event.ackId == tracker.pendingAckId
+        else {
+            return false
+        }
+        tracker.pendingAckId = nil
+        tracker.missedAckCount = 0
+        return true
+    }
+
+    private func handleStreamStatusForKeepAlive(_ status: StreamStatus) {
+        if let streamId = status.streamId,
+           activeStreamKeepAlive?.streamId != streamId
+        {
+            return
+        }
+
+        switch status.state {
+        case .stopped, .stopping, .error, .reconnectFailed:
+            stopStreamKeepAliveMonitor()
+        default:
+            break
+        }
+    }
+
     private func dispatchStoreUpdate(_ category: String, _ changes: [String: Any]) {
         switch ObservableStore.normalizeCategory(category) {
         case "glasses":
@@ -596,10 +695,17 @@ public final class MentraBluetoothSDK {
             delegate?.mentraBluetoothSDK(self, didReceive: .hotspotError(HotspotErrorEvent(values: data)))
         case "photo_response":
             delegate?.mentraBluetoothSDK(self, didReceive: .photoResponse(PhotoResponseEvent(values: data)))
+        case "photo_status":
+            delegate?.mentraBluetoothSDK(self, didReceive: .photoStatus(PhotoStatusEvent(values: data)))
         case "stream_status":
-            delegate?.mentraBluetoothSDK(self, didReceive: .streamStatus(StreamStatusEvent(values: data)))
+            let event = StreamStatusEvent(values: data)
+            handleStreamStatusForKeepAlive(event.status)
+            delegate?.mentraBluetoothSDK(self, didReceive: .streamStatus(event))
         case "keep_alive_ack":
-            delegate?.mentraBluetoothSDK(self, didReceive: .keepAliveAck(KeepAliveAckEvent(values: data)))
+            let event = KeepAliveAckEvent(values: data)
+            if !handleStreamKeepAliveAck(event) {
+                delegate?.mentraBluetoothSDK(self, didReceive: .keepAliveAck(event))
+            }
         case "compatible_glasses_search_stop":
             delegate?.mentraBluetoothSDK(self, didStopScan: .completed)
         case "pair_failure":