@mentra/bluetooth-sdk 0.1.6 → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/README.md +26 -22
  2. package/android/.project +28 -0
  3. package/android/build.gradle +5 -9
  4. package/android/lc3Lib/.project +28 -0
  5. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +54 -33
  6. package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +48 -10
  7. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +27 -36
  8. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt +8 -4
  9. package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +170 -14
  10. package/android/src/main/java/com/mentra/bluetoothsdk/audio/AudioModels.kt +6 -6
  11. package/android/src/main/java/com/mentra/bluetoothsdk/camera/CameraModels.kt +69 -10
  12. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/ControllerManager.kt +5 -3
  13. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/R1.kt +2 -1
  14. package/android/src/main/java/com/mentra/bluetoothsdk/debug/BleTraceLogger.kt +149 -0
  15. package/android/src/main/java/com/mentra/bluetoothsdk/events/BluetoothEvents.kt +13 -0
  16. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoUploadServer.kt +297 -0
  17. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/MentraPhotoReceiverModule.kt +120 -0
  18. package/android/src/main/java/com/mentra/bluetoothsdk/services/Foreground.kt +13 -0
  19. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G1.java +1 -1
  20. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt +3 -1
  21. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Mach1.java +1 -1
  22. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraLive.java +269 -28
  23. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraNex.kt +1 -1
  24. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt +8 -3
  25. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Simulated.kt +4 -1
  26. package/android/src/main/java/com/mentra/bluetoothsdk/status/DeviceStatus.kt +14 -12
  27. package/android/src/main/java/com/mentra/bluetoothsdk/status/RuntimeState.kt +4 -2
  28. package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +131 -6
  29. package/android/src/main/java/com/mentra/bluetoothsdk/stt/SherpaOnnxTranscriber.kt +1 -3
  30. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunkReassembler.java +105 -0
  31. package/build/BluetoothSdk.types.d.ts +107 -48
  32. package/build/BluetoothSdk.types.d.ts.map +1 -1
  33. package/build/BluetoothSdk.types.js +4 -0
  34. package/build/BluetoothSdk.types.js.map +1 -1
  35. package/build/_private/BluetoothSdkModule.d.ts +8 -6
  36. package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
  37. package/build/_private/BluetoothSdkModule.js +58 -9
  38. package/build/_private/BluetoothSdkModule.js.map +1 -1
  39. package/build/index.d.ts +2 -2
  40. package/build/index.d.ts.map +1 -1
  41. package/build/index.js +6 -5
  42. package/build/index.js.map +1 -1
  43. package/build/photo-receiver/MentraPhotoReceiver.types.d.ts +18 -0
  44. package/build/photo-receiver/MentraPhotoReceiver.types.d.ts.map +1 -0
  45. package/build/photo-receiver/MentraPhotoReceiver.types.js +2 -0
  46. package/build/photo-receiver/MentraPhotoReceiver.types.js.map +1 -0
  47. package/build/photo-receiver/MentraPhotoReceiverModule.d.ts +10 -0
  48. package/build/photo-receiver/MentraPhotoReceiverModule.d.ts.map +1 -0
  49. package/build/photo-receiver/MentraPhotoReceiverModule.js +3 -0
  50. package/build/photo-receiver/MentraPhotoReceiverModule.js.map +1 -0
  51. package/build/photo-receiver/MentraPhotoReceiverModule.web.d.ts +10 -0
  52. package/build/photo-receiver/MentraPhotoReceiverModule.web.d.ts.map +1 -0
  53. package/build/photo-receiver/MentraPhotoReceiverModule.web.js +12 -0
  54. package/build/photo-receiver/MentraPhotoReceiverModule.web.js.map +1 -0
  55. package/build/photo-receiver/index.d.ts +3 -0
  56. package/build/photo-receiver/index.d.ts.map +1 -0
  57. package/build/photo-receiver/index.js +3 -0
  58. package/build/photo-receiver/index.js.map +1 -0
  59. package/build/react/useMentraBluetooth.d.ts +5 -3
  60. package/build/react/useMentraBluetooth.d.ts.map +1 -1
  61. package/build/react/useMentraBluetooth.js +15 -4
  62. package/build/react/useMentraBluetooth.js.map +1 -1
  63. package/expo-module.config.json +5 -2
  64. package/ios/BluetoothSdkModule.swift +59 -26
  65. package/ios/LocalPhotoUploadServer.swift +411 -0
  66. package/ios/MentraBluetoothSDK.podspec +8 -5
  67. package/ios/MentraPhotoReceiverModule.swift +147 -0
  68. package/ios/Source/Audio/AudioModels.swift +10 -10
  69. package/ios/Source/Bridge.swift +42 -12
  70. package/ios/Source/Camera/CameraModels.swift +74 -19
  71. package/ios/Source/DeviceManager.swift +23 -98
  72. package/ios/Source/DeviceStore.swift +12 -4
  73. package/ios/Source/Events/BluetoothEvents.swift +37 -0
  74. package/ios/Source/MentraBluetoothSDK.swift +135 -14
  75. package/ios/Source/Status/DeviceStatus.swift +12 -4
  76. package/ios/Source/Status/RuntimeState.swift +22 -13
  77. package/ios/Source/Streaming/StreamModels.swift +239 -45
  78. package/ios/Source/controllers/ControllerManager.swift +3 -3
  79. package/ios/Source/controllers/R1.swift +9 -3
  80. package/ios/Source/sgcs/G1.swift +5 -4
  81. package/ios/Source/sgcs/G2.swift +5 -6
  82. package/ios/Source/sgcs/Mach1.swift +1 -1
  83. package/ios/Source/sgcs/MentraLive.swift +183 -22
  84. package/ios/Source/sgcs/MentraNex.swift +1 -1
  85. package/ios/Source/sgcs/SGCManager.swift +11 -3
  86. package/ios/Source/sgcs/Simulated.swift +2 -2
  87. package/ios/Source/stt/SherpaOnnxTranscriber.swift +1 -4
  88. package/ios/Source/utils/MessageChunkReassembler.swift +89 -0
  89. package/package.json +6 -1
  90. package/src/BluetoothSdk.types.ts +133 -62
  91. package/src/_private/BluetoothSdkModule.ts +79 -26
  92. package/src/index.ts +15 -8
  93. package/src/photo-receiver/MentraPhotoReceiver.types.ts +20 -0
  94. package/src/photo-receiver/MentraPhotoReceiverModule.ts +14 -0
  95. package/src/photo-receiver/MentraPhotoReceiverModule.web.ts +20 -0
  96. package/src/photo-receiver/index.ts +2 -0
  97. package/src/react/useMentraBluetooth.ts +19 -7
  98. package/android/src/main/java/com/mentra/bluetoothsdk/stt/VadGateSpeechPolicy.kt +0 -221
  99. package/ios/Packages/VAD/Common/VAD/VADQuality.swift +0 -41
  100. package/ios/Packages/VAD/Common/VAD/VADState.swift +0 -26
  101. package/ios/Packages/VAD/Common/VAD/VADStrategy.swift +0 -29
  102. package/ios/Packages/VAD/Common/VAD/VADType.swift +0 -14
  103. package/ios/Packages/VAD/Data/Configuration.swift +0 -68
  104. package/ios/Packages/VAD/Data/FrameSize.swift +0 -39
  105. package/ios/Packages/VAD/Data/Record.swift +0 -13
  106. package/ios/Packages/VAD/Data/Result.swift +0 -22
  107. package/ios/Packages/VAD/Data/SampleRate.swift +0 -48
  108. package/ios/Packages/VAD/Silero/Model/silero_vad.onnx +0 -0
  109. package/ios/Packages/VAD/Silero/SileroVAD.swift +0 -284
  110. package/ios/Packages/VAD/Silero/SileroVADStrategy.swift +0 -64
@@ -768,9 +768,12 @@ extension MentraLive: CBPeripheralDelegate {
768
768
  // Keep state as connecting until glasses are ready
769
769
  updateConnectionState(ConnTypes.CONNECTING)
770
770
 
771
- // Request MTU size
772
- let mtuSize = peripheral.maximumWriteValueLength(for: .withResponse)
773
- Bridge.log("LIVE: Current MTU size: \(mtuSize + 3) bytes")
771
+ let withResponseMtu = peripheral.maximumWriteValueLength(for: .withResponse) + 3
772
+ let withoutResponseMtu = peripheral.maximumWriteValueLength(for: .withoutResponse) + 3
773
+ currentMtu = max(23, min(withResponseMtu, withoutResponseMtu))
774
+ Bridge.log(
775
+ "LIVE: Current MTU estimate: withResponse=\(withResponseMtu), withoutResponse=\(withoutResponseMtu), selected=\(currentMtu)"
776
+ )
774
777
 
775
778
  // Enable notifications on RX characteristic
776
779
  peripheral.setNotifyValue(true, for: rx)
@@ -903,6 +906,7 @@ class MentraLive: NSObject, SGCManager {
903
906
  // BLOCK_AUDIO_DUPLEX: When true, suspends LC3 mic while phone is playing audio via A2DP
904
907
  // to avoid overloading the MCU. Set to false to allow simultaneous A2DP + LC3 mic.
905
908
  private let BLOCK_AUDIO_DUPLEX = false
909
+ private static let voiceActivityDetectionSwitchType = 8
906
910
 
907
911
  var connectionState: String = ConnTypes.DISCONNECTED
908
912
 
@@ -1071,6 +1075,7 @@ class MentraLive: NSObject, SGCManager {
1071
1075
  private var connectedPeripheral: CBPeripheral?
1072
1076
  private var txCharacteristic: CBCharacteristic?
1073
1077
  private var rxCharacteristic: CBCharacteristic?
1078
+ private let bes2700MtuLimit = 256
1074
1079
  private var currentMtu: Int = 23 // Default BLE MTU
1075
1080
 
1076
1081
  // State Tracking
@@ -1095,6 +1100,7 @@ class MentraLive: NSObject, SGCManager {
1095
1100
  // Queue Management
1096
1101
  private let commandQueue = CommandQueue()
1097
1102
  private let bluetoothQueue = DispatchQueue(label: "MentraLiveBluetooth", qos: .userInitiated)
1103
+ private let incomingChunkReassembler = MessageChunkReassembler()
1098
1104
  private var lastSendTimeMs: TimeInterval = 0
1099
1105
 
1100
1106
  // Timers
@@ -1311,9 +1317,11 @@ class MentraLive: NSObject, SGCManager {
1311
1317
 
1312
1318
  func requestPhoto(
1313
1319
  _ requestId: String, appId: String, size: String?, webhookUrl: String?, authToken: String?,
1314
- compress: String?, flash: Bool, sound: Bool
1320
+ compress: String?, flash: Bool, sound: Bool, exposureTimeNs: Double?, iso: Int?
1315
1321
  ) {
1316
- Bridge.log("Requesting photo: \(requestId) for app: \(appId), flash: \(flash), sound: \(sound)")
1322
+ Bridge.log(
1323
+ "LIVE: PHOTO PIPELINE [5/6] requestPhoto() entry requestId=\(requestId) appId=\(appId) flash=\(flash) sound=\(sound) iso=\(iso.map { String($0) } ?? "auto")"
1324
+ )
1317
1325
 
1318
1326
  var json: [String: Any] = [
1319
1327
  "type": "take_photo",
@@ -1360,7 +1368,17 @@ class MentraLive: NSObject, SGCManager {
1360
1368
  json["flash"] = flash
1361
1369
  json["sound"] = sound
1362
1370
 
1363
- Bridge.log("Using auto transfer mode with BLE fallback ID: \(bleImgId)")
1371
+ if let e = exposureTimeNs, e.isFinite, e > 0, e <= Double(Int64.max) {
1372
+ Bridge.log("LIVE: Using manual exposure time for photo request \(requestId): \(Int64(e)) ns")
1373
+ json["exposureTimeNs"] = Int64(e)
1374
+ }
1375
+ if let iso, iso > 0 {
1376
+ Bridge.log("LIVE: Using manual ISO for photo request \(requestId): ISO \(iso)")
1377
+ json["iso"] = iso
1378
+ }
1379
+
1380
+ Bridge.log("LIVE: PHOTO PIPELINE [5b/6] take_photo JSON ready bleImgId=\(bleImgId) transferMethod=auto")
1381
+ Bridge.log("LIVE: PHOTO PIPELINE [6/6] Dispatching take_photo to sendJson()")
1364
1382
 
1365
1383
  sendJson(json, wakeUp: true)
1366
1384
  }
@@ -1768,6 +1786,11 @@ class MentraLive: NSObject, SGCManager {
1768
1786
  // Log ALL incoming JSON objects for debugging
1769
1787
  // Bridge.log("LIVE: DEBUG: processJsonObject: \(json)")
1770
1788
 
1789
+ if MessageChunker.isChunkedMessage(json) {
1790
+ processChunkedJsonObject(json)
1791
+ return
1792
+ }
1793
+
1771
1794
  // Check for K900 command format
1772
1795
  if let command = json["C"] as? String {
1773
1796
  processK900JsonMessage(json)
@@ -1814,6 +1837,14 @@ class MentraLive: NSObject, SGCManager {
1814
1837
  let isCharging = json["charging"] as? Bool ?? charging
1815
1838
  updateBatteryStatus(level: level, isCharging: isCharging)
1816
1839
 
1840
+ case "voice_activity_detection_status":
1841
+ let enabled = json["voiceActivityDetectionEnabled"] as? Bool ?? true
1842
+ handleVoiceActivityDetectionStatus(enabled: enabled)
1843
+
1844
+ case "speaking_status":
1845
+ let speaking = json["speaking"] as? Bool ?? false
1846
+ handleSpeakingStatus(speaking: speaking)
1847
+
1817
1848
  case "wifi_status":
1818
1849
  let connected = json["connected"] as? Bool ?? false
1819
1850
  let ssid = json["ssid"] as? String ?? ""
@@ -1839,6 +1870,9 @@ class MentraLive: NSObject, SGCManager {
1839
1870
  case "stream_status":
1840
1871
  emitRtmpStreamStatus(json)
1841
1872
 
1873
+ case "photo_status":
1874
+ emitPhotoStatus(json)
1875
+
1842
1876
  case "gallery_status":
1843
1877
  let photoCount = json["photos"] as? Int ?? 0
1844
1878
  let videoCount = json["videos"] as? Int ?? 0
@@ -1893,9 +1927,7 @@ class MentraLive: NSObject, SGCManager {
1893
1927
  let switchType = (json["switch_type"] as? Int) ?? (json["switchType"] as? Int) ?? -1
1894
1928
  let switchValue = (json["switch_value"] as? Int) ?? (json["switchValue"] as? Int) ?? -1
1895
1929
  let timestamp = parseTimestamp(json["timestamp"])
1896
- Bridge.sendSwitchStatus(
1897
- switchType: switchType, value: switchValue, timestamp: timestamp
1898
- )
1930
+ handleSwitchStatus(switchType: switchType, value: switchValue, timestamp: timestamp)
1899
1931
 
1900
1932
  case "rgb_led_control_response":
1901
1933
  let requestId = json["requestId"] as? String ?? ""
@@ -2087,6 +2119,9 @@ class MentraLive: NSObject, SGCManager {
2087
2119
  // Note: Stored separately from BES version for OTA patch matching
2088
2120
  DeviceStore.shared.apply("glasses", "mtkFirmwareVersion", mtkFirmwareVersion)
2089
2121
  }
2122
+ if let systemTimeMs = fields["system_time_ms"] as? NSNumber {
2123
+ DeviceStore.shared.apply("glasses", "systemTimeMs", systemTimeMs.int64Value)
2124
+ }
2090
2125
  if let bluetoothMacAddress = fields["bt_mac_address"] as? String {
2091
2126
  DeviceStore.shared.apply("glasses", "bluetoothMacAddress", bluetoothMacAddress)
2092
2127
  }
@@ -2102,6 +2137,26 @@ class MentraLive: NSObject, SGCManager {
2102
2137
  }
2103
2138
  }
2104
2139
 
2140
+ private func processChunkedJsonObject(_ json: [String: Any]) {
2141
+ guard let info = MessageChunker.getChunkInfo(json) else {
2142
+ Bridge.log("LIVE: Received malformed chunked message from glasses")
2143
+ return
2144
+ }
2145
+
2146
+ guard let reassembled = incomingChunkReassembler.addChunk(info) else {
2147
+ return
2148
+ }
2149
+
2150
+ guard let data = reassembled.data(using: .utf8),
2151
+ let reassembledJson = try? JSONSerialization.jsonObject(with: data) as? [String: Any]
2152
+ else {
2153
+ Bridge.log("LIVE: Failed to parse reassembled chunked message")
2154
+ return
2155
+ }
2156
+
2157
+ processJsonObject(reassembledJson)
2158
+ }
2159
+
2105
2160
  /// Maps K900 gesture type codes to gesture names
2106
2161
  /// Compute the weighted overall OTA percentage for a BES progress event arriving via sr_adota.
2107
2162
  /// Mirrors the weight table in OtaSessionManager.computeStepWeights() on the glasses side.
@@ -2241,6 +2296,25 @@ class MentraLive: NSObject, SGCManager {
2241
2296
  case "sr_vol":
2242
2297
  handleSrVol(json)
2243
2298
 
2299
+ case "sr_vad":
2300
+ if let bodyObj = k900ParseBody(json["B"]),
2301
+ let on = k900JsonInt(bodyObj, "on"),
2302
+ on == 0 || on == 1
2303
+ {
2304
+ handleSpeakingStatus(speaking: on == 1)
2305
+ }
2306
+
2307
+ case "sr_swit":
2308
+ if let body = k900ParseBody(json["B"]) {
2309
+ let switchType = k900JsonInt(body, "type") ?? -1
2310
+ let switchValue = k900JsonInt(body, "switch") ?? -1
2311
+ handleSwitchStatus(
2312
+ switchType: switchType,
2313
+ value: switchValue,
2314
+ timestamp: Int64(Date().timeIntervalSince1970 * 1000)
2315
+ )
2316
+ }
2317
+
2244
2318
  case "sr_shut":
2245
2319
  Bridge.log("K900 shutdown command received - glasses shutting down")
2246
2320
  // Mark as killed to prevent reconnection attempts
@@ -2461,7 +2535,7 @@ class MentraLive: NSObject, SGCManager {
2461
2535
  }
2462
2536
 
2463
2537
  func sendGalleryMode() {
2464
- let active = DeviceStore.shared.get("bluetooth", "galleryModeAuto") as! Bool
2538
+ let active = DeviceStore.shared.get("bluetooth", "gallery_mode") as! Bool
2465
2539
  Bridge.log("LIVE: 📸 Sending gallery mode active to glasses: \(active)")
2466
2540
 
2467
2541
  let json: [String: Any] = [
@@ -2515,6 +2589,7 @@ class MentraLive: NSObject, SGCManager {
2515
2589
  Bridge.log("LIVE: 🎉 Received glasses_ready message - SOC is booted and ready!")
2516
2590
 
2517
2591
  stopReadinessCheckLoop()
2592
+ sendBleMtuConfig()
2518
2593
 
2519
2594
  // Invalidate any version fields from a prior link session so the next version_info
2520
2595
  // cannot leave a stale build number in RN (ASG is source of truth for PackageInfo).
@@ -2656,7 +2731,7 @@ class MentraLive: NSObject, SGCManager {
2656
2731
  // return
2657
2732
  // }
2658
2733
 
2659
- // // Forward PCM data to DeviceManager for VAD and server transmission (same as Android)
2734
+ // // Forward PCM data to DeviceManager for audio events and server transmission (same as Android)
2660
2735
  // DeviceManager.shared.handlePcm(pcmData)
2661
2736
 
2662
2737
  // Bridge.log(
@@ -2762,8 +2837,12 @@ class MentraLive: NSObject, SGCManager {
2762
2837
  activeFileTransfers.removeValue(forKey: fileName)
2763
2838
 
2764
2839
  let bleImgId = fileName.split(separator: ".").first.map(String.init) ?? ""
2765
- if blePhotoTransfers.removeValue(forKey: bleImgId) != nil {
2840
+ if let transfer = blePhotoTransfers.removeValue(forKey: bleImgId) {
2766
2841
  Bridge.log("LIVE: 🧹 Cleaned up timed out BLE photo transfer for: \(bleImgId)")
2842
+ Bridge.sendPhotoError(
2843
+ requestId: transfer.requestId, errorCode: "TRANSFER_TIMEOUT",
2844
+ errorMessage: "Transfer timed out for: \(fileName)"
2845
+ )
2767
2846
  }
2768
2847
  if bleIncidentLogRelays.removeValue(forKey: bleImgId) != nil {
2769
2848
  Bridge.log("LIVE: 🧹 Cleaned up timed out BLE incident log relay for: \(bleImgId)")
@@ -2785,8 +2864,11 @@ class MentraLive: NSObject, SGCManager {
2785
2864
  }
2786
2865
 
2787
2866
  Bridge.log("LIVE: ❌ Transfer failed for: \(fileName) (reason: \(reason))")
2867
+ let bleImgId = fileName.split(separator: ".").first.map(String.init) ?? ""
2868
+ let transfer = blePhotoTransfers[bleImgId]
2869
+ let effectiveRequestId = requestId.isEmpty ? transfer?.requestId ?? "" : requestId
2788
2870
  Bridge.sendPhotoError(
2789
- requestId: requestId, errorCode: "TRANSFER_FAILED",
2871
+ requestId: effectiveRequestId, errorCode: "TRANSFER_FAILED",
2790
2872
  errorMessage: "Transfer failed for: \(fileName) (reason: \(reason))"
2791
2873
  )
2792
2874
 
@@ -2796,7 +2878,6 @@ class MentraLive: NSObject, SGCManager {
2796
2878
  )
2797
2879
  }
2798
2880
 
2799
- let bleImgId = fileName.split(separator: ".").first.map(String.init) ?? ""
2800
2881
  if let transfer = blePhotoTransfers.removeValue(forKey: bleImgId) {
2801
2882
  Bridge.log(
2802
2883
  "LIVE: 🧹 Cleaned up failed BLE photo transfer for: \(bleImgId) (requestId: \(transfer.requestId))"
@@ -2933,11 +3014,13 @@ class MentraLive: NSObject, SGCManager {
2933
3014
  )
2934
3015
  Bridge.log("❌ Telling glasses to retry entire transfer")
2935
3016
 
2936
- // Tell glasses transfer failed, they will retry
3017
+ // Tell glasses transfer failed, they will retry. Keep the photo
3018
+ // transfer entry so the retry maps back to the original requestId.
2937
3019
  sendTransferCompleteConfirmation(
2938
3020
  fileName: packetInfo.fileName, success: false
2939
3021
  )
2940
- blePhotoTransfers.removeValue(forKey: bleImgId)
3022
+ photoTransfer.session = nil
3023
+ blePhotoTransfers[bleImgId] = photoTransfer
2941
3024
  }
2942
3025
  }
2943
3026
  }
@@ -3170,6 +3253,20 @@ class MentraLive: NSObject, SGCManager {
3170
3253
  )
3171
3254
  }
3172
3255
 
3256
+ private func sendBleMtuConfig() {
3257
+ let effectiveMtu = min(currentMtu, bes2700MtuLimit)
3258
+ let json: [String: Any] = [
3259
+ "type": "set_ble_mtu",
3260
+ "mtu": effectiveMtu,
3261
+ "timestamp": Int64(Date().timeIntervalSince1970 * 1000),
3262
+ ]
3263
+
3264
+ sendJson(json)
3265
+ Bridge.log(
3266
+ "LIVE: Sent BLE MTU config to glasses: negotiated=\(currentMtu), BES2700 limit=\(bes2700MtuLimit), effective=\(effectiveMtu)"
3267
+ )
3268
+ }
3269
+
3173
3270
  // MARK: - Sending Data
3174
3271
 
3175
3272
  func queueSend(_ data: Data, id: String) {
@@ -3237,6 +3334,9 @@ class MentraLive: NSObject, SGCManager {
3237
3334
  Bridge.log("LIVE: All chunks queued for transmission")
3238
3335
  } else {
3239
3336
  // Normal single message transmission
3337
+ if (json["type"] as? String) == "take_photo" {
3338
+ Bridge.log("LIVE: PHOTO PIPELINE BLE handoff — sendJson -> queueSend take_photo")
3339
+ }
3240
3340
  Bridge.log("LIVE: Sending data to glasses: \(jsonString)")
3241
3341
  let packedData = packJson(jsonString, wakeUp: wakeUp) ?? Data()
3242
3342
  queueSend(packedData, id: trackingId)
@@ -3508,6 +3608,27 @@ class MentraLive: NSObject, SGCManager {
3508
3608
  }
3509
3609
  }
3510
3610
 
3611
+ private func handleVoiceActivityDetectionStatus(enabled: Bool) {
3612
+ Bridge.log("LIVE: Voice Activity Detection \(enabled ? "enabled" : "disabled")")
3613
+ Bridge.sendVoiceActivityDetectionStatus(enabled)
3614
+ }
3615
+
3616
+ private func handleSpeakingStatus(speaking: Bool) {
3617
+ guard voiceActivityDetectionEnabled else {
3618
+ Bridge.log("LIVE: Ignoring speaking status because Voice Activity Detection is disabled")
3619
+ return
3620
+ }
3621
+ Bridge.log("LIVE: Speaking status \(speaking ? "speaking" : "not speaking")")
3622
+ Bridge.sendSpeakingStatus(speaking)
3623
+ }
3624
+
3625
+ private func handleSwitchStatus(switchType: Int, value: Int, timestamp: Int64) {
3626
+ Bridge.sendSwitchStatus(switchType: switchType, value: value, timestamp: timestamp)
3627
+ if switchType == Self.voiceActivityDetectionSwitchType, value == 0 || value == 1 {
3628
+ handleVoiceActivityDetectionStatus(enabled: value == 1)
3629
+ }
3630
+ }
3631
+
3511
3632
  private func updateWifiStatus(connected: Bool, ssid: String, ip: String) {
3512
3633
  Bridge.log("LIVE: 🌐 Updating WiFi status - connected: \(connected), ssid: \(ssid)")
3513
3634
  DeviceStore.shared.apply("glasses", "wifiConnected", connected)
@@ -3796,6 +3917,10 @@ class MentraLive: NSObject, SGCManager {
3796
3917
  Bridge.sendTypedMessage("stream_status", body: json)
3797
3918
  }
3798
3919
 
3920
+ private func emitPhotoStatus(_ json: [String: Any]) {
3921
+ Bridge.sendPhotoStatus(json)
3922
+ }
3923
+
3799
3924
  private func emitButtonPress(buttonId: String, pressType: String, timestamp: Int64) {
3800
3925
  let eventBody: [String: Any] = [
3801
3926
  "device_model": "Mentra Live",
@@ -4493,6 +4618,42 @@ extension MentraLive {
4493
4618
 
4494
4619
  // Send gallery mode state (camera app running status)
4495
4620
  sendGalleryMode()
4621
+
4622
+ // Send glasses-side Voice Activity Detection setting.
4623
+ sendVoiceActivityDetectionSetting()
4624
+ }
4625
+
4626
+ func sendVoiceActivityDetectionSetting() {
4627
+ let enabled = DeviceStore.shared.get("bluetooth", "voice_activity_detection_enabled") as? Bool ?? true
4628
+ Bridge.log("LIVE: 🎤 Sending Voice Activity Detection setting to glasses: \(enabled)")
4629
+
4630
+ guard connectedPeripheral != nil, txCharacteristic != nil else {
4631
+ Bridge.log("Cannot send Voice Activity Detection setting - BLE write path not ready")
4632
+ return
4633
+ }
4634
+
4635
+ do {
4636
+ let bodyData = try JSONSerialization.data(withJSONObject: [
4637
+ "type": Self.voiceActivityDetectionSwitchType,
4638
+ "switch": enabled ? 1 : 0,
4639
+ ])
4640
+ guard let bodyString = String(data: bodyData, encoding: .utf8) else {
4641
+ Bridge.log("LIVE: Failed to encode Voice Activity Detection payload")
4642
+ return
4643
+ }
4644
+ let command: [String: Any] = [
4645
+ "C": "cs_swit",
4646
+ "V": 1,
4647
+ "B": bodyString,
4648
+ ]
4649
+ if sendRawK900Command(command, wakeUp: true) {
4650
+ Bridge.sendVoiceActivityDetectionStatus(enabled)
4651
+ } else {
4652
+ Bridge.log("LIVE: Failed to send Voice Activity Detection setting command")
4653
+ }
4654
+ } catch {
4655
+ Bridge.log("LIVE: Error encoding Voice Activity Detection payload: \(error)")
4656
+ }
4496
4657
  }
4497
4658
 
4498
4659
  func sendButtonVideoRecordingSettings() {
@@ -4500,11 +4661,11 @@ extension MentraLive {
4500
4661
  DeviceStore.shared.get("bluetooth", "button_video_settings") as? [String: Any] ?? [
4501
4662
  "width": 1280,
4502
4663
  "height": 720,
4503
- "frameRate": 30,
4664
+ "fps": 30,
4504
4665
  ]
4505
4666
  let width = settings["width"] as? Int ?? 1280
4506
4667
  let height = settings["height"] as? Int ?? 720
4507
- let fps = settings["frameRate"] as? Int ?? 30
4668
+ let fps = settings["fps"] as? Int ?? 30
4508
4669
 
4509
4670
  // Use defaults if not set
4510
4671
  let finalWidth = width > 0 ? width : 1280
@@ -4582,9 +4743,9 @@ extension MentraLive {
4582
4743
  }
4583
4744
 
4584
4745
  func sendCameraFovSetting() {
4585
- let settings = DeviceStore.shared.get("bluetooth", "camera_fov") as? [String: Any] ?? ["fov": 118, "roiPosition": 0]
4746
+ let settings = DeviceStore.shared.get("bluetooth", "camera_fov") as? [String: Any] ?? ["fov": 118, "roi_position": 0]
4586
4747
  let fov = settings["fov"] as? Int ?? 118
4587
- let roiPosition = settings["roiPosition"] as? Int ?? 0
4748
+ let roiPosition = settings["roi_position"] as? Int ?? 0
4588
4749
 
4589
4750
  Bridge.log("Sending camera FOV setting: fov=\(fov), roiPosition=\(roiPosition)")
4590
4751
 
@@ -4642,7 +4803,7 @@ extension MentraLive {
4642
4803
 
4643
4804
  var json: [String: Any] = [
4644
4805
  "type": "start_video_recording",
4645
- "request_id": requestId,
4806
+ "requestId": requestId,
4646
4807
  "save": save,
4647
4808
  "flash": flash,
4648
4809
  "sound": sound,
@@ -4669,7 +4830,7 @@ extension MentraLive {
4669
4830
 
4670
4831
  let json: [String: Any] = [
4671
4832
  "type": "stop_video_recording",
4672
- "request_id": requestId,
4833
+ "requestId": requestId,
4673
4834
  ]
4674
4835
  sendJson(json)
4675
4836
  }
@@ -46,7 +46,7 @@ class MentraNexSGC: NSObject, CBCentralManagerDelegate, CBPeripheralDelegate, SG
46
46
 
47
47
  func requestPhoto(
48
48
  _: String, appId _: String, size _: String?, webhookUrl _: String?, authToken _: String?,
49
- compress _: String?, flash _: Bool, sound _: Bool
49
+ compress _: String?, flash _: Bool, sound _: Bool, exposureTimeNs _: Double?, iso _: Int?
50
50
  ) {}
51
51
 
52
52
  func startStream(_: [String: Any]) {}
@@ -20,7 +20,7 @@ protocol SGCManager {
20
20
 
21
21
  func requestPhoto(
22
22
  _ requestId: String, appId: String, size: String?, webhookUrl: String?, authToken: String?,
23
- compress: String?, flash: Bool, sound: Bool
23
+ compress: String?, flash: Bool, sound: Bool, exposureTimeNs: Double?, iso: Int?
24
24
  )
25
25
  func startStream(_ message: [String: Any])
26
26
  func stopStream()
@@ -103,6 +103,10 @@ protocol SGCManager {
103
103
  func queryGalleryStatus()
104
104
  func sendGalleryMode()
105
105
 
106
+ // MARK: - Voice Activity Detection
107
+
108
+ func sendVoiceActivityDetectionSetting()
109
+
106
110
  // MARK: - Version Info
107
111
 
108
112
  func requestVersionInfo()
@@ -127,6 +131,10 @@ extension SGCManager {
127
131
 
128
132
  func setDashboardMenu(_: [[String: Any]]) {}
129
133
 
134
+ // MARK: - Voice Activity Detection (default no-op — Mentra Live supports this)
135
+
136
+ func sendVoiceActivityDetectionSetting() {}
137
+
130
138
  // MARK: - Default DeviceStore-backed property implementations
131
139
 
132
140
  var fullyBooted: Bool {
@@ -181,8 +189,8 @@ extension SGCManager {
181
189
  DeviceStore.shared.get("glasses", "micEnabled") as? Bool ?? false
182
190
  }
183
191
 
184
- var vadEnabled: Bool {
185
- DeviceStore.shared.get("glasses", "vadEnabled") as? Bool ?? false
192
+ var voiceActivityDetectionEnabled: Bool {
193
+ DeviceStore.shared.get("glasses", "voiceActivityDetectionEnabled") as? Bool ?? true
186
194
  }
187
195
 
188
196
  var batteryLevel: Int {
@@ -12,7 +12,7 @@ class Simulated: SGCManager {
12
12
  DeviceStore.shared.apply("glasses", "connected", true)
13
13
  DeviceStore.shared.apply("glasses", "connectionState", ConnTypes.CONNECTED)
14
14
  DeviceStore.shared.apply("glasses", "micEnabled", false)
15
- DeviceStore.shared.apply("glasses", "vadEnabled", false)
15
+ DeviceStore.shared.apply("glasses", "voiceActivityDetectionEnabled", true)
16
16
  DeviceStore.shared.apply("glasses", "bluetoothClassicConnected", false)
17
17
  }
18
18
 
@@ -75,7 +75,7 @@ class Simulated: SGCManager {
75
75
 
76
76
  // MARK: - Camera & Media
77
77
 
78
- func requestPhoto(_: String, appId _: String, size _: String?, webhookUrl _: String?, authToken _: String?, compress _: String?, flash _: Bool, sound _: Bool) {
78
+ func requestPhoto(_: String, appId _: String, size _: String?, webhookUrl _: String?, authToken _: String?, compress _: String?, flash _: Bool, sound _: Bool, exposureTimeNs _: Double?, iso _: Int?) {
79
79
  Bridge.log("requestPhoto")
80
80
  }
81
81
 
@@ -214,16 +214,13 @@ class SherpaOnnxTranscriber {
214
214
  * Feed PCM audio data (16-bit little endian) into the transcriber.
215
215
  * This method should be called continuously with short chunks (e.g., 100-300ms).
216
216
  *
217
- * Note: Audio passed to this method is assumed to have already passed VAD elsewhere,
218
- * so it's directly queued for processing without additional VAD checks.
217
+ * Audio is queued directly; microphone VAD gating is not applied in the SDK.
219
218
  */
220
219
  func acceptAudio(pcm16le: Data) {
221
220
  guard isRunning else {
222
221
  return
223
222
  }
224
223
 
225
- // Directly queue the audio data for processing
226
- // No VAD check here as it's assumed to be done upstream
227
224
  queueAudioData(pcm16le)
228
225
  }
229
226
 
@@ -0,0 +1,89 @@
1
+ import Foundation
2
+
3
+ final class MessageChunkReassembler {
4
+ private static let chunkTimeoutMs: TimeInterval = 30_000
5
+ private static let maxConcurrentSessions = 10
6
+
7
+ private var activeSessions: [String: ChunkSession] = [:]
8
+
9
+ func addChunk(_ info: MessageChunker.ChunkInfo) -> String? {
10
+ cleanupTimedOutSessions()
11
+
12
+ if activeSessions.count >= Self.maxConcurrentSessions,
13
+ activeSessions[info.chunkId] == nil
14
+ {
15
+ removeOldestSession()
16
+ }
17
+
18
+ let session = activeSessions[info.chunkId] ?? ChunkSession(
19
+ chunkId: info.chunkId,
20
+ totalChunks: info.totalChunks
21
+ )
22
+ activeSessions[info.chunkId] = session
23
+
24
+ guard session.addChunk(index: info.chunkIndex, data: info.data) else {
25
+ print("MessageChunkReassembler: Failed to add chunk \(info.chunkIndex) for \(info.chunkId)")
26
+ return nil
27
+ }
28
+
29
+ guard session.isComplete else {
30
+ return nil
31
+ }
32
+
33
+ let reassembled = session.reassemble()
34
+ activeSessions.removeValue(forKey: info.chunkId)
35
+ print("MessageChunkReassembler: Reassembled \(reassembled.count) bytes from \(info.totalChunks) chunks")
36
+ return reassembled
37
+ }
38
+
39
+ func clear() {
40
+ activeSessions.removeAll()
41
+ }
42
+
43
+ private func cleanupTimedOutSessions() {
44
+ let now = Date().timeIntervalSince1970 * 1000
45
+ activeSessions = activeSessions.filter { entry in
46
+ now - entry.value.createdAtMs <= Self.chunkTimeoutMs
47
+ }
48
+ }
49
+
50
+ private func removeOldestSession() {
51
+ guard let oldest = activeSessions.min(by: { $0.value.createdAtMs < $1.value.createdAtMs }) else {
52
+ return
53
+ }
54
+ activeSessions.removeValue(forKey: oldest.key)
55
+ }
56
+
57
+ private final class ChunkSession {
58
+ let chunkId: String
59
+ let totalChunks: Int
60
+ let createdAtMs: TimeInterval
61
+ private var chunks: [Int: String] = [:]
62
+
63
+ init(chunkId: String, totalChunks: Int) {
64
+ self.chunkId = chunkId
65
+ self.totalChunks = totalChunks
66
+ self.createdAtMs = Date().timeIntervalSince1970 * 1000
67
+ }
68
+
69
+ func addChunk(index: Int, data: String) -> Bool {
70
+ guard index >= 0, index < totalChunks else {
71
+ return false
72
+ }
73
+ chunks[index] = data
74
+ return true
75
+ }
76
+
77
+ var isComplete: Bool {
78
+ chunks.count == totalChunks
79
+ }
80
+
81
+ func reassemble() -> String {
82
+ var result = ""
83
+ for index in 0 ..< totalChunks {
84
+ result += chunks[index] ?? ""
85
+ }
86
+ return result
87
+ }
88
+ }
89
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@mentra/bluetooth-sdk",
3
- "version": "0.1.6",
3
+ "version": "0.1.8",
4
4
  "description": "SDK for communicating with smart glasses",
5
5
  "main": "build/index.js",
6
6
  "react-native": "src/index.ts",
@@ -17,6 +17,11 @@
17
17
  "react-native": "./src/react/index.ts",
18
18
  "default": "./build/react/index.js"
19
19
  },
20
+ "./photo-receiver": {
21
+ "types": "./build/photo-receiver/index.d.ts",
22
+ "react-native": "./src/photo-receiver/index.ts",
23
+ "default": "./build/photo-receiver/index.js"
24
+ },
20
25
  "./_private/*": null,
21
26
  "./build/*": null,
22
27
  "./src/*": null,