@mentra/bluetooth-sdk 0.1.5 → 0.1.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/README.md +45 -27
  2. package/android/build.gradle +2 -9
  3. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +53 -36
  4. package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +22 -9
  5. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +25 -36
  6. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt +9 -4
  7. package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +68 -56
  8. package/android/src/main/java/com/mentra/bluetoothsdk/audio/AudioModels.kt +125 -0
  9. package/android/src/main/java/com/mentra/bluetoothsdk/camera/CameraModels.kt +209 -0
  10. package/android/src/main/java/com/mentra/bluetoothsdk/connection/ConnectionModels.kt +78 -0
  11. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/ControllerManager.kt +4 -3
  12. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/R1.kt +2 -1
  13. package/android/src/main/java/com/mentra/bluetoothsdk/events/BluetoothEvents.kt +72 -0
  14. package/android/src/main/java/com/mentra/bluetoothsdk/internal/MapParsing.kt +132 -0
  15. package/android/src/main/java/com/mentra/bluetoothsdk/requests/DisplayRequests.kt +39 -0
  16. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G1.java +1 -1
  17. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt +2 -1
  18. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Mach1.java +1 -1
  19. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraLive.java +168 -10
  20. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraNex.kt +1 -1
  21. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt +7 -3
  22. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Simulated.kt +3 -1
  23. package/android/src/main/java/com/mentra/bluetoothsdk/status/DeviceStatus.kt +607 -0
  24. package/android/src/main/java/com/mentra/bluetoothsdk/status/RuntimeState.kt +201 -0
  25. package/android/src/main/java/com/mentra/bluetoothsdk/status/WifiHotspotStatus.kt +173 -0
  26. package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +348 -0
  27. package/android/src/main/java/com/mentra/bluetoothsdk/stt/SherpaOnnxTranscriber.kt +1 -3
  28. package/android/src/main/java/com/mentra/bluetoothsdk/types/DeviceModels.kt +75 -0
  29. package/build/BluetoothSdk.types.d.ts +46 -20
  30. package/build/BluetoothSdk.types.d.ts.map +1 -1
  31. package/build/BluetoothSdk.types.js +1 -0
  32. package/build/BluetoothSdk.types.js.map +1 -1
  33. package/build/_private/BluetoothSdkModule.d.ts +6 -5
  34. package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
  35. package/build/_private/BluetoothSdkModule.js +40 -5
  36. package/build/_private/BluetoothSdkModule.js.map +1 -1
  37. package/build/index.d.ts +1 -1
  38. package/build/index.d.ts.map +1 -1
  39. package/build/index.js +4 -2
  40. package/build/index.js.map +1 -1
  41. package/build/react/useMentraBluetooth.d.ts +5 -3
  42. package/build/react/useMentraBluetooth.d.ts.map +1 -1
  43. package/build/react/useMentraBluetooth.js +15 -4
  44. package/build/react/useMentraBluetooth.js.map +1 -1
  45. package/ios/BluetoothSdkModule.swift +45 -28
  46. package/ios/MentraBluetoothSDK.podspec +0 -3
  47. package/ios/Source/Audio/AudioModels.swift +159 -0
  48. package/ios/Source/Bridge.swift +22 -11
  49. package/ios/Source/Camera/CameraModels.swift +248 -0
  50. package/ios/Source/Connection/ScanSession.swift +27 -0
  51. package/ios/Source/DeviceManager.swift +20 -97
  52. package/ios/Source/DeviceStore.swift +13 -4
  53. package/ios/Source/Errors/BluetoothError.swift +19 -0
  54. package/ios/Source/Events/BluetoothEvents.swift +149 -0
  55. package/ios/Source/Internal/BluetoothAvailability.swift +58 -0
  56. package/ios/Source/Internal/ValueParsing.swift +90 -0
  57. package/ios/Source/MentraBluetoothSDK.swift +50 -2150
  58. package/ios/Source/Requests/DisplayRequests.swift +58 -0
  59. package/ios/Source/Status/DeviceStatus.swift +471 -0
  60. package/ios/Source/Status/RuntimeState.swift +359 -0
  61. package/ios/Source/Status/WifiHotspotStatus.swift +326 -0
  62. package/ios/Source/Streaming/StreamModels.swift +436 -0
  63. package/ios/Source/Types/DeviceModels.swift +134 -0
  64. package/ios/Source/controllers/ControllerManager.swift +3 -3
  65. package/ios/Source/controllers/R1.swift +1 -1
  66. package/ios/Source/sgcs/G1.swift +1 -1
  67. package/ios/Source/sgcs/G2.swift +99 -39
  68. package/ios/Source/sgcs/Mach1.swift +1 -1
  69. package/ios/Source/sgcs/MentraLive.swift +109 -12
  70. package/ios/Source/sgcs/MentraNex.swift +1 -1
  71. package/ios/Source/sgcs/SGCManager.swift +11 -3
  72. package/ios/Source/sgcs/Simulated.swift +2 -2
  73. package/ios/Source/stt/SherpaOnnxTranscriber.swift +1 -4
  74. package/package.json +1 -1
  75. package/src/BluetoothSdk.types.ts +48 -28
  76. package/src/_private/BluetoothSdkModule.ts +47 -21
  77. package/src/index.ts +6 -4
  78. package/src/react/useMentraBluetooth.ts +19 -7
  79. package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothModels.kt +0 -1810
  80. package/android/src/main/java/com/mentra/bluetoothsdk/stt/VadGateSpeechPolicy.kt +0 -221
  81. package/ios/Packages/VAD/Common/VAD/VADQuality.swift +0 -41
  82. package/ios/Packages/VAD/Common/VAD/VADState.swift +0 -26
  83. package/ios/Packages/VAD/Common/VAD/VADStrategy.swift +0 -29
  84. package/ios/Packages/VAD/Common/VAD/VADType.swift +0 -14
  85. package/ios/Packages/VAD/Data/Configuration.swift +0 -68
  86. package/ios/Packages/VAD/Data/FrameSize.swift +0 -39
  87. package/ios/Packages/VAD/Data/Record.swift +0 -13
  88. package/ios/Packages/VAD/Data/Result.swift +0 -22
  89. package/ios/Packages/VAD/Data/SampleRate.swift +0 -48
  90. package/ios/Packages/VAD/Silero/Model/silero_vad.onnx +0 -0
  91. package/ios/Packages/VAD/Silero/SileroVAD.swift +0 -284
  92. package/ios/Packages/VAD/Silero/SileroVADStrategy.swift +0 -64
@@ -1062,6 +1062,8 @@ class G2: NSObject, SGCManager {
1062
1062
  private var rightAuthenticated: Bool = false
1063
1063
  private var isDisconnecting = false
1064
1064
  private var pairingTimeoutTimer: DispatchWorkItem?
1065
+ private var useEvenDashboard = true
1066
+ private var dashboardShowing = 0
1065
1067
 
1066
1068
  /// Device search
1067
1069
  var DEVICE_SEARCH_ID = "NOT_SET"
@@ -1452,8 +1454,7 @@ class G2: NSObject, SGCManager {
1452
1454
  DeviceStore.shared.apply("glasses", "bluetoothName", btName)
1453
1455
  DeviceStore.shared.apply("glasses", "deviceModel", DeviceTypes.G2)
1454
1456
 
1455
- DeviceStore.shared.apply("glasses", "connected", true)
1456
- DeviceStore.shared.apply("glasses", "fullyBooted", true)
1457
+ self.setFullyConnected()
1457
1458
 
1458
1459
  // connnect a controller if we have one:
1459
1460
  self.connectController()
@@ -1598,11 +1599,11 @@ class G2: NSObject, SGCManager {
1598
1599
  func sendTextWall(_ text: String) {
1599
1600
  // Bridge.log("G2: sendTextWall(\(text.prefix(50))...)")
1600
1601
 
1601
- // ignore events while the dashboard is open:
1602
- // let isHeadUp = DeviceStore.shared.get("glasses", "headUp") as? Bool ?? false
1603
- // if isHeadUp {
1604
- // return
1605
- // }
1602
+ // ignore events while the ER dashboard is open:
1603
+ let useNativeDashboard = DeviceStore.shared.get("bluetooth", "use_native_dashboard") as? Bool ?? false
1604
+ if useNativeDashboard && dashboardShowing > 0 {
1605
+ return
1606
+ }
1606
1607
 
1607
1608
  if text.isEmpty {
1608
1609
  clearDisplay()
@@ -2191,8 +2192,24 @@ class G2: NSObject, SGCManager {
2191
2192
  return bmp
2192
2193
  }
2193
2194
 
2195
+ /// Bring the Even Realities dashboard (the OS-level home/idle screen) to
2196
+ /// the foreground by tearing down whatever EvenHub page we currently own.
2197
+ /// The glasses fall back to the dashboard automatically when no page is up.
2194
2198
  func showDashboard() {
2195
- // G2 doesn't have a native dashboard concept via EvenHub
2199
+ Bridge.log("G2: showDashboard")
2200
+ dashboardShowing += 2
2201
+ let msg = EvenHubProto.shutdownMessage()
2202
+ sendEvenHubCommand(msg)
2203
+ pageCreated = false
2204
+ pageHasTextContainer = false
2205
+ currentTextContent = ""
2206
+ currentBitmapBase64 = ""
2207
+ DispatchQueue.main.asyncAfter(deadline: .now() + 0.5) { [weak self] in
2208
+ guard let self = self else { return }
2209
+ // activate the dashboard by setting dept to the current setting:
2210
+ let currentDepth = DeviceStore.shared.get("bluetooth", "dashboard_depth") as? Int ?? 0
2211
+ self.setDashboardDepthOnly(currentDepth)
2212
+ }
2196
2213
  }
2197
2214
 
2198
2215
  func setDashboardPosition(_ height: Int, _ depth: Int) {
@@ -2302,21 +2319,33 @@ class G2: NSObject, SGCManager {
2302
2319
  sendEvenHubCommand(toSend)
2303
2320
  }
2304
2321
 
2322
+ func restartMic() {
2323
+ // if already enabled, set to disabled, then send enabled after 500ms:
2324
+ DeviceStore.shared.apply("glasses", "micEnabled", true)
2325
+ let msg = EvenHubProto.audioControlMessage(enable: false)
2326
+ sendEvenHubCommand(msg)
2327
+ DispatchQueue.main.asyncAfter(deadline: .now() + 0.5) { [weak self] in
2328
+ guard let self = self else { return }
2329
+ let useNativeDashboard = DeviceStore.shared.get("bluetooth", "use_native_dashboard") as? Bool ?? false
2330
+ Bridge.log("G2: setMicEnabled - useNativeDashboard=\(useNativeDashboard), dashboardShowing=\(dashboardShowing)")
2331
+ if useNativeDashboard && dashboardShowing > 0 {
2332
+ return
2333
+ }
2334
+ if (!pageCreated || !pageHasTextContainer) {
2335
+ DeviceManager.shared.sendCurrentState()// should re-create the page if needed
2336
+ }
2337
+ let msg = EvenHubProto.audioControlMessage(enable: true)
2338
+ self.sendEvenHubCommand(msg)
2339
+ }
2340
+ }
2341
+
2305
2342
  // MARK: - SGCManager: Audio Control
2306
2343
 
2307
2344
  func setMicEnabled(_ enabled: Bool) {
2308
2345
  Bridge.log("G2: setMicEnabled(\(enabled))")
2309
2346
  let currentEnabled = DeviceStore.shared.get("glasses", "micEnabled") as? Bool ?? false
2310
2347
  if currentEnabled && enabled {
2311
- // if already enabled, set to disabled, then send enabled after 500ms:
2312
- DeviceStore.shared.apply("glasses", "micEnabled", true)
2313
- let msg = EvenHubProto.audioControlMessage(enable: false)
2314
- sendEvenHubCommand(msg)
2315
- DispatchQueue.main.asyncAfter(deadline: .now() + 0.5) { [weak self] in
2316
- guard let self = self else { return }
2317
- let msg = EvenHubProto.audioControlMessage(enable: true)
2318
- self.sendEvenHubCommand(msg)
2319
- }
2348
+ restartMic()
2320
2349
  return
2321
2350
  }
2322
2351
 
@@ -2505,7 +2534,8 @@ class G2: NSObject, SGCManager {
2505
2534
  // }
2506
2535
 
2507
2536
  // connectController("1B:08:26:8E:0E:E6")
2508
- connectController()
2537
+ // connectController()
2538
+ showDashboard()
2509
2539
  }
2510
2540
 
2511
2541
  func dbg2() {
@@ -2532,6 +2562,8 @@ class G2: NSObject, SGCManager {
2532
2562
  // // update the text
2533
2563
  // Bridge.log("G2: sendTextWall() - updating text container")
2534
2564
  // updateText("test2")
2565
+ let currentDepth = DeviceStore.shared.get("bluetooth", "dashboard_depth") as? Int ?? 0
2566
+ setDashboardDepthOnly(currentDepth)
2535
2567
  }
2536
2568
 
2537
2569
  // MARK: - SGCManager: Device Control
@@ -2615,7 +2647,7 @@ class G2: NSObject, SGCManager {
2615
2647
 
2616
2648
  func requestPhoto(
2617
2649
  _: String, appId _: String, size _: String?, webhookUrl _: String?, authToken _: String?,
2618
- compress _: String?, flash _: Bool, sound _: Bool
2650
+ compress _: String?, flash _: Bool, sound _: Bool, exposureTimeNs _: Double?
2619
2651
  ) {}
2620
2652
  func startVideoRecording(requestId _: String, save _: Bool, flash _: Bool, sound _: Bool) {}
2621
2653
  func startStream(_: [String: Any]) {}
@@ -3021,15 +3053,22 @@ class G2: NSObject, SGCManager {
3021
3053
  if eventType == .doubleClick {
3022
3054
  // trigger dashboard:
3023
3055
  let isHeadUp = DeviceStore.shared.get("glasses", "headUp") as? Bool ?? false
3024
- // toggle head up:
3025
- DeviceStore.shared.apply("glasses", "headUp", !isHeadUp)
3026
- if isHeadUp {
3027
- // Bridge.log("G2: going back to home, clearing display")
3028
- // clear the display after a delay:
3029
- DispatchQueue.main.asyncAfter(deadline: .now() + 0.5) {
3030
- self.clearDisplay()
3031
- }
3056
+
3057
+ let useNativeDashboard = DeviceStore.shared.get("bluetooth", "use_native_dashboard") as? Bool ?? false
3058
+ if useNativeDashboard {
3059
+ showDashboard()
3060
+ } else {
3061
+ // toggle head up:
3062
+ DeviceStore.shared.apply("glasses", "headUp", !isHeadUp)
3032
3063
  }
3064
+
3065
+ // if isHeadUp {
3066
+ // // Bridge.log("G2: going back to home, clearing display")
3067
+ // // clear the display after a delay:
3068
+ // DispatchQueue.main.asyncAfter(deadline: .now() + 0.5) {
3069
+ // self.clearDisplay()
3070
+ // }
3071
+ // }
3033
3072
  // sendDashboardCommand(DashboardCommand.trigger)
3034
3073
 
3035
3074
  // toggle head up:
@@ -3373,18 +3412,39 @@ class G2: NSObject, SGCManager {
3373
3412
  // if we got 08011A00 that means we closed the dashboard, which means the mic is probably dead,
3374
3413
  // so we need to revive it:
3375
3414
  if data == Data([0x08, 0x01, 0x1A, 0x00]) {
3376
- Bridge.log("G2: gesture_ctrl response: dashboard closed")
3377
- // re-send mic on / update mic state:
3378
- DeviceStore.shared.apply("glasses", "micEnabled", false)
3379
- DeviceManager.shared.updateMicState() // should set the mic back on if it should be on
3380
- // // let isHeadUp = DeviceStore.shared.get("glasses", "headUp") as? Bool ?? false
3381
-
3382
- // // toggle head up:
3383
- // DeviceStore.shared.apply("glasses", "headUp", false)
3384
- // // send the current state to the glasses
3385
- // DeviceManager.shared.sendCurrentState()
3386
- // reset the text container (different from clearDisplay())
3387
- sendTextWall(" ")
3415
+ Bridge.log("G2: dashboard closed / shutdown - dashboardShowing=\(dashboardShowing)")
3416
+ let useNativeDashboard = DeviceStore.shared.get("bluetooth", "use_native_dashboard") as? Bool ?? false
3417
+ if !useNativeDashboard {
3418
+ // make sure the container exists:
3419
+ DeviceManager.shared.sendCurrentState()
3420
+ // re-send mic on / if it's enabled:
3421
+ let micEnabled = DeviceStore.shared.get("glasses", "micEnabled") as? Bool ?? false
3422
+ if micEnabled {
3423
+ restartMic()
3424
+ }
3425
+ // reset the text container (different from clearDisplay())
3426
+ // sendTextWall(" ")
3427
+ // createPageWithText(" ")
3428
+ } else {
3429
+ // if we aren't trying to show the dashboard
3430
+ // then we need to turn the mic back on and display the mentra main page:
3431
+ if dashboardShowing <= 1 {
3432
+ dashboardShowing = 0
3433
+ // make sure the container exists:
3434
+ DeviceManager.shared.sendCurrentState()
3435
+ // set the mic back on if it should be on
3436
+ let micEnabled = DeviceStore.shared.get("glasses", "micEnabled") as? Bool ?? false
3437
+ if micEnabled {
3438
+ restartMic()
3439
+ }
3440
+ return
3441
+ }
3442
+ // do nothing this time since we just closed the dashboard
3443
+ dashboardShowing -= 1
3444
+ if (dashboardShowing < 0) {
3445
+ dashboardShowing = 0
3446
+ }
3447
+ }
3388
3448
  }
3389
3449
 
3390
3450
  // if we got 08011097012200 that means we selected a menu item:
@@ -17,7 +17,7 @@ class Mach1: UltraliteBaseViewController, SGCManager {
17
17
 
18
18
  func requestPhoto(
19
19
  _: String, appId _: String, size _: String?, webhookUrl _: String?, authToken _: String?,
20
- compress _: String?, flash _: Bool, sound _: Bool
20
+ compress _: String?, flash _: Bool, sound _: Bool, exposureTimeNs _: Double?
21
21
  ) {}
22
22
 
23
23
  func sendGalleryMode() {}
@@ -903,6 +903,7 @@ class MentraLive: NSObject, SGCManager {
903
903
  // BLOCK_AUDIO_DUPLEX: When true, suspends LC3 mic while phone is playing audio via A2DP
904
904
  // to avoid overloading the MCU. Set to false to allow simultaneous A2DP + LC3 mic.
905
905
  private let BLOCK_AUDIO_DUPLEX = false
906
+ private static let voiceActivityDetectionSwitchType = 8
906
907
 
907
908
  var connectionState: String = ConnTypes.DISCONNECTED
908
909
 
@@ -1311,9 +1312,11 @@ class MentraLive: NSObject, SGCManager {
1311
1312
 
1312
1313
  func requestPhoto(
1313
1314
  _ requestId: String, appId: String, size: String?, webhookUrl: String?, authToken: String?,
1314
- compress: String?, flash: Bool, sound: Bool
1315
+ compress: String?, flash: Bool, sound: Bool, exposureTimeNs: Double?
1315
1316
  ) {
1316
- Bridge.log("Requesting photo: \(requestId) for app: \(appId), flash: \(flash), sound: \(sound)")
1317
+ Bridge.log(
1318
+ "LIVE: PHOTO PIPELINE [5/6] requestPhoto() entry requestId=\(requestId) appId=\(appId) flash=\(flash) sound=\(sound)"
1319
+ )
1317
1320
 
1318
1321
  var json: [String: Any] = [
1319
1322
  "type": "take_photo",
@@ -1360,7 +1363,13 @@ class MentraLive: NSObject, SGCManager {
1360
1363
  json["flash"] = flash
1361
1364
  json["sound"] = sound
1362
1365
 
1363
- Bridge.log("Using auto transfer mode with BLE fallback ID: \(bleImgId)")
1366
+ if let e = exposureTimeNs, e.isFinite, e > 0, e <= Double(Int64.max) {
1367
+ Bridge.log("LIVE: Using manual exposure time for photo request \(requestId): \(Int64(e)) ns")
1368
+ json["exposureTimeNs"] = Int64(e)
1369
+ }
1370
+
1371
+ Bridge.log("LIVE: PHOTO PIPELINE [5b/6] take_photo JSON ready bleImgId=\(bleImgId) transferMethod=auto")
1372
+ Bridge.log("LIVE: PHOTO PIPELINE [6/6] Dispatching take_photo to sendJson()")
1364
1373
 
1365
1374
  sendJson(json, wakeUp: true)
1366
1375
  }
@@ -1814,6 +1823,14 @@ class MentraLive: NSObject, SGCManager {
1814
1823
  let isCharging = json["charging"] as? Bool ?? charging
1815
1824
  updateBatteryStatus(level: level, isCharging: isCharging)
1816
1825
 
1826
+ case "voice_activity_detection_status":
1827
+ let enabled = json["voiceActivityDetectionEnabled"] as? Bool ?? true
1828
+ handleVoiceActivityDetectionStatus(enabled: enabled)
1829
+
1830
+ case "speaking_status":
1831
+ let speaking = json["speaking"] as? Bool ?? false
1832
+ handleSpeakingStatus(speaking: speaking)
1833
+
1817
1834
  case "wifi_status":
1818
1835
  let connected = json["connected"] as? Bool ?? false
1819
1836
  let ssid = json["ssid"] as? String ?? ""
@@ -1893,9 +1910,7 @@ class MentraLive: NSObject, SGCManager {
1893
1910
  let switchType = (json["switch_type"] as? Int) ?? (json["switchType"] as? Int) ?? -1
1894
1911
  let switchValue = (json["switch_value"] as? Int) ?? (json["switchValue"] as? Int) ?? -1
1895
1912
  let timestamp = parseTimestamp(json["timestamp"])
1896
- Bridge.sendSwitchStatus(
1897
- switchType: switchType, value: switchValue, timestamp: timestamp
1898
- )
1913
+ handleSwitchStatus(switchType: switchType, value: switchValue, timestamp: timestamp)
1899
1914
 
1900
1915
  case "rgb_led_control_response":
1901
1916
  let requestId = json["requestId"] as? String ?? ""
@@ -2087,6 +2102,9 @@ class MentraLive: NSObject, SGCManager {
2087
2102
  // Note: Stored separately from BES version for OTA patch matching
2088
2103
  DeviceStore.shared.apply("glasses", "mtkFirmwareVersion", mtkFirmwareVersion)
2089
2104
  }
2105
+ if let systemTimeMs = fields["system_time_ms"] as? NSNumber {
2106
+ DeviceStore.shared.apply("glasses", "systemTimeMs", systemTimeMs.int64Value)
2107
+ }
2090
2108
  if let bluetoothMacAddress = fields["bt_mac_address"] as? String {
2091
2109
  DeviceStore.shared.apply("glasses", "bluetoothMacAddress", bluetoothMacAddress)
2092
2110
  }
@@ -2241,6 +2259,25 @@ class MentraLive: NSObject, SGCManager {
2241
2259
  case "sr_vol":
2242
2260
  handleSrVol(json)
2243
2261
 
2262
+ case "sr_vad":
2263
+ if let bodyObj = k900ParseBody(json["B"]),
2264
+ let on = k900JsonInt(bodyObj, "on"),
2265
+ on == 0 || on == 1
2266
+ {
2267
+ handleSpeakingStatus(speaking: on == 1)
2268
+ }
2269
+
2270
+ case "sr_swit":
2271
+ if let body = k900ParseBody(json["B"]) {
2272
+ let switchType = k900JsonInt(body, "type") ?? -1
2273
+ let switchValue = k900JsonInt(body, "switch") ?? -1
2274
+ handleSwitchStatus(
2275
+ switchType: switchType,
2276
+ value: switchValue,
2277
+ timestamp: Int64(Date().timeIntervalSince1970 * 1000)
2278
+ )
2279
+ }
2280
+
2244
2281
  case "sr_shut":
2245
2282
  Bridge.log("K900 shutdown command received - glasses shutting down")
2246
2283
  // Mark as killed to prevent reconnection attempts
@@ -2461,7 +2498,7 @@ class MentraLive: NSObject, SGCManager {
2461
2498
  }
2462
2499
 
2463
2500
  func sendGalleryMode() {
2464
- let active = DeviceStore.shared.get("bluetooth", "galleryModeAuto") as! Bool
2501
+ let active = DeviceStore.shared.get("bluetooth", "gallery_mode") as! Bool
2465
2502
  Bridge.log("LIVE: 📸 Sending gallery mode active to glasses: \(active)")
2466
2503
 
2467
2504
  let json: [String: Any] = [
@@ -2656,7 +2693,7 @@ class MentraLive: NSObject, SGCManager {
2656
2693
  // return
2657
2694
  // }
2658
2695
 
2659
- // // Forward PCM data to DeviceManager for VAD and server transmission (same as Android)
2696
+ // // Forward PCM data to DeviceManager for audio events and server transmission (same as Android)
2660
2697
  // DeviceManager.shared.handlePcm(pcmData)
2661
2698
 
2662
2699
  // Bridge.log(
@@ -3237,6 +3274,9 @@ class MentraLive: NSObject, SGCManager {
3237
3274
  Bridge.log("LIVE: All chunks queued for transmission")
3238
3275
  } else {
3239
3276
  // Normal single message transmission
3277
+ if (json["type"] as? String) == "take_photo" {
3278
+ Bridge.log("LIVE: PHOTO PIPELINE BLE handoff — sendJson -> queueSend take_photo")
3279
+ }
3240
3280
  Bridge.log("LIVE: Sending data to glasses: \(jsonString)")
3241
3281
  let packedData = packJson(jsonString, wakeUp: wakeUp) ?? Data()
3242
3282
  queueSend(packedData, id: trackingId)
@@ -3508,6 +3548,27 @@ class MentraLive: NSObject, SGCManager {
3508
3548
  }
3509
3549
  }
3510
3550
 
3551
+ private func handleVoiceActivityDetectionStatus(enabled: Bool) {
3552
+ Bridge.log("LIVE: Voice Activity Detection \(enabled ? "enabled" : "disabled")")
3553
+ Bridge.sendVoiceActivityDetectionStatus(enabled)
3554
+ }
3555
+
3556
+ private func handleSpeakingStatus(speaking: Bool) {
3557
+ guard voiceActivityDetectionEnabled else {
3558
+ Bridge.log("LIVE: Ignoring speaking status because Voice Activity Detection is disabled")
3559
+ return
3560
+ }
3561
+ Bridge.log("LIVE: Speaking status \(speaking ? "speaking" : "not speaking")")
3562
+ Bridge.sendSpeakingStatus(speaking)
3563
+ }
3564
+
3565
+ private func handleSwitchStatus(switchType: Int, value: Int, timestamp: Int64) {
3566
+ Bridge.sendSwitchStatus(switchType: switchType, value: value, timestamp: timestamp)
3567
+ if switchType == Self.voiceActivityDetectionSwitchType, value == 0 || value == 1 {
3568
+ handleVoiceActivityDetectionStatus(enabled: value == 1)
3569
+ }
3570
+ }
3571
+
3511
3572
  private func updateWifiStatus(connected: Bool, ssid: String, ip: String) {
3512
3573
  Bridge.log("LIVE: 🌐 Updating WiFi status - connected: \(connected), ssid: \(ssid)")
3513
3574
  DeviceStore.shared.apply("glasses", "wifiConnected", connected)
@@ -4493,6 +4554,42 @@ extension MentraLive {
4493
4554
 
4494
4555
  // Send gallery mode state (camera app running status)
4495
4556
  sendGalleryMode()
4557
+
4558
+ // Send glasses-side Voice Activity Detection setting.
4559
+ sendVoiceActivityDetectionSetting()
4560
+ }
4561
+
4562
+ func sendVoiceActivityDetectionSetting() {
4563
+ let enabled = DeviceStore.shared.get("bluetooth", "voice_activity_detection_enabled") as? Bool ?? true
4564
+ Bridge.log("LIVE: 🎤 Sending Voice Activity Detection setting to glasses: \(enabled)")
4565
+
4566
+ guard connectedPeripheral != nil, txCharacteristic != nil else {
4567
+ Bridge.log("Cannot send Voice Activity Detection setting - BLE write path not ready")
4568
+ return
4569
+ }
4570
+
4571
+ do {
4572
+ let bodyData = try JSONSerialization.data(withJSONObject: [
4573
+ "type": Self.voiceActivityDetectionSwitchType,
4574
+ "switch": enabled ? 1 : 0,
4575
+ ])
4576
+ guard let bodyString = String(data: bodyData, encoding: .utf8) else {
4577
+ Bridge.log("LIVE: Failed to encode Voice Activity Detection payload")
4578
+ return
4579
+ }
4580
+ let command: [String: Any] = [
4581
+ "C": "cs_swit",
4582
+ "V": 1,
4583
+ "B": bodyString,
4584
+ ]
4585
+ if sendRawK900Command(command, wakeUp: true) {
4586
+ Bridge.sendVoiceActivityDetectionStatus(enabled)
4587
+ } else {
4588
+ Bridge.log("LIVE: Failed to send Voice Activity Detection setting command")
4589
+ }
4590
+ } catch {
4591
+ Bridge.log("LIVE: Error encoding Voice Activity Detection payload: \(error)")
4592
+ }
4496
4593
  }
4497
4594
 
4498
4595
  func sendButtonVideoRecordingSettings() {
@@ -4500,11 +4597,11 @@ extension MentraLive {
4500
4597
  DeviceStore.shared.get("bluetooth", "button_video_settings") as? [String: Any] ?? [
4501
4598
  "width": 1280,
4502
4599
  "height": 720,
4503
- "frameRate": 30,
4600
+ "fps": 30,
4504
4601
  ]
4505
4602
  let width = settings["width"] as? Int ?? 1280
4506
4603
  let height = settings["height"] as? Int ?? 720
4507
- let fps = settings["frameRate"] as? Int ?? 30
4604
+ let fps = settings["fps"] as? Int ?? 30
4508
4605
 
4509
4606
  // Use defaults if not set
4510
4607
  let finalWidth = width > 0 ? width : 1280
@@ -4582,9 +4679,9 @@ extension MentraLive {
4582
4679
  }
4583
4680
 
4584
4681
  func sendCameraFovSetting() {
4585
- let settings = DeviceStore.shared.get("bluetooth", "camera_fov") as? [String: Any] ?? ["fov": 118, "roiPosition": 0]
4682
+ let settings = DeviceStore.shared.get("bluetooth", "camera_fov") as? [String: Any] ?? ["fov": 118, "roi_position": 0]
4586
4683
  let fov = settings["fov"] as? Int ?? 118
4587
- let roiPosition = settings["roiPosition"] as? Int ?? 0
4684
+ let roiPosition = settings["roi_position"] as? Int ?? 0
4588
4685
 
4589
4686
  Bridge.log("Sending camera FOV setting: fov=\(fov), roiPosition=\(roiPosition)")
4590
4687
 
@@ -46,7 +46,7 @@ class MentraNexSGC: NSObject, CBCentralManagerDelegate, CBPeripheralDelegate, SG
46
46
 
47
47
  func requestPhoto(
48
48
  _: String, appId _: String, size _: String?, webhookUrl _: String?, authToken _: String?,
49
- compress _: String?, flash _: Bool, sound _: Bool
49
+ compress _: String?, flash _: Bool, sound _: Bool, exposureTimeNs _: Double?
50
50
  ) {}
51
51
 
52
52
  func startStream(_: [String: Any]) {}
@@ -20,7 +20,7 @@ protocol SGCManager {
20
20
 
21
21
  func requestPhoto(
22
22
  _ requestId: String, appId: String, size: String?, webhookUrl: String?, authToken: String?,
23
- compress: String?, flash: Bool, sound: Bool
23
+ compress: String?, flash: Bool, sound: Bool, exposureTimeNs: Double?
24
24
  )
25
25
  func startStream(_ message: [String: Any])
26
26
  func stopStream()
@@ -103,6 +103,10 @@ protocol SGCManager {
103
103
  func queryGalleryStatus()
104
104
  func sendGalleryMode()
105
105
 
106
+ // MARK: - Voice Activity Detection
107
+
108
+ func sendVoiceActivityDetectionSetting()
109
+
106
110
  // MARK: - Version Info
107
111
 
108
112
  func requestVersionInfo()
@@ -127,6 +131,10 @@ extension SGCManager {
127
131
 
128
132
  func setDashboardMenu(_: [[String: Any]]) {}
129
133
 
134
+ // MARK: - Voice Activity Detection (default no-op — Mentra Live supports this)
135
+
136
+ func sendVoiceActivityDetectionSetting() {}
137
+
130
138
  // MARK: - Default DeviceStore-backed property implementations
131
139
 
132
140
  var fullyBooted: Bool {
@@ -181,8 +189,8 @@ extension SGCManager {
181
189
  DeviceStore.shared.get("glasses", "micEnabled") as? Bool ?? false
182
190
  }
183
191
 
184
- var vadEnabled: Bool {
185
- DeviceStore.shared.get("glasses", "vadEnabled") as? Bool ?? false
192
+ var voiceActivityDetectionEnabled: Bool {
193
+ DeviceStore.shared.get("glasses", "voiceActivityDetectionEnabled") as? Bool ?? true
186
194
  }
187
195
 
188
196
  var batteryLevel: Int {
@@ -12,7 +12,7 @@ class Simulated: SGCManager {
12
12
  DeviceStore.shared.apply("glasses", "connected", true)
13
13
  DeviceStore.shared.apply("glasses", "connectionState", ConnTypes.CONNECTED)
14
14
  DeviceStore.shared.apply("glasses", "micEnabled", false)
15
- DeviceStore.shared.apply("glasses", "vadEnabled", false)
15
+ DeviceStore.shared.apply("glasses", "voiceActivityDetectionEnabled", true)
16
16
  DeviceStore.shared.apply("glasses", "bluetoothClassicConnected", false)
17
17
  }
18
18
 
@@ -75,7 +75,7 @@ class Simulated: SGCManager {
75
75
 
76
76
  // MARK: - Camera & Media
77
77
 
78
- func requestPhoto(_: String, appId _: String, size _: String?, webhookUrl _: String?, authToken _: String?, compress _: String?, flash _: Bool, sound _: Bool) {
78
+ func requestPhoto(_: String, appId _: String, size _: String?, webhookUrl _: String?, authToken _: String?, compress _: String?, flash _: Bool, sound _: Bool, exposureTimeNs _: Double?) {
79
79
  Bridge.log("requestPhoto")
80
80
  }
81
81
 
@@ -214,16 +214,13 @@ class SherpaOnnxTranscriber {
214
214
  * Feed PCM audio data (16-bit little endian) into the transcriber.
215
215
  * This method should be called continuously with short chunks (e.g., 100-300ms).
216
216
  *
217
- * Note: Audio passed to this method is assumed to have already passed VAD elsewhere,
218
- * so it's directly queued for processing without additional VAD checks.
217
+ * Audio is queued directly; microphone VAD gating is not applied in the SDK.
219
218
  */
220
219
  func acceptAudio(pcm16le: Data) {
221
220
  guard isRunning else {
222
221
  return
223
222
  }
224
223
 
225
- // Directly queue the audio data for processing
226
- // No VAD check here as it's assumed to be done upstream
227
224
  queueAudioData(pcm16le)
228
225
  }
229
226
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@mentra/bluetooth-sdk",
3
- "version": "0.1.5",
3
+ "version": "0.1.7",
4
4
  "description": "SDK for communicating with smart glasses",
5
5
  "main": "build/index.js",
6
6
  "react-native": "src/index.ts",