@mentra/bluetooth-sdk 0.1.6 → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/README.md +26 -22
  2. package/android/.project +28 -0
  3. package/android/build.gradle +5 -9
  4. package/android/lc3Lib/.project +28 -0
  5. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +54 -33
  6. package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +48 -10
  7. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +27 -36
  8. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt +8 -4
  9. package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +170 -14
  10. package/android/src/main/java/com/mentra/bluetoothsdk/audio/AudioModels.kt +6 -6
  11. package/android/src/main/java/com/mentra/bluetoothsdk/camera/CameraModels.kt +69 -10
  12. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/ControllerManager.kt +5 -3
  13. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/R1.kt +2 -1
  14. package/android/src/main/java/com/mentra/bluetoothsdk/debug/BleTraceLogger.kt +149 -0
  15. package/android/src/main/java/com/mentra/bluetoothsdk/events/BluetoothEvents.kt +13 -0
  16. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoUploadServer.kt +297 -0
  17. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/MentraPhotoReceiverModule.kt +120 -0
  18. package/android/src/main/java/com/mentra/bluetoothsdk/services/Foreground.kt +13 -0
  19. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G1.java +1 -1
  20. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt +3 -1
  21. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Mach1.java +1 -1
  22. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraLive.java +269 -28
  23. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraNex.kt +1 -1
  24. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt +8 -3
  25. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Simulated.kt +4 -1
  26. package/android/src/main/java/com/mentra/bluetoothsdk/status/DeviceStatus.kt +14 -12
  27. package/android/src/main/java/com/mentra/bluetoothsdk/status/RuntimeState.kt +4 -2
  28. package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +131 -6
  29. package/android/src/main/java/com/mentra/bluetoothsdk/stt/SherpaOnnxTranscriber.kt +1 -3
  30. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunkReassembler.java +105 -0
  31. package/build/BluetoothSdk.types.d.ts +107 -48
  32. package/build/BluetoothSdk.types.d.ts.map +1 -1
  33. package/build/BluetoothSdk.types.js +4 -0
  34. package/build/BluetoothSdk.types.js.map +1 -1
  35. package/build/_private/BluetoothSdkModule.d.ts +8 -6
  36. package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
  37. package/build/_private/BluetoothSdkModule.js +58 -9
  38. package/build/_private/BluetoothSdkModule.js.map +1 -1
  39. package/build/index.d.ts +2 -2
  40. package/build/index.d.ts.map +1 -1
  41. package/build/index.js +6 -5
  42. package/build/index.js.map +1 -1
  43. package/build/photo-receiver/MentraPhotoReceiver.types.d.ts +18 -0
  44. package/build/photo-receiver/MentraPhotoReceiver.types.d.ts.map +1 -0
  45. package/build/photo-receiver/MentraPhotoReceiver.types.js +2 -0
  46. package/build/photo-receiver/MentraPhotoReceiver.types.js.map +1 -0
  47. package/build/photo-receiver/MentraPhotoReceiverModule.d.ts +10 -0
  48. package/build/photo-receiver/MentraPhotoReceiverModule.d.ts.map +1 -0
  49. package/build/photo-receiver/MentraPhotoReceiverModule.js +3 -0
  50. package/build/photo-receiver/MentraPhotoReceiverModule.js.map +1 -0
  51. package/build/photo-receiver/MentraPhotoReceiverModule.web.d.ts +10 -0
  52. package/build/photo-receiver/MentraPhotoReceiverModule.web.d.ts.map +1 -0
  53. package/build/photo-receiver/MentraPhotoReceiverModule.web.js +12 -0
  54. package/build/photo-receiver/MentraPhotoReceiverModule.web.js.map +1 -0
  55. package/build/photo-receiver/index.d.ts +3 -0
  56. package/build/photo-receiver/index.d.ts.map +1 -0
  57. package/build/photo-receiver/index.js +3 -0
  58. package/build/photo-receiver/index.js.map +1 -0
  59. package/build/react/useMentraBluetooth.d.ts +5 -3
  60. package/build/react/useMentraBluetooth.d.ts.map +1 -1
  61. package/build/react/useMentraBluetooth.js +15 -4
  62. package/build/react/useMentraBluetooth.js.map +1 -1
  63. package/expo-module.config.json +5 -2
  64. package/ios/BluetoothSdkModule.swift +59 -26
  65. package/ios/LocalPhotoUploadServer.swift +411 -0
  66. package/ios/MentraBluetoothSDK.podspec +8 -5
  67. package/ios/MentraPhotoReceiverModule.swift +147 -0
  68. package/ios/Source/Audio/AudioModels.swift +10 -10
  69. package/ios/Source/Bridge.swift +42 -12
  70. package/ios/Source/Camera/CameraModels.swift +74 -19
  71. package/ios/Source/DeviceManager.swift +23 -98
  72. package/ios/Source/DeviceStore.swift +12 -4
  73. package/ios/Source/Events/BluetoothEvents.swift +37 -0
  74. package/ios/Source/MentraBluetoothSDK.swift +135 -14
  75. package/ios/Source/Status/DeviceStatus.swift +12 -4
  76. package/ios/Source/Status/RuntimeState.swift +22 -13
  77. package/ios/Source/Streaming/StreamModels.swift +239 -45
  78. package/ios/Source/controllers/ControllerManager.swift +3 -3
  79. package/ios/Source/controllers/R1.swift +9 -3
  80. package/ios/Source/sgcs/G1.swift +5 -4
  81. package/ios/Source/sgcs/G2.swift +5 -6
  82. package/ios/Source/sgcs/Mach1.swift +1 -1
  83. package/ios/Source/sgcs/MentraLive.swift +183 -22
  84. package/ios/Source/sgcs/MentraNex.swift +1 -1
  85. package/ios/Source/sgcs/SGCManager.swift +11 -3
  86. package/ios/Source/sgcs/Simulated.swift +2 -2
  87. package/ios/Source/stt/SherpaOnnxTranscriber.swift +1 -4
  88. package/ios/Source/utils/MessageChunkReassembler.swift +89 -0
  89. package/package.json +6 -1
  90. package/src/BluetoothSdk.types.ts +133 -62
  91. package/src/_private/BluetoothSdkModule.ts +79 -26
  92. package/src/index.ts +15 -8
  93. package/src/photo-receiver/MentraPhotoReceiver.types.ts +20 -0
  94. package/src/photo-receiver/MentraPhotoReceiverModule.ts +14 -0
  95. package/src/photo-receiver/MentraPhotoReceiverModule.web.ts +20 -0
  96. package/src/photo-receiver/index.ts +2 -0
  97. package/src/react/useMentraBluetooth.ts +19 -7
  98. package/android/src/main/java/com/mentra/bluetoothsdk/stt/VadGateSpeechPolicy.kt +0 -221
  99. package/ios/Packages/VAD/Common/VAD/VADQuality.swift +0 -41
  100. package/ios/Packages/VAD/Common/VAD/VADState.swift +0 -26
  101. package/ios/Packages/VAD/Common/VAD/VADStrategy.swift +0 -29
  102. package/ios/Packages/VAD/Common/VAD/VADType.swift +0 -14
  103. package/ios/Packages/VAD/Data/Configuration.swift +0 -68
  104. package/ios/Packages/VAD/Data/FrameSize.swift +0 -39
  105. package/ios/Packages/VAD/Data/Record.swift +0 -13
  106. package/ios/Packages/VAD/Data/Result.swift +0 -22
  107. package/ios/Packages/VAD/Data/SampleRate.swift +0 -48
  108. package/ios/Packages/VAD/Silero/Model/silero_vad.onnx +0 -0
  109. package/ios/Packages/VAD/Silero/SileroVAD.swift +0 -284
  110. package/ios/Packages/VAD/Silero/SileroVADStrategy.swift +0 -64
@@ -66,6 +66,7 @@ object DeviceStore {
66
66
  store.set("glasses", "deviceModel", "")
67
67
  store.set("glasses", "firmwareVersion", "")
68
68
  store.set("glasses", "micEnabled", false)
69
+ store.set("glasses", "voiceActivityDetectionEnabled", true)
69
70
  store.set("glasses", "bluetoothClassicConnected", false)
70
71
  store.set("glasses", "caseRemoved", true)
71
72
  store.set("glasses", "caseOpen", true)
@@ -121,12 +122,13 @@ object DeviceStore {
121
122
  store.set("bluetooth", "dashboard_depth", 2)
122
123
  store.set("bluetooth", "head_up_angle", 30)
123
124
  store.set("bluetooth", "contextual_dashboard", true)
124
- store.set("bluetooth", "galleryModeAuto", true)
125
+ store.set("bluetooth", "gallery_mode", true)
126
+ store.set("bluetooth", "voice_activity_detection_enabled", true)
125
127
  store.set("bluetooth", "screen_disabled", false)
126
128
  store.set("bluetooth", "button_photo_size", "medium")
127
129
  store.set("bluetooth", "button_camera_led", true)
128
130
  store.set("bluetooth", "button_max_recording_time", 10)
129
- store.set("bluetooth", "camera_fov", mapOf("fov" to 118, "roiPosition" to 0))
131
+ store.set("bluetooth", "camera_fov", mapOf("fov" to 118, "roi_position" to 0))
130
132
  store.set("bluetooth", "button_video_width", 1280)
131
133
  store.set("bluetooth", "button_video_height", 720)
132
134
  store.set("bluetooth", "button_video_fps", 30)
@@ -137,7 +139,6 @@ object DeviceStore {
137
139
  store.set("bluetooth", "should_send_pcm", false)
138
140
  store.set("bluetooth", "should_send_lc3", false)
139
141
  store.set("bluetooth", "should_send_transcript", false)
140
- store.set("bluetooth", "bypass_vad", true)
141
142
  store.set("bluetooth", "use_native_dashboard", false)
142
143
  }
143
144
 
@@ -240,9 +241,12 @@ object DeviceStore {
240
241
  DeviceManager.getInstance().sgc?.setDashboardMenu(items)
241
242
  }
242
243
  }
243
- "bluetooth" to "galleryModeAuto" -> {
244
+ "bluetooth" to "gallery_mode" -> {
244
245
  DeviceManager.getInstance().sgc?.sendGalleryMode()
245
246
  }
247
+ "bluetooth" to "voice_activity_detection_enabled" -> {
248
+ DeviceManager.getInstance().sgc?.sendVoiceActivityDetectionSetting()
249
+ }
246
250
  "bluetooth" to "screen_disabled" -> {
247
251
  (value as? Boolean)?.let { disabled ->
248
252
  if (disabled) {
@@ -7,6 +7,7 @@ import android.os.Looper
7
7
  import com.mentra.bluetoothsdk.utils.ControllerTypes
8
8
  import com.mentra.bluetoothsdk.utils.PhoneAudioMonitor
9
9
  import java.util.Collections
10
+ import java.util.UUID
10
11
  import java.util.concurrent.atomic.AtomicBoolean
11
12
 
12
13
  class MentraBluetoothSdk private constructor(
@@ -23,6 +24,8 @@ class MentraBluetoothSdk private constructor(
23
24
  private val bridgeEventSinkId: String
24
25
  private val storeListenerId: String
25
26
  private var suppressDefaultDeviceEvents = false
27
+ private val streamKeepAliveLock = Any()
28
+ private var activeStreamKeepAlive: ActiveStreamKeepAlive? = null
26
29
 
27
30
  init {
28
31
  listeners.add(listener)
@@ -36,6 +39,8 @@ class MentraBluetoothSdk private constructor(
36
39
  private val DEFAULT_DEVICE_KEYS = setOf("default_wearable", "device_name", "device_address")
37
40
  private val SCAN_STATE_KEYS = setOf("searching", "searchingController", "searchResults")
38
41
  private const val DEFAULT_SCAN_TIMEOUT_MS = 15_000L
42
+ private const val DEFAULT_STREAM_KEEP_ALIVE_INTERVAL_SECONDS = 5
43
+ private const val MAX_MISSED_STREAM_KEEP_ALIVE_ACKS = 3
39
44
 
40
45
  @JvmStatic
41
46
  fun create(
@@ -51,6 +56,14 @@ class MentraBluetoothSdk private constructor(
51
56
  ): MentraBluetoothSdk = MentraBluetoothSdk(context, config, listener)
52
57
  }
53
58
 
59
+ private data class ActiveStreamKeepAlive(
60
+ val streamId: String,
61
+ val intervalMs: Long,
62
+ var pendingAckId: String? = null,
63
+ var missedAckCount: Int = 0,
64
+ var nextTick: Runnable? = null,
65
+ )
66
+
54
67
  fun addListener(listener: MentraBluetoothSdkListener) {
55
68
  listeners.add(listener)
56
69
  }
@@ -308,8 +321,12 @@ class MentraBluetoothSdk private constructor(
308
321
  DeviceStore.apply(ObservableStore.BLUETOOTH_CATEGORY, "screen_disabled", disabled)
309
322
  }
310
323
 
311
- fun setGalleryMode(mode: GalleryMode) {
312
- DeviceStore.apply(ObservableStore.BLUETOOTH_CATEGORY, "galleryModeAuto", mode == GalleryMode.AUTO)
324
+ fun setGalleryModeEnabled(enabled: Boolean) {
325
+ DeviceStore.apply(ObservableStore.BLUETOOTH_CATEGORY, "gallery_mode", enabled)
326
+ }
327
+
328
+ fun setVoiceActivityDetectionEnabled(enabled: Boolean) {
329
+ DeviceStore.apply(ObservableStore.BLUETOOTH_CATEGORY, "voice_activity_detection_enabled", enabled)
313
330
  }
314
331
 
315
332
  fun setButtonPhotoSettings(size: ButtonPhotoSize) {
@@ -320,14 +337,14 @@ class MentraBluetoothSdk private constructor(
320
337
  setButtonPhotoSettings(size = settings.size)
321
338
  }
322
339
 
323
- fun setButtonVideoRecordingSettings(width: Int, height: Int, frameRate: Int) {
340
+ fun setButtonVideoRecordingSettings(width: Int, height: Int, fps: Int) {
324
341
  DeviceStore.apply(ObservableStore.BLUETOOTH_CATEGORY, "button_video_width", width)
325
342
  DeviceStore.apply(ObservableStore.BLUETOOTH_CATEGORY, "button_video_height", height)
326
- DeviceStore.apply(ObservableStore.BLUETOOTH_CATEGORY, "button_video_fps", frameRate)
343
+ DeviceStore.apply(ObservableStore.BLUETOOTH_CATEGORY, "button_video_fps", fps)
327
344
  }
328
345
 
329
346
  fun setButtonVideoRecordingSettings(settings: ButtonVideoRecordingSettings) {
330
- setButtonVideoRecordingSettings(width = settings.width, height = settings.height, frameRate = settings.frameRate)
347
+ setButtonVideoRecordingSettings(width = settings.width, height = settings.height, fps = settings.fps)
331
348
  }
332
349
 
333
350
  fun setButtonCameraLed(enabled: Boolean) {
@@ -342,14 +359,13 @@ class MentraBluetoothSdk private constructor(
342
359
  DeviceStore.apply(
343
360
  ObservableStore.BLUETOOTH_CATEGORY,
344
361
  "camera_fov",
345
- mapOf("fov" to fov.fov, "roiPosition" to fov.roiPosition),
362
+ mapOf("fov" to fov.fov, "roi_position" to fov.roiPosition),
346
363
  )
347
364
  }
348
365
 
349
366
  fun setMicState(
350
367
  enabled: Boolean,
351
368
  useGlassesMic: Boolean = true,
352
- bypassVad: Boolean = true,
353
369
  sendTranscript: Boolean = false,
354
370
  sendLc3Data: Boolean = false,
355
371
  ) {
@@ -363,7 +379,6 @@ class MentraBluetoothSdk private constructor(
363
379
  applyMicState(
364
380
  sendPcmData = enabled,
365
381
  sendTranscript = enabled && sendTranscript,
366
- bypassVad = bypassVad,
367
382
  sendLc3Data = enabled && sendLc3Data,
368
383
  )
369
384
  }
@@ -371,13 +386,11 @@ class MentraBluetoothSdk private constructor(
371
386
  private fun applyMicState(
372
387
  sendPcmData: Boolean,
373
388
  sendTranscript: Boolean,
374
- bypassVad: Boolean,
375
389
  sendLc3Data: Boolean,
376
390
  ) {
377
391
  DeviceStore.apply(ObservableStore.BLUETOOTH_CATEGORY, "should_send_pcm", sendPcmData)
378
392
  DeviceStore.apply(ObservableStore.BLUETOOTH_CATEGORY, "should_send_lc3", sendLc3Data)
379
393
  DeviceStore.apply(ObservableStore.BLUETOOTH_CATEGORY, "should_send_transcript", sendTranscript)
380
- DeviceStore.apply(ObservableStore.BLUETOOTH_CATEGORY, "bypass_vad", bypassVad)
381
394
  deviceManager.setMicState()
382
395
  }
383
396
 
@@ -414,6 +427,9 @@ class MentraBluetoothSdk private constructor(
414
427
  }
415
428
 
416
429
  fun requestPhoto(request: PhotoRequest) {
430
+ Bridge.log(
431
+ "NATIVE: PHOTO PIPELINE [3b/6] MentraBluetoothSdk.requestPhoto requestId=${request.requestId} appId=${request.appId}"
432
+ )
417
433
  deviceManager.requestPhoto(
418
434
  request.requestId,
419
435
  request.appId,
@@ -421,7 +437,10 @@ class MentraBluetoothSdk private constructor(
421
437
  request.webhookUrl,
422
438
  request.authToken,
423
439
  request.compress.value,
440
+ request.flash,
424
441
  request.sound,
442
+ request.exposureTimeNs,
443
+ request.iso,
425
444
  )
426
445
  }
427
446
 
@@ -430,10 +449,18 @@ class MentraBluetoothSdk private constructor(
430
449
  }
431
450
 
432
451
  fun startStream(request: StreamRequest) {
433
- deviceManager.startStream(request.toMap().toMutableMap())
452
+ val message = request.toMap().toMutableMap()
453
+ val streamId = (message["streamId"] as? String)?.takeIf { it.isNotBlank() }
454
+ ?: "sdk-${UUID.randomUUID()}"
455
+ message["streamId"] = streamId
456
+ stopStreamKeepAliveMonitor()
457
+ deviceManager.startStream(message)
458
+ if (request.keepAlive && !request.isExternallyManagedKeepAlive()) {
459
+ startStreamKeepAliveMonitor(streamId, request.keepAliveIntervalSeconds)
460
+ }
434
461
  }
435
462
 
436
- fun keepStreamAlive(request: StreamKeepAliveRequest) {
463
+ internal fun sendCloudStreamKeepAlive(request: StreamKeepAliveRequest) {
437
464
  deviceManager.keepStreamAlive(request.toMap().toMutableMap())
438
465
  }
439
466
 
@@ -450,6 +477,7 @@ class MentraBluetoothSdk private constructor(
450
477
  }
451
478
 
452
479
  fun stopStream() {
480
+ stopStreamKeepAliveMonitor()
453
481
  deviceManager.stopStream()
454
482
  }
455
483
 
@@ -486,6 +514,7 @@ class MentraBluetoothSdk private constructor(
486
514
  }
487
515
 
488
516
  override fun close() {
517
+ stopStreamKeepAliveMonitor()
489
518
  Bridge.removeEventSink(bridgeEventSinkId)
490
519
  DeviceStore.store.removeListener(storeListenerId)
491
520
  listeners.clear()
@@ -605,6 +634,25 @@ class MentraBluetoothSdk private constructor(
605
634
  }
606
635
  }
607
636
  "head_up" -> dispatchToListeners { it.onHeadUpChanged(data["up"] as? Boolean ?: false) }
637
+ "voice_activity_detection_status" ->
638
+ dispatchToListeners {
639
+ it.onVoiceActivityDetectionStatus(
640
+ VoiceActivityDetectionStatusEvent(
641
+ voiceActivityDetectionEnabled =
642
+ data["voiceActivityDetectionEnabled"] as? Boolean ?: true,
643
+ values = data,
644
+ )
645
+ )
646
+ }
647
+ "speaking_status" ->
648
+ dispatchToListeners {
649
+ it.onSpeakingStatus(
650
+ SpeakingStatusEvent(
651
+ speaking = data["speaking"] as? Boolean ?: false,
652
+ values = data,
653
+ )
654
+ )
655
+ }
608
656
  "battery_status" ->
609
657
  dispatchToListeners {
610
658
  it.onBatteryStatus(
@@ -620,8 +668,18 @@ class MentraBluetoothSdk private constructor(
620
668
  "hotspot_error" -> dispatchToListeners { it.onHotspotError(HotspotErrorEvent(data)) }
621
669
  "gallery_status" -> dispatchToListeners { it.onGalleryStatus(GalleryStatusEvent(data)) }
622
670
  "photo_response" -> dispatchToListeners { it.onPhotoResponse(PhotoResponseEvent(data)) }
623
- "stream_status" -> dispatchToListeners { it.onStreamStatus(StreamStatusEvent(data)) }
624
- "keep_alive_ack" -> dispatchToListeners { it.onKeepAliveAck(KeepAliveAckEvent(data)) }
671
+ "photo_status" -> dispatchToListeners { it.onPhotoStatus(PhotoStatusEvent(data)) }
672
+ "stream_status" -> {
673
+ val event = StreamStatusEvent(data)
674
+ handleStreamStatusForKeepAlive(event.status)
675
+ dispatchToListeners { it.onStreamStatus(event) }
676
+ }
677
+ "keep_alive_ack" -> {
678
+ val event = KeepAliveAckEvent(data)
679
+ if (!handleStreamKeepAliveAck(event)) {
680
+ dispatchToListeners { it.onKeepAliveAck(event) }
681
+ }
682
+ }
625
683
  "mic_pcm" -> {
626
684
  val event = MicPcmEvent(data)
627
685
  if (event.pcm.isNotEmpty()) {
@@ -676,4 +734,102 @@ class MentraBluetoothSdk private constructor(
676
734
  deliver()
677
735
  }
678
736
  }
737
+
738
+ private fun startStreamKeepAliveMonitor(streamId: String, requestedIntervalSeconds: Int) {
739
+ val intervalSeconds =
740
+ requestedIntervalSeconds.takeIf { it > 0 } ?: DEFAULT_STREAM_KEEP_ALIVE_INTERVAL_SECONDS
741
+ val tracker = ActiveStreamKeepAlive(
742
+ streamId = streamId,
743
+ intervalMs = intervalSeconds * 1_000L,
744
+ )
745
+ synchronized(streamKeepAliveLock) {
746
+ activeStreamKeepAlive = tracker
747
+ }
748
+ sendNextStreamKeepAlive(tracker)
749
+ }
750
+
751
+ private fun stopStreamKeepAliveMonitor() {
752
+ val tracker =
753
+ synchronized(streamKeepAliveLock) {
754
+ val current = activeStreamKeepAlive
755
+ activeStreamKeepAlive = null
756
+ current
757
+ }
758
+ tracker?.nextTick?.let { mainHandler.removeCallbacks(it) }
759
+ }
760
+
761
+ private fun sendNextStreamKeepAlive(tracker: ActiveStreamKeepAlive) {
762
+ var timeoutEvent: StreamStatusEvent? = null
763
+ var request: StreamKeepAliveRequest? = null
764
+
765
+ synchronized(streamKeepAliveLock) {
766
+ if (activeStreamKeepAlive !== tracker) {
767
+ return
768
+ }
769
+
770
+ if (tracker.pendingAckId != null) {
771
+ tracker.missedAckCount += 1
772
+ if (tracker.missedAckCount >= MAX_MISSED_STREAM_KEEP_ALIVE_ACKS) {
773
+ activeStreamKeepAlive = null
774
+ tracker.nextTick?.let { mainHandler.removeCallbacks(it) }
775
+ timeoutEvent =
776
+ StreamStatusEvent(
777
+ StreamStatus.Error(
778
+ streamId = tracker.streamId,
779
+ errorDetails =
780
+ "Stream keep-alive timed out after ${tracker.missedAckCount} missed ACKs",
781
+ timestamp = System.currentTimeMillis(),
782
+ resolvedConfig = null,
783
+ )
784
+ )
785
+ return@synchronized
786
+ }
787
+ }
788
+
789
+ val ackId = "ack-${System.currentTimeMillis()}"
790
+ tracker.pendingAckId = ackId
791
+ request = StreamKeepAliveRequest(streamId = tracker.streamId, ackId = ackId)
792
+ val nextTick = Runnable { sendNextStreamKeepAlive(tracker) }
793
+ tracker.nextTick = nextTick
794
+ mainHandler.postDelayed(nextTick, tracker.intervalMs)
795
+ }
796
+
797
+ timeoutEvent?.let { event ->
798
+ dispatchToListeners { it.onStreamStatus(event) }
799
+ return
800
+ }
801
+
802
+ request?.let { keepAlive ->
803
+ deviceManager.keepStreamAlive(keepAlive.toMap().toMutableMap())
804
+ }
805
+ }
806
+
807
+ private fun handleStreamKeepAliveAck(event: KeepAliveAckEvent): Boolean {
808
+ synchronized(streamKeepAliveLock) {
809
+ val tracker = activeStreamKeepAlive ?: return false
810
+ if (event.streamId != tracker.streamId || event.ackId != tracker.pendingAckId) {
811
+ return false
812
+ }
813
+ tracker.pendingAckId = null
814
+ tracker.missedAckCount = 0
815
+ return true
816
+ }
817
+ }
818
+
819
+ private fun handleStreamStatusForKeepAlive(status: StreamStatus) {
820
+ val streamId = status.streamId
821
+ if (
822
+ streamId != null &&
823
+ synchronized(streamKeepAliveLock) { activeStreamKeepAlive?.streamId } != streamId
824
+ ) {
825
+ return
826
+ }
827
+ when (status.state) {
828
+ StreamState.STOPPED,
829
+ StreamState.STOPPING,
830
+ StreamState.ERROR,
831
+ StreamState.RECONNECT_FAILED -> stopStreamKeepAliveMonitor()
832
+ else -> Unit
833
+ }
834
+ }
679
835
  }
@@ -13,7 +13,7 @@ data class MicPcmEvent(
13
13
  val bitsPerSample: Int,
14
14
  val channels: Int,
15
15
  val encoding: String,
16
- val vadGated: Boolean,
16
+ val voiceActivityDetectionEnabled: Boolean,
17
17
  ) {
18
18
  constructor(values: Map<String, Any>) : this(
19
19
  pcm = values["pcm"] as? ByteArray ?: ByteArray(0),
@@ -21,7 +21,7 @@ data class MicPcmEvent(
21
21
  bitsPerSample = numberValue(values, "bitsPerSample") ?: BITS_PER_SAMPLE,
22
22
  channels = numberValue(values, "channels") ?: CHANNELS,
23
23
  encoding = stringValue(values, "encoding") ?: ENCODING,
24
- vadGated = boolValue(values, "vadGated") ?: false,
24
+ voiceActivityDetectionEnabled = boolValue(values, "voiceActivityDetectionEnabled") ?: true,
25
25
  )
26
26
 
27
27
  fun toMap(): Map<String, Any> =
@@ -32,7 +32,7 @@ data class MicPcmEvent(
32
32
  "bitsPerSample" to bitsPerSample,
33
33
  "channels" to channels,
34
34
  "encoding" to encoding,
35
- "vadGated" to vadGated,
35
+ "voiceActivityDetectionEnabled" to voiceActivityDetectionEnabled,
36
36
  )
37
37
 
38
38
  companion object {
@@ -52,7 +52,7 @@ data class MicLc3Event(
52
52
  val frameSizeBytes: Int,
53
53
  val bitrate: Int,
54
54
  val packetizedFromGlasses: Boolean,
55
- val vadGated: Boolean,
55
+ val voiceActivityDetectionEnabled: Boolean,
56
56
  ) {
57
57
  constructor(values: Map<String, Any>) : this(
58
58
  lc3 = values["lc3"] as? ByteArray ?: ByteArray(0),
@@ -63,7 +63,7 @@ data class MicLc3Event(
63
63
  frameSizeBytes = numberValue(values, "frameSizeBytes") ?: DEFAULT_FRAME_SIZE_BYTES,
64
64
  bitrate = numberValue(values, "bitrate") ?: DEFAULT_BITRATE,
65
65
  packetizedFromGlasses = boolValue(values, "packetizedFromGlasses") ?: false,
66
- vadGated = boolValue(values, "vadGated") ?: false,
66
+ voiceActivityDetectionEnabled = boolValue(values, "voiceActivityDetectionEnabled") ?: true,
67
67
  )
68
68
 
69
69
  fun toMap(): Map<String, Any> =
@@ -77,7 +77,7 @@ data class MicLc3Event(
77
77
  "frameSizeBytes" to frameSizeBytes,
78
78
  "bitrate" to bitrate,
79
79
  "packetizedFromGlasses" to packetizedFromGlasses,
80
- "vadGated" to vadGated,
80
+ "voiceActivityDetectionEnabled" to voiceActivityDetectionEnabled,
81
81
  )
82
82
 
83
83
  companion object {
@@ -1,10 +1,5 @@
1
1
  package com.mentra.bluetoothsdk
2
2
 
3
- enum class GalleryMode {
4
- AUTO,
5
- MANUAL,
6
- }
7
-
8
3
  enum class PhotoSize(val value: String) {
9
4
  SMALL("small"),
10
5
  MEDIUM("medium"),
@@ -49,12 +44,24 @@ data class ButtonPhotoSettings(
49
44
  data class ButtonVideoRecordingSettings(
50
45
  val width: Int,
51
46
  val height: Int,
52
- val frameRate: Int,
47
+ val fps: Int,
53
48
  )
54
49
 
55
- enum class CameraFov(val fov: Int, val roiPosition: Int) {
56
- STANDARD(118, 0),
57
- WIDE(118, 0),
50
+ class CameraFov @JvmOverloads constructor(
51
+ fov: Int = DEFAULT_FOV,
52
+ roiPosition: Int = DEFAULT_ROI_POSITION,
53
+ ) {
54
+ val fov: Int = fov.coerceIn(MIN_FOV, MAX_FOV)
55
+ val roiPosition: Int = roiPosition.coerceIn(MIN_ROI_POSITION, MAX_ROI_POSITION)
56
+
57
+ companion object {
58
+ const val MIN_FOV = 82
59
+ const val MAX_FOV = 118
60
+ const val DEFAULT_FOV = 102
61
+ const val MIN_ROI_POSITION = 0
62
+ const val MAX_ROI_POSITION = 2
63
+ const val DEFAULT_ROI_POSITION = 0
64
+ }
58
65
  }
59
66
 
60
67
  data class PhotoRequest @JvmOverloads constructor(
@@ -64,8 +71,49 @@ data class PhotoRequest @JvmOverloads constructor(
64
71
  val webhookUrl: String,
65
72
  val authToken: String? = null,
66
73
  val compress: PhotoCompression = PhotoCompression.MEDIUM,
74
+ val flash: Boolean = true,
67
75
  val sound: Boolean = true,
68
- )
76
+ /** Sensor exposure time for this capture only (ns), or null for auto exposure */
77
+ val exposureTimeNs: Double? = null,
78
+ /** Sensor ISO for this capture only. Only used when exposureTimeNs enables manual exposure. */
79
+ val iso: Int? = null,
80
+ ) {
81
+ companion object {
82
+ /** Mirrors iOS `BluetoothSdkModule` defaults for keys omitted from the JS bridge. */
83
+ @JvmStatic
84
+ fun fromMap(values: Map<String, Any>): PhotoRequest {
85
+ val rawExp = values["exposureTimeNs"] ?: values["exposure_time_ns"]
86
+ val exposureTimeNs: Double? =
87
+ when (rawExp) {
88
+ is Number -> rawExp.toDouble().takeIf { it.isFinite() && it > 0 }
89
+ else -> null
90
+ }
91
+ val rawIso = values["iso"]
92
+ val iso: Int? =
93
+ when (rawIso) {
94
+ is Number -> rawIso.toDouble().takeIf { it.isFinite() && it > 0 }?.let { value ->
95
+ when {
96
+ value > Int.MAX_VALUE.toDouble() -> Int.MAX_VALUE
97
+ else -> value.toInt()
98
+ }
99
+ }
100
+ else -> null
101
+ }
102
+ return PhotoRequest(
103
+ requestId = stringValue(values, "requestId", "request_id").orEmpty(),
104
+ appId = stringValue(values, "appId", "app_id").orEmpty(),
105
+ size = PhotoSize.fromValue(stringValue(values, "size") ?: "medium"),
106
+ webhookUrl = stringValue(values, "webhookUrl", "webhook_url").orEmpty(),
107
+ authToken = stringValue(values, "authToken", "auth_token")?.takeIf { it.isNotBlank() },
108
+ compress = PhotoCompression.fromValue(stringValue(values, "compress") ?: "none"),
109
+ flash = boolValue(values, "flash") ?: true,
110
+ sound = boolValue(values, "sound") ?: true,
111
+ exposureTimeNs = exposureTimeNs,
112
+ iso = iso,
113
+ )
114
+ }
115
+ }
116
+ }
69
117
 
70
118
  enum class RgbLedAction(val value: String) {
71
119
  ON("on"),
@@ -185,3 +233,14 @@ data class PhotoResponseEvent(
185
233
  val requestId: String get() = response.requestId
186
234
  val values: Map<String, Any> get() = response.toEventMap()
187
235
  }
236
+
237
+ data class PhotoStatusEvent(
238
+ val values: Map<String, Any>,
239
+ ) {
240
+ val requestId: String get() = stringValue(values, "requestId").orEmpty()
241
+ val status: String get() = stringValue(values, "status").orEmpty()
242
+ val timestamp: Long get() = longValue(values, "timestamp") ?: System.currentTimeMillis()
243
+ val resolvedConfig: Map<String, Any>? get() = stringMapValue(values["resolvedConfig"])
244
+ val errorCode: String? get() = stringValue(values, "errorCode")
245
+ val errorMessage: String? get() = stringValue(values, "errorMessage")
246
+ }
@@ -22,7 +22,9 @@ abstract class ControllerManager {
22
22
  authToken: String?,
23
23
  compress: String?,
24
24
  flash: Boolean,
25
- sound: Boolean
25
+ sound: Boolean,
26
+ exposureTimeNs: Long?,
27
+ iso: Int?,
26
28
  )
27
29
  abstract fun startStream(message: Map<String, Any>)
28
30
  abstract fun stopStream()
@@ -132,8 +134,8 @@ abstract class ControllerManager {
132
134
  val micEnabled: Boolean
133
135
  get() = DeviceStore.get("glasses", "micEnabled") as? Boolean ?: false
134
136
 
135
- val vadEnabled: Boolean
136
- get() = DeviceStore.get("glasses", "vadEnabled") as? Boolean ?: false
137
+ val voiceActivityDetectionEnabled: Boolean
138
+ get() = DeviceStore.get("glasses", "voiceActivityDetectionEnabled") as? Boolean ?: true
137
139
 
138
140
  val batteryLevel: Int
139
141
  get() = DeviceStore.get("glasses", "batteryLevel") as? Int ?: -1
@@ -863,7 +863,8 @@ class R1 : ControllerManager() {
863
863
  override fun sendJson(jsonOriginal: Map<String, Any>, wakeUp: Boolean, requireAck: Boolean) {}
864
864
  override fun requestPhoto(
865
865
  requestId: String, appId: String, size: String?, webhookUrl: String?,
866
- authToken: String?, compress: String?, flash: Boolean, sound: Boolean
866
+ authToken: String?, compress: String?, flash: Boolean, sound: Boolean,
867
+ exposureTimeNs: Long?, iso: Int?,
867
868
  ) {}
868
869
  override fun startVideoRecording(requestId: String, save: Boolean, flash: Boolean, sound: Boolean) {}
869
870
  override fun stopVideoRecording(requestId: String) {}