@mentra/bluetooth-sdk 0.1.8 → 0.1.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (147) hide show
  1. package/README.md +116 -18
  2. package/android/build.gradle +289 -16
  3. package/android/gradle.properties.example +9 -0
  4. package/android/lc3Lib/build.gradle +95 -6
  5. package/android/settings.gradle +3 -0
  6. package/android/silero/LICENSE.md +17 -0
  7. package/android/silero/build.gradle +42 -0
  8. package/android/silero/proguard-rules.pro +24 -0
  9. package/android/silero/src/androidTest/assets/hello.wav +0 -0
  10. package/android/silero/src/androidTest/java/com/konovalov/vad/silero/VadSileroTest.kt +79 -0
  11. package/android/silero/src/main/assets/LICENSE +21 -0
  12. package/android/silero/src/main/assets/silero_vad.onnx +0 -0
  13. package/android/silero/src/main/java/com/konovalov/vad/silero/Vad.kt +160 -0
  14. package/android/silero/src/main/java/com/konovalov/vad/silero/VadSilero.kt +449 -0
  15. package/android/silero/src/main/java/com/konovalov/vad/silero/config/FrameSize.kt +15 -0
  16. package/android/silero/src/main/java/com/konovalov/vad/silero/config/Mode.kt +14 -0
  17. package/android/silero/src/main/java/com/konovalov/vad/silero/config/SampleRate.kt +12 -0
  18. package/android/silero/src/main/java/com/konovalov/vad/silero/utils/AudioUtils.kt +47 -0
  19. package/android/silero/src/main/java/com/konovalov/vad/silero/utils/TensorMap.kt +41 -0
  20. package/android/silero/src/test/java/com/konovalov/vad/silero/utils/AudioUtilsTest.kt +44 -0
  21. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkDefaults.kt +6 -0
  22. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +165 -27
  23. package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +89 -21
  24. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +189 -16
  25. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt +22 -2
  26. package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +787 -85
  27. package/android/src/main/java/com/mentra/bluetoothsdk/audio/AudioModels.kt +6 -2
  28. package/android/src/main/java/com/mentra/bluetoothsdk/camera/CameraModels.kt +123 -10
  29. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/ControllerManager.kt +12 -2
  30. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/R1.kt +8 -2
  31. package/android/src/main/java/com/mentra/bluetoothsdk/debug/BleTraceLogger.kt +33 -17
  32. package/android/src/main/java/com/mentra/bluetoothsdk/events/BluetoothEvents.kt +101 -0
  33. package/android/src/main/java/com/mentra/bluetoothsdk/internal/MapParsing.kt +8 -0
  34. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistry.java +108 -0
  35. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoUploadServer.kt +399 -74
  36. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/MentraPhotoReceiverModule.kt +100 -17
  37. package/android/src/main/java/com/mentra/bluetoothsdk/requests/DisplayRequests.kt +17 -0
  38. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G1.java +2 -2
  39. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt +569 -220
  40. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Mach1.java +2 -2
  41. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraLive.java +271 -66
  42. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraNex.kt +176 -104
  43. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraosBle.java +12753 -27036
  44. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt +54 -2
  45. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Simulated.kt +16 -4
  46. package/android/src/main/java/com/mentra/bluetoothsdk/status/DeviceStatus.kt +60 -3
  47. package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +12 -3
  48. package/android/src/main/java/com/mentra/bluetoothsdk/stt/STTTools.kt +87 -36
  49. package/android/src/main/java/com/mentra/bluetoothsdk/stt/SherpaOnnxTranscriber.kt +13 -4
  50. package/android/src/main/java/com/mentra/bluetoothsdk/stt/VadGateSpeechPolicy.kt +229 -0
  51. package/android/src/main/java/com/mentra/bluetoothsdk/tts/TTSTools.kt +180 -0
  52. package/android/src/main/java/com/mentra/bluetoothsdk/utils/AvifExifStripper.java +573 -0
  53. package/android/src/main/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadService.java +488 -50
  54. package/android/src/main/java/com/mentra/bluetoothsdk/utils/HeifExifTagReader.java +199 -0
  55. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunkReassembler.java +18 -1
  56. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunker.java +64 -18
  57. package/android/src/main/java/com/mentra/bluetoothsdk/utils/NexSGCUtils.kt +48 -39
  58. package/android/src/test/java/com/mentra/bluetoothsdk/camera/PhotoRequestTest.kt +23 -0
  59. package/android/src/test/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistryTest.java +75 -0
  60. package/android/src/test/java/com/mentra/bluetoothsdk/utils/AvifExifStripperTest.java +30 -0
  61. package/android/src/test/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadServiceTest.java +98 -0
  62. package/android/src/test/resources/avif_with_exif.avif +0 -0
  63. package/build/BluetoothSdk.types.d.ts +241 -43
  64. package/build/BluetoothSdk.types.d.ts.map +1 -1
  65. package/build/BluetoothSdk.types.js +11 -10
  66. package/build/BluetoothSdk.types.js.map +1 -1
  67. package/build/_private/BluetoothSdkModule.d.ts +36 -23
  68. package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
  69. package/build/_private/BluetoothSdkModule.js +30 -52
  70. package/build/_private/BluetoothSdkModule.js.map +1 -1
  71. package/build/_private/photoRequestPayload.d.ts +4 -0
  72. package/build/_private/photoRequestPayload.d.ts.map +1 -0
  73. package/build/_private/photoRequestPayload.js +25 -0
  74. package/build/_private/photoRequestPayload.js.map +1 -0
  75. package/build/index.d.ts +1 -1
  76. package/build/index.d.ts.map +1 -1
  77. package/build/index.js +23 -0
  78. package/build/index.js.map +1 -1
  79. package/build/react/useMentraBluetooth.d.ts +2 -2
  80. package/build/react/useMentraBluetooth.d.ts.map +1 -1
  81. package/build/react/useMentraBluetooth.js +3 -3
  82. package/build/react/useMentraBluetooth.js.map +1 -1
  83. package/ios/BluetoothSdkModule.swift +174 -70
  84. package/ios/LocalPhotoUploadServer.swift +370 -105
  85. package/ios/MentraBluetoothSDK.podspec +3 -1
  86. package/ios/MentraPhotoReceiverModule.swift +62 -10
  87. package/ios/Packages/CoreObjC/include/PcmConverter.h +22 -0
  88. package/ios/Packages/SherpaOnnx/SherpaOnnx.swift +668 -26
  89. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Info.plist +13 -9
  90. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
  91. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
  92. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/libsherpa-onnx.a +0 -0
  93. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
  94. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
  95. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/libsherpa-onnx.a +0 -0
  96. package/ios/Source/BluetoothSdkDefaults.swift +6 -0
  97. package/ios/Source/Bridge.swift +58 -19
  98. package/ios/Source/DeviceManager.swift +149 -17
  99. package/ios/Source/DeviceStore.swift +16 -6
  100. package/ios/Source/MentraBluetoothSDK.swift +803 -47
  101. package/ios/Source/ObservableStore.swift +8 -2
  102. package/ios/Source/{Audio → audio}/AudioModels.swift +9 -7
  103. package/ios/Source/{Camera → camera}/CameraModels.swift +225 -16
  104. package/ios/Source/controllers/ControllerManager.swift +4 -3
  105. package/ios/Source/controllers/R1.swift +3 -3
  106. package/ios/Source/{Events → events}/BluetoothEvents.swift +183 -1
  107. package/ios/Source/{Internal → internal}/ValueParsing.swift +10 -1
  108. package/ios/Source/{Requests → requests}/DisplayRequests.swift +28 -4
  109. package/ios/Source/services/PhoneMic.swift +1 -0
  110. package/ios/Source/sgcs/G1.swift +2 -2
  111. package/ios/Source/sgcs/G2.swift +645 -403
  112. package/ios/Source/sgcs/Mach1.swift +2 -2
  113. package/ios/Source/sgcs/MentraLive.swift +812 -319
  114. package/ios/Source/sgcs/MentraNex.swift +412 -182
  115. package/ios/Source/sgcs/SGCManager.swift +57 -3
  116. package/ios/Source/sgcs/Simulated.swift +7 -3
  117. package/ios/Source/sgcs/mentraos_ble.pb.swift +3073 -3134
  118. package/ios/Source/status/DeviceStatus.swift +974 -0
  119. package/ios/Source/{Status → status}/WifiHotspotStatus.swift +4 -4
  120. package/ios/Source/{Streaming → streaming}/StreamModels.swift +24 -9
  121. package/ios/Source/stt/STTTools.swift +27 -12
  122. package/ios/Source/stt/SherpaOnnxTranscriber.swift +31 -8
  123. package/ios/Source/tts/TTSTools.swift +164 -0
  124. package/ios/Source/utils/JSCExperiment.swift +7 -7
  125. package/ios/Source/utils/MessageChunkReassembler.swift +20 -1
  126. package/ios/Source/utils/MessageChunker.swift +78 -21
  127. package/ios/Source/utils/TarBz2Extractor.swift +53 -2
  128. package/package.json +1 -1
  129. package/plugin/build/withAndroid.js +68 -4
  130. package/src/BluetoothSdk.types.ts +331 -59
  131. package/src/_private/BluetoothSdkModule.ts +113 -102
  132. package/src/_private/photoRequestPayload.ts +28 -0
  133. package/src/index.ts +58 -15
  134. package/src/react/useMentraBluetooth.ts +6 -4
  135. package/android/.project +0 -28
  136. package/android/lc3Lib/.project +0 -28
  137. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/cargs.h +0 -162
  138. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/c-api.h +0 -1852
  139. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/cxx-api.h +0 -674
  140. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/sherpa-onnx.a +0 -0
  141. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/sherpa-onnx.a +0 -0
  142. package/ios/Source/Status/DeviceStatus.swift +0 -471
  143. /package/ios/Source/{Connection → connection}/ScanSession.swift +0 -0
  144. /package/ios/Source/{Errors → errors}/BluetoothError.swift +0 -0
  145. /package/ios/Source/{Internal → internal}/BluetoothAvailability.swift +0 -0
  146. /package/ios/Source/{Status → status}/RuntimeState.swift +0 -0
  147. /package/ios/Source/{Types → types}/DeviceModels.swift +0 -0
@@ -1,5 +1,6 @@
1
1
  package com.mentra.bluetoothsdk.sgcs
2
2
 
3
+ import com.mentra.bluetoothsdk.BluetoothSdkDefaults
3
4
  import com.mentra.bluetoothsdk.DeviceManager
4
5
  import com.mentra.bluetoothsdk.DeviceStore
5
6
 
@@ -27,7 +28,6 @@ import mentraos.ble.MentraosBle.GlassesToPhone
27
28
  import mentraos.ble.MentraosBle.PhoneToGlasses
28
29
  import mentraos.ble.MentraosBle.ChargingState
29
30
  import mentraos.ble.MentraosBle.BatteryStatus
30
- import mentraos.ble.MentraosBle.VersionResponse
31
31
  import mentraos.ble.MentraosBle.HeadGesture
32
32
  import mentraos.ble.MentraosBle.ButtonEvent
33
33
  import mentraos.ble.MentraosBle.ImuData
@@ -70,8 +70,14 @@ class MentraNex : SGCManager() {
70
70
 
71
71
  private const val MAX_CHUNK_SIZE_DEFAULT: Int = 176 // Maximum chunk size for BLE packets
72
72
  private const val DELAY_BETWEEN_CHUNKS_SEND: Long = 10 // Adjust this value as needed
73
+ // Upper bound on waiting for onCharacteristicWrite. A no-response write's
74
+ // callback is normally near-instant; this only guards against a callback that
75
+ // never arrives, so a dropped write can't wedge the send queue forever.
76
+ private const val WRITE_CALLBACK_TIMEOUT_MS: Long = 2000
73
77
 
74
78
  private const val INITIAL_CONNECTION_DELAY_MS = 350L // Adjust this value as needed
79
+ // Window to let a queued DisconnectRequest flush to the glasses before we close GATT.
80
+ private const val DISCONNECT_FLUSH_DELAY_MS = 250L
75
81
  private const val MICBEAT_INTERVAL_MS: Long = (1000 * 60) * 30; // micbeat every 30 minutes
76
82
 
77
83
  private const val MAIN_TASK_HANDLER_CODE_GATT_STATUS_CHANGED: Int = 110
@@ -98,7 +104,10 @@ class MentraNex : SGCManager() {
98
104
  private var context: Context? = null
99
105
  // private var isDebug: Boolean = true
100
106
 
101
- private var isLc3AudioEnabled: Boolean = true
107
+ // Off by default; toggled from Nex Developer Settings via the nex_audio_playback flag.
108
+ private val isLc3AudioEnabled: Boolean
109
+ // get() = DeviceStore.get("bluetooth", "nex_audio_playback") as? Boolean ?: false
110
+ get() = false
102
111
  private var lc3AudioPlayer: Lc3Player? = null
103
112
 
104
113
  private var lc3DecoderPtr: Long = 0
@@ -132,6 +141,7 @@ class MentraNex : SGCManager() {
132
141
 
133
142
  private var maxChunkSize: Int = MAX_CHUNK_SIZE_DEFAULT // Maximum chunk size for BLE packets
134
143
  private var bmpChunkSize: Int = 194 // BMP chunk size
144
+ private var protobufSeq: Int = 0 // Rolling sequence for fragmented control messages
135
145
 
136
146
  @Volatile private var isWorkerRunning = false
137
147
  // Queue to hold pending requests
@@ -149,8 +159,6 @@ class MentraNex : SGCManager() {
149
159
 
150
160
  private var protobufVersionPosted: Boolean = false
151
161
 
152
- private var hasConnectedThisSession: Boolean = false
153
-
154
162
  private var bleScanCallback: ScanCallback? = null
155
163
 
156
164
  private val modernScanCallback = object : ScanCallback() {
@@ -212,6 +220,17 @@ class MentraNex : SGCManager() {
212
220
  }
213
221
  }
214
222
 
223
+ /** Start/stop the LC3 player when the nex_audio_playback flag changes. */
224
+ override fun applyNexAudioPlaybackSetting() {
225
+ if (isLc3AudioEnabled) {
226
+ Bridge.log("Nex: LC3 audio playback enabled - starting player")
227
+ lc3AudioPlayer?.startPlay()
228
+ } else {
229
+ Bridge.log("Nex: LC3 audio playback disabled - stopping player")
230
+ lc3AudioPlayer?.stopPlay()
231
+ }
232
+ }
233
+
215
234
  private val random: Random = Random()
216
235
 
217
236
  private val mainGattCallback: BluetoothGattCallback = createGattCallback()
@@ -264,6 +283,17 @@ class MentraNex : SGCManager() {
264
283
  }
265
284
  }
266
285
 
286
+ /** Bounded wait: returns early if the flag is still set after [timeoutMs]. */
287
+ @Synchronized
288
+ fun waitWhileTrue(timeoutMs: Long) {
289
+ val deadline = System.currentTimeMillis() + timeoutMs
290
+ while (flag) {
291
+ val remaining = deadline - System.currentTimeMillis()
292
+ if (remaining <= 0) break
293
+ (this as Object).wait(remaining)
294
+ }
295
+ }
296
+
267
297
  @Synchronized
268
298
  fun setTrue() {
269
299
  flag = true
@@ -286,11 +316,27 @@ class MentraNex : SGCManager() {
286
316
  override fun queryGalleryStatus() { Bridge.log("Nex: queryGalleryStatus operation not supported") }
287
317
  override fun sendGalleryMode() { Bridge.log("Nex: sendGalleryMode operation not supported") }
288
318
 
319
+ override fun sendVoiceActivityDetectionSetting() {
320
+ val enabled =
321
+ DeviceStore.get("bluetooth", "voice_activity_detection_enabled") as? Boolean
322
+ ?: BluetoothSdkDefaults.VOICE_ACTIVITY_DETECTION_ENABLED
323
+ Bridge.log("Nex: 🎤 Sending Voice Activity Detection setting to glasses: $enabled")
324
+
325
+ if (connectionState != ConnTypes.CONNECTED) {
326
+ Bridge.log("Nex: Cannot send VAD setting - not connected")
327
+ return
328
+ }
329
+
330
+ val bytes = NexProtobufUtils.generateVadEnabledRequestCommandBytes(enabled)
331
+ sendDataSequentially(bytes)
332
+ Bridge.sendVoiceActivityDetectionStatus(enabled)
333
+ }
334
+
289
335
  // Version info: Not supported on Nex (uses protobuf for version info)
290
336
  override fun requestVersionInfo() { Bridge.log("Nex: requestVersionInfo operation not supported") }
291
337
 
292
338
  // Camera & Media: Not supported on Nex (No camera)
293
- override fun requestPhoto(requestId: String, appId: String, size: String, webhookUrl: String?, authToken: String?, compress: String?, flash: Boolean, sound: Boolean, exposureTimeNs: Long?, iso: Int?) { Bridge.log("Nex: requestPhoto operation not supported") }
339
+ override fun requestPhoto(requestId: String, appId: String, size: String, webhookUrl: String?, authToken: String?, compress: String?, flash: Boolean, save: Boolean, sound: Boolean, exposureTimeNs: Long?, iso: Int?) { Bridge.log("Nex: requestPhoto operation not supported") }
294
340
  override fun startStream(message: MutableMap<String, Any>) { Bridge.log("Nex: startStream operation not supported") }
295
341
  override fun stopStream() { Bridge.log("Nex: stopStream operation not supported") }
296
342
  override fun sendStreamKeepAlive(message: MutableMap<String, Any>) { Bridge.log("Nex: sendStreamKeepAlive operation not supported") }
@@ -391,12 +437,12 @@ class MentraNex : SGCManager() {
391
437
 
392
438
  override fun disconnect() {
393
439
  DeviceStore.apply("glasses", "fullyBooted", false)
394
- destroy();
440
+ destroy()
395
441
  }
396
442
 
397
443
  override fun forget() {
398
444
  DeviceStore.apply("glasses", "fullyBooted", false)
399
- destroy();
445
+ destroy()
400
446
  }
401
447
 
402
448
  override fun cleanup() {
@@ -411,7 +457,7 @@ class MentraNex : SGCManager() {
411
457
  Bridge.log("Nex: === SENDING HEAD UP ANGLE COMMAND TO GLASSES ===")
412
458
  Bridge.log("Nex: Head Up Angle: $validatedAngle degrees (validated range: 0-60)")
413
459
  val cmdBytes = NexProtobufUtils.generateHeadUpAngleConfigCommandBytes(angle)
414
- sendDataSequentially(cmdBytes, 10)
460
+ sendProtobuf(cmdBytes, 10)
415
461
  }
416
462
 
417
463
  override fun getBatteryStatus() {
@@ -444,16 +490,16 @@ class MentraNex : SGCManager() {
444
490
  // TODO: test this logic. Is it correct? Should we send both or just one?
445
491
  Bridge.log("Nex: setBrightness() - level: " + level + "%, autoMode: " + autoMode);
446
492
  val brightnessCmdBytes = NexProtobufUtils.generateBrightnessConfigCommandBytes(level)
447
- sendDataSequentially(brightnessCmdBytes, 10)
493
+ sendProtobuf(brightnessCmdBytes, 10)
448
494
 
449
495
  val autoBrightnessCmdBytes = NexProtobufUtils.generateAutoBrightnessConfigCommandBytes(autoMode)
450
- sendDataSequentially(autoBrightnessCmdBytes, 10)
496
+ sendProtobuf(autoBrightnessCmdBytes, 10)
451
497
  }
452
498
 
453
499
  override fun clearDisplay() {
454
500
  Bridge.log("Nex: clearDisplay() - sending clear display request command bytes");
455
501
  val clearDisplayPackets = NexProtobufUtils.generateClearDisplayRequestCommandBytes()
456
- sendDataSequentially(clearDisplayPackets, 10)
502
+ sendProtobuf(clearDisplayPackets, 10)
457
503
  // sendTextWall(" ")
458
504
  Bridge.log("Nex: clearDisplay() - sent clear display request command bytes");
459
505
  }
@@ -461,7 +507,7 @@ class MentraNex : SGCManager() {
461
507
  override fun sendTextWall(text: String) {
462
508
  Bridge.log("Nex: sendTextWall() - text: " + text);
463
509
  val textChunks: ByteArray = createTextWallChunksForNex(text)
464
- sendDataSequentially(textChunks);
510
+ sendProtobuf(textChunks)
465
511
  }
466
512
 
467
513
  override fun sendDoubleTextWall(top: String, bottom: String) {
@@ -472,10 +518,16 @@ class MentraNex : SGCManager() {
472
518
  bottom?.let { append(it) }
473
519
  }
474
520
  val textChunks = createTextWallChunksForNex(finalText)
475
- sendDataSequentially(textChunks)
521
+ sendProtobuf(textChunks)
476
522
  }
477
523
 
478
- override fun displayBitmap(base64ImageData: String): Boolean {
524
+ override fun displayBitmap(
525
+ base64ImageData: String,
526
+ x: Int?,
527
+ y: Int?,
528
+ width: Int?,
529
+ height: Int?
530
+ ): Boolean {
479
531
  try {
480
532
  val bmpData: ByteArray? = android.util.Base64.decode(base64ImageData, android.util.Base64.DEFAULT)
481
533
  if (bmpData == null || bmpData.isEmpty()) {
@@ -501,23 +553,23 @@ class MentraNex : SGCManager() {
501
553
  Bridge.log("Nex: setDashboardPosition() - height: " + height + ", depth: " + depth)
502
554
  // Send display_height then display_distance; adjust order if Nex firmware requires otherwise.
503
555
  val heightBytes = NexProtobufUtils.generateDisplayHeightCommandBytes(height)
504
- sendDataSequentially(heightBytes, 10)
556
+ sendProtobuf(heightBytes, 10)
505
557
  val distanceCm = NexProtobufUtils.dashboardDepthToDistanceCm(depth)
506
558
  val distanceBytes = NexProtobufUtils.generateDisplayDistanceCommandBytes(distanceCm)
507
- sendDataSequentially(distanceBytes, 10)
559
+ sendProtobuf(distanceBytes, 10)
508
560
  }
509
561
 
510
562
  override fun setDashboardHeightOnly(height: Int) {
511
563
  Bridge.log("Nex: setDashboardHeightOnly() - height: $height")
512
564
  val heightBytes = NexProtobufUtils.generateDisplayHeightCommandBytes(height)
513
- sendDataSequentially(heightBytes, 10)
565
+ sendProtobuf(heightBytes, 10)
514
566
  }
515
567
 
516
568
  override fun setDashboardDepthOnly(depth: Int) {
517
569
  Bridge.log("Nex: setDashboardDepthOnly() - depth: $depth")
518
570
  val distanceCm = NexProtobufUtils.dashboardDepthToDistanceCm(depth)
519
571
  val distanceBytes = NexProtobufUtils.generateDisplayDistanceCommandBytes(distanceCm)
520
- sendDataSequentially(distanceBytes, 10)
572
+ sendProtobuf(distanceBytes, 10)
521
573
  }
522
574
 
523
575
  override fun ping() {
@@ -704,12 +756,16 @@ class MentraNex : SGCManager() {
704
756
  val packetHex = values.joinToString("") { "%02X".format(it) }
705
757
  Bridge.log("onCharacteristicWrite Values - $packetHex")
706
758
 
707
- if (values.isNotEmpty()) {
708
- val packetType = values[0]
709
- val protobufData = values.copyOfRange(1, values.size)
710
-
711
- if (packetType == NexBluetoothPacketTypes.PACKET_TYPE_PROTOBUF) {
712
- // just for test
759
+ if (values.size >= 4 && values[0] == NexBluetoothPacketTypes.PACKET_TYPE_PROTOBUF) {
760
+ // Debug-only re-decode of our own outgoing write. Frames are now
761
+ // [0x02][seq][total][index] + protobuf, so strip the full 4-byte
762
+ // transport header. Only a single-fragment write (total==1) holds a
763
+ // complete, independently decodable protobuf; fragments of a
764
+ // multi-part message are partial and would fail to decode.
765
+ val total = values[2].toInt() and 0xFF
766
+ val index = values[3].toInt() and 0xFF
767
+ if (total == 1 && index == 0) {
768
+ val protobufData = values.copyOfRange(4, values.size)
713
769
  decodeProtobufsByWrite(protobufData, packetHex)
714
770
  }
715
771
  }
@@ -871,14 +927,6 @@ class MentraNex : SGCManager() {
871
927
  showHomeScreen() // Turn on the NexGlasses display
872
928
  updateConnectionState()
873
929
 
874
- if (hasConnectedThisSession) {
875
- Bridge.log("Nex: BLE reconnect detected — showing reconnected banner")
876
- sendTextWall("// MentraOS Reconnected")
877
- mainTaskHandler.postDelayed({ clearDisplay() }, 3000)
878
- } else {
879
- hasConnectedThisSession = true
880
- }
881
-
882
930
  // Post protobuf schema version information (only once)
883
931
  if (!protobufVersionPosted) {
884
932
  postProtobufSchemaVersionInfo()
@@ -888,8 +936,15 @@ class MentraNex : SGCManager() {
888
936
  // Query glasses protobuf version from firmware
889
937
  Bridge.log("=== SENDING GLASSES PROTOBUF VERSION REQUEST ===")
890
938
  val versionQueryPacket = NexProtobufUtils.generateVersionRequestCommandBytes()
891
- sendDataSequentially(versionQueryPacket, 100)
892
- Bridge.log("Sent glasses protobuf version request")
939
+ if (versionQueryPacket.isNotEmpty()) {
940
+ sendProtobuf(versionQueryPacket, 100)
941
+ Bridge.log("Sent glasses protobuf version request")
942
+ } else {
943
+ Bridge.log("Skipping version request: schema removed VersionRequest")
944
+ }
945
+
946
+ // Push current glasses-side Voice Activity Detection setting
947
+ sendVoiceActivityDetectionSetting()
893
948
  }
894
949
  } else {
895
950
  Log.e(TAG, " glass UART service not found")
@@ -903,9 +958,13 @@ class MentraNex : SGCManager() {
903
958
  val result = gatt.setCharacteristicNotification(characteristic, true)
904
959
  Bridge.log("Nex: PROC_QUEUE - setCharacteristicNotification result: $result")
905
960
 
906
- // Set write type for the characteristic
907
- characteristic.writeType = BluetoothGattCharacteristic.WRITE_TYPE_DEFAULT
908
- Bridge.log("Nex: PROC_QUEUE - write type set")
961
+ // Write-without-response: the onCharacteristicWrite callback fires when the
962
+ // local stack accepts the buffer, not after a remote ATT round-trip, so TX is
963
+ // no longer throttled to ~1 packet per connection interval. This is what keeps
964
+ // captions flowing when the phone is asleep and the interval is relaxed (the
965
+ // firmware RX characteristic advertises WRITE_WITHOUT_RESP). Matches G1.
966
+ characteristic.writeType = BluetoothGattCharacteristic.WRITE_TYPE_NO_RESPONSE
967
+ Bridge.log("Nex: PROC_QUEUE - write type set (NO_RESPONSE)")
909
968
 
910
969
  // Add delay
911
970
  Bridge.log("Nex: PROC_QUEUE - waiting to enable notification...")
@@ -1085,6 +1144,21 @@ class MentraNex : SGCManager() {
1085
1144
  }
1086
1145
 
1087
1146
  private fun destroy() {
1147
+ // Tell the glasses this teardown is intentional so they return to the welcome
1148
+ // screen immediately instead of holding the last frame through the firmware's
1149
+ // unexpected-disconnect grace period. Best-effort: if the write can't be sent
1150
+ // or doesn't flush in time, the firmware grace period is the safety net.
1151
+ val canSend = mainGlassGatt != null && mainWriteChar != null && isMainConnected
1152
+ if (!canSend) {
1153
+ performDestroy()
1154
+ return
1155
+ }
1156
+ Bridge.log("Nex: Sending DisconnectRequest before teardown")
1157
+ sendProtobuf(NexProtobufUtils.generateDisconnectRequestCommandBytes(), 100)
1158
+ mainTaskHandler.postDelayed({ performDestroy() }, DISCONNECT_FLUSH_DELAY_MS)
1159
+ }
1160
+
1161
+ private fun performDestroy() {
1088
1162
  Bridge.log("Nex: MentraNexSGC ONDESTROY")
1089
1163
  showHomeScreen()
1090
1164
  isKilled = true
@@ -1136,7 +1210,6 @@ class MentraNex : SGCManager() {
1136
1210
  sendQueue.offer(emptyArray()) // is this needed?
1137
1211
  isWorkerRunning = false
1138
1212
  isMainConnected = false
1139
- hasConnectedThisSession = false
1140
1213
  Bridge.log("Nex: MentraNexSGC cleanup complete")
1141
1214
  }
1142
1215
 
@@ -1200,7 +1273,7 @@ class MentraNex : SGCManager() {
1200
1273
  val streamId = "%04X".format(random.nextInt(0x10000)) // 4-digit hex format
1201
1274
 
1202
1275
  val startImageSendingBytes = NexProtobufUtils.generateDisplayImageCommandBytes(streamId, totalChunks, width, height)
1203
- sendDataSequentially(startImageSendingBytes)
1276
+ sendProtobuf(startImageSendingBytes)
1204
1277
 
1205
1278
  // Send all chunks with proper stream ID parsing
1206
1279
  val chunks = NexProtobufUtils.createBmpChunksForNexGlasses(streamId, bmpData, totalChunks, bmpChunkSize)
@@ -1382,11 +1455,6 @@ class MentraNex : SGCManager() {
1382
1455
  val headUpAngleResponse: HeadUpAngleResponse = glassesToPhone.headUpAngleSet
1383
1456
  Bridge.log("headUpAngleResponse: $headUpAngleResponse")
1384
1457
  }
1385
- GlassesToPhone.PayloadCase.PING -> {
1386
- lastHeartbeatReceivedTime = System.currentTimeMillis()
1387
- Bridge.log("=== RECEIVED PING FROM GLASSES === (Time: $lastHeartbeatReceivedTime)")
1388
- sendPongResponse()
1389
- }
1390
1458
  GlassesToPhone.PayloadCase.VAD_EVENT -> {
1391
1459
  // val vadEvent = glassesToPhone.vadEvent
1392
1460
  // EventBus.getDefault().post(VadEvent(vadEvent.vad))
@@ -1429,28 +1497,6 @@ class MentraNex : SGCManager() {
1429
1497
  // EventBus.getDefault().post(GlassesHeadDownEvent())
1430
1498
  // EventBus.getDefault().post(GlassesTapOutputEvent(2, isRight, System.currentTimeMillis()))
1431
1499
  }
1432
- GlassesToPhone.PayloadCase.VERSION_RESPONSE -> {
1433
- val versionResponse: VersionResponse = glassesToPhone.versionResponse
1434
- Bridge.log("=== RECEIVED GLASSES PROTOBUF VERSION RESPONSE ===")
1435
- Bridge.log("Glasses Protobuf Version: ${versionResponse.version}")
1436
- Bridge.log("Message ID: ${versionResponse.msgId}")
1437
- DeviceStore.apply("glasses", "protobufVersion", versionResponse.version.toString())
1438
-
1439
- if (versionResponse.commit.isNotEmpty()) {
1440
- Bridge.log("Commit: ${versionResponse.commit}")
1441
- }
1442
- if (versionResponse.buildDate.isNotEmpty()) {
1443
- Bridge.log("Build Date: ${versionResponse.buildDate}")
1444
- }
1445
-
1446
- // Post glasses protobuf version event to update UI
1447
- // EventBus.getDefault().post(ProtocolVersionResponseEvent(
1448
- // versionResponse.version,
1449
- // versionResponse.commit,
1450
- // versionResponse.buildDate,
1451
- // versionResponse.msgId
1452
- // ))
1453
- }
1454
1500
  GlassesToPhone.PayloadCase.PAYLOAD_NOT_SET,
1455
1501
  null -> {
1456
1502
  // Do nothing
@@ -1490,38 +1536,6 @@ class MentraNex : SGCManager() {
1490
1536
  sendDataSequentially(missingChunks)
1491
1537
  }
1492
1538
 
1493
- private fun sendPongResponse() {
1494
- // Respond to ping from glasses with pong
1495
- lastHeartbeatReceivedTime = System.currentTimeMillis()
1496
- Bridge.log("=== SENDING PONG RESPONSE TO GLASSES === (Time: $lastHeartbeatReceivedTime)")
1497
-
1498
- val pongPacket = NexProtobufUtils.constructPongResponse()
1499
-
1500
- // Send the pong response
1501
- if (pongPacket != null) {
1502
- sendDataSequentially(pongPacket, 100)
1503
- Bridge.log("Pong response sent successfully")
1504
-
1505
- // Notify mobile app about pong sent
1506
- lastHeartbeatSentTime = System.currentTimeMillis()
1507
- NexEventUtils.sendHeartbeatSentEvent(lastHeartbeatSentTime)
1508
- // EventBus.getDefault().post(HeartbeatSentEvent(timestamp))
1509
- } else {
1510
- Log.e(TAG,"Failed to construct pong response packet")
1511
- }
1512
-
1513
- // Still query battery periodically (every 10 pings received)
1514
- if (batteryLevel == -1 || heartbeatCount % 10 == 0) {
1515
- mainTaskHandler.sendEmptyMessageDelayed(MAIN_TASK_HANDLER_CODE_BATTERY_QUERY, 500)
1516
- }
1517
-
1518
- heartbeatCount++
1519
-
1520
- // Notify mobile app about heartbeat received
1521
- NexEventUtils.sendHeartbeatReceivedEvent(lastHeartbeatReceivedTime)
1522
- // EventBus.getDefault().post(HeartbeatReceivedEvent(lastHeartbeatReceivedTime))
1523
- }
1524
-
1525
1539
  private fun createTextWallChunksForNex(text: String): ByteArray {
1526
1540
  // Create the PhoneToGlasses using its builder and set the DisplayText
1527
1541
  return NexProtobufUtils.generateDisplayTextCommandBytes(text)
@@ -1540,14 +1554,14 @@ class MentraNex : SGCManager() {
1540
1554
  }
1541
1555
  Bridge.log("Nex: === SENDING MICROPHONE STATE COMMAND TO GLASSES ===")
1542
1556
  val micConfigBytes = NexProtobufUtils.generateMicStateConfigCommandBytes(enable)
1543
- sendDataSequentially(micConfigBytes, 10) // wait some time to setup the mic
1557
+ sendProtobuf(micConfigBytes, 10) // wait some time to setup the mic
1544
1558
  Bridge.log("Nex: Sent MIC command: ${micConfigBytes.joinToString("") { "%02x".format(it) }}")
1545
1559
  }, delay)
1546
1560
  }
1547
1561
 
1548
1562
  private fun queryBatteryStatus() {
1549
1563
  val batteryQueryPacket = NexProtobufUtils.generateBatteryStateRequestCommandBytes()
1550
- sendDataSequentially(batteryQueryPacket, 250)
1564
+ sendProtobuf(batteryQueryPacket, 250)
1551
1565
  }
1552
1566
 
1553
1567
  ///// PROCESSING THREAD /////////////////
@@ -1581,6 +1595,51 @@ class MentraNex : SGCManager() {
1581
1595
  startWorkerIfNeeded()
1582
1596
  }
1583
1597
 
1598
+ /**
1599
+ * Frames a serialized protobuf message onto the 0x02 control channel and sends it.
1600
+ *
1601
+ * The message is split into fragments that each fit the negotiated MTU, prefixed with a
1602
+ * 4-byte transport header: [0x02][seq][totalChunks][chunkIndex]. The firmware reassembles
1603
+ * fragments by seq/index before decoding. Single-fragment messages (totalChunks == 1) are
1604
+ * decoded directly by the firmware fast path.
1605
+ *
1606
+ * @param protobufBytes raw serialized PhoneToGlasses bytes (no framing)
1607
+ * @param waitTime optional post-message delay in ms applied after the last fragment (-1 = none)
1608
+ */
1609
+ private fun sendProtobuf(protobufBytes: ByteArray, waitTime: Int = -1) {
1610
+ val headerSize = 4 // [0x02][seq][totalChunks][chunkIndex]
1611
+ val maxFragment = (maxChunkSize - headerSize).coerceAtLeast(1)
1612
+ val totalChunks =
1613
+ if (protobufBytes.isEmpty()) 1 else (protobufBytes.size + maxFragment - 1) / maxFragment
1614
+
1615
+ if (totalChunks > 255) {
1616
+ Log.e(TAG, "Nex: Protobuf message too large to fragment ($totalChunks chunks) - dropping")
1617
+ return
1618
+ }
1619
+
1620
+ val seq = protobufSeq
1621
+ protobufSeq = (protobufSeq + 1) and 0xFF
1622
+
1623
+ val frames = Array(totalChunks) { index ->
1624
+ val start = index * maxFragment
1625
+ val end = minOf(start + maxFragment, protobufBytes.size)
1626
+ val fragLen = end - start
1627
+ val frame = ByteArray(headerSize + fragLen)
1628
+ frame[0] = NexBluetoothPacketTypes.PACKET_TYPE_PROTOBUF
1629
+ frame[1] = seq.toByte()
1630
+ frame[2] = totalChunks.toByte()
1631
+ frame[3] = index.toByte()
1632
+ if (fragLen > 0) {
1633
+ System.arraycopy(protobufBytes, start, frame, headerSize, fragLen)
1634
+ }
1635
+ // Carry the post-message delay on the final fragment only.
1636
+ SendRequest(frame, if (index == totalChunks - 1) waitTime else -1)
1637
+ }
1638
+
1639
+ sendQueue.offer(frames)
1640
+ startWorkerIfNeeded()
1641
+ }
1642
+
1584
1643
  private fun processQueue() {
1585
1644
  // First wait until the services are ready to receive data
1586
1645
  Bridge.log("[BLE] PROC_QUEUE started — waiting for descriptor write (mainServicesWaiter)")
@@ -1617,17 +1676,30 @@ class MentraNex : SGCManager() {
1617
1676
  // Send to main glass
1618
1677
  val canSend = mainGlassGatt != null && mainWriteChar != null && isMainConnected
1619
1678
  Bridge.log("[BLE] PROC_QUEUE send check: gatt=${mainGlassGatt != null} writeChar=${mainWriteChar != null} connected=$isMainConnected → canSend=$canSend len=${request.data.size}")
1679
+ var writeStarted = false
1620
1680
  if (canSend) {
1621
1681
  mainWaiter.setTrue()
1622
1682
  mainWriteChar?.value = request.data
1623
- mainGlassGatt?.writeCharacteristic(mainWriteChar)
1624
- lastSendTimestamp = System.currentTimeMillis()
1625
- Bridge.log("[BLE] PROC_QUEUE writeCharacteristic issued, waiting for onCharacteristicWrite callback")
1683
+ // Honor the return value: writeCharacteristic returns false when the
1684
+ // stack is busy / out of buffers, and in that case no callback fires.
1685
+ // Only wait for the callback if the write actually started, otherwise
1686
+ // the waiter would block until the next disconnect.
1687
+ val issued = mainGlassGatt?.writeCharacteristic(mainWriteChar) ?: false
1688
+ if (issued) {
1689
+ writeStarted = true
1690
+ lastSendTimestamp = System.currentTimeMillis()
1691
+ Bridge.log("[BLE] PROC_QUEUE writeCharacteristic issued, awaiting onCharacteristicWrite")
1692
+ } else {
1693
+ mainWaiter.setFalse()
1694
+ Log.e(TAG, "[BLE] PROC_QUEUE writeCharacteristic returned false — stack busy, dropping fragment")
1695
+ }
1626
1696
  } else {
1627
1697
  Bridge.log("[BLE] PROC_QUEUE skipping write — not ready, packet dropped")
1628
1698
  }
1629
1699
 
1630
- mainWaiter.waitWhileTrue()
1700
+ if (writeStarted) {
1701
+ mainWaiter.waitWhileTrue(WRITE_CALLBACK_TIMEOUT_MS)
1702
+ }
1631
1703
 
1632
1704
  Thread.sleep(DELAY_BETWEEN_CHUNKS_SEND)
1633
1705