@mentra/bluetooth-sdk 0.1.9 → 0.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +116 -18
- package/android/build.gradle +289 -16
- package/android/gradle.properties.example +9 -0
- package/android/lc3Lib/build.gradle +95 -6
- package/android/settings.gradle +3 -0
- package/android/silero/LICENSE.md +17 -0
- package/android/silero/build.gradle +42 -0
- package/android/silero/proguard-rules.pro +24 -0
- package/android/silero/src/androidTest/assets/hello.wav +0 -0
- package/android/silero/src/androidTest/java/com/konovalov/vad/silero/VadSileroTest.kt +79 -0
- package/android/silero/src/main/assets/LICENSE +21 -0
- package/android/silero/src/main/assets/silero_vad.onnx +0 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/Vad.kt +160 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/VadSilero.kt +449 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/config/FrameSize.kt +15 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/config/Mode.kt +14 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/config/SampleRate.kt +12 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/utils/AudioUtils.kt +47 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/utils/TensorMap.kt +41 -0
- package/android/silero/src/test/java/com/konovalov/vad/silero/utils/AudioUtilsTest.kt +44 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkDefaults.kt +6 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +165 -27
- package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +89 -21
- package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +189 -16
- package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt +22 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +787 -85
- package/android/src/main/java/com/mentra/bluetoothsdk/audio/AudioModels.kt +6 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/camera/CameraModels.kt +123 -10
- package/android/src/main/java/com/mentra/bluetoothsdk/controllers/ControllerManager.kt +12 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/controllers/R1.kt +8 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/debug/BleTraceLogger.kt +33 -17
- package/android/src/main/java/com/mentra/bluetoothsdk/events/BluetoothEvents.kt +101 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/internal/MapParsing.kt +8 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistry.java +108 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoUploadServer.kt +399 -74
- package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/MentraPhotoReceiverModule.kt +100 -17
- package/android/src/main/java/com/mentra/bluetoothsdk/requests/DisplayRequests.kt +17 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G1.java +2 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt +569 -220
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Mach1.java +2 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraLive.java +271 -66
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraNex.kt +176 -104
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraosBle.java +12753 -27036
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt +54 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Simulated.kt +16 -4
- package/android/src/main/java/com/mentra/bluetoothsdk/status/DeviceStatus.kt +60 -3
- package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +12 -3
- package/android/src/main/java/com/mentra/bluetoothsdk/stt/STTTools.kt +87 -36
- package/android/src/main/java/com/mentra/bluetoothsdk/stt/SherpaOnnxTranscriber.kt +13 -4
- package/android/src/main/java/com/mentra/bluetoothsdk/stt/VadGateSpeechPolicy.kt +229 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/tts/TTSTools.kt +180 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/AvifExifStripper.java +573 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadService.java +488 -50
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/HeifExifTagReader.java +199 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunkReassembler.java +18 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunker.java +64 -18
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/NexSGCUtils.kt +48 -39
- package/android/src/test/java/com/mentra/bluetoothsdk/camera/PhotoRequestTest.kt +23 -0
- package/android/src/test/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistryTest.java +75 -0
- package/android/src/test/java/com/mentra/bluetoothsdk/utils/AvifExifStripperTest.java +30 -0
- package/android/src/test/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadServiceTest.java +98 -0
- package/android/src/test/resources/avif_with_exif.avif +0 -0
- package/build/BluetoothSdk.types.d.ts +241 -43
- package/build/BluetoothSdk.types.d.ts.map +1 -1
- package/build/BluetoothSdk.types.js +11 -10
- package/build/BluetoothSdk.types.js.map +1 -1
- package/build/_private/BluetoothSdkModule.d.ts +36 -23
- package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
- package/build/_private/BluetoothSdkModule.js +30 -52
- package/build/_private/BluetoothSdkModule.js.map +1 -1
- package/build/_private/photoRequestPayload.d.ts +4 -0
- package/build/_private/photoRequestPayload.d.ts.map +1 -0
- package/build/_private/photoRequestPayload.js +25 -0
- package/build/_private/photoRequestPayload.js.map +1 -0
- package/build/index.d.ts +1 -1
- package/build/index.d.ts.map +1 -1
- package/build/index.js +23 -0
- package/build/index.js.map +1 -1
- package/build/react/useMentraBluetooth.d.ts +2 -2
- package/build/react/useMentraBluetooth.d.ts.map +1 -1
- package/build/react/useMentraBluetooth.js +3 -3
- package/build/react/useMentraBluetooth.js.map +1 -1
- package/ios/BluetoothSdkModule.swift +174 -70
- package/ios/LocalPhotoUploadServer.swift +370 -105
- package/ios/MentraBluetoothSDK.podspec +3 -1
- package/ios/MentraPhotoReceiverModule.swift +62 -10
- package/ios/Packages/CoreObjC/include/PcmConverter.h +22 -0
- package/ios/Packages/SherpaOnnx/SherpaOnnx.swift +668 -26
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Info.plist +13 -9
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/libsherpa-onnx.a +0 -0
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/libsherpa-onnx.a +0 -0
- package/ios/Source/BluetoothSdkDefaults.swift +6 -0
- package/ios/Source/Bridge.swift +58 -19
- package/ios/Source/DeviceManager.swift +149 -17
- package/ios/Source/DeviceStore.swift +16 -6
- package/ios/Source/MentraBluetoothSDK.swift +803 -47
- package/ios/Source/ObservableStore.swift +6 -0
- package/ios/Source/{Audio → audio}/AudioModels.swift +9 -7
- package/ios/Source/{Camera → camera}/CameraModels.swift +225 -16
- package/ios/Source/controllers/ControllerManager.swift +4 -3
- package/ios/Source/controllers/R1.swift +3 -3
- package/ios/Source/{Events → events}/BluetoothEvents.swift +183 -1
- package/ios/Source/{Internal → internal}/ValueParsing.swift +10 -1
- package/ios/Source/{Requests → requests}/DisplayRequests.swift +28 -4
- package/ios/Source/services/PhoneMic.swift +1 -0
- package/ios/Source/sgcs/G1.swift +2 -2
- package/ios/Source/sgcs/G2.swift +645 -403
- package/ios/Source/sgcs/Mach1.swift +2 -2
- package/ios/Source/sgcs/MentraLive.swift +812 -319
- package/ios/Source/sgcs/MentraNex.swift +412 -182
- package/ios/Source/sgcs/SGCManager.swift +57 -3
- package/ios/Source/sgcs/Simulated.swift +7 -3
- package/ios/Source/sgcs/mentraos_ble.pb.swift +3073 -3134
- package/ios/Source/status/DeviceStatus.swift +974 -0
- package/ios/Source/{Status → status}/WifiHotspotStatus.swift +4 -4
- package/ios/Source/{Streaming → streaming}/StreamModels.swift +24 -9
- package/ios/Source/stt/STTTools.swift +27 -12
- package/ios/Source/stt/SherpaOnnxTranscriber.swift +31 -8
- package/ios/Source/tts/TTSTools.swift +164 -0
- package/ios/Source/utils/JSCExperiment.swift +7 -7
- package/ios/Source/utils/MessageChunkReassembler.swift +20 -1
- package/ios/Source/utils/MessageChunker.swift +78 -21
- package/ios/Source/utils/TarBz2Extractor.swift +53 -2
- package/package.json +1 -1
- package/plugin/build/withAndroid.js +68 -4
- package/src/BluetoothSdk.types.ts +331 -59
- package/src/_private/BluetoothSdkModule.ts +113 -102
- package/src/_private/photoRequestPayload.ts +28 -0
- package/src/index.ts +58 -15
- package/src/react/useMentraBluetooth.ts +6 -4
- package/android/.project +0 -28
- package/android/lc3Lib/.project +0 -28
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/cargs.h +0 -162
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/c-api.h +0 -1852
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/cxx-api.h +0 -674
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/sherpa-onnx.a +0 -0
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/sherpa-onnx.a +0 -0
- package/ios/Source/Status/DeviceStatus.swift +0 -471
- /package/ios/Source/{Connection → connection}/ScanSession.swift +0 -0
- /package/ios/Source/{Errors → errors}/BluetoothError.swift +0 -0
- /package/ios/Source/{Internal → internal}/BluetoothAvailability.swift +0 -0
- /package/ios/Source/{Status → status}/RuntimeState.swift +0 -0
- /package/ios/Source/{Types → types}/DeviceModels.swift +0 -0
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
package com.mentra.bluetoothsdk.sgcs
|
|
2
2
|
|
|
3
|
+
import com.mentra.bluetoothsdk.BluetoothSdkDefaults
|
|
3
4
|
import com.mentra.bluetoothsdk.DeviceManager
|
|
4
5
|
import com.mentra.bluetoothsdk.DeviceStore
|
|
5
6
|
|
|
@@ -27,7 +28,6 @@ import mentraos.ble.MentraosBle.GlassesToPhone
|
|
|
27
28
|
import mentraos.ble.MentraosBle.PhoneToGlasses
|
|
28
29
|
import mentraos.ble.MentraosBle.ChargingState
|
|
29
30
|
import mentraos.ble.MentraosBle.BatteryStatus
|
|
30
|
-
import mentraos.ble.MentraosBle.VersionResponse
|
|
31
31
|
import mentraos.ble.MentraosBle.HeadGesture
|
|
32
32
|
import mentraos.ble.MentraosBle.ButtonEvent
|
|
33
33
|
import mentraos.ble.MentraosBle.ImuData
|
|
@@ -70,8 +70,14 @@ class MentraNex : SGCManager() {
|
|
|
70
70
|
|
|
71
71
|
private const val MAX_CHUNK_SIZE_DEFAULT: Int = 176 // Maximum chunk size for BLE packets
|
|
72
72
|
private const val DELAY_BETWEEN_CHUNKS_SEND: Long = 10 // Adjust this value as needed
|
|
73
|
+
// Upper bound on waiting for onCharacteristicWrite. A no-response write's
|
|
74
|
+
// callback is normally near-instant; this only guards against a callback that
|
|
75
|
+
// never arrives, so a dropped write can't wedge the send queue forever.
|
|
76
|
+
private const val WRITE_CALLBACK_TIMEOUT_MS: Long = 2000
|
|
73
77
|
|
|
74
78
|
private const val INITIAL_CONNECTION_DELAY_MS = 350L // Adjust this value as needed
|
|
79
|
+
// Window to let a queued DisconnectRequest flush to the glasses before we close GATT.
|
|
80
|
+
private const val DISCONNECT_FLUSH_DELAY_MS = 250L
|
|
75
81
|
private const val MICBEAT_INTERVAL_MS: Long = (1000 * 60) * 30; // micbeat every 30 minutes
|
|
76
82
|
|
|
77
83
|
private const val MAIN_TASK_HANDLER_CODE_GATT_STATUS_CHANGED: Int = 110
|
|
@@ -98,7 +104,10 @@ class MentraNex : SGCManager() {
|
|
|
98
104
|
private var context: Context? = null
|
|
99
105
|
// private var isDebug: Boolean = true
|
|
100
106
|
|
|
101
|
-
|
|
107
|
+
// Off by default; toggled from Nex Developer Settings via the nex_audio_playback flag.
|
|
108
|
+
private val isLc3AudioEnabled: Boolean
|
|
109
|
+
// get() = DeviceStore.get("bluetooth", "nex_audio_playback") as? Boolean ?: false
|
|
110
|
+
get() = false
|
|
102
111
|
private var lc3AudioPlayer: Lc3Player? = null
|
|
103
112
|
|
|
104
113
|
private var lc3DecoderPtr: Long = 0
|
|
@@ -132,6 +141,7 @@ class MentraNex : SGCManager() {
|
|
|
132
141
|
|
|
133
142
|
private var maxChunkSize: Int = MAX_CHUNK_SIZE_DEFAULT // Maximum chunk size for BLE packets
|
|
134
143
|
private var bmpChunkSize: Int = 194 // BMP chunk size
|
|
144
|
+
private var protobufSeq: Int = 0 // Rolling sequence for fragmented control messages
|
|
135
145
|
|
|
136
146
|
@Volatile private var isWorkerRunning = false
|
|
137
147
|
// Queue to hold pending requests
|
|
@@ -149,8 +159,6 @@ class MentraNex : SGCManager() {
|
|
|
149
159
|
|
|
150
160
|
private var protobufVersionPosted: Boolean = false
|
|
151
161
|
|
|
152
|
-
private var hasConnectedThisSession: Boolean = false
|
|
153
|
-
|
|
154
162
|
private var bleScanCallback: ScanCallback? = null
|
|
155
163
|
|
|
156
164
|
private val modernScanCallback = object : ScanCallback() {
|
|
@@ -212,6 +220,17 @@ class MentraNex : SGCManager() {
|
|
|
212
220
|
}
|
|
213
221
|
}
|
|
214
222
|
|
|
223
|
+
/** Start/stop the LC3 player when the nex_audio_playback flag changes. */
|
|
224
|
+
override fun applyNexAudioPlaybackSetting() {
|
|
225
|
+
if (isLc3AudioEnabled) {
|
|
226
|
+
Bridge.log("Nex: LC3 audio playback enabled - starting player")
|
|
227
|
+
lc3AudioPlayer?.startPlay()
|
|
228
|
+
} else {
|
|
229
|
+
Bridge.log("Nex: LC3 audio playback disabled - stopping player")
|
|
230
|
+
lc3AudioPlayer?.stopPlay()
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
|
|
215
234
|
private val random: Random = Random()
|
|
216
235
|
|
|
217
236
|
private val mainGattCallback: BluetoothGattCallback = createGattCallback()
|
|
@@ -264,6 +283,17 @@ class MentraNex : SGCManager() {
|
|
|
264
283
|
}
|
|
265
284
|
}
|
|
266
285
|
|
|
286
|
+
/** Bounded wait: returns early if the flag is still set after [timeoutMs]. */
|
|
287
|
+
@Synchronized
|
|
288
|
+
fun waitWhileTrue(timeoutMs: Long) {
|
|
289
|
+
val deadline = System.currentTimeMillis() + timeoutMs
|
|
290
|
+
while (flag) {
|
|
291
|
+
val remaining = deadline - System.currentTimeMillis()
|
|
292
|
+
if (remaining <= 0) break
|
|
293
|
+
(this as Object).wait(remaining)
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
|
|
267
297
|
@Synchronized
|
|
268
298
|
fun setTrue() {
|
|
269
299
|
flag = true
|
|
@@ -286,11 +316,27 @@ class MentraNex : SGCManager() {
|
|
|
286
316
|
override fun queryGalleryStatus() { Bridge.log("Nex: queryGalleryStatus operation not supported") }
|
|
287
317
|
override fun sendGalleryMode() { Bridge.log("Nex: sendGalleryMode operation not supported") }
|
|
288
318
|
|
|
319
|
+
override fun sendVoiceActivityDetectionSetting() {
|
|
320
|
+
val enabled =
|
|
321
|
+
DeviceStore.get("bluetooth", "voice_activity_detection_enabled") as? Boolean
|
|
322
|
+
?: BluetoothSdkDefaults.VOICE_ACTIVITY_DETECTION_ENABLED
|
|
323
|
+
Bridge.log("Nex: 🎤 Sending Voice Activity Detection setting to glasses: $enabled")
|
|
324
|
+
|
|
325
|
+
if (connectionState != ConnTypes.CONNECTED) {
|
|
326
|
+
Bridge.log("Nex: Cannot send VAD setting - not connected")
|
|
327
|
+
return
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
val bytes = NexProtobufUtils.generateVadEnabledRequestCommandBytes(enabled)
|
|
331
|
+
sendDataSequentially(bytes)
|
|
332
|
+
Bridge.sendVoiceActivityDetectionStatus(enabled)
|
|
333
|
+
}
|
|
334
|
+
|
|
289
335
|
// Version info: Not supported on Nex (uses protobuf for version info)
|
|
290
336
|
override fun requestVersionInfo() { Bridge.log("Nex: requestVersionInfo operation not supported") }
|
|
291
337
|
|
|
292
338
|
// Camera & Media: Not supported on Nex (No camera)
|
|
293
|
-
override fun requestPhoto(requestId: String, appId: String, size: String, webhookUrl: String?, authToken: String?, compress: String?, flash: Boolean, sound: Boolean, exposureTimeNs: Long?, iso: Int?) { Bridge.log("Nex: requestPhoto operation not supported") }
|
|
339
|
+
override fun requestPhoto(requestId: String, appId: String, size: String, webhookUrl: String?, authToken: String?, compress: String?, flash: Boolean, save: Boolean, sound: Boolean, exposureTimeNs: Long?, iso: Int?) { Bridge.log("Nex: requestPhoto operation not supported") }
|
|
294
340
|
override fun startStream(message: MutableMap<String, Any>) { Bridge.log("Nex: startStream operation not supported") }
|
|
295
341
|
override fun stopStream() { Bridge.log("Nex: stopStream operation not supported") }
|
|
296
342
|
override fun sendStreamKeepAlive(message: MutableMap<String, Any>) { Bridge.log("Nex: sendStreamKeepAlive operation not supported") }
|
|
@@ -391,12 +437,12 @@ class MentraNex : SGCManager() {
|
|
|
391
437
|
|
|
392
438
|
override fun disconnect() {
|
|
393
439
|
DeviceStore.apply("glasses", "fullyBooted", false)
|
|
394
|
-
destroy()
|
|
440
|
+
destroy()
|
|
395
441
|
}
|
|
396
442
|
|
|
397
443
|
override fun forget() {
|
|
398
444
|
DeviceStore.apply("glasses", "fullyBooted", false)
|
|
399
|
-
destroy()
|
|
445
|
+
destroy()
|
|
400
446
|
}
|
|
401
447
|
|
|
402
448
|
override fun cleanup() {
|
|
@@ -411,7 +457,7 @@ class MentraNex : SGCManager() {
|
|
|
411
457
|
Bridge.log("Nex: === SENDING HEAD UP ANGLE COMMAND TO GLASSES ===")
|
|
412
458
|
Bridge.log("Nex: Head Up Angle: $validatedAngle degrees (validated range: 0-60)")
|
|
413
459
|
val cmdBytes = NexProtobufUtils.generateHeadUpAngleConfigCommandBytes(angle)
|
|
414
|
-
|
|
460
|
+
sendProtobuf(cmdBytes, 10)
|
|
415
461
|
}
|
|
416
462
|
|
|
417
463
|
override fun getBatteryStatus() {
|
|
@@ -444,16 +490,16 @@ class MentraNex : SGCManager() {
|
|
|
444
490
|
// TODO: test this logic. Is it correct? Should we send both or just one?
|
|
445
491
|
Bridge.log("Nex: setBrightness() - level: " + level + "%, autoMode: " + autoMode);
|
|
446
492
|
val brightnessCmdBytes = NexProtobufUtils.generateBrightnessConfigCommandBytes(level)
|
|
447
|
-
|
|
493
|
+
sendProtobuf(brightnessCmdBytes, 10)
|
|
448
494
|
|
|
449
495
|
val autoBrightnessCmdBytes = NexProtobufUtils.generateAutoBrightnessConfigCommandBytes(autoMode)
|
|
450
|
-
|
|
496
|
+
sendProtobuf(autoBrightnessCmdBytes, 10)
|
|
451
497
|
}
|
|
452
498
|
|
|
453
499
|
override fun clearDisplay() {
|
|
454
500
|
Bridge.log("Nex: clearDisplay() - sending clear display request command bytes");
|
|
455
501
|
val clearDisplayPackets = NexProtobufUtils.generateClearDisplayRequestCommandBytes()
|
|
456
|
-
|
|
502
|
+
sendProtobuf(clearDisplayPackets, 10)
|
|
457
503
|
// sendTextWall(" ")
|
|
458
504
|
Bridge.log("Nex: clearDisplay() - sent clear display request command bytes");
|
|
459
505
|
}
|
|
@@ -461,7 +507,7 @@ class MentraNex : SGCManager() {
|
|
|
461
507
|
override fun sendTextWall(text: String) {
|
|
462
508
|
Bridge.log("Nex: sendTextWall() - text: " + text);
|
|
463
509
|
val textChunks: ByteArray = createTextWallChunksForNex(text)
|
|
464
|
-
|
|
510
|
+
sendProtobuf(textChunks)
|
|
465
511
|
}
|
|
466
512
|
|
|
467
513
|
override fun sendDoubleTextWall(top: String, bottom: String) {
|
|
@@ -472,10 +518,16 @@ class MentraNex : SGCManager() {
|
|
|
472
518
|
bottom?.let { append(it) }
|
|
473
519
|
}
|
|
474
520
|
val textChunks = createTextWallChunksForNex(finalText)
|
|
475
|
-
|
|
521
|
+
sendProtobuf(textChunks)
|
|
476
522
|
}
|
|
477
523
|
|
|
478
|
-
override fun displayBitmap(
|
|
524
|
+
override fun displayBitmap(
|
|
525
|
+
base64ImageData: String,
|
|
526
|
+
x: Int?,
|
|
527
|
+
y: Int?,
|
|
528
|
+
width: Int?,
|
|
529
|
+
height: Int?
|
|
530
|
+
): Boolean {
|
|
479
531
|
try {
|
|
480
532
|
val bmpData: ByteArray? = android.util.Base64.decode(base64ImageData, android.util.Base64.DEFAULT)
|
|
481
533
|
if (bmpData == null || bmpData.isEmpty()) {
|
|
@@ -501,23 +553,23 @@ class MentraNex : SGCManager() {
|
|
|
501
553
|
Bridge.log("Nex: setDashboardPosition() - height: " + height + ", depth: " + depth)
|
|
502
554
|
// Send display_height then display_distance; adjust order if Nex firmware requires otherwise.
|
|
503
555
|
val heightBytes = NexProtobufUtils.generateDisplayHeightCommandBytes(height)
|
|
504
|
-
|
|
556
|
+
sendProtobuf(heightBytes, 10)
|
|
505
557
|
val distanceCm = NexProtobufUtils.dashboardDepthToDistanceCm(depth)
|
|
506
558
|
val distanceBytes = NexProtobufUtils.generateDisplayDistanceCommandBytes(distanceCm)
|
|
507
|
-
|
|
559
|
+
sendProtobuf(distanceBytes, 10)
|
|
508
560
|
}
|
|
509
561
|
|
|
510
562
|
override fun setDashboardHeightOnly(height: Int) {
|
|
511
563
|
Bridge.log("Nex: setDashboardHeightOnly() - height: $height")
|
|
512
564
|
val heightBytes = NexProtobufUtils.generateDisplayHeightCommandBytes(height)
|
|
513
|
-
|
|
565
|
+
sendProtobuf(heightBytes, 10)
|
|
514
566
|
}
|
|
515
567
|
|
|
516
568
|
override fun setDashboardDepthOnly(depth: Int) {
|
|
517
569
|
Bridge.log("Nex: setDashboardDepthOnly() - depth: $depth")
|
|
518
570
|
val distanceCm = NexProtobufUtils.dashboardDepthToDistanceCm(depth)
|
|
519
571
|
val distanceBytes = NexProtobufUtils.generateDisplayDistanceCommandBytes(distanceCm)
|
|
520
|
-
|
|
572
|
+
sendProtobuf(distanceBytes, 10)
|
|
521
573
|
}
|
|
522
574
|
|
|
523
575
|
override fun ping() {
|
|
@@ -704,12 +756,16 @@ class MentraNex : SGCManager() {
|
|
|
704
756
|
val packetHex = values.joinToString("") { "%02X".format(it) }
|
|
705
757
|
Bridge.log("onCharacteristicWrite Values - $packetHex")
|
|
706
758
|
|
|
707
|
-
if (values.
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
759
|
+
if (values.size >= 4 && values[0] == NexBluetoothPacketTypes.PACKET_TYPE_PROTOBUF) {
|
|
760
|
+
// Debug-only re-decode of our own outgoing write. Frames are now
|
|
761
|
+
// [0x02][seq][total][index] + protobuf, so strip the full 4-byte
|
|
762
|
+
// transport header. Only a single-fragment write (total==1) holds a
|
|
763
|
+
// complete, independently decodable protobuf; fragments of a
|
|
764
|
+
// multi-part message are partial and would fail to decode.
|
|
765
|
+
val total = values[2].toInt() and 0xFF
|
|
766
|
+
val index = values[3].toInt() and 0xFF
|
|
767
|
+
if (total == 1 && index == 0) {
|
|
768
|
+
val protobufData = values.copyOfRange(4, values.size)
|
|
713
769
|
decodeProtobufsByWrite(protobufData, packetHex)
|
|
714
770
|
}
|
|
715
771
|
}
|
|
@@ -871,14 +927,6 @@ class MentraNex : SGCManager() {
|
|
|
871
927
|
showHomeScreen() // Turn on the NexGlasses display
|
|
872
928
|
updateConnectionState()
|
|
873
929
|
|
|
874
|
-
if (hasConnectedThisSession) {
|
|
875
|
-
Bridge.log("Nex: BLE reconnect detected — showing reconnected banner")
|
|
876
|
-
sendTextWall("// MentraOS Reconnected")
|
|
877
|
-
mainTaskHandler.postDelayed({ clearDisplay() }, 3000)
|
|
878
|
-
} else {
|
|
879
|
-
hasConnectedThisSession = true
|
|
880
|
-
}
|
|
881
|
-
|
|
882
930
|
// Post protobuf schema version information (only once)
|
|
883
931
|
if (!protobufVersionPosted) {
|
|
884
932
|
postProtobufSchemaVersionInfo()
|
|
@@ -888,8 +936,15 @@ class MentraNex : SGCManager() {
|
|
|
888
936
|
// Query glasses protobuf version from firmware
|
|
889
937
|
Bridge.log("=== SENDING GLASSES PROTOBUF VERSION REQUEST ===")
|
|
890
938
|
val versionQueryPacket = NexProtobufUtils.generateVersionRequestCommandBytes()
|
|
891
|
-
|
|
892
|
-
|
|
939
|
+
if (versionQueryPacket.isNotEmpty()) {
|
|
940
|
+
sendProtobuf(versionQueryPacket, 100)
|
|
941
|
+
Bridge.log("Sent glasses protobuf version request")
|
|
942
|
+
} else {
|
|
943
|
+
Bridge.log("Skipping version request: schema removed VersionRequest")
|
|
944
|
+
}
|
|
945
|
+
|
|
946
|
+
// Push current glasses-side Voice Activity Detection setting
|
|
947
|
+
sendVoiceActivityDetectionSetting()
|
|
893
948
|
}
|
|
894
949
|
} else {
|
|
895
950
|
Log.e(TAG, " glass UART service not found")
|
|
@@ -903,9 +958,13 @@ class MentraNex : SGCManager() {
|
|
|
903
958
|
val result = gatt.setCharacteristicNotification(characteristic, true)
|
|
904
959
|
Bridge.log("Nex: PROC_QUEUE - setCharacteristicNotification result: $result")
|
|
905
960
|
|
|
906
|
-
//
|
|
907
|
-
|
|
908
|
-
|
|
961
|
+
// Write-without-response: the onCharacteristicWrite callback fires when the
|
|
962
|
+
// local stack accepts the buffer, not after a remote ATT round-trip, so TX is
|
|
963
|
+
// no longer throttled to ~1 packet per connection interval. This is what keeps
|
|
964
|
+
// captions flowing when the phone is asleep and the interval is relaxed (the
|
|
965
|
+
// firmware RX characteristic advertises WRITE_WITHOUT_RESP). Matches G1.
|
|
966
|
+
characteristic.writeType = BluetoothGattCharacteristic.WRITE_TYPE_NO_RESPONSE
|
|
967
|
+
Bridge.log("Nex: PROC_QUEUE - write type set (NO_RESPONSE)")
|
|
909
968
|
|
|
910
969
|
// Add delay
|
|
911
970
|
Bridge.log("Nex: PROC_QUEUE - waiting to enable notification...")
|
|
@@ -1085,6 +1144,21 @@ class MentraNex : SGCManager() {
|
|
|
1085
1144
|
}
|
|
1086
1145
|
|
|
1087
1146
|
private fun destroy() {
|
|
1147
|
+
// Tell the glasses this teardown is intentional so they return to the welcome
|
|
1148
|
+
// screen immediately instead of holding the last frame through the firmware's
|
|
1149
|
+
// unexpected-disconnect grace period. Best-effort: if the write can't be sent
|
|
1150
|
+
// or doesn't flush in time, the firmware grace period is the safety net.
|
|
1151
|
+
val canSend = mainGlassGatt != null && mainWriteChar != null && isMainConnected
|
|
1152
|
+
if (!canSend) {
|
|
1153
|
+
performDestroy()
|
|
1154
|
+
return
|
|
1155
|
+
}
|
|
1156
|
+
Bridge.log("Nex: Sending DisconnectRequest before teardown")
|
|
1157
|
+
sendProtobuf(NexProtobufUtils.generateDisconnectRequestCommandBytes(), 100)
|
|
1158
|
+
mainTaskHandler.postDelayed({ performDestroy() }, DISCONNECT_FLUSH_DELAY_MS)
|
|
1159
|
+
}
|
|
1160
|
+
|
|
1161
|
+
private fun performDestroy() {
|
|
1088
1162
|
Bridge.log("Nex: MentraNexSGC ONDESTROY")
|
|
1089
1163
|
showHomeScreen()
|
|
1090
1164
|
isKilled = true
|
|
@@ -1136,7 +1210,6 @@ class MentraNex : SGCManager() {
|
|
|
1136
1210
|
sendQueue.offer(emptyArray()) // is this needed?
|
|
1137
1211
|
isWorkerRunning = false
|
|
1138
1212
|
isMainConnected = false
|
|
1139
|
-
hasConnectedThisSession = false
|
|
1140
1213
|
Bridge.log("Nex: MentraNexSGC cleanup complete")
|
|
1141
1214
|
}
|
|
1142
1215
|
|
|
@@ -1200,7 +1273,7 @@ class MentraNex : SGCManager() {
|
|
|
1200
1273
|
val streamId = "%04X".format(random.nextInt(0x10000)) // 4-digit hex format
|
|
1201
1274
|
|
|
1202
1275
|
val startImageSendingBytes = NexProtobufUtils.generateDisplayImageCommandBytes(streamId, totalChunks, width, height)
|
|
1203
|
-
|
|
1276
|
+
sendProtobuf(startImageSendingBytes)
|
|
1204
1277
|
|
|
1205
1278
|
// Send all chunks with proper stream ID parsing
|
|
1206
1279
|
val chunks = NexProtobufUtils.createBmpChunksForNexGlasses(streamId, bmpData, totalChunks, bmpChunkSize)
|
|
@@ -1382,11 +1455,6 @@ class MentraNex : SGCManager() {
|
|
|
1382
1455
|
val headUpAngleResponse: HeadUpAngleResponse = glassesToPhone.headUpAngleSet
|
|
1383
1456
|
Bridge.log("headUpAngleResponse: $headUpAngleResponse")
|
|
1384
1457
|
}
|
|
1385
|
-
GlassesToPhone.PayloadCase.PING -> {
|
|
1386
|
-
lastHeartbeatReceivedTime = System.currentTimeMillis()
|
|
1387
|
-
Bridge.log("=== RECEIVED PING FROM GLASSES === (Time: $lastHeartbeatReceivedTime)")
|
|
1388
|
-
sendPongResponse()
|
|
1389
|
-
}
|
|
1390
1458
|
GlassesToPhone.PayloadCase.VAD_EVENT -> {
|
|
1391
1459
|
// val vadEvent = glassesToPhone.vadEvent
|
|
1392
1460
|
// EventBus.getDefault().post(VadEvent(vadEvent.vad))
|
|
@@ -1429,28 +1497,6 @@ class MentraNex : SGCManager() {
|
|
|
1429
1497
|
// EventBus.getDefault().post(GlassesHeadDownEvent())
|
|
1430
1498
|
// EventBus.getDefault().post(GlassesTapOutputEvent(2, isRight, System.currentTimeMillis()))
|
|
1431
1499
|
}
|
|
1432
|
-
GlassesToPhone.PayloadCase.VERSION_RESPONSE -> {
|
|
1433
|
-
val versionResponse: VersionResponse = glassesToPhone.versionResponse
|
|
1434
|
-
Bridge.log("=== RECEIVED GLASSES PROTOBUF VERSION RESPONSE ===")
|
|
1435
|
-
Bridge.log("Glasses Protobuf Version: ${versionResponse.version}")
|
|
1436
|
-
Bridge.log("Message ID: ${versionResponse.msgId}")
|
|
1437
|
-
DeviceStore.apply("glasses", "protobufVersion", versionResponse.version.toString())
|
|
1438
|
-
|
|
1439
|
-
if (versionResponse.commit.isNotEmpty()) {
|
|
1440
|
-
Bridge.log("Commit: ${versionResponse.commit}")
|
|
1441
|
-
}
|
|
1442
|
-
if (versionResponse.buildDate.isNotEmpty()) {
|
|
1443
|
-
Bridge.log("Build Date: ${versionResponse.buildDate}")
|
|
1444
|
-
}
|
|
1445
|
-
|
|
1446
|
-
// Post glasses protobuf version event to update UI
|
|
1447
|
-
// EventBus.getDefault().post(ProtocolVersionResponseEvent(
|
|
1448
|
-
// versionResponse.version,
|
|
1449
|
-
// versionResponse.commit,
|
|
1450
|
-
// versionResponse.buildDate,
|
|
1451
|
-
// versionResponse.msgId
|
|
1452
|
-
// ))
|
|
1453
|
-
}
|
|
1454
1500
|
GlassesToPhone.PayloadCase.PAYLOAD_NOT_SET,
|
|
1455
1501
|
null -> {
|
|
1456
1502
|
// Do nothing
|
|
@@ -1490,38 +1536,6 @@ class MentraNex : SGCManager() {
|
|
|
1490
1536
|
sendDataSequentially(missingChunks)
|
|
1491
1537
|
}
|
|
1492
1538
|
|
|
1493
|
-
private fun sendPongResponse() {
|
|
1494
|
-
// Respond to ping from glasses with pong
|
|
1495
|
-
lastHeartbeatReceivedTime = System.currentTimeMillis()
|
|
1496
|
-
Bridge.log("=== SENDING PONG RESPONSE TO GLASSES === (Time: $lastHeartbeatReceivedTime)")
|
|
1497
|
-
|
|
1498
|
-
val pongPacket = NexProtobufUtils.constructPongResponse()
|
|
1499
|
-
|
|
1500
|
-
// Send the pong response
|
|
1501
|
-
if (pongPacket != null) {
|
|
1502
|
-
sendDataSequentially(pongPacket, 100)
|
|
1503
|
-
Bridge.log("Pong response sent successfully")
|
|
1504
|
-
|
|
1505
|
-
// Notify mobile app about pong sent
|
|
1506
|
-
lastHeartbeatSentTime = System.currentTimeMillis()
|
|
1507
|
-
NexEventUtils.sendHeartbeatSentEvent(lastHeartbeatSentTime)
|
|
1508
|
-
// EventBus.getDefault().post(HeartbeatSentEvent(timestamp))
|
|
1509
|
-
} else {
|
|
1510
|
-
Log.e(TAG,"Failed to construct pong response packet")
|
|
1511
|
-
}
|
|
1512
|
-
|
|
1513
|
-
// Still query battery periodically (every 10 pings received)
|
|
1514
|
-
if (batteryLevel == -1 || heartbeatCount % 10 == 0) {
|
|
1515
|
-
mainTaskHandler.sendEmptyMessageDelayed(MAIN_TASK_HANDLER_CODE_BATTERY_QUERY, 500)
|
|
1516
|
-
}
|
|
1517
|
-
|
|
1518
|
-
heartbeatCount++
|
|
1519
|
-
|
|
1520
|
-
// Notify mobile app about heartbeat received
|
|
1521
|
-
NexEventUtils.sendHeartbeatReceivedEvent(lastHeartbeatReceivedTime)
|
|
1522
|
-
// EventBus.getDefault().post(HeartbeatReceivedEvent(lastHeartbeatReceivedTime))
|
|
1523
|
-
}
|
|
1524
|
-
|
|
1525
1539
|
private fun createTextWallChunksForNex(text: String): ByteArray {
|
|
1526
1540
|
// Create the PhoneToGlasses using its builder and set the DisplayText
|
|
1527
1541
|
return NexProtobufUtils.generateDisplayTextCommandBytes(text)
|
|
@@ -1540,14 +1554,14 @@ class MentraNex : SGCManager() {
|
|
|
1540
1554
|
}
|
|
1541
1555
|
Bridge.log("Nex: === SENDING MICROPHONE STATE COMMAND TO GLASSES ===")
|
|
1542
1556
|
val micConfigBytes = NexProtobufUtils.generateMicStateConfigCommandBytes(enable)
|
|
1543
|
-
|
|
1557
|
+
sendProtobuf(micConfigBytes, 10) // wait some time to setup the mic
|
|
1544
1558
|
Bridge.log("Nex: Sent MIC command: ${micConfigBytes.joinToString("") { "%02x".format(it) }}")
|
|
1545
1559
|
}, delay)
|
|
1546
1560
|
}
|
|
1547
1561
|
|
|
1548
1562
|
private fun queryBatteryStatus() {
|
|
1549
1563
|
val batteryQueryPacket = NexProtobufUtils.generateBatteryStateRequestCommandBytes()
|
|
1550
|
-
|
|
1564
|
+
sendProtobuf(batteryQueryPacket, 250)
|
|
1551
1565
|
}
|
|
1552
1566
|
|
|
1553
1567
|
///// PROCESSING THREAD /////////////////
|
|
@@ -1581,6 +1595,51 @@ class MentraNex : SGCManager() {
|
|
|
1581
1595
|
startWorkerIfNeeded()
|
|
1582
1596
|
}
|
|
1583
1597
|
|
|
1598
|
+
/**
|
|
1599
|
+
* Frames a serialized protobuf message onto the 0x02 control channel and sends it.
|
|
1600
|
+
*
|
|
1601
|
+
* The message is split into fragments that each fit the negotiated MTU, prefixed with a
|
|
1602
|
+
* 4-byte transport header: [0x02][seq][totalChunks][chunkIndex]. The firmware reassembles
|
|
1603
|
+
* fragments by seq/index before decoding. Single-fragment messages (totalChunks == 1) are
|
|
1604
|
+
* decoded directly by the firmware fast path.
|
|
1605
|
+
*
|
|
1606
|
+
* @param protobufBytes raw serialized PhoneToGlasses bytes (no framing)
|
|
1607
|
+
* @param waitTime optional post-message delay in ms applied after the last fragment (-1 = none)
|
|
1608
|
+
*/
|
|
1609
|
+
private fun sendProtobuf(protobufBytes: ByteArray, waitTime: Int = -1) {
|
|
1610
|
+
val headerSize = 4 // [0x02][seq][totalChunks][chunkIndex]
|
|
1611
|
+
val maxFragment = (maxChunkSize - headerSize).coerceAtLeast(1)
|
|
1612
|
+
val totalChunks =
|
|
1613
|
+
if (protobufBytes.isEmpty()) 1 else (protobufBytes.size + maxFragment - 1) / maxFragment
|
|
1614
|
+
|
|
1615
|
+
if (totalChunks > 255) {
|
|
1616
|
+
Log.e(TAG, "Nex: Protobuf message too large to fragment ($totalChunks chunks) - dropping")
|
|
1617
|
+
return
|
|
1618
|
+
}
|
|
1619
|
+
|
|
1620
|
+
val seq = protobufSeq
|
|
1621
|
+
protobufSeq = (protobufSeq + 1) and 0xFF
|
|
1622
|
+
|
|
1623
|
+
val frames = Array(totalChunks) { index ->
|
|
1624
|
+
val start = index * maxFragment
|
|
1625
|
+
val end = minOf(start + maxFragment, protobufBytes.size)
|
|
1626
|
+
val fragLen = end - start
|
|
1627
|
+
val frame = ByteArray(headerSize + fragLen)
|
|
1628
|
+
frame[0] = NexBluetoothPacketTypes.PACKET_TYPE_PROTOBUF
|
|
1629
|
+
frame[1] = seq.toByte()
|
|
1630
|
+
frame[2] = totalChunks.toByte()
|
|
1631
|
+
frame[3] = index.toByte()
|
|
1632
|
+
if (fragLen > 0) {
|
|
1633
|
+
System.arraycopy(protobufBytes, start, frame, headerSize, fragLen)
|
|
1634
|
+
}
|
|
1635
|
+
// Carry the post-message delay on the final fragment only.
|
|
1636
|
+
SendRequest(frame, if (index == totalChunks - 1) waitTime else -1)
|
|
1637
|
+
}
|
|
1638
|
+
|
|
1639
|
+
sendQueue.offer(frames)
|
|
1640
|
+
startWorkerIfNeeded()
|
|
1641
|
+
}
|
|
1642
|
+
|
|
1584
1643
|
private fun processQueue() {
|
|
1585
1644
|
// First wait until the services are ready to receive data
|
|
1586
1645
|
Bridge.log("[BLE] PROC_QUEUE started — waiting for descriptor write (mainServicesWaiter)")
|
|
@@ -1617,17 +1676,30 @@ class MentraNex : SGCManager() {
|
|
|
1617
1676
|
// Send to main glass
|
|
1618
1677
|
val canSend = mainGlassGatt != null && mainWriteChar != null && isMainConnected
|
|
1619
1678
|
Bridge.log("[BLE] PROC_QUEUE send check: gatt=${mainGlassGatt != null} writeChar=${mainWriteChar != null} connected=$isMainConnected → canSend=$canSend len=${request.data.size}")
|
|
1679
|
+
var writeStarted = false
|
|
1620
1680
|
if (canSend) {
|
|
1621
1681
|
mainWaiter.setTrue()
|
|
1622
1682
|
mainWriteChar?.value = request.data
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1683
|
+
// Honor the return value: writeCharacteristic returns false when the
|
|
1684
|
+
// stack is busy / out of buffers, and in that case no callback fires.
|
|
1685
|
+
// Only wait for the callback if the write actually started, otherwise
|
|
1686
|
+
// the waiter would block until the next disconnect.
|
|
1687
|
+
val issued = mainGlassGatt?.writeCharacteristic(mainWriteChar) ?: false
|
|
1688
|
+
if (issued) {
|
|
1689
|
+
writeStarted = true
|
|
1690
|
+
lastSendTimestamp = System.currentTimeMillis()
|
|
1691
|
+
Bridge.log("[BLE] PROC_QUEUE writeCharacteristic issued, awaiting onCharacteristicWrite")
|
|
1692
|
+
} else {
|
|
1693
|
+
mainWaiter.setFalse()
|
|
1694
|
+
Log.e(TAG, "[BLE] PROC_QUEUE writeCharacteristic returned false — stack busy, dropping fragment")
|
|
1695
|
+
}
|
|
1626
1696
|
} else {
|
|
1627
1697
|
Bridge.log("[BLE] PROC_QUEUE skipping write — not ready, packet dropped")
|
|
1628
1698
|
}
|
|
1629
1699
|
|
|
1630
|
-
|
|
1700
|
+
if (writeStarted) {
|
|
1701
|
+
mainWaiter.waitWhileTrue(WRITE_CALLBACK_TIMEOUT_MS)
|
|
1702
|
+
}
|
|
1631
1703
|
|
|
1632
1704
|
Thread.sleep(DELAY_BETWEEN_CHUNKS_SEND)
|
|
1633
1705
|
|