@mentra/bluetooth-sdk 0.1.9 → 0.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (147) hide show
  1. package/README.md +116 -18
  2. package/android/build.gradle +289 -16
  3. package/android/gradle.properties.example +9 -0
  4. package/android/lc3Lib/build.gradle +95 -6
  5. package/android/settings.gradle +3 -0
  6. package/android/silero/LICENSE.md +17 -0
  7. package/android/silero/build.gradle +42 -0
  8. package/android/silero/proguard-rules.pro +24 -0
  9. package/android/silero/src/androidTest/assets/hello.wav +0 -0
  10. package/android/silero/src/androidTest/java/com/konovalov/vad/silero/VadSileroTest.kt +79 -0
  11. package/android/silero/src/main/assets/LICENSE +21 -0
  12. package/android/silero/src/main/assets/silero_vad.onnx +0 -0
  13. package/android/silero/src/main/java/com/konovalov/vad/silero/Vad.kt +160 -0
  14. package/android/silero/src/main/java/com/konovalov/vad/silero/VadSilero.kt +449 -0
  15. package/android/silero/src/main/java/com/konovalov/vad/silero/config/FrameSize.kt +15 -0
  16. package/android/silero/src/main/java/com/konovalov/vad/silero/config/Mode.kt +14 -0
  17. package/android/silero/src/main/java/com/konovalov/vad/silero/config/SampleRate.kt +12 -0
  18. package/android/silero/src/main/java/com/konovalov/vad/silero/utils/AudioUtils.kt +47 -0
  19. package/android/silero/src/main/java/com/konovalov/vad/silero/utils/TensorMap.kt +41 -0
  20. package/android/silero/src/test/java/com/konovalov/vad/silero/utils/AudioUtilsTest.kt +44 -0
  21. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkDefaults.kt +6 -0
  22. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +195 -34
  23. package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +103 -21
  24. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +210 -18
  25. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt +30 -2
  26. package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +787 -85
  27. package/android/src/main/java/com/mentra/bluetoothsdk/audio/AudioModels.kt +6 -2
  28. package/android/src/main/java/com/mentra/bluetoothsdk/camera/CameraModels.kt +125 -11
  29. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/ControllerManager.kt +12 -2
  30. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/R1.kt +8 -2
  31. package/android/src/main/java/com/mentra/bluetoothsdk/debug/BleTraceLogger.kt +33 -17
  32. package/android/src/main/java/com/mentra/bluetoothsdk/events/BluetoothEvents.kt +101 -0
  33. package/android/src/main/java/com/mentra/bluetoothsdk/internal/MapParsing.kt +8 -0
  34. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistry.java +108 -0
  35. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoUploadServer.kt +399 -74
  36. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/MentraPhotoReceiverModule.kt +100 -17
  37. package/android/src/main/java/com/mentra/bluetoothsdk/requests/DisplayRequests.kt +17 -0
  38. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G1.java +2 -2
  39. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt +1293 -758
  40. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Mach1.java +2 -2
  41. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraLive.java +271 -66
  42. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraNex.kt +197 -149
  43. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraosBle.java +12753 -27036
  44. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt +64 -2
  45. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Simulated.kt +16 -4
  46. package/android/src/main/java/com/mentra/bluetoothsdk/status/DeviceStatus.kt +60 -3
  47. package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +12 -3
  48. package/android/src/main/java/com/mentra/bluetoothsdk/stt/STTTools.kt +87 -36
  49. package/android/src/main/java/com/mentra/bluetoothsdk/stt/SherpaOnnxTranscriber.kt +13 -4
  50. package/android/src/main/java/com/mentra/bluetoothsdk/stt/VadGateSpeechPolicy.kt +229 -0
  51. package/android/src/main/java/com/mentra/bluetoothsdk/tts/TTSTools.kt +180 -0
  52. package/android/src/main/java/com/mentra/bluetoothsdk/utils/AvifExifStripper.java +573 -0
  53. package/android/src/main/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadService.java +488 -50
  54. package/android/src/main/java/com/mentra/bluetoothsdk/utils/HeifExifTagReader.java +199 -0
  55. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunkReassembler.java +18 -1
  56. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunker.java +64 -18
  57. package/android/src/main/java/com/mentra/bluetoothsdk/utils/NexSGCUtils.kt +48 -39
  58. package/android/src/test/java/com/mentra/bluetoothsdk/camera/PhotoRequestTest.kt +23 -0
  59. package/android/src/test/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistryTest.java +75 -0
  60. package/android/src/test/java/com/mentra/bluetoothsdk/utils/AvifExifStripperTest.java +30 -0
  61. package/android/src/test/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadServiceTest.java +98 -0
  62. package/android/src/test/resources/avif_with_exif.avif +0 -0
  63. package/build/BluetoothSdk.types.d.ts +253 -44
  64. package/build/BluetoothSdk.types.d.ts.map +1 -1
  65. package/build/BluetoothSdk.types.js +11 -10
  66. package/build/BluetoothSdk.types.js.map +1 -1
  67. package/build/_private/BluetoothSdkModule.d.ts +37 -23
  68. package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
  69. package/build/_private/BluetoothSdkModule.js +43 -52
  70. package/build/_private/BluetoothSdkModule.js.map +1 -1
  71. package/build/_private/photoRequestPayload.d.ts +4 -0
  72. package/build/_private/photoRequestPayload.d.ts.map +1 -0
  73. package/build/_private/photoRequestPayload.js +25 -0
  74. package/build/_private/photoRequestPayload.js.map +1 -0
  75. package/build/index.d.ts +1 -1
  76. package/build/index.d.ts.map +1 -1
  77. package/build/index.js +25 -0
  78. package/build/index.js.map +1 -1
  79. package/build/react/useMentraBluetooth.d.ts +2 -2
  80. package/build/react/useMentraBluetooth.d.ts.map +1 -1
  81. package/build/react/useMentraBluetooth.js +3 -3
  82. package/build/react/useMentraBluetooth.js.map +1 -1
  83. package/ios/BluetoothSdkModule.swift +177 -70
  84. package/ios/LocalPhotoUploadServer.swift +370 -105
  85. package/ios/MentraBluetoothSDK.podspec +3 -1
  86. package/ios/MentraPhotoReceiverModule.swift +62 -10
  87. package/ios/Packages/CoreObjC/include/PcmConverter.h +22 -0
  88. package/ios/Packages/SherpaOnnx/SherpaOnnx.swift +668 -26
  89. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Info.plist +13 -9
  90. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
  91. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
  92. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/libsherpa-onnx.a +0 -0
  93. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
  94. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
  95. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/libsherpa-onnx.a +0 -0
  96. package/ios/Source/BluetoothSdkDefaults.swift +6 -0
  97. package/ios/Source/Bridge.swift +68 -19
  98. package/ios/Source/DeviceManager.swift +167 -32
  99. package/ios/Source/DeviceStore.swift +24 -8
  100. package/ios/Source/MentraBluetoothSDK.swift +803 -47
  101. package/ios/Source/ObservableStore.swift +6 -0
  102. package/ios/Source/{Audio → audio}/AudioModels.swift +9 -7
  103. package/ios/Source/{Camera → camera}/CameraModels.swift +226 -16
  104. package/ios/Source/controllers/ControllerManager.swift +4 -3
  105. package/ios/Source/controllers/R1.swift +3 -3
  106. package/ios/Source/{Events → events}/BluetoothEvents.swift +183 -1
  107. package/ios/Source/{Internal → internal}/ValueParsing.swift +10 -1
  108. package/ios/Source/{Requests → requests}/DisplayRequests.swift +28 -4
  109. package/ios/Source/services/PhoneMic.swift +1 -0
  110. package/ios/Source/sgcs/G1.swift +5 -5
  111. package/ios/Source/sgcs/G2.swift +1494 -870
  112. package/ios/Source/sgcs/Mach1.swift +4 -4
  113. package/ios/Source/sgcs/MentraLive.swift +814 -321
  114. package/ios/Source/sgcs/MentraNex.swift +604 -244
  115. package/ios/Source/sgcs/SGCManager.swift +68 -5
  116. package/ios/Source/sgcs/Simulated.swift +9 -5
  117. package/ios/Source/sgcs/mentraos_ble.pb.swift +3073 -3134
  118. package/ios/Source/status/DeviceStatus.swift +974 -0
  119. package/ios/Source/{Status → status}/WifiHotspotStatus.swift +4 -4
  120. package/ios/Source/{Streaming → streaming}/StreamModels.swift +24 -9
  121. package/ios/Source/stt/STTTools.swift +27 -12
  122. package/ios/Source/stt/SherpaOnnxTranscriber.swift +31 -8
  123. package/ios/Source/tts/TTSTools.swift +164 -0
  124. package/ios/Source/utils/JSCExperiment.swift +7 -7
  125. package/ios/Source/utils/MessageChunkReassembler.swift +20 -1
  126. package/ios/Source/utils/MessageChunker.swift +78 -21
  127. package/ios/Source/utils/TarBz2Extractor.swift +53 -2
  128. package/package.json +1 -1
  129. package/plugin/build/withAndroid.js +68 -4
  130. package/src/BluetoothSdk.types.ts +344 -60
  131. package/src/_private/BluetoothSdkModule.ts +132 -102
  132. package/src/_private/photoRequestPayload.ts +28 -0
  133. package/src/index.ts +61 -15
  134. package/src/react/useMentraBluetooth.ts +6 -4
  135. package/android/.project +0 -28
  136. package/android/lc3Lib/.project +0 -28
  137. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/cargs.h +0 -162
  138. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/c-api.h +0 -1852
  139. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/cxx-api.h +0 -674
  140. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/sherpa-onnx.a +0 -0
  141. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/sherpa-onnx.a +0 -0
  142. package/ios/Source/Status/DeviceStatus.swift +0 -471
  143. /package/ios/Source/{Connection → connection}/ScanSession.swift +0 -0
  144. /package/ios/Source/{Errors → errors}/BluetoothError.swift +0 -0
  145. /package/ios/Source/{Internal → internal}/BluetoothAvailability.swift +0 -0
  146. /package/ios/Source/{Status → status}/RuntimeState.swift +0 -0
  147. /package/ios/Source/{Types → types}/DeviceModels.swift +0 -0
@@ -2,8 +2,11 @@ package com.mentra.bluetoothsdk
2
2
 
3
3
  import com.mentra.bluetoothsdk.debug.BleTraceLogger
4
4
  import com.mentra.bluetoothsdk.utils.DeviceTypes
5
+ import expo.modules.kotlin.functions.Coroutine
5
6
  import expo.modules.kotlin.modules.Module
6
7
  import expo.modules.kotlin.modules.ModuleDefinition
8
+ import kotlinx.coroutines.Dispatchers
9
+ import kotlinx.coroutines.withContext
7
10
 
8
11
  class BluetoothSdkModule : Module() {
9
12
  private var sdk: MentraBluetoothSdk? = null
@@ -113,6 +116,14 @@ class BluetoothSdkModule : Module() {
113
116
  sendEvent("photo_status", event.values)
114
117
  }
115
118
 
119
+ override fun onVideoRecordingStatus(event: VideoRecordingStatusEvent) {
120
+ sendEvent("video_recording_status", event.values)
121
+ }
122
+
123
+ override fun onRgbLedControlResponse(event: RgbLedControlResponseEvent) {
124
+ sendEvent("rgb_led_control_response", event.values)
125
+ }
126
+
116
127
  override fun onStreamStatus(event: StreamStatusEvent) {
117
128
  sendEvent("stream_status", event.values)
118
129
  }
@@ -121,6 +132,26 @@ class BluetoothSdkModule : Module() {
121
132
  sendEvent("keep_alive_ack", event.values)
122
133
  }
123
134
 
135
+ override fun onOtaUpdateAvailable(event: OtaUpdateAvailableEvent) {
136
+ sendEvent("ota_update_available", event.values)
137
+ }
138
+
139
+ override fun onOtaStartAck(event: OtaStartAckEvent) {
140
+ sendEvent("ota_start_ack", event.values)
141
+ }
142
+
143
+ override fun onOtaStatus(event: OtaStatusEvent) {
144
+ sendEvent("ota_status", event.values)
145
+ }
146
+
147
+ override fun onSettingsAck(event: SettingsAckEvent) {
148
+ sendEvent("settings_ack", event.values)
149
+ }
150
+
151
+ override fun onVersionInfo(event: VersionInfoResult) {
152
+ sendEvent("version_info", event.toMap() + mapOf("type" to "version_info"))
153
+ }
154
+
124
155
  override fun onMicPcm(event: MicPcmEvent) {
125
156
  sendEvent("mic_pcm", event.toMap())
126
157
  }
@@ -144,7 +175,14 @@ class BluetoothSdkModule : Module() {
144
175
  override fun onRawEvent(eventName: String, values: Map<String, Any>) {
145
176
  sendEvent(eventName, values)
146
177
  }
147
- }
178
+ }
179
+
180
+ private fun requireSdk(): MentraBluetoothSdk =
181
+ sdk
182
+ ?: throw BluetoothException(
183
+ "sdk_not_initialized",
184
+ "Bluetooth SDK is not initialized.",
185
+ )
148
186
 
149
187
  override fun definition() = ModuleDefinition {
150
188
  Name("BluetoothSdk")
@@ -160,16 +198,21 @@ class BluetoothSdkModule : Module() {
160
198
  "glasses_not_ready",
161
199
  "button_press",
162
200
  "touch_event",
201
+ "accel_event",
202
+ "CompassHeadingEvent",
203
+ "CompassCalibrationEvent",
163
204
  "head_up",
164
205
  "voice_activity_detection_status",
165
206
  "speaking_status",
166
207
  "battery_status",
167
208
  "local_transcription",
168
209
  "wifi_status_change",
210
+ "wifi_scan_result",
169
211
  "hotspot_status_change",
170
212
  "hotspot_error",
171
213
  "photo_response",
172
214
  "photo_status",
215
+ "video_recording_status",
173
216
  "gallery_status",
174
217
  "compatible_glasses_search_stop",
175
218
  "heartbeat_sent",
@@ -179,6 +222,8 @@ class BluetoothSdkModule : Module() {
179
222
  "swipe_volume_status",
180
223
  "switch_status",
181
224
  "rgb_led_control_response",
225
+ "settings_ack",
226
+ "version_info",
182
227
  "pair_failure",
183
228
  "audio_pairing_needed",
184
229
  "audio_connected",
@@ -202,6 +247,7 @@ class BluetoothSdkModule : Module() {
202
247
  "receive_command_from_ble",
203
248
  "miniapp_selected",
204
249
  "captions_tester_incident",
250
+ "extraction_progress",
205
251
  )
206
252
 
207
253
  OnCreate {
@@ -356,29 +402,58 @@ class BluetoothSdkModule : Module() {
356
402
 
357
403
  // MARK: - WiFi Commands
358
404
 
359
- AsyncFunction("requestWifiScan") { sdk?.requestWifiScan() }
405
+ AsyncFunction("requestWifiScan") { requireSdk().requestWifiScan().map { it.toMap() } }
360
406
 
361
407
  AsyncFunction("sendWifiCredentials") { ssid: String, password: String ->
362
- sdk?.sendWifiCredentials(ssid, password)
408
+ requireSdk().sendWifiCredentials(ssid, password).values
363
409
  }
364
410
 
365
- AsyncFunction("forgetWifiNetwork") { ssid: String -> sdk?.forgetWifiNetwork(ssid) }
411
+ AsyncFunction("forgetWifiNetwork") { ssid: String -> requireSdk().forgetWifiNetwork(ssid).values }
366
412
 
367
413
  AsyncFunction("setHotspotState") { enabled: Boolean ->
368
- sdk?.setHotspotState(enabled)
414
+ requireSdk().setHotspotState(enabled).values
415
+ }
416
+
417
+ AsyncFunction("setSystemTime") { timestampMs: Double ->
418
+ sdk?.setSystemTime(timestampMs.toLong())
369
419
  }
370
420
 
371
421
  // MARK: - Gallery Commands
372
422
 
373
423
  AsyncFunction("setGalleryModeEnabled") { enabled: Boolean ->
374
- sdk?.setGalleryModeEnabled(enabled)
424
+ requireSdk().setGalleryModeEnabled(enabled).values
375
425
  }
376
426
 
377
427
  AsyncFunction("setVoiceActivityDetectionEnabled") { enabled: Boolean ->
378
428
  sdk?.setVoiceActivityDetectionEnabled(enabled)
379
429
  }
380
430
 
381
- AsyncFunction("queryGalleryStatus") { sdk?.queryGalleryStatus() }
431
+ AsyncFunction("setButtonPhotoSettings") { size: String ->
432
+ requireSdk().setButtonPhotoSettings(ButtonPhotoSize.fromValue(size)).values
433
+ }
434
+
435
+ AsyncFunction("setButtonVideoRecordingSettings") { width: Int, height: Int, fps: Int ->
436
+ requireSdk().setButtonVideoRecordingSettings(width, height, fps).values
437
+ }
438
+
439
+ AsyncFunction("setButtonCameraLed") { enabled: Boolean ->
440
+ requireSdk().setButtonCameraLed(enabled).values
441
+ }
442
+
443
+ AsyncFunction("setButtonMaxRecordingTime") { minutes: Int ->
444
+ requireSdk().setButtonMaxRecordingTime(minutes).values
445
+ }
446
+
447
+ AsyncFunction("setCameraFov") { fov: Map<String, Any> ->
448
+ val value = (fov["fov"] as? Number)?.toInt() ?: CameraFov.DEFAULT_FOV
449
+ val roiPosition = CameraRoiPosition.fromValue(
450
+ (fov["roiPosition"] as? Number)?.toInt()
451
+ ?: (fov["roi_position"] as? Number)?.toInt(),
452
+ )
453
+ requireSdk().setCameraFov(CameraFov(value, roiPosition)).values
454
+ }
455
+
456
+ AsyncFunction("queryGalleryStatus") { requireSdk().queryGalleryStatus().values }
382
457
 
383
458
  AsyncFunction("requestPhoto") { params: Map<String, Any?> ->
384
459
  // JS may pass null for optional fields; Map<String, Any> rejects null values at the bridge.
@@ -390,25 +465,20 @@ class BluetoothSdkModule : Module() {
390
465
  Bridge.log(
391
466
  "NATIVE: PHOTO PIPELINE [3/6] BluetoothSdk.requestPhoto requestId=${req.requestId} appId=${req.appId} size=${req.size} compress=${req.compress} flash=${req.flash} sound=${req.sound} exposureTimeNs=${req.exposureTimeNs} iso=${req.iso}"
392
467
  )
393
- val activeSdk = sdk
394
- if (activeSdk == null) {
395
- Bridge.log(
396
- "NATIVE: PHOTO PIPELINE — sdk is null; requestPhoto dropped requestId=${req.requestId}"
397
- )
398
- } else {
399
- activeSdk.requestPhoto(req)
400
- }
468
+ requireSdk().requestPhoto(req).values
401
469
  }
402
470
 
403
471
  // MARK: - OTA Commands
404
472
 
405
- AsyncFunction("sendOtaStart") { sdk?.sendOtaStart() }
473
+ AsyncFunction("sendOtaStart") { requireSdk().sendOtaStart().values }
406
474
 
407
- AsyncFunction("sendOtaQueryStatus") { sdk?.sendOtaQueryStatus() }
475
+ AsyncFunction("sendOtaQueryStatus") { requireSdk().sendOtaQueryStatus().values }
476
+
477
+ AsyncFunction("retryOtaVersionCheck") { requireSdk().retryOtaVersionCheck().values }
408
478
 
409
479
  // MARK: - Version Info Commands
410
480
 
411
- AsyncFunction("requestVersionInfo") { sdk?.requestVersionInfo() }
481
+ AsyncFunction("requestVersionInfo") { requireSdk().requestVersionInfo().toMap() }
412
482
 
413
483
  // MARK: - Power Control Commands
414
484
 
@@ -418,24 +488,41 @@ class BluetoothSdkModule : Module() {
418
488
 
419
489
  // MARK: - Video Recording Commands
420
490
 
421
- AsyncFunction("startVideoRecording") { requestId: String, save: Boolean, sound: Boolean ->
422
- sdk?.startVideoRecording(VideoRecordingRequest(requestId, save, sound))
491
+ AsyncFunction("startVideoRecording") {
492
+ requestId: String,
493
+ save: Boolean,
494
+ sound: Boolean,
495
+ settings: Map<String, Any?>? ->
496
+ // Optional per-recording {width,height,fps}. Absent fields stay 0, which
497
+ // the glasses treat as "use the saved button-video default". JS numbers
498
+ // arrive as Double across the bridge, so coerce to Int.
499
+ fun dim(key: String): Int = (settings?.get(key) as? Number)?.toInt() ?: 0
500
+ requireSdk().startVideoRecording(
501
+ VideoRecordingRequest(
502
+ requestId,
503
+ save,
504
+ sound,
505
+ dim("width"),
506
+ dim("height"),
507
+ dim("fps"),
508
+ )
509
+ ).values
423
510
  }
424
511
 
425
512
  AsyncFunction("stopVideoRecording") { requestId: String ->
426
- sdk?.stopVideoRecording(requestId)
513
+ requireSdk().stopVideoRecording(requestId).values
427
514
  }
428
515
 
429
516
  // MARK: - Stream Commands
430
517
 
431
518
  AsyncFunction("startStream") { params: Map<String, Any> ->
432
- sdk?.startStream(StreamRequest.fromMap(params))
519
+ requireSdk().startStream(StreamRequest.fromMap(params)).values
433
520
  }
434
521
 
435
- AsyncFunction("stopStream") { sdk?.stopStream() }
522
+ AsyncFunction("stopStream") { requireSdk().stopStream().values }
436
523
 
437
- AsyncFunction("sendCloudStreamKeepAlive") { params: Map<String, Any> ->
438
- sdk?.sendCloudStreamKeepAlive(StreamKeepAliveRequest.fromMap(params))
524
+ AsyncFunction("sendExternallyManagedStreamKeepAlive") { params: Map<String, Any> ->
525
+ sdk?.sendExternallyManagedStreamKeepAlive(StreamKeepAliveRequest.fromMap(params))
439
526
  }
440
527
 
441
528
  // MARK: - Microphone Commands
@@ -453,7 +540,12 @@ class BluetoothSdkModule : Module() {
453
540
  )
454
541
  }
455
542
 
456
- AsyncFunction("restartTranscriber") { deviceManager?.restartTranscriber() }
543
+ // Runs on Dispatchers.IO, not the shared Expo AsyncFunctionQueue: restart()
544
+ // does a synchronous JNI model reload that would otherwise block every other
545
+ // native call in the app until it completes.
546
+ AsyncFunction("restartTranscriber") Coroutine { ->
547
+ withContext(Dispatchers.IO) { deviceManager?.restartTranscriber() }
548
+ }
457
549
 
458
550
  // MARK: - Audio Playback Monitoring
459
551
 
@@ -461,14 +553,17 @@ class BluetoothSdkModule : Module() {
461
553
  sdk?.setOwnAppAudioPlaying(playing)
462
554
  }
463
555
 
464
- AsyncFunction("getGlassesMediaVolume") {
556
+ // *Blocking on Dispatchers.IO, not the shared AsyncFunctionQueue: these wait on
557
+ // a CountDownLatch (up to 5s) for a BLE round-trip, which would otherwise stall
558
+ // every other native call queued behind them.
559
+ AsyncFunction("getGlassesMediaVolume") Coroutine { ->
465
560
  val cm = deviceManager ?: throw IllegalStateException("device_manager_null")
466
- cm.getGlassesMediaVolumeBlocking()
561
+ withContext(Dispatchers.IO) { cm.getGlassesMediaVolumeBlocking() }
467
562
  }
468
563
 
469
- AsyncFunction("setGlassesMediaVolume") { level: Int ->
564
+ AsyncFunction("setGlassesMediaVolume") Coroutine { level: Int ->
470
565
  val cm = deviceManager ?: throw IllegalStateException("device_manager_null")
471
- cm.setGlassesMediaVolumeBlocking(level)
566
+ withContext(Dispatchers.IO) { cm.setGlassesMediaVolumeBlocking(level) }
472
567
  }
473
568
 
474
569
  // MARK: - RGB LED Control
@@ -481,7 +576,7 @@ class BluetoothSdkModule : Module() {
481
576
  onDurationMs: Int,
482
577
  offDurationMs: Int,
483
578
  count: Int ->
484
- sdk?.rgbLedControl(
579
+ requireSdk().rgbLedControl(
485
580
  RgbLedRequest(
486
581
  requestId = requestId,
487
582
  packageName = packageName,
@@ -491,7 +586,7 @@ class BluetoothSdkModule : Module() {
491
586
  offDurationMs = offDurationMs,
492
587
  count = count,
493
588
  )
494
- )
589
+ ).values
495
590
  }
496
591
 
497
592
  // MARK: - STT Commands
@@ -524,10 +619,76 @@ class BluetoothSdkModule : Module() {
524
619
  com.mentra.bluetoothsdk.stt.STTTools.validateSTTModel(path)
525
620
  }
526
621
 
527
- AsyncFunction("extractTarBz2") { sourcePath: String, destinationPath: String ->
528
- com.mentra.bluetoothsdk.stt.STTTools.extractTarBz2(sourcePath, destinationPath)
622
+ // Runs on Dispatchers.IO, not the shared Expo AsyncFunctionQueue: bz2/tar
623
+ // extraction of the 100–350MB model is a multi-minute, CPU-bound job. On the
624
+ // shared queue it froze every other native call in the app until it finished.
625
+ AsyncFunction("extractTarBz2") Coroutine { sourcePath: String, destinationPath: String ->
626
+ withContext(Dispatchers.IO) {
627
+ com.mentra.bluetoothsdk.stt.STTTools.extractTarBz2(sourcePath, destinationPath)
628
+ }
629
+ }
630
+
631
+ // MARK: - TTS Commands
632
+
633
+ AsyncFunction("setTtsModelDetails") { path: String, languageCode: String ->
634
+ val context =
635
+ appContext.reactContext
636
+ ?: appContext.currentActivity
637
+ ?: throw IllegalStateException("No context available")
638
+ com.mentra.core.tts.TTSTools.setTtsModelDetails(context, path, languageCode)
639
+ }
640
+
641
+ AsyncFunction("getTtsModelPath") { ->
642
+ val context =
643
+ appContext.reactContext
644
+ ?: appContext.currentActivity
645
+ ?: throw IllegalStateException("No context available")
646
+ com.mentra.core.tts.TTSTools.getTtsModelPath(context)
647
+ }
648
+
649
+ AsyncFunction("getTtsModelLanguage") { ->
650
+ val context =
651
+ appContext.reactContext
652
+ ?: appContext.currentActivity
653
+ ?: throw IllegalStateException("No context available")
654
+ com.mentra.core.tts.TTSTools.getTtsModelLanguage(context)
655
+ }
656
+
657
+ AsyncFunction("checkTtsModelAvailable") { ->
658
+ val context =
659
+ appContext.reactContext
660
+ ?: appContext.currentActivity
661
+ ?: throw IllegalStateException("No context available")
662
+ com.mentra.core.tts.TTSTools.checkTTSModelAvailable(context)
663
+ }
664
+
665
+ AsyncFunction("validateTtsModel") { path: String ->
666
+ com.mentra.core.tts.TTSTools.validateTTSModel(path)
529
667
  }
530
668
 
669
+ // Runs on Dispatchers.IO, not the shared Expo AsyncFunctionQueue: TTS synthesis
670
+ // is a synchronous JNI call that would otherwise block other native calls.
671
+ AsyncFunction("generateTtsAudio") Coroutine {
672
+ text: String,
673
+ modelPath: String,
674
+ outputPath: String,
675
+ speakerId: Int,
676
+ speed: Double ->
677
+ val context =
678
+ appContext.reactContext
679
+ ?: appContext.currentActivity
680
+ ?: throw IllegalStateException("No context available")
681
+ withContext(Dispatchers.IO) {
682
+ com.mentra.core.tts.TTSTools.generateTtsAudio(
683
+ context,
684
+ text,
685
+ modelPath,
686
+ outputPath,
687
+ speakerId,
688
+ speed.toFloat()
689
+ )
690
+ }
691
+ }
531
692
  }
532
693
  }
533
694
 
@@ -107,6 +107,16 @@ public class Bridge private constructor() {
107
107
  sendTypedMessage("log", data as Map<String, Any>)
108
108
  }
109
109
 
110
+ /** Report tar.bz2 extraction progress to JavaScript. */
111
+ @JvmStatic
112
+ fun sendExtractionProgress(percentage: Int, bytesRead: Long, totalBytes: Long) {
113
+ val data = HashMap<String, Any>()
114
+ data["percentage"] = percentage
115
+ data["bytesRead"] = bytesRead
116
+ data["totalBytes"] = totalBytes
117
+ sendTypedMessage("extraction_progress", data as Map<String, Any>)
118
+ }
119
+
110
120
  /** Send head position event */
111
121
  @JvmStatic
112
122
  fun sendHeadUp(isUp: Boolean) {
@@ -153,7 +163,7 @@ public class Bridge private constructor() {
153
163
  private fun micPcmEventBody(data: ByteArray): HashMap<String, Any> {
154
164
  val voiceActivityDetectionEnabled =
155
165
  DeviceStore.get("glasses", "voiceActivityDetectionEnabled") as? Boolean
156
- ?: true
166
+ ?: BluetoothSdkDefaults.VOICE_ACTIVITY_DETECTION_ENABLED
157
167
  val body = HashMap<String, Any>()
158
168
  body["pcm"] = data
159
169
  body["sampleRate"] = MIC_SAMPLE_RATE
@@ -167,7 +177,7 @@ public class Bridge private constructor() {
167
177
  private fun micLc3EventBody(data: ByteArray): HashMap<String, Any> {
168
178
  val voiceActivityDetectionEnabled =
169
179
  DeviceStore.get("glasses", "voiceActivityDetectionEnabled") as? Boolean
170
- ?: true
180
+ ?: BluetoothSdkDefaults.VOICE_ACTIVITY_DETECTION_ENABLED
171
181
  val frameSizeBytes =
172
182
  (DeviceStore.store.get("bluetooth", "lc3_frame_size") as? Number)?.toInt()
173
183
  ?: DEFAULT_LC3_FRAME_SIZE_BYTES
@@ -357,15 +367,6 @@ public class Bridge private constructor() {
357
367
  event["errorMessage"] = errorMessage
358
368
  event["timestamp"] = timestamp
359
369
  sendTypedMessage("photo_response", event as Map<String, Any>)
360
-
361
- val status = HashMap<String, Any>()
362
- status["type"] = "photo_status"
363
- status["status"] = "failed"
364
- status["requestId"] = requestId
365
- status["errorCode"] = errorCode
366
- status["errorMessage"] = errorMessage
367
- status["timestamp"] = timestamp
368
- sendTypedMessage("photo_status", status as Map<String, Any>)
369
370
  }
370
371
 
371
372
  @JvmStatic
@@ -373,6 +374,11 @@ public class Bridge private constructor() {
373
374
  sendTypedMessage("photo_status", statusJson)
374
375
  }
375
376
 
377
+ @JvmStatic
378
+ fun sendPhotoResponse(responseJson: Map<String, Any>) {
379
+ sendTypedMessage("photo_response", responseJson)
380
+ }
381
+
376
382
  /** Send RGB LED control response */
377
383
  @JvmStatic
378
384
  fun sendRgbLedControlResponse(requestId: String, success: Boolean, error: String?) {
@@ -391,6 +397,45 @@ public class Bridge private constructor() {
391
397
  }
392
398
  }
393
399
 
400
+ @JvmStatic
401
+ fun sendSettingsAck(values: Map<String, Any>) {
402
+ val body = HashMap<String, Any>()
403
+ body["type"] = "settings_ack"
404
+ values.forEach { (key, value) ->
405
+ body[key] = value
406
+ }
407
+ sendTypedMessage("settings_ack", body)
408
+ }
409
+
410
+ @JvmStatic
411
+ fun sendVideoRecordingStatus(values: Map<String, Any>) {
412
+ val body = HashMap<String, Any>()
413
+ body["type"] = "video_recording_status"
414
+ values.forEach { (key, value) ->
415
+ body[key] = value
416
+ }
417
+ sendTypedMessage("video_recording_status", body)
418
+ }
419
+
420
+ @JvmStatic
421
+ fun sendVersionInfo(values: Map<String, Any>) {
422
+ fun stringField(vararg keys: String): String =
423
+ keys.firstNotNullOfOrNull { key -> values[key] as? String } ?: ""
424
+ val body = HashMap<String, Any>()
425
+ body["type"] = "version_info"
426
+ body["androidVersion"] = stringField("androidVersion", "android_version")
427
+ body["firmwareVersion"] = stringField("firmwareVersion", "firmware_version")
428
+ body["besFirmwareVersion"] = stringField("besFirmwareVersion", "bes_fw_version")
429
+ body["mtkFirmwareVersion"] = stringField("mtkFirmwareVersion", "mtk_fw_version")
430
+ body["buildNumber"] = stringField("buildNumber", "build_number")
431
+ (values["systemTimeMs"] as? Number ?: values["system_time_ms"] as? Number)?.let {
432
+ body["systemTimeMs"] = it.toLong()
433
+ }
434
+ body["otaVersionUrl"] = stringField("otaVersionUrl", "ota_version_url")
435
+ body["appVersion"] = stringField("appVersion", "app_version")
436
+ sendTypedMessage("version_info", body)
437
+ }
438
+
394
439
  /**
395
440
  * Send transcription result to server Used by AOSManager to send pre-formatted
396
441
  * transcription results Matches the Swift structure exactly
@@ -457,7 +502,7 @@ public class Bridge private constructor() {
457
502
 
458
503
  /** Send WiFi scan results */
459
504
  @JvmStatic
460
- fun updateWifiScanResults(networks: List<Map<String, Any>>) {
505
+ fun updateWifiScanResults(networks: List<Map<String, Any>>, scanComplete: Boolean) {
461
506
  var storedNetworks: List<Map<String, Any>> =
462
507
  DeviceStore.get("bluetooth", "wifiScanResults") as? List<Map<String, Any>>
463
508
  ?: emptyList()
@@ -469,6 +514,10 @@ public class Bridge private constructor() {
469
514
  }
470
515
  }
471
516
  DeviceStore.apply("bluetooth", "wifiScanResults", updatedNetworks)
517
+ val body = HashMap<String, Any>()
518
+ body["networks"] = updatedNetworks
519
+ body["scanComplete"] = scanComplete
520
+ sendTypedMessage("wifi_scan_result", body)
472
521
  }
473
522
 
474
523
  /** Send gallery status - matches iOS MentraLive.swift handleGalleryStatus pattern */
@@ -478,14 +527,21 @@ public class Bridge private constructor() {
478
527
  videoCount: Int,
479
528
  totalCount: Int,
480
529
  totalSize: Long,
481
- hasContent: Boolean
530
+ hasContent: Boolean,
531
+ cameraBusy: Boolean,
532
+ cameraBusyReason: String?
482
533
  ) {
483
534
  val galleryData = HashMap<String, Any>()
535
+ galleryData["type"] = "gallery_status"
484
536
  galleryData["photos"] = photoCount
485
537
  galleryData["videos"] = videoCount
486
538
  galleryData["total"] = totalCount
487
539
  galleryData["totalSize"] = totalSize
488
540
  galleryData["hasContent"] = hasContent
541
+ galleryData["cameraBusy"] = cameraBusy
542
+ if (!cameraBusyReason.isNullOrBlank()) {
543
+ galleryData["cameraBusyReason"] = cameraBusyReason
544
+ }
489
545
 
490
546
  sendTypedMessage("gallery_status", galleryData as Map<String, Any>)
491
547
  }
@@ -550,6 +606,7 @@ public class Bridge private constructor() {
550
606
  }
551
607
 
552
608
  @JvmStatic
609
+ @JvmOverloads
553
610
  fun sendOtaStatus(
554
611
  sessionId: String,
555
612
  totalSteps: Int,
@@ -559,7 +616,8 @@ public class Bridge private constructor() {
559
616
  stepPercent: Int,
560
617
  overallPercent: Int,
561
618
  status: String,
562
- errorMessage: String?
619
+ errorMessage: String? = null,
620
+ glassesTimeMs: Long? = null,
563
621
  ) {
564
622
  val eventBody = HashMap<String, Any>()
565
623
  eventBody["session_id"] = sessionId
@@ -571,6 +629,9 @@ public class Bridge private constructor() {
571
629
  eventBody["overall_percent"] = overallPercent
572
630
  eventBody["status"] = status
573
631
  errorMessage?.let { eventBody["error_message"] = it }
632
+ if (glassesTimeMs != null && glassesTimeMs > 0) {
633
+ eventBody["glasses_time_ms"] = glassesTimeMs
634
+ }
574
635
 
575
636
  Log.d(TAG, "Bridge: sendOtaStatus: $eventBody")
576
637
 
@@ -620,6 +681,20 @@ public class Bridge private constructor() {
620
681
  sendTypedMessage("imu_gesture_event", eventBody as Map<String, Any>)
621
682
  }
622
683
 
684
+ /**
685
+ * Send a single accelerometer reading from the glasses IMU - matches iOS
686
+ * Bridge.sendAccelEvent. A richer combined IMU event (gyro + magnetometer) is future work.
687
+ */
688
+ @JvmStatic
689
+ fun sendAccelEvent(x: Float, y: Float, z: Float, timestamp: Long) {
690
+ val body = HashMap<String, Any>()
691
+ body["x"] = x
692
+ body["y"] = y
693
+ body["z"] = z
694
+ body["timestamp"] = timestamp
695
+ sendTypedMessage("accel_event", body as Map<String, Any>)
696
+ }
697
+
623
698
  // Arbitrary WS Comms (don't use these, make a dedicated function for your use case):
624
699
 
625
700
  /** Send WebSocket text message */
@@ -661,13 +736,17 @@ public class Bridge private constructor() {
661
736
  return
662
737
  }
663
738
 
664
- if (type != "log") {
665
- BleTraceLogger.logMap(
666
- "phone_to_app",
667
- "sdk_event_dispatch",
668
- type,
669
- mutableBody as Map<String, Any>,
670
- )
739
+ if (shouldTraceTypedMessage(type)) {
740
+ try {
741
+ BleTraceLogger.logMap(
742
+ "phone_to_app",
743
+ "sdk_event_dispatch",
744
+ type,
745
+ mutableBody as Map<String, Any>,
746
+ )
747
+ } catch (e: Exception) {
748
+ Log.d(TAG, "BLE trace logging failed for typed message '$type'", e)
749
+ }
671
750
  }
672
751
 
673
752
  // Send directly using type as event name - no JSON serialization
@@ -687,6 +766,9 @@ public class Bridge private constructor() {
687
766
  Log.e(TAG, "Error sending typed message of type '$type'", e)
688
767
  }
689
768
  }
769
+
770
+ private fun shouldTraceTypedMessage(type: String): Boolean =
771
+ type != "log" && type != "mic_pcm" && type != "mic_lc3"
690
772
  }
691
773
 
692
774
  init {