@mentra/bluetooth-sdk 0.1.9 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +116 -18
- package/android/build.gradle +289 -16
- package/android/gradle.properties.example +9 -0
- package/android/lc3Lib/build.gradle +95 -6
- package/android/settings.gradle +3 -0
- package/android/silero/LICENSE.md +17 -0
- package/android/silero/build.gradle +42 -0
- package/android/silero/proguard-rules.pro +24 -0
- package/android/silero/src/androidTest/assets/hello.wav +0 -0
- package/android/silero/src/androidTest/java/com/konovalov/vad/silero/VadSileroTest.kt +79 -0
- package/android/silero/src/main/assets/LICENSE +21 -0
- package/android/silero/src/main/assets/silero_vad.onnx +0 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/Vad.kt +160 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/VadSilero.kt +449 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/config/FrameSize.kt +15 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/config/Mode.kt +14 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/config/SampleRate.kt +12 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/utils/AudioUtils.kt +47 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/utils/TensorMap.kt +41 -0
- package/android/silero/src/test/java/com/konovalov/vad/silero/utils/AudioUtilsTest.kt +44 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkDefaults.kt +6 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +195 -34
- package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +103 -21
- package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +210 -18
- package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt +30 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +787 -85
- package/android/src/main/java/com/mentra/bluetoothsdk/audio/AudioModels.kt +6 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/camera/CameraModels.kt +125 -11
- package/android/src/main/java/com/mentra/bluetoothsdk/controllers/ControllerManager.kt +12 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/controllers/R1.kt +8 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/debug/BleTraceLogger.kt +33 -17
- package/android/src/main/java/com/mentra/bluetoothsdk/events/BluetoothEvents.kt +101 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/internal/MapParsing.kt +8 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistry.java +108 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoUploadServer.kt +399 -74
- package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/MentraPhotoReceiverModule.kt +100 -17
- package/android/src/main/java/com/mentra/bluetoothsdk/requests/DisplayRequests.kt +17 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G1.java +2 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt +1293 -758
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Mach1.java +2 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraLive.java +271 -66
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraNex.kt +197 -149
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraosBle.java +12753 -27036
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt +64 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Simulated.kt +16 -4
- package/android/src/main/java/com/mentra/bluetoothsdk/status/DeviceStatus.kt +60 -3
- package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +12 -3
- package/android/src/main/java/com/mentra/bluetoothsdk/stt/STTTools.kt +87 -36
- package/android/src/main/java/com/mentra/bluetoothsdk/stt/SherpaOnnxTranscriber.kt +13 -4
- package/android/src/main/java/com/mentra/bluetoothsdk/stt/VadGateSpeechPolicy.kt +229 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/tts/TTSTools.kt +180 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/AvifExifStripper.java +573 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadService.java +488 -50
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/HeifExifTagReader.java +199 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunkReassembler.java +18 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunker.java +64 -18
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/NexSGCUtils.kt +48 -39
- package/android/src/test/java/com/mentra/bluetoothsdk/camera/PhotoRequestTest.kt +23 -0
- package/android/src/test/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistryTest.java +75 -0
- package/android/src/test/java/com/mentra/bluetoothsdk/utils/AvifExifStripperTest.java +30 -0
- package/android/src/test/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadServiceTest.java +98 -0
- package/android/src/test/resources/avif_with_exif.avif +0 -0
- package/build/BluetoothSdk.types.d.ts +253 -44
- package/build/BluetoothSdk.types.d.ts.map +1 -1
- package/build/BluetoothSdk.types.js +11 -10
- package/build/BluetoothSdk.types.js.map +1 -1
- package/build/_private/BluetoothSdkModule.d.ts +37 -23
- package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
- package/build/_private/BluetoothSdkModule.js +43 -52
- package/build/_private/BluetoothSdkModule.js.map +1 -1
- package/build/_private/photoRequestPayload.d.ts +4 -0
- package/build/_private/photoRequestPayload.d.ts.map +1 -0
- package/build/_private/photoRequestPayload.js +25 -0
- package/build/_private/photoRequestPayload.js.map +1 -0
- package/build/index.d.ts +1 -1
- package/build/index.d.ts.map +1 -1
- package/build/index.js +25 -0
- package/build/index.js.map +1 -1
- package/build/react/useMentraBluetooth.d.ts +2 -2
- package/build/react/useMentraBluetooth.d.ts.map +1 -1
- package/build/react/useMentraBluetooth.js +3 -3
- package/build/react/useMentraBluetooth.js.map +1 -1
- package/ios/BluetoothSdkModule.swift +177 -70
- package/ios/LocalPhotoUploadServer.swift +370 -105
- package/ios/MentraBluetoothSDK.podspec +3 -1
- package/ios/MentraPhotoReceiverModule.swift +62 -10
- package/ios/Packages/CoreObjC/include/PcmConverter.h +22 -0
- package/ios/Packages/SherpaOnnx/SherpaOnnx.swift +668 -26
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Info.plist +13 -9
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/libsherpa-onnx.a +0 -0
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/libsherpa-onnx.a +0 -0
- package/ios/Source/BluetoothSdkDefaults.swift +6 -0
- package/ios/Source/Bridge.swift +68 -19
- package/ios/Source/DeviceManager.swift +167 -32
- package/ios/Source/DeviceStore.swift +24 -8
- package/ios/Source/MentraBluetoothSDK.swift +803 -47
- package/ios/Source/ObservableStore.swift +6 -0
- package/ios/Source/{Audio → audio}/AudioModels.swift +9 -7
- package/ios/Source/{Camera → camera}/CameraModels.swift +226 -16
- package/ios/Source/controllers/ControllerManager.swift +4 -3
- package/ios/Source/controllers/R1.swift +3 -3
- package/ios/Source/{Events → events}/BluetoothEvents.swift +183 -1
- package/ios/Source/{Internal → internal}/ValueParsing.swift +10 -1
- package/ios/Source/{Requests → requests}/DisplayRequests.swift +28 -4
- package/ios/Source/services/PhoneMic.swift +1 -0
- package/ios/Source/sgcs/G1.swift +5 -5
- package/ios/Source/sgcs/G2.swift +1494 -870
- package/ios/Source/sgcs/Mach1.swift +4 -4
- package/ios/Source/sgcs/MentraLive.swift +814 -321
- package/ios/Source/sgcs/MentraNex.swift +604 -244
- package/ios/Source/sgcs/SGCManager.swift +68 -5
- package/ios/Source/sgcs/Simulated.swift +9 -5
- package/ios/Source/sgcs/mentraos_ble.pb.swift +3073 -3134
- package/ios/Source/status/DeviceStatus.swift +974 -0
- package/ios/Source/{Status → status}/WifiHotspotStatus.swift +4 -4
- package/ios/Source/{Streaming → streaming}/StreamModels.swift +24 -9
- package/ios/Source/stt/STTTools.swift +27 -12
- package/ios/Source/stt/SherpaOnnxTranscriber.swift +31 -8
- package/ios/Source/tts/TTSTools.swift +164 -0
- package/ios/Source/utils/JSCExperiment.swift +7 -7
- package/ios/Source/utils/MessageChunkReassembler.swift +20 -1
- package/ios/Source/utils/MessageChunker.swift +78 -21
- package/ios/Source/utils/TarBz2Extractor.swift +53 -2
- package/package.json +1 -1
- package/plugin/build/withAndroid.js +68 -4
- package/src/BluetoothSdk.types.ts +344 -60
- package/src/_private/BluetoothSdkModule.ts +132 -102
- package/src/_private/photoRequestPayload.ts +28 -0
- package/src/index.ts +61 -15
- package/src/react/useMentraBluetooth.ts +6 -4
- package/android/.project +0 -28
- package/android/lc3Lib/.project +0 -28
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/cargs.h +0 -162
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/c-api.h +0 -1852
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/cxx-api.h +0 -674
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/sherpa-onnx.a +0 -0
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/sherpa-onnx.a +0 -0
- package/ios/Source/Status/DeviceStatus.swift +0 -471
- /package/ios/Source/{Connection → connection}/ScanSession.swift +0 -0
- /package/ios/Source/{Errors → errors}/BluetoothError.swift +0 -0
- /package/ios/Source/{Internal → internal}/BluetoothAvailability.swift +0 -0
- /package/ios/Source/{Status → status}/RuntimeState.swift +0 -0
- /package/ios/Source/{Types → types}/DeviceModels.swift +0 -0
|
@@ -2,8 +2,11 @@ package com.mentra.bluetoothsdk
|
|
|
2
2
|
|
|
3
3
|
import com.mentra.bluetoothsdk.debug.BleTraceLogger
|
|
4
4
|
import com.mentra.bluetoothsdk.utils.DeviceTypes
|
|
5
|
+
import expo.modules.kotlin.functions.Coroutine
|
|
5
6
|
import expo.modules.kotlin.modules.Module
|
|
6
7
|
import expo.modules.kotlin.modules.ModuleDefinition
|
|
8
|
+
import kotlinx.coroutines.Dispatchers
|
|
9
|
+
import kotlinx.coroutines.withContext
|
|
7
10
|
|
|
8
11
|
class BluetoothSdkModule : Module() {
|
|
9
12
|
private var sdk: MentraBluetoothSdk? = null
|
|
@@ -113,6 +116,14 @@ class BluetoothSdkModule : Module() {
|
|
|
113
116
|
sendEvent("photo_status", event.values)
|
|
114
117
|
}
|
|
115
118
|
|
|
119
|
+
override fun onVideoRecordingStatus(event: VideoRecordingStatusEvent) {
|
|
120
|
+
sendEvent("video_recording_status", event.values)
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
override fun onRgbLedControlResponse(event: RgbLedControlResponseEvent) {
|
|
124
|
+
sendEvent("rgb_led_control_response", event.values)
|
|
125
|
+
}
|
|
126
|
+
|
|
116
127
|
override fun onStreamStatus(event: StreamStatusEvent) {
|
|
117
128
|
sendEvent("stream_status", event.values)
|
|
118
129
|
}
|
|
@@ -121,6 +132,26 @@ class BluetoothSdkModule : Module() {
|
|
|
121
132
|
sendEvent("keep_alive_ack", event.values)
|
|
122
133
|
}
|
|
123
134
|
|
|
135
|
+
override fun onOtaUpdateAvailable(event: OtaUpdateAvailableEvent) {
|
|
136
|
+
sendEvent("ota_update_available", event.values)
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
override fun onOtaStartAck(event: OtaStartAckEvent) {
|
|
140
|
+
sendEvent("ota_start_ack", event.values)
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
override fun onOtaStatus(event: OtaStatusEvent) {
|
|
144
|
+
sendEvent("ota_status", event.values)
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
override fun onSettingsAck(event: SettingsAckEvent) {
|
|
148
|
+
sendEvent("settings_ack", event.values)
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
override fun onVersionInfo(event: VersionInfoResult) {
|
|
152
|
+
sendEvent("version_info", event.toMap() + mapOf("type" to "version_info"))
|
|
153
|
+
}
|
|
154
|
+
|
|
124
155
|
override fun onMicPcm(event: MicPcmEvent) {
|
|
125
156
|
sendEvent("mic_pcm", event.toMap())
|
|
126
157
|
}
|
|
@@ -144,7 +175,14 @@ class BluetoothSdkModule : Module() {
|
|
|
144
175
|
override fun onRawEvent(eventName: String, values: Map<String, Any>) {
|
|
145
176
|
sendEvent(eventName, values)
|
|
146
177
|
}
|
|
147
|
-
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
private fun requireSdk(): MentraBluetoothSdk =
|
|
181
|
+
sdk
|
|
182
|
+
?: throw BluetoothException(
|
|
183
|
+
"sdk_not_initialized",
|
|
184
|
+
"Bluetooth SDK is not initialized.",
|
|
185
|
+
)
|
|
148
186
|
|
|
149
187
|
override fun definition() = ModuleDefinition {
|
|
150
188
|
Name("BluetoothSdk")
|
|
@@ -160,16 +198,21 @@ class BluetoothSdkModule : Module() {
|
|
|
160
198
|
"glasses_not_ready",
|
|
161
199
|
"button_press",
|
|
162
200
|
"touch_event",
|
|
201
|
+
"accel_event",
|
|
202
|
+
"CompassHeadingEvent",
|
|
203
|
+
"CompassCalibrationEvent",
|
|
163
204
|
"head_up",
|
|
164
205
|
"voice_activity_detection_status",
|
|
165
206
|
"speaking_status",
|
|
166
207
|
"battery_status",
|
|
167
208
|
"local_transcription",
|
|
168
209
|
"wifi_status_change",
|
|
210
|
+
"wifi_scan_result",
|
|
169
211
|
"hotspot_status_change",
|
|
170
212
|
"hotspot_error",
|
|
171
213
|
"photo_response",
|
|
172
214
|
"photo_status",
|
|
215
|
+
"video_recording_status",
|
|
173
216
|
"gallery_status",
|
|
174
217
|
"compatible_glasses_search_stop",
|
|
175
218
|
"heartbeat_sent",
|
|
@@ -179,6 +222,8 @@ class BluetoothSdkModule : Module() {
|
|
|
179
222
|
"swipe_volume_status",
|
|
180
223
|
"switch_status",
|
|
181
224
|
"rgb_led_control_response",
|
|
225
|
+
"settings_ack",
|
|
226
|
+
"version_info",
|
|
182
227
|
"pair_failure",
|
|
183
228
|
"audio_pairing_needed",
|
|
184
229
|
"audio_connected",
|
|
@@ -202,6 +247,7 @@ class BluetoothSdkModule : Module() {
|
|
|
202
247
|
"receive_command_from_ble",
|
|
203
248
|
"miniapp_selected",
|
|
204
249
|
"captions_tester_incident",
|
|
250
|
+
"extraction_progress",
|
|
205
251
|
)
|
|
206
252
|
|
|
207
253
|
OnCreate {
|
|
@@ -356,29 +402,58 @@ class BluetoothSdkModule : Module() {
|
|
|
356
402
|
|
|
357
403
|
// MARK: - WiFi Commands
|
|
358
404
|
|
|
359
|
-
AsyncFunction("requestWifiScan") {
|
|
405
|
+
AsyncFunction("requestWifiScan") { requireSdk().requestWifiScan().map { it.toMap() } }
|
|
360
406
|
|
|
361
407
|
AsyncFunction("sendWifiCredentials") { ssid: String, password: String ->
|
|
362
|
-
|
|
408
|
+
requireSdk().sendWifiCredentials(ssid, password).values
|
|
363
409
|
}
|
|
364
410
|
|
|
365
|
-
AsyncFunction("forgetWifiNetwork") { ssid: String ->
|
|
411
|
+
AsyncFunction("forgetWifiNetwork") { ssid: String -> requireSdk().forgetWifiNetwork(ssid).values }
|
|
366
412
|
|
|
367
413
|
AsyncFunction("setHotspotState") { enabled: Boolean ->
|
|
368
|
-
|
|
414
|
+
requireSdk().setHotspotState(enabled).values
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
AsyncFunction("setSystemTime") { timestampMs: Double ->
|
|
418
|
+
sdk?.setSystemTime(timestampMs.toLong())
|
|
369
419
|
}
|
|
370
420
|
|
|
371
421
|
// MARK: - Gallery Commands
|
|
372
422
|
|
|
373
423
|
AsyncFunction("setGalleryModeEnabled") { enabled: Boolean ->
|
|
374
|
-
|
|
424
|
+
requireSdk().setGalleryModeEnabled(enabled).values
|
|
375
425
|
}
|
|
376
426
|
|
|
377
427
|
AsyncFunction("setVoiceActivityDetectionEnabled") { enabled: Boolean ->
|
|
378
428
|
sdk?.setVoiceActivityDetectionEnabled(enabled)
|
|
379
429
|
}
|
|
380
430
|
|
|
381
|
-
AsyncFunction("
|
|
431
|
+
AsyncFunction("setButtonPhotoSettings") { size: String ->
|
|
432
|
+
requireSdk().setButtonPhotoSettings(ButtonPhotoSize.fromValue(size)).values
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
AsyncFunction("setButtonVideoRecordingSettings") { width: Int, height: Int, fps: Int ->
|
|
436
|
+
requireSdk().setButtonVideoRecordingSettings(width, height, fps).values
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
AsyncFunction("setButtonCameraLed") { enabled: Boolean ->
|
|
440
|
+
requireSdk().setButtonCameraLed(enabled).values
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
AsyncFunction("setButtonMaxRecordingTime") { minutes: Int ->
|
|
444
|
+
requireSdk().setButtonMaxRecordingTime(minutes).values
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
AsyncFunction("setCameraFov") { fov: Map<String, Any> ->
|
|
448
|
+
val value = (fov["fov"] as? Number)?.toInt() ?: CameraFov.DEFAULT_FOV
|
|
449
|
+
val roiPosition = CameraRoiPosition.fromValue(
|
|
450
|
+
(fov["roiPosition"] as? Number)?.toInt()
|
|
451
|
+
?: (fov["roi_position"] as? Number)?.toInt(),
|
|
452
|
+
)
|
|
453
|
+
requireSdk().setCameraFov(CameraFov(value, roiPosition)).values
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
AsyncFunction("queryGalleryStatus") { requireSdk().queryGalleryStatus().values }
|
|
382
457
|
|
|
383
458
|
AsyncFunction("requestPhoto") { params: Map<String, Any?> ->
|
|
384
459
|
// JS may pass null for optional fields; Map<String, Any> rejects null values at the bridge.
|
|
@@ -390,25 +465,20 @@ class BluetoothSdkModule : Module() {
|
|
|
390
465
|
Bridge.log(
|
|
391
466
|
"NATIVE: PHOTO PIPELINE [3/6] BluetoothSdk.requestPhoto requestId=${req.requestId} appId=${req.appId} size=${req.size} compress=${req.compress} flash=${req.flash} sound=${req.sound} exposureTimeNs=${req.exposureTimeNs} iso=${req.iso}"
|
|
392
467
|
)
|
|
393
|
-
|
|
394
|
-
if (activeSdk == null) {
|
|
395
|
-
Bridge.log(
|
|
396
|
-
"NATIVE: PHOTO PIPELINE — sdk is null; requestPhoto dropped requestId=${req.requestId}"
|
|
397
|
-
)
|
|
398
|
-
} else {
|
|
399
|
-
activeSdk.requestPhoto(req)
|
|
400
|
-
}
|
|
468
|
+
requireSdk().requestPhoto(req).values
|
|
401
469
|
}
|
|
402
470
|
|
|
403
471
|
// MARK: - OTA Commands
|
|
404
472
|
|
|
405
|
-
AsyncFunction("sendOtaStart") {
|
|
473
|
+
AsyncFunction("sendOtaStart") { requireSdk().sendOtaStart().values }
|
|
406
474
|
|
|
407
|
-
AsyncFunction("sendOtaQueryStatus") {
|
|
475
|
+
AsyncFunction("sendOtaQueryStatus") { requireSdk().sendOtaQueryStatus().values }
|
|
476
|
+
|
|
477
|
+
AsyncFunction("retryOtaVersionCheck") { requireSdk().retryOtaVersionCheck().values }
|
|
408
478
|
|
|
409
479
|
// MARK: - Version Info Commands
|
|
410
480
|
|
|
411
|
-
AsyncFunction("requestVersionInfo") {
|
|
481
|
+
AsyncFunction("requestVersionInfo") { requireSdk().requestVersionInfo().toMap() }
|
|
412
482
|
|
|
413
483
|
// MARK: - Power Control Commands
|
|
414
484
|
|
|
@@ -418,24 +488,41 @@ class BluetoothSdkModule : Module() {
|
|
|
418
488
|
|
|
419
489
|
// MARK: - Video Recording Commands
|
|
420
490
|
|
|
421
|
-
AsyncFunction("startVideoRecording") {
|
|
422
|
-
|
|
491
|
+
AsyncFunction("startVideoRecording") {
|
|
492
|
+
requestId: String,
|
|
493
|
+
save: Boolean,
|
|
494
|
+
sound: Boolean,
|
|
495
|
+
settings: Map<String, Any?>? ->
|
|
496
|
+
// Optional per-recording {width,height,fps}. Absent fields stay 0, which
|
|
497
|
+
// the glasses treat as "use the saved button-video default". JS numbers
|
|
498
|
+
// arrive as Double across the bridge, so coerce to Int.
|
|
499
|
+
fun dim(key: String): Int = (settings?.get(key) as? Number)?.toInt() ?: 0
|
|
500
|
+
requireSdk().startVideoRecording(
|
|
501
|
+
VideoRecordingRequest(
|
|
502
|
+
requestId,
|
|
503
|
+
save,
|
|
504
|
+
sound,
|
|
505
|
+
dim("width"),
|
|
506
|
+
dim("height"),
|
|
507
|
+
dim("fps"),
|
|
508
|
+
)
|
|
509
|
+
).values
|
|
423
510
|
}
|
|
424
511
|
|
|
425
512
|
AsyncFunction("stopVideoRecording") { requestId: String ->
|
|
426
|
-
|
|
513
|
+
requireSdk().stopVideoRecording(requestId).values
|
|
427
514
|
}
|
|
428
515
|
|
|
429
516
|
// MARK: - Stream Commands
|
|
430
517
|
|
|
431
518
|
AsyncFunction("startStream") { params: Map<String, Any> ->
|
|
432
|
-
|
|
519
|
+
requireSdk().startStream(StreamRequest.fromMap(params)).values
|
|
433
520
|
}
|
|
434
521
|
|
|
435
|
-
AsyncFunction("stopStream") {
|
|
522
|
+
AsyncFunction("stopStream") { requireSdk().stopStream().values }
|
|
436
523
|
|
|
437
|
-
AsyncFunction("
|
|
438
|
-
sdk?.
|
|
524
|
+
AsyncFunction("sendExternallyManagedStreamKeepAlive") { params: Map<String, Any> ->
|
|
525
|
+
sdk?.sendExternallyManagedStreamKeepAlive(StreamKeepAliveRequest.fromMap(params))
|
|
439
526
|
}
|
|
440
527
|
|
|
441
528
|
// MARK: - Microphone Commands
|
|
@@ -453,7 +540,12 @@ class BluetoothSdkModule : Module() {
|
|
|
453
540
|
)
|
|
454
541
|
}
|
|
455
542
|
|
|
456
|
-
|
|
543
|
+
// Runs on Dispatchers.IO, not the shared Expo AsyncFunctionQueue: restart()
|
|
544
|
+
// does a synchronous JNI model reload that would otherwise block every other
|
|
545
|
+
// native call in the app until it completes.
|
|
546
|
+
AsyncFunction("restartTranscriber") Coroutine { ->
|
|
547
|
+
withContext(Dispatchers.IO) { deviceManager?.restartTranscriber() }
|
|
548
|
+
}
|
|
457
549
|
|
|
458
550
|
// MARK: - Audio Playback Monitoring
|
|
459
551
|
|
|
@@ -461,14 +553,17 @@ class BluetoothSdkModule : Module() {
|
|
|
461
553
|
sdk?.setOwnAppAudioPlaying(playing)
|
|
462
554
|
}
|
|
463
555
|
|
|
464
|
-
|
|
556
|
+
// *Blocking on Dispatchers.IO, not the shared AsyncFunctionQueue: these wait on
|
|
557
|
+
// a CountDownLatch (up to 5s) for a BLE round-trip, which would otherwise stall
|
|
558
|
+
// every other native call queued behind them.
|
|
559
|
+
AsyncFunction("getGlassesMediaVolume") Coroutine { ->
|
|
465
560
|
val cm = deviceManager ?: throw IllegalStateException("device_manager_null")
|
|
466
|
-
cm.getGlassesMediaVolumeBlocking()
|
|
561
|
+
withContext(Dispatchers.IO) { cm.getGlassesMediaVolumeBlocking() }
|
|
467
562
|
}
|
|
468
563
|
|
|
469
|
-
AsyncFunction("setGlassesMediaVolume") { level: Int ->
|
|
564
|
+
AsyncFunction("setGlassesMediaVolume") Coroutine { level: Int ->
|
|
470
565
|
val cm = deviceManager ?: throw IllegalStateException("device_manager_null")
|
|
471
|
-
cm.setGlassesMediaVolumeBlocking(level)
|
|
566
|
+
withContext(Dispatchers.IO) { cm.setGlassesMediaVolumeBlocking(level) }
|
|
472
567
|
}
|
|
473
568
|
|
|
474
569
|
// MARK: - RGB LED Control
|
|
@@ -481,7 +576,7 @@ class BluetoothSdkModule : Module() {
|
|
|
481
576
|
onDurationMs: Int,
|
|
482
577
|
offDurationMs: Int,
|
|
483
578
|
count: Int ->
|
|
484
|
-
|
|
579
|
+
requireSdk().rgbLedControl(
|
|
485
580
|
RgbLedRequest(
|
|
486
581
|
requestId = requestId,
|
|
487
582
|
packageName = packageName,
|
|
@@ -491,7 +586,7 @@ class BluetoothSdkModule : Module() {
|
|
|
491
586
|
offDurationMs = offDurationMs,
|
|
492
587
|
count = count,
|
|
493
588
|
)
|
|
494
|
-
)
|
|
589
|
+
).values
|
|
495
590
|
}
|
|
496
591
|
|
|
497
592
|
// MARK: - STT Commands
|
|
@@ -524,10 +619,76 @@ class BluetoothSdkModule : Module() {
|
|
|
524
619
|
com.mentra.bluetoothsdk.stt.STTTools.validateSTTModel(path)
|
|
525
620
|
}
|
|
526
621
|
|
|
527
|
-
|
|
528
|
-
|
|
622
|
+
// Runs on Dispatchers.IO, not the shared Expo AsyncFunctionQueue: bz2/tar
|
|
623
|
+
// extraction of the 100–350MB model is a multi-minute, CPU-bound job. On the
|
|
624
|
+
// shared queue it froze every other native call in the app until it finished.
|
|
625
|
+
AsyncFunction("extractTarBz2") Coroutine { sourcePath: String, destinationPath: String ->
|
|
626
|
+
withContext(Dispatchers.IO) {
|
|
627
|
+
com.mentra.bluetoothsdk.stt.STTTools.extractTarBz2(sourcePath, destinationPath)
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
// MARK: - TTS Commands
|
|
632
|
+
|
|
633
|
+
AsyncFunction("setTtsModelDetails") { path: String, languageCode: String ->
|
|
634
|
+
val context =
|
|
635
|
+
appContext.reactContext
|
|
636
|
+
?: appContext.currentActivity
|
|
637
|
+
?: throw IllegalStateException("No context available")
|
|
638
|
+
com.mentra.core.tts.TTSTools.setTtsModelDetails(context, path, languageCode)
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
AsyncFunction("getTtsModelPath") { ->
|
|
642
|
+
val context =
|
|
643
|
+
appContext.reactContext
|
|
644
|
+
?: appContext.currentActivity
|
|
645
|
+
?: throw IllegalStateException("No context available")
|
|
646
|
+
com.mentra.core.tts.TTSTools.getTtsModelPath(context)
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
AsyncFunction("getTtsModelLanguage") { ->
|
|
650
|
+
val context =
|
|
651
|
+
appContext.reactContext
|
|
652
|
+
?: appContext.currentActivity
|
|
653
|
+
?: throw IllegalStateException("No context available")
|
|
654
|
+
com.mentra.core.tts.TTSTools.getTtsModelLanguage(context)
|
|
655
|
+
}
|
|
656
|
+
|
|
657
|
+
AsyncFunction("checkTtsModelAvailable") { ->
|
|
658
|
+
val context =
|
|
659
|
+
appContext.reactContext
|
|
660
|
+
?: appContext.currentActivity
|
|
661
|
+
?: throw IllegalStateException("No context available")
|
|
662
|
+
com.mentra.core.tts.TTSTools.checkTTSModelAvailable(context)
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
AsyncFunction("validateTtsModel") { path: String ->
|
|
666
|
+
com.mentra.core.tts.TTSTools.validateTTSModel(path)
|
|
529
667
|
}
|
|
530
668
|
|
|
669
|
+
// Runs on Dispatchers.IO, not the shared Expo AsyncFunctionQueue: TTS synthesis
|
|
670
|
+
// is a synchronous JNI call that would otherwise block other native calls.
|
|
671
|
+
AsyncFunction("generateTtsAudio") Coroutine {
|
|
672
|
+
text: String,
|
|
673
|
+
modelPath: String,
|
|
674
|
+
outputPath: String,
|
|
675
|
+
speakerId: Int,
|
|
676
|
+
speed: Double ->
|
|
677
|
+
val context =
|
|
678
|
+
appContext.reactContext
|
|
679
|
+
?: appContext.currentActivity
|
|
680
|
+
?: throw IllegalStateException("No context available")
|
|
681
|
+
withContext(Dispatchers.IO) {
|
|
682
|
+
com.mentra.core.tts.TTSTools.generateTtsAudio(
|
|
683
|
+
context,
|
|
684
|
+
text,
|
|
685
|
+
modelPath,
|
|
686
|
+
outputPath,
|
|
687
|
+
speakerId,
|
|
688
|
+
speed.toFloat()
|
|
689
|
+
)
|
|
690
|
+
}
|
|
691
|
+
}
|
|
531
692
|
}
|
|
532
693
|
}
|
|
533
694
|
|
|
@@ -107,6 +107,16 @@ public class Bridge private constructor() {
|
|
|
107
107
|
sendTypedMessage("log", data as Map<String, Any>)
|
|
108
108
|
}
|
|
109
109
|
|
|
110
|
+
/** Report tar.bz2 extraction progress to JavaScript. */
|
|
111
|
+
@JvmStatic
|
|
112
|
+
fun sendExtractionProgress(percentage: Int, bytesRead: Long, totalBytes: Long) {
|
|
113
|
+
val data = HashMap<String, Any>()
|
|
114
|
+
data["percentage"] = percentage
|
|
115
|
+
data["bytesRead"] = bytesRead
|
|
116
|
+
data["totalBytes"] = totalBytes
|
|
117
|
+
sendTypedMessage("extraction_progress", data as Map<String, Any>)
|
|
118
|
+
}
|
|
119
|
+
|
|
110
120
|
/** Send head position event */
|
|
111
121
|
@JvmStatic
|
|
112
122
|
fun sendHeadUp(isUp: Boolean) {
|
|
@@ -153,7 +163,7 @@ public class Bridge private constructor() {
|
|
|
153
163
|
private fun micPcmEventBody(data: ByteArray): HashMap<String, Any> {
|
|
154
164
|
val voiceActivityDetectionEnabled =
|
|
155
165
|
DeviceStore.get("glasses", "voiceActivityDetectionEnabled") as? Boolean
|
|
156
|
-
?:
|
|
166
|
+
?: BluetoothSdkDefaults.VOICE_ACTIVITY_DETECTION_ENABLED
|
|
157
167
|
val body = HashMap<String, Any>()
|
|
158
168
|
body["pcm"] = data
|
|
159
169
|
body["sampleRate"] = MIC_SAMPLE_RATE
|
|
@@ -167,7 +177,7 @@ public class Bridge private constructor() {
|
|
|
167
177
|
private fun micLc3EventBody(data: ByteArray): HashMap<String, Any> {
|
|
168
178
|
val voiceActivityDetectionEnabled =
|
|
169
179
|
DeviceStore.get("glasses", "voiceActivityDetectionEnabled") as? Boolean
|
|
170
|
-
?:
|
|
180
|
+
?: BluetoothSdkDefaults.VOICE_ACTIVITY_DETECTION_ENABLED
|
|
171
181
|
val frameSizeBytes =
|
|
172
182
|
(DeviceStore.store.get("bluetooth", "lc3_frame_size") as? Number)?.toInt()
|
|
173
183
|
?: DEFAULT_LC3_FRAME_SIZE_BYTES
|
|
@@ -357,15 +367,6 @@ public class Bridge private constructor() {
|
|
|
357
367
|
event["errorMessage"] = errorMessage
|
|
358
368
|
event["timestamp"] = timestamp
|
|
359
369
|
sendTypedMessage("photo_response", event as Map<String, Any>)
|
|
360
|
-
|
|
361
|
-
val status = HashMap<String, Any>()
|
|
362
|
-
status["type"] = "photo_status"
|
|
363
|
-
status["status"] = "failed"
|
|
364
|
-
status["requestId"] = requestId
|
|
365
|
-
status["errorCode"] = errorCode
|
|
366
|
-
status["errorMessage"] = errorMessage
|
|
367
|
-
status["timestamp"] = timestamp
|
|
368
|
-
sendTypedMessage("photo_status", status as Map<String, Any>)
|
|
369
370
|
}
|
|
370
371
|
|
|
371
372
|
@JvmStatic
|
|
@@ -373,6 +374,11 @@ public class Bridge private constructor() {
|
|
|
373
374
|
sendTypedMessage("photo_status", statusJson)
|
|
374
375
|
}
|
|
375
376
|
|
|
377
|
+
@JvmStatic
|
|
378
|
+
fun sendPhotoResponse(responseJson: Map<String, Any>) {
|
|
379
|
+
sendTypedMessage("photo_response", responseJson)
|
|
380
|
+
}
|
|
381
|
+
|
|
376
382
|
/** Send RGB LED control response */
|
|
377
383
|
@JvmStatic
|
|
378
384
|
fun sendRgbLedControlResponse(requestId: String, success: Boolean, error: String?) {
|
|
@@ -391,6 +397,45 @@ public class Bridge private constructor() {
|
|
|
391
397
|
}
|
|
392
398
|
}
|
|
393
399
|
|
|
400
|
+
@JvmStatic
|
|
401
|
+
fun sendSettingsAck(values: Map<String, Any>) {
|
|
402
|
+
val body = HashMap<String, Any>()
|
|
403
|
+
body["type"] = "settings_ack"
|
|
404
|
+
values.forEach { (key, value) ->
|
|
405
|
+
body[key] = value
|
|
406
|
+
}
|
|
407
|
+
sendTypedMessage("settings_ack", body)
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
@JvmStatic
|
|
411
|
+
fun sendVideoRecordingStatus(values: Map<String, Any>) {
|
|
412
|
+
val body = HashMap<String, Any>()
|
|
413
|
+
body["type"] = "video_recording_status"
|
|
414
|
+
values.forEach { (key, value) ->
|
|
415
|
+
body[key] = value
|
|
416
|
+
}
|
|
417
|
+
sendTypedMessage("video_recording_status", body)
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
@JvmStatic
|
|
421
|
+
fun sendVersionInfo(values: Map<String, Any>) {
|
|
422
|
+
fun stringField(vararg keys: String): String =
|
|
423
|
+
keys.firstNotNullOfOrNull { key -> values[key] as? String } ?: ""
|
|
424
|
+
val body = HashMap<String, Any>()
|
|
425
|
+
body["type"] = "version_info"
|
|
426
|
+
body["androidVersion"] = stringField("androidVersion", "android_version")
|
|
427
|
+
body["firmwareVersion"] = stringField("firmwareVersion", "firmware_version")
|
|
428
|
+
body["besFirmwareVersion"] = stringField("besFirmwareVersion", "bes_fw_version")
|
|
429
|
+
body["mtkFirmwareVersion"] = stringField("mtkFirmwareVersion", "mtk_fw_version")
|
|
430
|
+
body["buildNumber"] = stringField("buildNumber", "build_number")
|
|
431
|
+
(values["systemTimeMs"] as? Number ?: values["system_time_ms"] as? Number)?.let {
|
|
432
|
+
body["systemTimeMs"] = it.toLong()
|
|
433
|
+
}
|
|
434
|
+
body["otaVersionUrl"] = stringField("otaVersionUrl", "ota_version_url")
|
|
435
|
+
body["appVersion"] = stringField("appVersion", "app_version")
|
|
436
|
+
sendTypedMessage("version_info", body)
|
|
437
|
+
}
|
|
438
|
+
|
|
394
439
|
/**
|
|
395
440
|
* Send transcription result to server Used by AOSManager to send pre-formatted
|
|
396
441
|
* transcription results Matches the Swift structure exactly
|
|
@@ -457,7 +502,7 @@ public class Bridge private constructor() {
|
|
|
457
502
|
|
|
458
503
|
/** Send WiFi scan results */
|
|
459
504
|
@JvmStatic
|
|
460
|
-
fun updateWifiScanResults(networks: List<Map<String, Any
|
|
505
|
+
fun updateWifiScanResults(networks: List<Map<String, Any>>, scanComplete: Boolean) {
|
|
461
506
|
var storedNetworks: List<Map<String, Any>> =
|
|
462
507
|
DeviceStore.get("bluetooth", "wifiScanResults") as? List<Map<String, Any>>
|
|
463
508
|
?: emptyList()
|
|
@@ -469,6 +514,10 @@ public class Bridge private constructor() {
|
|
|
469
514
|
}
|
|
470
515
|
}
|
|
471
516
|
DeviceStore.apply("bluetooth", "wifiScanResults", updatedNetworks)
|
|
517
|
+
val body = HashMap<String, Any>()
|
|
518
|
+
body["networks"] = updatedNetworks
|
|
519
|
+
body["scanComplete"] = scanComplete
|
|
520
|
+
sendTypedMessage("wifi_scan_result", body)
|
|
472
521
|
}
|
|
473
522
|
|
|
474
523
|
/** Send gallery status - matches iOS MentraLive.swift handleGalleryStatus pattern */
|
|
@@ -478,14 +527,21 @@ public class Bridge private constructor() {
|
|
|
478
527
|
videoCount: Int,
|
|
479
528
|
totalCount: Int,
|
|
480
529
|
totalSize: Long,
|
|
481
|
-
hasContent: Boolean
|
|
530
|
+
hasContent: Boolean,
|
|
531
|
+
cameraBusy: Boolean,
|
|
532
|
+
cameraBusyReason: String?
|
|
482
533
|
) {
|
|
483
534
|
val galleryData = HashMap<String, Any>()
|
|
535
|
+
galleryData["type"] = "gallery_status"
|
|
484
536
|
galleryData["photos"] = photoCount
|
|
485
537
|
galleryData["videos"] = videoCount
|
|
486
538
|
galleryData["total"] = totalCount
|
|
487
539
|
galleryData["totalSize"] = totalSize
|
|
488
540
|
galleryData["hasContent"] = hasContent
|
|
541
|
+
galleryData["cameraBusy"] = cameraBusy
|
|
542
|
+
if (!cameraBusyReason.isNullOrBlank()) {
|
|
543
|
+
galleryData["cameraBusyReason"] = cameraBusyReason
|
|
544
|
+
}
|
|
489
545
|
|
|
490
546
|
sendTypedMessage("gallery_status", galleryData as Map<String, Any>)
|
|
491
547
|
}
|
|
@@ -550,6 +606,7 @@ public class Bridge private constructor() {
|
|
|
550
606
|
}
|
|
551
607
|
|
|
552
608
|
@JvmStatic
|
|
609
|
+
@JvmOverloads
|
|
553
610
|
fun sendOtaStatus(
|
|
554
611
|
sessionId: String,
|
|
555
612
|
totalSteps: Int,
|
|
@@ -559,7 +616,8 @@ public class Bridge private constructor() {
|
|
|
559
616
|
stepPercent: Int,
|
|
560
617
|
overallPercent: Int,
|
|
561
618
|
status: String,
|
|
562
|
-
errorMessage: String?
|
|
619
|
+
errorMessage: String? = null,
|
|
620
|
+
glassesTimeMs: Long? = null,
|
|
563
621
|
) {
|
|
564
622
|
val eventBody = HashMap<String, Any>()
|
|
565
623
|
eventBody["session_id"] = sessionId
|
|
@@ -571,6 +629,9 @@ public class Bridge private constructor() {
|
|
|
571
629
|
eventBody["overall_percent"] = overallPercent
|
|
572
630
|
eventBody["status"] = status
|
|
573
631
|
errorMessage?.let { eventBody["error_message"] = it }
|
|
632
|
+
if (glassesTimeMs != null && glassesTimeMs > 0) {
|
|
633
|
+
eventBody["glasses_time_ms"] = glassesTimeMs
|
|
634
|
+
}
|
|
574
635
|
|
|
575
636
|
Log.d(TAG, "Bridge: sendOtaStatus: $eventBody")
|
|
576
637
|
|
|
@@ -620,6 +681,20 @@ public class Bridge private constructor() {
|
|
|
620
681
|
sendTypedMessage("imu_gesture_event", eventBody as Map<String, Any>)
|
|
621
682
|
}
|
|
622
683
|
|
|
684
|
+
/**
|
|
685
|
+
* Send a single accelerometer reading from the glasses IMU - matches iOS
|
|
686
|
+
* Bridge.sendAccelEvent. A richer combined IMU event (gyro + magnetometer) is future work.
|
|
687
|
+
*/
|
|
688
|
+
@JvmStatic
|
|
689
|
+
fun sendAccelEvent(x: Float, y: Float, z: Float, timestamp: Long) {
|
|
690
|
+
val body = HashMap<String, Any>()
|
|
691
|
+
body["x"] = x
|
|
692
|
+
body["y"] = y
|
|
693
|
+
body["z"] = z
|
|
694
|
+
body["timestamp"] = timestamp
|
|
695
|
+
sendTypedMessage("accel_event", body as Map<String, Any>)
|
|
696
|
+
}
|
|
697
|
+
|
|
623
698
|
// Arbitrary WS Comms (don't use these, make a dedicated function for your use case):
|
|
624
699
|
|
|
625
700
|
/** Send WebSocket text message */
|
|
@@ -661,13 +736,17 @@ public class Bridge private constructor() {
|
|
|
661
736
|
return
|
|
662
737
|
}
|
|
663
738
|
|
|
664
|
-
if (type
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
739
|
+
if (shouldTraceTypedMessage(type)) {
|
|
740
|
+
try {
|
|
741
|
+
BleTraceLogger.logMap(
|
|
742
|
+
"phone_to_app",
|
|
743
|
+
"sdk_event_dispatch",
|
|
744
|
+
type,
|
|
745
|
+
mutableBody as Map<String, Any>,
|
|
746
|
+
)
|
|
747
|
+
} catch (e: Exception) {
|
|
748
|
+
Log.d(TAG, "BLE trace logging failed for typed message '$type'", e)
|
|
749
|
+
}
|
|
671
750
|
}
|
|
672
751
|
|
|
673
752
|
// Send directly using type as event name - no JSON serialization
|
|
@@ -687,6 +766,9 @@ public class Bridge private constructor() {
|
|
|
687
766
|
Log.e(TAG, "Error sending typed message of type '$type'", e)
|
|
688
767
|
}
|
|
689
768
|
}
|
|
769
|
+
|
|
770
|
+
private fun shouldTraceTypedMessage(type: String): Boolean =
|
|
771
|
+
type != "log" && type != "mic_pcm" && type != "mic_lc3"
|
|
690
772
|
}
|
|
691
773
|
|
|
692
774
|
init {
|