@mentra/bluetooth-sdk 0.1.6 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -22
- package/android/.project +28 -0
- package/android/build.gradle +5 -9
- package/android/lc3Lib/.project +28 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +54 -33
- package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +48 -10
- package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +27 -36
- package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt +8 -4
- package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +170 -14
- package/android/src/main/java/com/mentra/bluetoothsdk/audio/AudioModels.kt +6 -6
- package/android/src/main/java/com/mentra/bluetoothsdk/camera/CameraModels.kt +69 -10
- package/android/src/main/java/com/mentra/bluetoothsdk/controllers/ControllerManager.kt +5 -3
- package/android/src/main/java/com/mentra/bluetoothsdk/controllers/R1.kt +2 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/debug/BleTraceLogger.kt +149 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/events/BluetoothEvents.kt +13 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoUploadServer.kt +297 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/MentraPhotoReceiverModule.kt +120 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/services/Foreground.kt +13 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G1.java +1 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt +3 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Mach1.java +1 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraLive.java +269 -28
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraNex.kt +1 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt +8 -3
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Simulated.kt +4 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/status/DeviceStatus.kt +14 -12
- package/android/src/main/java/com/mentra/bluetoothsdk/status/RuntimeState.kt +4 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +131 -6
- package/android/src/main/java/com/mentra/bluetoothsdk/stt/SherpaOnnxTranscriber.kt +1 -3
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunkReassembler.java +105 -0
- package/build/BluetoothSdk.types.d.ts +107 -48
- package/build/BluetoothSdk.types.d.ts.map +1 -1
- package/build/BluetoothSdk.types.js +4 -0
- package/build/BluetoothSdk.types.js.map +1 -1
- package/build/_private/BluetoothSdkModule.d.ts +8 -6
- package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
- package/build/_private/BluetoothSdkModule.js +58 -9
- package/build/_private/BluetoothSdkModule.js.map +1 -1
- package/build/index.d.ts +2 -2
- package/build/index.d.ts.map +1 -1
- package/build/index.js +6 -5
- package/build/index.js.map +1 -1
- package/build/photo-receiver/MentraPhotoReceiver.types.d.ts +18 -0
- package/build/photo-receiver/MentraPhotoReceiver.types.d.ts.map +1 -0
- package/build/photo-receiver/MentraPhotoReceiver.types.js +2 -0
- package/build/photo-receiver/MentraPhotoReceiver.types.js.map +1 -0
- package/build/photo-receiver/MentraPhotoReceiverModule.d.ts +10 -0
- package/build/photo-receiver/MentraPhotoReceiverModule.d.ts.map +1 -0
- package/build/photo-receiver/MentraPhotoReceiverModule.js +3 -0
- package/build/photo-receiver/MentraPhotoReceiverModule.js.map +1 -0
- package/build/photo-receiver/MentraPhotoReceiverModule.web.d.ts +10 -0
- package/build/photo-receiver/MentraPhotoReceiverModule.web.d.ts.map +1 -0
- package/build/photo-receiver/MentraPhotoReceiverModule.web.js +12 -0
- package/build/photo-receiver/MentraPhotoReceiverModule.web.js.map +1 -0
- package/build/photo-receiver/index.d.ts +3 -0
- package/build/photo-receiver/index.d.ts.map +1 -0
- package/build/photo-receiver/index.js +3 -0
- package/build/photo-receiver/index.js.map +1 -0
- package/build/react/useMentraBluetooth.d.ts +5 -3
- package/build/react/useMentraBluetooth.d.ts.map +1 -1
- package/build/react/useMentraBluetooth.js +15 -4
- package/build/react/useMentraBluetooth.js.map +1 -1
- package/expo-module.config.json +5 -2
- package/ios/BluetoothSdkModule.swift +59 -26
- package/ios/LocalPhotoUploadServer.swift +411 -0
- package/ios/MentraBluetoothSDK.podspec +8 -5
- package/ios/MentraPhotoReceiverModule.swift +147 -0
- package/ios/Source/Audio/AudioModels.swift +10 -10
- package/ios/Source/Bridge.swift +42 -12
- package/ios/Source/Camera/CameraModels.swift +74 -19
- package/ios/Source/DeviceManager.swift +23 -98
- package/ios/Source/DeviceStore.swift +12 -4
- package/ios/Source/Events/BluetoothEvents.swift +37 -0
- package/ios/Source/MentraBluetoothSDK.swift +135 -14
- package/ios/Source/Status/DeviceStatus.swift +12 -4
- package/ios/Source/Status/RuntimeState.swift +22 -13
- package/ios/Source/Streaming/StreamModels.swift +239 -45
- package/ios/Source/controllers/ControllerManager.swift +3 -3
- package/ios/Source/controllers/R1.swift +9 -3
- package/ios/Source/sgcs/G1.swift +5 -4
- package/ios/Source/sgcs/G2.swift +5 -6
- package/ios/Source/sgcs/Mach1.swift +1 -1
- package/ios/Source/sgcs/MentraLive.swift +183 -22
- package/ios/Source/sgcs/MentraNex.swift +1 -1
- package/ios/Source/sgcs/SGCManager.swift +11 -3
- package/ios/Source/sgcs/Simulated.swift +2 -2
- package/ios/Source/stt/SherpaOnnxTranscriber.swift +1 -4
- package/ios/Source/utils/MessageChunkReassembler.swift +89 -0
- package/package.json +6 -1
- package/src/BluetoothSdk.types.ts +133 -62
- package/src/_private/BluetoothSdkModule.ts +79 -26
- package/src/index.ts +15 -8
- package/src/photo-receiver/MentraPhotoReceiver.types.ts +20 -0
- package/src/photo-receiver/MentraPhotoReceiverModule.ts +14 -0
- package/src/photo-receiver/MentraPhotoReceiverModule.web.ts +20 -0
- package/src/photo-receiver/index.ts +2 -0
- package/src/react/useMentraBluetooth.ts +19 -7
- package/android/src/main/java/com/mentra/bluetoothsdk/stt/VadGateSpeechPolicy.kt +0 -221
- package/ios/Packages/VAD/Common/VAD/VADQuality.swift +0 -41
- package/ios/Packages/VAD/Common/VAD/VADState.swift +0 -26
- package/ios/Packages/VAD/Common/VAD/VADStrategy.swift +0 -29
- package/ios/Packages/VAD/Common/VAD/VADType.swift +0 -14
- package/ios/Packages/VAD/Data/Configuration.swift +0 -68
- package/ios/Packages/VAD/Data/FrameSize.swift +0 -39
- package/ios/Packages/VAD/Data/Record.swift +0 -13
- package/ios/Packages/VAD/Data/Result.swift +0 -22
- package/ios/Packages/VAD/Data/SampleRate.swift +0 -48
- package/ios/Packages/VAD/Silero/Model/silero_vad.onnx +0 -0
- package/ios/Packages/VAD/Silero/SileroVAD.swift +0 -284
- package/ios/Packages/VAD/Silero/SileroVADStrategy.swift +0 -64
|
@@ -66,6 +66,7 @@ object DeviceStore {
|
|
|
66
66
|
store.set("glasses", "deviceModel", "")
|
|
67
67
|
store.set("glasses", "firmwareVersion", "")
|
|
68
68
|
store.set("glasses", "micEnabled", false)
|
|
69
|
+
store.set("glasses", "voiceActivityDetectionEnabled", true)
|
|
69
70
|
store.set("glasses", "bluetoothClassicConnected", false)
|
|
70
71
|
store.set("glasses", "caseRemoved", true)
|
|
71
72
|
store.set("glasses", "caseOpen", true)
|
|
@@ -121,12 +122,13 @@ object DeviceStore {
|
|
|
121
122
|
store.set("bluetooth", "dashboard_depth", 2)
|
|
122
123
|
store.set("bluetooth", "head_up_angle", 30)
|
|
123
124
|
store.set("bluetooth", "contextual_dashboard", true)
|
|
124
|
-
store.set("bluetooth", "
|
|
125
|
+
store.set("bluetooth", "gallery_mode", true)
|
|
126
|
+
store.set("bluetooth", "voice_activity_detection_enabled", true)
|
|
125
127
|
store.set("bluetooth", "screen_disabled", false)
|
|
126
128
|
store.set("bluetooth", "button_photo_size", "medium")
|
|
127
129
|
store.set("bluetooth", "button_camera_led", true)
|
|
128
130
|
store.set("bluetooth", "button_max_recording_time", 10)
|
|
129
|
-
store.set("bluetooth", "camera_fov", mapOf("fov" to 118, "
|
|
131
|
+
store.set("bluetooth", "camera_fov", mapOf("fov" to 118, "roi_position" to 0))
|
|
130
132
|
store.set("bluetooth", "button_video_width", 1280)
|
|
131
133
|
store.set("bluetooth", "button_video_height", 720)
|
|
132
134
|
store.set("bluetooth", "button_video_fps", 30)
|
|
@@ -137,7 +139,6 @@ object DeviceStore {
|
|
|
137
139
|
store.set("bluetooth", "should_send_pcm", false)
|
|
138
140
|
store.set("bluetooth", "should_send_lc3", false)
|
|
139
141
|
store.set("bluetooth", "should_send_transcript", false)
|
|
140
|
-
store.set("bluetooth", "bypass_vad", true)
|
|
141
142
|
store.set("bluetooth", "use_native_dashboard", false)
|
|
142
143
|
}
|
|
143
144
|
|
|
@@ -240,9 +241,12 @@ object DeviceStore {
|
|
|
240
241
|
DeviceManager.getInstance().sgc?.setDashboardMenu(items)
|
|
241
242
|
}
|
|
242
243
|
}
|
|
243
|
-
"bluetooth" to "
|
|
244
|
+
"bluetooth" to "gallery_mode" -> {
|
|
244
245
|
DeviceManager.getInstance().sgc?.sendGalleryMode()
|
|
245
246
|
}
|
|
247
|
+
"bluetooth" to "voice_activity_detection_enabled" -> {
|
|
248
|
+
DeviceManager.getInstance().sgc?.sendVoiceActivityDetectionSetting()
|
|
249
|
+
}
|
|
246
250
|
"bluetooth" to "screen_disabled" -> {
|
|
247
251
|
(value as? Boolean)?.let { disabled ->
|
|
248
252
|
if (disabled) {
|
|
@@ -7,6 +7,7 @@ import android.os.Looper
|
|
|
7
7
|
import com.mentra.bluetoothsdk.utils.ControllerTypes
|
|
8
8
|
import com.mentra.bluetoothsdk.utils.PhoneAudioMonitor
|
|
9
9
|
import java.util.Collections
|
|
10
|
+
import java.util.UUID
|
|
10
11
|
import java.util.concurrent.atomic.AtomicBoolean
|
|
11
12
|
|
|
12
13
|
class MentraBluetoothSdk private constructor(
|
|
@@ -23,6 +24,8 @@ class MentraBluetoothSdk private constructor(
|
|
|
23
24
|
private val bridgeEventSinkId: String
|
|
24
25
|
private val storeListenerId: String
|
|
25
26
|
private var suppressDefaultDeviceEvents = false
|
|
27
|
+
private val streamKeepAliveLock = Any()
|
|
28
|
+
private var activeStreamKeepAlive: ActiveStreamKeepAlive? = null
|
|
26
29
|
|
|
27
30
|
init {
|
|
28
31
|
listeners.add(listener)
|
|
@@ -36,6 +39,8 @@ class MentraBluetoothSdk private constructor(
|
|
|
36
39
|
private val DEFAULT_DEVICE_KEYS = setOf("default_wearable", "device_name", "device_address")
|
|
37
40
|
private val SCAN_STATE_KEYS = setOf("searching", "searchingController", "searchResults")
|
|
38
41
|
private const val DEFAULT_SCAN_TIMEOUT_MS = 15_000L
|
|
42
|
+
private const val DEFAULT_STREAM_KEEP_ALIVE_INTERVAL_SECONDS = 5
|
|
43
|
+
private const val MAX_MISSED_STREAM_KEEP_ALIVE_ACKS = 3
|
|
39
44
|
|
|
40
45
|
@JvmStatic
|
|
41
46
|
fun create(
|
|
@@ -51,6 +56,14 @@ class MentraBluetoothSdk private constructor(
|
|
|
51
56
|
): MentraBluetoothSdk = MentraBluetoothSdk(context, config, listener)
|
|
52
57
|
}
|
|
53
58
|
|
|
59
|
+
private data class ActiveStreamKeepAlive(
|
|
60
|
+
val streamId: String,
|
|
61
|
+
val intervalMs: Long,
|
|
62
|
+
var pendingAckId: String? = null,
|
|
63
|
+
var missedAckCount: Int = 0,
|
|
64
|
+
var nextTick: Runnable? = null,
|
|
65
|
+
)
|
|
66
|
+
|
|
54
67
|
fun addListener(listener: MentraBluetoothSdkListener) {
|
|
55
68
|
listeners.add(listener)
|
|
56
69
|
}
|
|
@@ -308,8 +321,12 @@ class MentraBluetoothSdk private constructor(
|
|
|
308
321
|
DeviceStore.apply(ObservableStore.BLUETOOTH_CATEGORY, "screen_disabled", disabled)
|
|
309
322
|
}
|
|
310
323
|
|
|
311
|
-
fun
|
|
312
|
-
DeviceStore.apply(ObservableStore.BLUETOOTH_CATEGORY, "
|
|
324
|
+
fun setGalleryModeEnabled(enabled: Boolean) {
|
|
325
|
+
DeviceStore.apply(ObservableStore.BLUETOOTH_CATEGORY, "gallery_mode", enabled)
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
fun setVoiceActivityDetectionEnabled(enabled: Boolean) {
|
|
329
|
+
DeviceStore.apply(ObservableStore.BLUETOOTH_CATEGORY, "voice_activity_detection_enabled", enabled)
|
|
313
330
|
}
|
|
314
331
|
|
|
315
332
|
fun setButtonPhotoSettings(size: ButtonPhotoSize) {
|
|
@@ -320,14 +337,14 @@ class MentraBluetoothSdk private constructor(
|
|
|
320
337
|
setButtonPhotoSettings(size = settings.size)
|
|
321
338
|
}
|
|
322
339
|
|
|
323
|
-
fun setButtonVideoRecordingSettings(width: Int, height: Int,
|
|
340
|
+
fun setButtonVideoRecordingSettings(width: Int, height: Int, fps: Int) {
|
|
324
341
|
DeviceStore.apply(ObservableStore.BLUETOOTH_CATEGORY, "button_video_width", width)
|
|
325
342
|
DeviceStore.apply(ObservableStore.BLUETOOTH_CATEGORY, "button_video_height", height)
|
|
326
|
-
DeviceStore.apply(ObservableStore.BLUETOOTH_CATEGORY, "button_video_fps",
|
|
343
|
+
DeviceStore.apply(ObservableStore.BLUETOOTH_CATEGORY, "button_video_fps", fps)
|
|
327
344
|
}
|
|
328
345
|
|
|
329
346
|
fun setButtonVideoRecordingSettings(settings: ButtonVideoRecordingSettings) {
|
|
330
|
-
setButtonVideoRecordingSettings(width = settings.width, height = settings.height,
|
|
347
|
+
setButtonVideoRecordingSettings(width = settings.width, height = settings.height, fps = settings.fps)
|
|
331
348
|
}
|
|
332
349
|
|
|
333
350
|
fun setButtonCameraLed(enabled: Boolean) {
|
|
@@ -342,14 +359,13 @@ class MentraBluetoothSdk private constructor(
|
|
|
342
359
|
DeviceStore.apply(
|
|
343
360
|
ObservableStore.BLUETOOTH_CATEGORY,
|
|
344
361
|
"camera_fov",
|
|
345
|
-
mapOf("fov" to fov.fov, "
|
|
362
|
+
mapOf("fov" to fov.fov, "roi_position" to fov.roiPosition),
|
|
346
363
|
)
|
|
347
364
|
}
|
|
348
365
|
|
|
349
366
|
fun setMicState(
|
|
350
367
|
enabled: Boolean,
|
|
351
368
|
useGlassesMic: Boolean = true,
|
|
352
|
-
bypassVad: Boolean = true,
|
|
353
369
|
sendTranscript: Boolean = false,
|
|
354
370
|
sendLc3Data: Boolean = false,
|
|
355
371
|
) {
|
|
@@ -363,7 +379,6 @@ class MentraBluetoothSdk private constructor(
|
|
|
363
379
|
applyMicState(
|
|
364
380
|
sendPcmData = enabled,
|
|
365
381
|
sendTranscript = enabled && sendTranscript,
|
|
366
|
-
bypassVad = bypassVad,
|
|
367
382
|
sendLc3Data = enabled && sendLc3Data,
|
|
368
383
|
)
|
|
369
384
|
}
|
|
@@ -371,13 +386,11 @@ class MentraBluetoothSdk private constructor(
|
|
|
371
386
|
private fun applyMicState(
|
|
372
387
|
sendPcmData: Boolean,
|
|
373
388
|
sendTranscript: Boolean,
|
|
374
|
-
bypassVad: Boolean,
|
|
375
389
|
sendLc3Data: Boolean,
|
|
376
390
|
) {
|
|
377
391
|
DeviceStore.apply(ObservableStore.BLUETOOTH_CATEGORY, "should_send_pcm", sendPcmData)
|
|
378
392
|
DeviceStore.apply(ObservableStore.BLUETOOTH_CATEGORY, "should_send_lc3", sendLc3Data)
|
|
379
393
|
DeviceStore.apply(ObservableStore.BLUETOOTH_CATEGORY, "should_send_transcript", sendTranscript)
|
|
380
|
-
DeviceStore.apply(ObservableStore.BLUETOOTH_CATEGORY, "bypass_vad", bypassVad)
|
|
381
394
|
deviceManager.setMicState()
|
|
382
395
|
}
|
|
383
396
|
|
|
@@ -414,6 +427,9 @@ class MentraBluetoothSdk private constructor(
|
|
|
414
427
|
}
|
|
415
428
|
|
|
416
429
|
fun requestPhoto(request: PhotoRequest) {
|
|
430
|
+
Bridge.log(
|
|
431
|
+
"NATIVE: PHOTO PIPELINE [3b/6] MentraBluetoothSdk.requestPhoto requestId=${request.requestId} appId=${request.appId}"
|
|
432
|
+
)
|
|
417
433
|
deviceManager.requestPhoto(
|
|
418
434
|
request.requestId,
|
|
419
435
|
request.appId,
|
|
@@ -421,7 +437,10 @@ class MentraBluetoothSdk private constructor(
|
|
|
421
437
|
request.webhookUrl,
|
|
422
438
|
request.authToken,
|
|
423
439
|
request.compress.value,
|
|
440
|
+
request.flash,
|
|
424
441
|
request.sound,
|
|
442
|
+
request.exposureTimeNs,
|
|
443
|
+
request.iso,
|
|
425
444
|
)
|
|
426
445
|
}
|
|
427
446
|
|
|
@@ -430,10 +449,18 @@ class MentraBluetoothSdk private constructor(
|
|
|
430
449
|
}
|
|
431
450
|
|
|
432
451
|
fun startStream(request: StreamRequest) {
|
|
433
|
-
|
|
452
|
+
val message = request.toMap().toMutableMap()
|
|
453
|
+
val streamId = (message["streamId"] as? String)?.takeIf { it.isNotBlank() }
|
|
454
|
+
?: "sdk-${UUID.randomUUID()}"
|
|
455
|
+
message["streamId"] = streamId
|
|
456
|
+
stopStreamKeepAliveMonitor()
|
|
457
|
+
deviceManager.startStream(message)
|
|
458
|
+
if (request.keepAlive && !request.isExternallyManagedKeepAlive()) {
|
|
459
|
+
startStreamKeepAliveMonitor(streamId, request.keepAliveIntervalSeconds)
|
|
460
|
+
}
|
|
434
461
|
}
|
|
435
462
|
|
|
436
|
-
fun
|
|
463
|
+
internal fun sendCloudStreamKeepAlive(request: StreamKeepAliveRequest) {
|
|
437
464
|
deviceManager.keepStreamAlive(request.toMap().toMutableMap())
|
|
438
465
|
}
|
|
439
466
|
|
|
@@ -450,6 +477,7 @@ class MentraBluetoothSdk private constructor(
|
|
|
450
477
|
}
|
|
451
478
|
|
|
452
479
|
fun stopStream() {
|
|
480
|
+
stopStreamKeepAliveMonitor()
|
|
453
481
|
deviceManager.stopStream()
|
|
454
482
|
}
|
|
455
483
|
|
|
@@ -486,6 +514,7 @@ class MentraBluetoothSdk private constructor(
|
|
|
486
514
|
}
|
|
487
515
|
|
|
488
516
|
override fun close() {
|
|
517
|
+
stopStreamKeepAliveMonitor()
|
|
489
518
|
Bridge.removeEventSink(bridgeEventSinkId)
|
|
490
519
|
DeviceStore.store.removeListener(storeListenerId)
|
|
491
520
|
listeners.clear()
|
|
@@ -605,6 +634,25 @@ class MentraBluetoothSdk private constructor(
|
|
|
605
634
|
}
|
|
606
635
|
}
|
|
607
636
|
"head_up" -> dispatchToListeners { it.onHeadUpChanged(data["up"] as? Boolean ?: false) }
|
|
637
|
+
"voice_activity_detection_status" ->
|
|
638
|
+
dispatchToListeners {
|
|
639
|
+
it.onVoiceActivityDetectionStatus(
|
|
640
|
+
VoiceActivityDetectionStatusEvent(
|
|
641
|
+
voiceActivityDetectionEnabled =
|
|
642
|
+
data["voiceActivityDetectionEnabled"] as? Boolean ?: true,
|
|
643
|
+
values = data,
|
|
644
|
+
)
|
|
645
|
+
)
|
|
646
|
+
}
|
|
647
|
+
"speaking_status" ->
|
|
648
|
+
dispatchToListeners {
|
|
649
|
+
it.onSpeakingStatus(
|
|
650
|
+
SpeakingStatusEvent(
|
|
651
|
+
speaking = data["speaking"] as? Boolean ?: false,
|
|
652
|
+
values = data,
|
|
653
|
+
)
|
|
654
|
+
)
|
|
655
|
+
}
|
|
608
656
|
"battery_status" ->
|
|
609
657
|
dispatchToListeners {
|
|
610
658
|
it.onBatteryStatus(
|
|
@@ -620,8 +668,18 @@ class MentraBluetoothSdk private constructor(
|
|
|
620
668
|
"hotspot_error" -> dispatchToListeners { it.onHotspotError(HotspotErrorEvent(data)) }
|
|
621
669
|
"gallery_status" -> dispatchToListeners { it.onGalleryStatus(GalleryStatusEvent(data)) }
|
|
622
670
|
"photo_response" -> dispatchToListeners { it.onPhotoResponse(PhotoResponseEvent(data)) }
|
|
623
|
-
"
|
|
624
|
-
"
|
|
671
|
+
"photo_status" -> dispatchToListeners { it.onPhotoStatus(PhotoStatusEvent(data)) }
|
|
672
|
+
"stream_status" -> {
|
|
673
|
+
val event = StreamStatusEvent(data)
|
|
674
|
+
handleStreamStatusForKeepAlive(event.status)
|
|
675
|
+
dispatchToListeners { it.onStreamStatus(event) }
|
|
676
|
+
}
|
|
677
|
+
"keep_alive_ack" -> {
|
|
678
|
+
val event = KeepAliveAckEvent(data)
|
|
679
|
+
if (!handleStreamKeepAliveAck(event)) {
|
|
680
|
+
dispatchToListeners { it.onKeepAliveAck(event) }
|
|
681
|
+
}
|
|
682
|
+
}
|
|
625
683
|
"mic_pcm" -> {
|
|
626
684
|
val event = MicPcmEvent(data)
|
|
627
685
|
if (event.pcm.isNotEmpty()) {
|
|
@@ -676,4 +734,102 @@ class MentraBluetoothSdk private constructor(
|
|
|
676
734
|
deliver()
|
|
677
735
|
}
|
|
678
736
|
}
|
|
737
|
+
|
|
738
|
+
private fun startStreamKeepAliveMonitor(streamId: String, requestedIntervalSeconds: Int) {
|
|
739
|
+
val intervalSeconds =
|
|
740
|
+
requestedIntervalSeconds.takeIf { it > 0 } ?: DEFAULT_STREAM_KEEP_ALIVE_INTERVAL_SECONDS
|
|
741
|
+
val tracker = ActiveStreamKeepAlive(
|
|
742
|
+
streamId = streamId,
|
|
743
|
+
intervalMs = intervalSeconds * 1_000L,
|
|
744
|
+
)
|
|
745
|
+
synchronized(streamKeepAliveLock) {
|
|
746
|
+
activeStreamKeepAlive = tracker
|
|
747
|
+
}
|
|
748
|
+
sendNextStreamKeepAlive(tracker)
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
private fun stopStreamKeepAliveMonitor() {
|
|
752
|
+
val tracker =
|
|
753
|
+
synchronized(streamKeepAliveLock) {
|
|
754
|
+
val current = activeStreamKeepAlive
|
|
755
|
+
activeStreamKeepAlive = null
|
|
756
|
+
current
|
|
757
|
+
}
|
|
758
|
+
tracker?.nextTick?.let { mainHandler.removeCallbacks(it) }
|
|
759
|
+
}
|
|
760
|
+
|
|
761
|
+
private fun sendNextStreamKeepAlive(tracker: ActiveStreamKeepAlive) {
|
|
762
|
+
var timeoutEvent: StreamStatusEvent? = null
|
|
763
|
+
var request: StreamKeepAliveRequest? = null
|
|
764
|
+
|
|
765
|
+
synchronized(streamKeepAliveLock) {
|
|
766
|
+
if (activeStreamKeepAlive !== tracker) {
|
|
767
|
+
return
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
if (tracker.pendingAckId != null) {
|
|
771
|
+
tracker.missedAckCount += 1
|
|
772
|
+
if (tracker.missedAckCount >= MAX_MISSED_STREAM_KEEP_ALIVE_ACKS) {
|
|
773
|
+
activeStreamKeepAlive = null
|
|
774
|
+
tracker.nextTick?.let { mainHandler.removeCallbacks(it) }
|
|
775
|
+
timeoutEvent =
|
|
776
|
+
StreamStatusEvent(
|
|
777
|
+
StreamStatus.Error(
|
|
778
|
+
streamId = tracker.streamId,
|
|
779
|
+
errorDetails =
|
|
780
|
+
"Stream keep-alive timed out after ${tracker.missedAckCount} missed ACKs",
|
|
781
|
+
timestamp = System.currentTimeMillis(),
|
|
782
|
+
resolvedConfig = null,
|
|
783
|
+
)
|
|
784
|
+
)
|
|
785
|
+
return@synchronized
|
|
786
|
+
}
|
|
787
|
+
}
|
|
788
|
+
|
|
789
|
+
val ackId = "ack-${System.currentTimeMillis()}"
|
|
790
|
+
tracker.pendingAckId = ackId
|
|
791
|
+
request = StreamKeepAliveRequest(streamId = tracker.streamId, ackId = ackId)
|
|
792
|
+
val nextTick = Runnable { sendNextStreamKeepAlive(tracker) }
|
|
793
|
+
tracker.nextTick = nextTick
|
|
794
|
+
mainHandler.postDelayed(nextTick, tracker.intervalMs)
|
|
795
|
+
}
|
|
796
|
+
|
|
797
|
+
timeoutEvent?.let { event ->
|
|
798
|
+
dispatchToListeners { it.onStreamStatus(event) }
|
|
799
|
+
return
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
request?.let { keepAlive ->
|
|
803
|
+
deviceManager.keepStreamAlive(keepAlive.toMap().toMutableMap())
|
|
804
|
+
}
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
private fun handleStreamKeepAliveAck(event: KeepAliveAckEvent): Boolean {
|
|
808
|
+
synchronized(streamKeepAliveLock) {
|
|
809
|
+
val tracker = activeStreamKeepAlive ?: return false
|
|
810
|
+
if (event.streamId != tracker.streamId || event.ackId != tracker.pendingAckId) {
|
|
811
|
+
return false
|
|
812
|
+
}
|
|
813
|
+
tracker.pendingAckId = null
|
|
814
|
+
tracker.missedAckCount = 0
|
|
815
|
+
return true
|
|
816
|
+
}
|
|
817
|
+
}
|
|
818
|
+
|
|
819
|
+
private fun handleStreamStatusForKeepAlive(status: StreamStatus) {
|
|
820
|
+
val streamId = status.streamId
|
|
821
|
+
if (
|
|
822
|
+
streamId != null &&
|
|
823
|
+
synchronized(streamKeepAliveLock) { activeStreamKeepAlive?.streamId } != streamId
|
|
824
|
+
) {
|
|
825
|
+
return
|
|
826
|
+
}
|
|
827
|
+
when (status.state) {
|
|
828
|
+
StreamState.STOPPED,
|
|
829
|
+
StreamState.STOPPING,
|
|
830
|
+
StreamState.ERROR,
|
|
831
|
+
StreamState.RECONNECT_FAILED -> stopStreamKeepAliveMonitor()
|
|
832
|
+
else -> Unit
|
|
833
|
+
}
|
|
834
|
+
}
|
|
679
835
|
}
|
|
@@ -13,7 +13,7 @@ data class MicPcmEvent(
|
|
|
13
13
|
val bitsPerSample: Int,
|
|
14
14
|
val channels: Int,
|
|
15
15
|
val encoding: String,
|
|
16
|
-
val
|
|
16
|
+
val voiceActivityDetectionEnabled: Boolean,
|
|
17
17
|
) {
|
|
18
18
|
constructor(values: Map<String, Any>) : this(
|
|
19
19
|
pcm = values["pcm"] as? ByteArray ?: ByteArray(0),
|
|
@@ -21,7 +21,7 @@ data class MicPcmEvent(
|
|
|
21
21
|
bitsPerSample = numberValue(values, "bitsPerSample") ?: BITS_PER_SAMPLE,
|
|
22
22
|
channels = numberValue(values, "channels") ?: CHANNELS,
|
|
23
23
|
encoding = stringValue(values, "encoding") ?: ENCODING,
|
|
24
|
-
|
|
24
|
+
voiceActivityDetectionEnabled = boolValue(values, "voiceActivityDetectionEnabled") ?: true,
|
|
25
25
|
)
|
|
26
26
|
|
|
27
27
|
fun toMap(): Map<String, Any> =
|
|
@@ -32,7 +32,7 @@ data class MicPcmEvent(
|
|
|
32
32
|
"bitsPerSample" to bitsPerSample,
|
|
33
33
|
"channels" to channels,
|
|
34
34
|
"encoding" to encoding,
|
|
35
|
-
"
|
|
35
|
+
"voiceActivityDetectionEnabled" to voiceActivityDetectionEnabled,
|
|
36
36
|
)
|
|
37
37
|
|
|
38
38
|
companion object {
|
|
@@ -52,7 +52,7 @@ data class MicLc3Event(
|
|
|
52
52
|
val frameSizeBytes: Int,
|
|
53
53
|
val bitrate: Int,
|
|
54
54
|
val packetizedFromGlasses: Boolean,
|
|
55
|
-
val
|
|
55
|
+
val voiceActivityDetectionEnabled: Boolean,
|
|
56
56
|
) {
|
|
57
57
|
constructor(values: Map<String, Any>) : this(
|
|
58
58
|
lc3 = values["lc3"] as? ByteArray ?: ByteArray(0),
|
|
@@ -63,7 +63,7 @@ data class MicLc3Event(
|
|
|
63
63
|
frameSizeBytes = numberValue(values, "frameSizeBytes") ?: DEFAULT_FRAME_SIZE_BYTES,
|
|
64
64
|
bitrate = numberValue(values, "bitrate") ?: DEFAULT_BITRATE,
|
|
65
65
|
packetizedFromGlasses = boolValue(values, "packetizedFromGlasses") ?: false,
|
|
66
|
-
|
|
66
|
+
voiceActivityDetectionEnabled = boolValue(values, "voiceActivityDetectionEnabled") ?: true,
|
|
67
67
|
)
|
|
68
68
|
|
|
69
69
|
fun toMap(): Map<String, Any> =
|
|
@@ -77,7 +77,7 @@ data class MicLc3Event(
|
|
|
77
77
|
"frameSizeBytes" to frameSizeBytes,
|
|
78
78
|
"bitrate" to bitrate,
|
|
79
79
|
"packetizedFromGlasses" to packetizedFromGlasses,
|
|
80
|
-
"
|
|
80
|
+
"voiceActivityDetectionEnabled" to voiceActivityDetectionEnabled,
|
|
81
81
|
)
|
|
82
82
|
|
|
83
83
|
companion object {
|
|
@@ -1,10 +1,5 @@
|
|
|
1
1
|
package com.mentra.bluetoothsdk
|
|
2
2
|
|
|
3
|
-
enum class GalleryMode {
|
|
4
|
-
AUTO,
|
|
5
|
-
MANUAL,
|
|
6
|
-
}
|
|
7
|
-
|
|
8
3
|
enum class PhotoSize(val value: String) {
|
|
9
4
|
SMALL("small"),
|
|
10
5
|
MEDIUM("medium"),
|
|
@@ -49,12 +44,24 @@ data class ButtonPhotoSettings(
|
|
|
49
44
|
data class ButtonVideoRecordingSettings(
|
|
50
45
|
val width: Int,
|
|
51
46
|
val height: Int,
|
|
52
|
-
val
|
|
47
|
+
val fps: Int,
|
|
53
48
|
)
|
|
54
49
|
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
50
|
+
class CameraFov @JvmOverloads constructor(
|
|
51
|
+
fov: Int = DEFAULT_FOV,
|
|
52
|
+
roiPosition: Int = DEFAULT_ROI_POSITION,
|
|
53
|
+
) {
|
|
54
|
+
val fov: Int = fov.coerceIn(MIN_FOV, MAX_FOV)
|
|
55
|
+
val roiPosition: Int = roiPosition.coerceIn(MIN_ROI_POSITION, MAX_ROI_POSITION)
|
|
56
|
+
|
|
57
|
+
companion object {
|
|
58
|
+
const val MIN_FOV = 82
|
|
59
|
+
const val MAX_FOV = 118
|
|
60
|
+
const val DEFAULT_FOV = 102
|
|
61
|
+
const val MIN_ROI_POSITION = 0
|
|
62
|
+
const val MAX_ROI_POSITION = 2
|
|
63
|
+
const val DEFAULT_ROI_POSITION = 0
|
|
64
|
+
}
|
|
58
65
|
}
|
|
59
66
|
|
|
60
67
|
data class PhotoRequest @JvmOverloads constructor(
|
|
@@ -64,8 +71,49 @@ data class PhotoRequest @JvmOverloads constructor(
|
|
|
64
71
|
val webhookUrl: String,
|
|
65
72
|
val authToken: String? = null,
|
|
66
73
|
val compress: PhotoCompression = PhotoCompression.MEDIUM,
|
|
74
|
+
val flash: Boolean = true,
|
|
67
75
|
val sound: Boolean = true,
|
|
68
|
-
)
|
|
76
|
+
/** Sensor exposure time for this capture only (ns), or null for auto exposure */
|
|
77
|
+
val exposureTimeNs: Double? = null,
|
|
78
|
+
/** Sensor ISO for this capture only. Only used when exposureTimeNs enables manual exposure. */
|
|
79
|
+
val iso: Int? = null,
|
|
80
|
+
) {
|
|
81
|
+
companion object {
|
|
82
|
+
/** Mirrors iOS `BluetoothSdkModule` defaults for keys omitted from the JS bridge. */
|
|
83
|
+
@JvmStatic
|
|
84
|
+
fun fromMap(values: Map<String, Any>): PhotoRequest {
|
|
85
|
+
val rawExp = values["exposureTimeNs"] ?: values["exposure_time_ns"]
|
|
86
|
+
val exposureTimeNs: Double? =
|
|
87
|
+
when (rawExp) {
|
|
88
|
+
is Number -> rawExp.toDouble().takeIf { it.isFinite() && it > 0 }
|
|
89
|
+
else -> null
|
|
90
|
+
}
|
|
91
|
+
val rawIso = values["iso"]
|
|
92
|
+
val iso: Int? =
|
|
93
|
+
when (rawIso) {
|
|
94
|
+
is Number -> rawIso.toDouble().takeIf { it.isFinite() && it > 0 }?.let { value ->
|
|
95
|
+
when {
|
|
96
|
+
value > Int.MAX_VALUE.toDouble() -> Int.MAX_VALUE
|
|
97
|
+
else -> value.toInt()
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
else -> null
|
|
101
|
+
}
|
|
102
|
+
return PhotoRequest(
|
|
103
|
+
requestId = stringValue(values, "requestId", "request_id").orEmpty(),
|
|
104
|
+
appId = stringValue(values, "appId", "app_id").orEmpty(),
|
|
105
|
+
size = PhotoSize.fromValue(stringValue(values, "size") ?: "medium"),
|
|
106
|
+
webhookUrl = stringValue(values, "webhookUrl", "webhook_url").orEmpty(),
|
|
107
|
+
authToken = stringValue(values, "authToken", "auth_token")?.takeIf { it.isNotBlank() },
|
|
108
|
+
compress = PhotoCompression.fromValue(stringValue(values, "compress") ?: "none"),
|
|
109
|
+
flash = boolValue(values, "flash") ?: true,
|
|
110
|
+
sound = boolValue(values, "sound") ?: true,
|
|
111
|
+
exposureTimeNs = exposureTimeNs,
|
|
112
|
+
iso = iso,
|
|
113
|
+
)
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
}
|
|
69
117
|
|
|
70
118
|
enum class RgbLedAction(val value: String) {
|
|
71
119
|
ON("on"),
|
|
@@ -185,3 +233,14 @@ data class PhotoResponseEvent(
|
|
|
185
233
|
val requestId: String get() = response.requestId
|
|
186
234
|
val values: Map<String, Any> get() = response.toEventMap()
|
|
187
235
|
}
|
|
236
|
+
|
|
237
|
+
data class PhotoStatusEvent(
|
|
238
|
+
val values: Map<String, Any>,
|
|
239
|
+
) {
|
|
240
|
+
val requestId: String get() = stringValue(values, "requestId").orEmpty()
|
|
241
|
+
val status: String get() = stringValue(values, "status").orEmpty()
|
|
242
|
+
val timestamp: Long get() = longValue(values, "timestamp") ?: System.currentTimeMillis()
|
|
243
|
+
val resolvedConfig: Map<String, Any>? get() = stringMapValue(values["resolvedConfig"])
|
|
244
|
+
val errorCode: String? get() = stringValue(values, "errorCode")
|
|
245
|
+
val errorMessage: String? get() = stringValue(values, "errorMessage")
|
|
246
|
+
}
|
|
@@ -22,7 +22,9 @@ abstract class ControllerManager {
|
|
|
22
22
|
authToken: String?,
|
|
23
23
|
compress: String?,
|
|
24
24
|
flash: Boolean,
|
|
25
|
-
sound: Boolean
|
|
25
|
+
sound: Boolean,
|
|
26
|
+
exposureTimeNs: Long?,
|
|
27
|
+
iso: Int?,
|
|
26
28
|
)
|
|
27
29
|
abstract fun startStream(message: Map<String, Any>)
|
|
28
30
|
abstract fun stopStream()
|
|
@@ -132,8 +134,8 @@ abstract class ControllerManager {
|
|
|
132
134
|
val micEnabled: Boolean
|
|
133
135
|
get() = DeviceStore.get("glasses", "micEnabled") as? Boolean ?: false
|
|
134
136
|
|
|
135
|
-
val
|
|
136
|
-
get() = DeviceStore.get("glasses", "
|
|
137
|
+
val voiceActivityDetectionEnabled: Boolean
|
|
138
|
+
get() = DeviceStore.get("glasses", "voiceActivityDetectionEnabled") as? Boolean ?: true
|
|
137
139
|
|
|
138
140
|
val batteryLevel: Int
|
|
139
141
|
get() = DeviceStore.get("glasses", "batteryLevel") as? Int ?: -1
|
|
@@ -863,7 +863,8 @@ class R1 : ControllerManager() {
|
|
|
863
863
|
override fun sendJson(jsonOriginal: Map<String, Any>, wakeUp: Boolean, requireAck: Boolean) {}
|
|
864
864
|
override fun requestPhoto(
|
|
865
865
|
requestId: String, appId: String, size: String?, webhookUrl: String?,
|
|
866
|
-
authToken: String?, compress: String?, flash: Boolean, sound: Boolean
|
|
866
|
+
authToken: String?, compress: String?, flash: Boolean, sound: Boolean,
|
|
867
|
+
exposureTimeNs: Long?, iso: Int?,
|
|
867
868
|
) {}
|
|
868
869
|
override fun startVideoRecording(requestId: String, save: Boolean, flash: Boolean, sound: Boolean) {}
|
|
869
870
|
override fun stopVideoRecording(requestId: String) {}
|