@mentra/bluetooth-sdk 0.1.6 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -22
- package/android/.project +28 -0
- package/android/build.gradle +5 -9
- package/android/lc3Lib/.project +28 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +54 -33
- package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +48 -10
- package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +27 -36
- package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt +8 -4
- package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +170 -14
- package/android/src/main/java/com/mentra/bluetoothsdk/audio/AudioModels.kt +6 -6
- package/android/src/main/java/com/mentra/bluetoothsdk/camera/CameraModels.kt +69 -10
- package/android/src/main/java/com/mentra/bluetoothsdk/controllers/ControllerManager.kt +5 -3
- package/android/src/main/java/com/mentra/bluetoothsdk/controllers/R1.kt +2 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/debug/BleTraceLogger.kt +149 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/events/BluetoothEvents.kt +13 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoUploadServer.kt +297 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/MentraPhotoReceiverModule.kt +120 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/services/Foreground.kt +13 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G1.java +1 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt +3 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Mach1.java +1 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraLive.java +269 -28
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraNex.kt +1 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt +8 -3
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Simulated.kt +4 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/status/DeviceStatus.kt +14 -12
- package/android/src/main/java/com/mentra/bluetoothsdk/status/RuntimeState.kt +4 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +131 -6
- package/android/src/main/java/com/mentra/bluetoothsdk/stt/SherpaOnnxTranscriber.kt +1 -3
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunkReassembler.java +105 -0
- package/build/BluetoothSdk.types.d.ts +107 -48
- package/build/BluetoothSdk.types.d.ts.map +1 -1
- package/build/BluetoothSdk.types.js +4 -0
- package/build/BluetoothSdk.types.js.map +1 -1
- package/build/_private/BluetoothSdkModule.d.ts +8 -6
- package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
- package/build/_private/BluetoothSdkModule.js +58 -9
- package/build/_private/BluetoothSdkModule.js.map +1 -1
- package/build/index.d.ts +2 -2
- package/build/index.d.ts.map +1 -1
- package/build/index.js +6 -5
- package/build/index.js.map +1 -1
- package/build/photo-receiver/MentraPhotoReceiver.types.d.ts +18 -0
- package/build/photo-receiver/MentraPhotoReceiver.types.d.ts.map +1 -0
- package/build/photo-receiver/MentraPhotoReceiver.types.js +2 -0
- package/build/photo-receiver/MentraPhotoReceiver.types.js.map +1 -0
- package/build/photo-receiver/MentraPhotoReceiverModule.d.ts +10 -0
- package/build/photo-receiver/MentraPhotoReceiverModule.d.ts.map +1 -0
- package/build/photo-receiver/MentraPhotoReceiverModule.js +3 -0
- package/build/photo-receiver/MentraPhotoReceiverModule.js.map +1 -0
- package/build/photo-receiver/MentraPhotoReceiverModule.web.d.ts +10 -0
- package/build/photo-receiver/MentraPhotoReceiverModule.web.d.ts.map +1 -0
- package/build/photo-receiver/MentraPhotoReceiverModule.web.js +12 -0
- package/build/photo-receiver/MentraPhotoReceiverModule.web.js.map +1 -0
- package/build/photo-receiver/index.d.ts +3 -0
- package/build/photo-receiver/index.d.ts.map +1 -0
- package/build/photo-receiver/index.js +3 -0
- package/build/photo-receiver/index.js.map +1 -0
- package/build/react/useMentraBluetooth.d.ts +5 -3
- package/build/react/useMentraBluetooth.d.ts.map +1 -1
- package/build/react/useMentraBluetooth.js +15 -4
- package/build/react/useMentraBluetooth.js.map +1 -1
- package/expo-module.config.json +5 -2
- package/ios/BluetoothSdkModule.swift +59 -26
- package/ios/LocalPhotoUploadServer.swift +411 -0
- package/ios/MentraBluetoothSDK.podspec +8 -5
- package/ios/MentraPhotoReceiverModule.swift +147 -0
- package/ios/Source/Audio/AudioModels.swift +10 -10
- package/ios/Source/Bridge.swift +42 -12
- package/ios/Source/Camera/CameraModels.swift +74 -19
- package/ios/Source/DeviceManager.swift +23 -98
- package/ios/Source/DeviceStore.swift +12 -4
- package/ios/Source/Events/BluetoothEvents.swift +37 -0
- package/ios/Source/MentraBluetoothSDK.swift +135 -14
- package/ios/Source/Status/DeviceStatus.swift +12 -4
- package/ios/Source/Status/RuntimeState.swift +22 -13
- package/ios/Source/Streaming/StreamModels.swift +239 -45
- package/ios/Source/controllers/ControllerManager.swift +3 -3
- package/ios/Source/controllers/R1.swift +9 -3
- package/ios/Source/sgcs/G1.swift +5 -4
- package/ios/Source/sgcs/G2.swift +5 -6
- package/ios/Source/sgcs/Mach1.swift +1 -1
- package/ios/Source/sgcs/MentraLive.swift +183 -22
- package/ios/Source/sgcs/MentraNex.swift +1 -1
- package/ios/Source/sgcs/SGCManager.swift +11 -3
- package/ios/Source/sgcs/Simulated.swift +2 -2
- package/ios/Source/stt/SherpaOnnxTranscriber.swift +1 -4
- package/ios/Source/utils/MessageChunkReassembler.swift +89 -0
- package/package.json +6 -1
- package/src/BluetoothSdk.types.ts +133 -62
- package/src/_private/BluetoothSdkModule.ts +79 -26
- package/src/index.ts +15 -8
- package/src/photo-receiver/MentraPhotoReceiver.types.ts +20 -0
- package/src/photo-receiver/MentraPhotoReceiverModule.ts +14 -0
- package/src/photo-receiver/MentraPhotoReceiverModule.web.ts +20 -0
- package/src/photo-receiver/index.ts +2 -0
- package/src/react/useMentraBluetooth.ts +19 -7
- package/android/src/main/java/com/mentra/bluetoothsdk/stt/VadGateSpeechPolicy.kt +0 -221
- package/ios/Packages/VAD/Common/VAD/VADQuality.swift +0 -41
- package/ios/Packages/VAD/Common/VAD/VADState.swift +0 -26
- package/ios/Packages/VAD/Common/VAD/VADStrategy.swift +0 -29
- package/ios/Packages/VAD/Common/VAD/VADType.swift +0 -14
- package/ios/Packages/VAD/Data/Configuration.swift +0 -68
- package/ios/Packages/VAD/Data/FrameSize.swift +0 -39
- package/ios/Packages/VAD/Data/Record.swift +0 -13
- package/ios/Packages/VAD/Data/Result.swift +0 -22
- package/ios/Packages/VAD/Data/SampleRate.swift +0 -48
- package/ios/Packages/VAD/Silero/Model/silero_vad.onnx +0 -0
- package/ios/Packages/VAD/Silero/SileroVAD.swift +0 -284
- package/ios/Packages/VAD/Silero/SileroVADStrategy.swift +0 -64
|
@@ -32,6 +32,7 @@ internal data class GlassesStatus(
|
|
|
32
32
|
val fullyBooted: Boolean,
|
|
33
33
|
val connected: Boolean,
|
|
34
34
|
val micEnabled: Boolean,
|
|
35
|
+
val voiceActivityDetectionEnabled: Boolean,
|
|
35
36
|
val connectionState: GlassesConnectionState,
|
|
36
37
|
val bluetoothClassicConnected: Boolean,
|
|
37
38
|
val signalStrength: Int,
|
|
@@ -72,6 +73,7 @@ internal data class GlassesStatus(
|
|
|
72
73
|
mapOf(
|
|
73
74
|
"connection" to connectionState.toStatusMap(connected, fullyBooted),
|
|
74
75
|
"micEnabled" to micEnabled,
|
|
76
|
+
"voiceActivityDetectionEnabled" to voiceActivityDetectionEnabled,
|
|
75
77
|
"bluetoothClassicConnected" to bluetoothClassicConnected,
|
|
76
78
|
"signalStrength" to signalStrength,
|
|
77
79
|
"signalStrengthUpdatedAt" to signalStrengthUpdatedAt,
|
|
@@ -114,6 +116,7 @@ internal data class GlassesStatus(
|
|
|
114
116
|
fullyBooted = boolValue(values, "fullyBooted") ?: false,
|
|
115
117
|
connected = boolValue(values, "connected") ?: false,
|
|
116
118
|
micEnabled = boolValue(values, "micEnabled") ?: false,
|
|
119
|
+
voiceActivityDetectionEnabled = boolValue(values, "voiceActivityDetectionEnabled") ?: true,
|
|
117
120
|
connectionState = GlassesConnectionState.fromValue(stringValue(values, "connectionState")),
|
|
118
121
|
bluetoothClassicConnected = boolValue(values, "bluetoothClassicConnected") ?: false,
|
|
119
122
|
signalStrength = numberValue(values, "signalStrength") ?: -1,
|
|
@@ -186,7 +189,7 @@ internal data class BluetoothStatus(
|
|
|
186
189
|
val dashboardDepth: Int,
|
|
187
190
|
val headUpAngle: Int,
|
|
188
191
|
val contextualDashboard: Boolean,
|
|
189
|
-
val
|
|
192
|
+
val galleryModeEnabled: Boolean,
|
|
190
193
|
val buttonPhotoSize: ButtonPhotoSize,
|
|
191
194
|
val buttonCameraLed: Boolean,
|
|
192
195
|
val buttonMaxRecordingTime: Int,
|
|
@@ -196,7 +199,6 @@ internal data class BluetoothStatus(
|
|
|
196
199
|
val shouldSendPcm: Boolean,
|
|
197
200
|
val shouldSendLc3: Boolean,
|
|
198
201
|
val shouldSendTranscript: Boolean,
|
|
199
|
-
val bypassVad: Boolean,
|
|
200
202
|
val offlineCaptionsRunning: Boolean,
|
|
201
203
|
val localSttFallbackActive: Boolean,
|
|
202
204
|
val shouldSendBootingMessage: Boolean,
|
|
@@ -240,7 +242,7 @@ internal data class BluetoothStatus(
|
|
|
240
242
|
"dashboard_depth" to dashboardDepth,
|
|
241
243
|
"head_up_angle" to headUpAngle,
|
|
242
244
|
"contextual_dashboard" to contextualDashboard,
|
|
243
|
-
"
|
|
245
|
+
"galleryModeEnabled" to galleryModeEnabled,
|
|
244
246
|
"button_photo_size" to buttonPhotoSize.value,
|
|
245
247
|
"button_camera_led" to buttonCameraLed,
|
|
246
248
|
"button_max_recording_time" to buttonMaxRecordingTime,
|
|
@@ -250,7 +252,6 @@ internal data class BluetoothStatus(
|
|
|
250
252
|
"should_send_pcm" to shouldSendPcm,
|
|
251
253
|
"should_send_lc3" to shouldSendLc3,
|
|
252
254
|
"should_send_transcript" to shouldSendTranscript,
|
|
253
|
-
"bypass_vad" to bypassVad,
|
|
254
255
|
"offline_captions_running" to offlineCaptionsRunning,
|
|
255
256
|
"local_stt_fallback_active" to localSttFallbackActive,
|
|
256
257
|
"shouldSendBootingMessage" to shouldSendBootingMessage,
|
|
@@ -288,7 +289,8 @@ internal data class BluetoothStatus(
|
|
|
288
289
|
dashboardDepth = numberValue(values, "dashboard_depth") ?: 2,
|
|
289
290
|
headUpAngle = numberValue(values, "head_up_angle") ?: 30,
|
|
290
291
|
contextualDashboard = boolValue(values, "contextual_dashboard") ?: true,
|
|
291
|
-
|
|
292
|
+
galleryModeEnabled =
|
|
293
|
+
boolValue(values, "gallery_mode") ?: boolValue(values, "galleryModeEnabled") ?: true,
|
|
292
294
|
buttonPhotoSize = ButtonPhotoSize.fromValue(stringValue(values, "button_photo_size")),
|
|
293
295
|
buttonCameraLed = boolValue(values, "button_camera_led") ?: true,
|
|
294
296
|
buttonMaxRecordingTime = numberValue(values, "button_max_recording_time") ?: 10,
|
|
@@ -298,7 +300,6 @@ internal data class BluetoothStatus(
|
|
|
298
300
|
shouldSendPcm = boolValue(values, "should_send_pcm") ?: false,
|
|
299
301
|
shouldSendLc3 = boolValue(values, "should_send_lc3") ?: false,
|
|
300
302
|
shouldSendTranscript = boolValue(values, "should_send_transcript") ?: false,
|
|
301
|
-
bypassVad = boolValue(values, "bypass_vad") ?: true,
|
|
302
303
|
offlineCaptionsRunning = boolValue(values, "offline_captions_running") ?: false,
|
|
303
304
|
localSttFallbackActive = boolValue(values, "local_stt_fallback_active") ?: false,
|
|
304
305
|
shouldSendBootingMessage = boolValue(values, "shouldSendBootingMessage") ?: true,
|
|
@@ -310,6 +311,7 @@ internal data class GlassesStatusUpdate(
|
|
|
310
311
|
val fullyBooted: Boolean? = null,
|
|
311
312
|
val connected: Boolean? = null,
|
|
312
313
|
val micEnabled: Boolean? = null,
|
|
314
|
+
val voiceActivityDetectionEnabled: Boolean? = null,
|
|
313
315
|
val connectionState: GlassesConnectionState? = null,
|
|
314
316
|
val bluetoothClassicConnected: Boolean? = null,
|
|
315
317
|
val signalStrength: Int? = null,
|
|
@@ -359,6 +361,7 @@ internal data class GlassesStatusUpdate(
|
|
|
359
361
|
put("connection", state.toStatusMap(connected == true, fullyBooted == true))
|
|
360
362
|
}
|
|
361
363
|
putIfNotNull("micEnabled", micEnabled)
|
|
364
|
+
putIfNotNull("voiceActivityDetectionEnabled", voiceActivityDetectionEnabled)
|
|
362
365
|
putIfNotNull("bluetoothClassicConnected", bluetoothClassicConnected)
|
|
363
366
|
putIfNotNull("signalStrength", signalStrength)
|
|
364
367
|
putIfNotNull("signalStrengthUpdatedAt", signalStrengthUpdatedAt)
|
|
@@ -405,6 +408,7 @@ internal data class GlassesStatusUpdate(
|
|
|
405
408
|
fullyBooted = optionalBoolValue(values, "fullyBooted"),
|
|
406
409
|
connected = optionalBoolValue(values, "connected"),
|
|
407
410
|
micEnabled = optionalBoolValue(values, "micEnabled"),
|
|
411
|
+
voiceActivityDetectionEnabled = optionalBoolValue(values, "voiceActivityDetectionEnabled"),
|
|
408
412
|
connectionState = GlassesConnectionState.optionalFromValue(optionalStringValue(values, "connectionState")),
|
|
409
413
|
bluetoothClassicConnected = optionalBoolValue(values, "bluetoothClassicConnected"),
|
|
410
414
|
signalStrength = optionalNumberValue(values, "signalStrength"),
|
|
@@ -491,7 +495,7 @@ internal data class BluetoothStatusUpdate(
|
|
|
491
495
|
val dashboardDepth: Int? = null,
|
|
492
496
|
val headUpAngle: Int? = null,
|
|
493
497
|
val contextualDashboard: Boolean? = null,
|
|
494
|
-
val
|
|
498
|
+
val galleryModeEnabled: Boolean? = null,
|
|
495
499
|
val buttonPhotoSize: ButtonPhotoSize? = null,
|
|
496
500
|
val buttonCameraLed: Boolean? = null,
|
|
497
501
|
val buttonMaxRecordingTime: Int? = null,
|
|
@@ -501,7 +505,6 @@ internal data class BluetoothStatusUpdate(
|
|
|
501
505
|
val shouldSendPcm: Boolean? = null,
|
|
502
506
|
val shouldSendLc3: Boolean? = null,
|
|
503
507
|
val shouldSendTranscript: Boolean? = null,
|
|
504
|
-
val bypassVad: Boolean? = null,
|
|
505
508
|
val offlineCaptionsRunning: Boolean? = null,
|
|
506
509
|
val localSttFallbackActive: Boolean? = null,
|
|
507
510
|
val shouldSendBootingMessage: Boolean? = null,
|
|
@@ -535,7 +538,7 @@ internal data class BluetoothStatusUpdate(
|
|
|
535
538
|
putIfNotNull("dashboard_depth", dashboardDepth)
|
|
536
539
|
putIfNotNull("head_up_angle", headUpAngle)
|
|
537
540
|
putIfNotNull("contextual_dashboard", contextualDashboard)
|
|
538
|
-
putIfNotNull("
|
|
541
|
+
putIfNotNull("galleryModeEnabled", galleryModeEnabled)
|
|
539
542
|
buttonPhotoSize?.let { put("button_photo_size", it.value) }
|
|
540
543
|
putIfNotNull("button_camera_led", buttonCameraLed)
|
|
541
544
|
putIfNotNull("button_max_recording_time", buttonMaxRecordingTime)
|
|
@@ -545,7 +548,6 @@ internal data class BluetoothStatusUpdate(
|
|
|
545
548
|
putIfNotNull("should_send_pcm", shouldSendPcm)
|
|
546
549
|
putIfNotNull("should_send_lc3", shouldSendLc3)
|
|
547
550
|
putIfNotNull("should_send_transcript", shouldSendTranscript)
|
|
548
|
-
putIfNotNull("bypass_vad", bypassVad)
|
|
549
551
|
putIfNotNull("offline_captions_running", offlineCaptionsRunning)
|
|
550
552
|
putIfNotNull("local_stt_fallback_active", localSttFallbackActive)
|
|
551
553
|
putIfNotNull("shouldSendBootingMessage", shouldSendBootingMessage)
|
|
@@ -585,7 +587,8 @@ internal data class BluetoothStatusUpdate(
|
|
|
585
587
|
dashboardDepth = optionalNumberValue(values, "dashboard_depth"),
|
|
586
588
|
headUpAngle = optionalNumberValue(values, "head_up_angle"),
|
|
587
589
|
contextualDashboard = optionalBoolValue(values, "contextual_dashboard"),
|
|
588
|
-
|
|
590
|
+
galleryModeEnabled =
|
|
591
|
+
optionalBoolValue(values, "gallery_mode") ?: optionalBoolValue(values, "galleryModeEnabled"),
|
|
589
592
|
buttonPhotoSize =
|
|
590
593
|
optionalStringValue(values, "button_photo_size")?.let(ButtonPhotoSize::fromValue),
|
|
591
594
|
buttonCameraLed = optionalBoolValue(values, "button_camera_led"),
|
|
@@ -596,7 +599,6 @@ internal data class BluetoothStatusUpdate(
|
|
|
596
599
|
shouldSendPcm = optionalBoolValue(values, "should_send_pcm"),
|
|
597
600
|
shouldSendLc3 = optionalBoolValue(values, "should_send_lc3"),
|
|
598
601
|
shouldSendTranscript = optionalBoolValue(values, "should_send_transcript"),
|
|
599
|
-
bypassVad = optionalBoolValue(values, "bypass_vad"),
|
|
600
602
|
offlineCaptionsRunning = optionalBoolValue(values, "offline_captions_running"),
|
|
601
603
|
localSttFallbackActive = optionalBoolValue(values, "local_stt_fallback_active"),
|
|
602
604
|
shouldSendBootingMessage = optionalBoolValue(values, "shouldSendBootingMessage"),
|
|
@@ -74,6 +74,7 @@ sealed interface GlassesRuntimeState {
|
|
|
74
74
|
val hotspot: HotspotStatus,
|
|
75
75
|
override val ready: Boolean,
|
|
76
76
|
val signal: SignalState,
|
|
77
|
+
val voiceActivityDetectionEnabled: Boolean,
|
|
77
78
|
val wifi: WifiStatus,
|
|
78
79
|
) : GlassesRuntimeState {
|
|
79
80
|
override val connected: Boolean = true
|
|
@@ -95,6 +96,7 @@ sealed interface GlassesRuntimeState {
|
|
|
95
96
|
strengthDbm = status.signalStrength.takeUnless { it == -1 },
|
|
96
97
|
updatedAt = status.signalStrengthUpdatedAt.takeUnless { it <= 0L },
|
|
97
98
|
),
|
|
99
|
+
voiceActivityDetectionEnabled = status.voiceActivityDetectionEnabled,
|
|
98
100
|
wifi = status.wifi,
|
|
99
101
|
)
|
|
100
102
|
}
|
|
@@ -149,7 +151,7 @@ enum class MicMode(val value: String) {
|
|
|
149
151
|
}
|
|
150
152
|
|
|
151
153
|
data class GalleryModeState(
|
|
152
|
-
val
|
|
154
|
+
val enabled: Boolean,
|
|
153
155
|
)
|
|
154
156
|
|
|
155
157
|
data class PhoneSdkRuntimeState(
|
|
@@ -169,7 +171,7 @@ data class PhoneSdkRuntimeState(
|
|
|
169
171
|
PhoneSdkRuntimeState(
|
|
170
172
|
currentMic = MicMode.fromValue(status.currentMic),
|
|
171
173
|
defaultDevice = status.defaultDevice,
|
|
172
|
-
galleryMode = GalleryModeState(
|
|
174
|
+
galleryMode = GalleryModeState(enabled = status.galleryModeEnabled),
|
|
173
175
|
lastLog = status.lastLog,
|
|
174
176
|
micRanking = status.micRanking.mapNotNull { MicMode.fromValue(it) },
|
|
175
177
|
otherBluetoothConnected = status.otherBtConnected,
|
|
@@ -4,14 +4,14 @@ data class StreamVideoConfig @JvmOverloads constructor(
|
|
|
4
4
|
val width: Int? = null,
|
|
5
5
|
val height: Int? = null,
|
|
6
6
|
val bitrate: Int? = null,
|
|
7
|
-
val
|
|
7
|
+
val fps: Int? = null,
|
|
8
8
|
) {
|
|
9
9
|
fun toMap(): Map<String, Any> =
|
|
10
10
|
listOfNotNull(
|
|
11
11
|
width?.let { "width" to it },
|
|
12
12
|
height?.let { "height" to it },
|
|
13
13
|
bitrate?.let { "bitrate" to it },
|
|
14
|
-
|
|
14
|
+
fps?.let { "frameRate" to it },
|
|
15
15
|
).toMap()
|
|
16
16
|
|
|
17
17
|
companion object {
|
|
@@ -22,7 +22,7 @@ data class StreamVideoConfig @JvmOverloads constructor(
|
|
|
22
22
|
width = numberValue(values, "width"),
|
|
23
23
|
height = numberValue(values, "height"),
|
|
24
24
|
bitrate = numberValue(values, "bitrate"),
|
|
25
|
-
|
|
25
|
+
fps = numberValue(values, "fps"),
|
|
26
26
|
)
|
|
27
27
|
}
|
|
28
28
|
}
|
|
@@ -56,11 +56,115 @@ data class StreamAudioConfig @JvmOverloads constructor(
|
|
|
56
56
|
}
|
|
57
57
|
}
|
|
58
58
|
|
|
59
|
+
data class StreamResolvedVideoConfig @JvmOverloads constructor(
|
|
60
|
+
val width: Int,
|
|
61
|
+
val height: Int,
|
|
62
|
+
val captureWidth: Int? = null,
|
|
63
|
+
val captureHeight: Int? = null,
|
|
64
|
+
val bitrate: Int,
|
|
65
|
+
val fps: Int,
|
|
66
|
+
) {
|
|
67
|
+
fun toMap(): Map<String, Any> =
|
|
68
|
+
buildMap {
|
|
69
|
+
put("width", width)
|
|
70
|
+
put("height", height)
|
|
71
|
+
captureWidth?.let { put("captureWidth", it) }
|
|
72
|
+
captureHeight?.let { put("captureHeight", it) }
|
|
73
|
+
put("bitrate", bitrate)
|
|
74
|
+
put("fps", fps)
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
companion object {
|
|
78
|
+
@JvmStatic
|
|
79
|
+
fun fromMap(values: Map<String, Any>?): StreamResolvedVideoConfig? {
|
|
80
|
+
values ?: return null
|
|
81
|
+
return StreamResolvedVideoConfig(
|
|
82
|
+
width = numberValue(values, "width") ?: return null,
|
|
83
|
+
height = numberValue(values, "height") ?: return null,
|
|
84
|
+
captureWidth = numberValue(values, "captureWidth"),
|
|
85
|
+
captureHeight = numberValue(values, "captureHeight"),
|
|
86
|
+
bitrate = numberValue(values, "bitrate") ?: return null,
|
|
87
|
+
fps = numberValue(values, "fps") ?: return null,
|
|
88
|
+
)
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
data class StreamResolvedAudioConfig @JvmOverloads constructor(
|
|
94
|
+
val bitrate: Int? = null,
|
|
95
|
+
val sampleRate: Int? = null,
|
|
96
|
+
val echoCancellation: Boolean? = null,
|
|
97
|
+
val noiseSuppression: Boolean? = null,
|
|
98
|
+
) {
|
|
99
|
+
fun toMap(): Map<String, Any> =
|
|
100
|
+
buildMap {
|
|
101
|
+
bitrate?.let { put("bitrate", it) }
|
|
102
|
+
sampleRate?.let { put("sampleRate", it) }
|
|
103
|
+
echoCancellation?.let { put("echoCancellation", it) }
|
|
104
|
+
noiseSuppression?.let { put("noiseSuppression", it) }
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
companion object {
|
|
108
|
+
@JvmStatic
|
|
109
|
+
fun fromMap(values: Map<String, Any>?): StreamResolvedAudioConfig? {
|
|
110
|
+
values ?: return null
|
|
111
|
+
return StreamResolvedAudioConfig(
|
|
112
|
+
bitrate = numberValue(values, "bitrate"),
|
|
113
|
+
sampleRate = numberValue(values, "sampleRate"),
|
|
114
|
+
echoCancellation = boolValue(values, "echoCancellation"),
|
|
115
|
+
noiseSuppression = boolValue(values, "noiseSuppression"),
|
|
116
|
+
)
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
enum class StreamTransport(val value: String) {
|
|
122
|
+
RTMP("rtmp"),
|
|
123
|
+
SRT("srt"),
|
|
124
|
+
WHIP("whip");
|
|
125
|
+
|
|
126
|
+
companion object {
|
|
127
|
+
@JvmStatic
|
|
128
|
+
fun fromValue(value: String?): StreamTransport? =
|
|
129
|
+
when (value?.lowercase()) {
|
|
130
|
+
"rtmp" -> RTMP
|
|
131
|
+
"srt" -> SRT
|
|
132
|
+
"whip" -> WHIP
|
|
133
|
+
else -> null
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
data class StreamResolvedConfig @JvmOverloads constructor(
|
|
139
|
+
val transport: StreamTransport? = null,
|
|
140
|
+
val video: StreamResolvedVideoConfig? = null,
|
|
141
|
+
val audio: StreamResolvedAudioConfig? = null,
|
|
142
|
+
) {
|
|
143
|
+
fun toMap(): Map<String, Any> =
|
|
144
|
+
buildMap {
|
|
145
|
+
transport?.let { put("transport", it.value) }
|
|
146
|
+
video?.let { put("video", it.toMap()) }
|
|
147
|
+
audio?.let { put("audio", it.toMap()) }
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
companion object {
|
|
151
|
+
@JvmStatic
|
|
152
|
+
fun fromMap(values: Map<String, Any>?): StreamResolvedConfig? {
|
|
153
|
+
values ?: return null
|
|
154
|
+
return StreamResolvedConfig(
|
|
155
|
+
transport = StreamTransport.fromValue(stringValue(values, "transport")),
|
|
156
|
+
video = StreamResolvedVideoConfig.fromMap(stringMapValue(values["video"])),
|
|
157
|
+
audio = StreamResolvedAudioConfig.fromMap(stringMapValue(values["audio"])),
|
|
158
|
+
)
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
59
163
|
data class StreamRequest @JvmOverloads constructor(
|
|
60
164
|
val streamUrl: String,
|
|
61
165
|
val streamId: String = "",
|
|
62
166
|
val keepAlive: Boolean = true,
|
|
63
|
-
val keepAliveIntervalSeconds: Int =
|
|
167
|
+
val keepAliveIntervalSeconds: Int = 5,
|
|
64
168
|
val sound: Boolean = true,
|
|
65
169
|
val video: StreamVideoConfig? = null,
|
|
66
170
|
val audio: StreamAudioConfig? = null,
|
|
@@ -68,6 +172,7 @@ data class StreamRequest @JvmOverloads constructor(
|
|
|
68
172
|
) {
|
|
69
173
|
fun toMap(): Map<String, Any> {
|
|
70
174
|
val values = extraValues.toMutableMap()
|
|
175
|
+
values.remove("keepAliveMode")
|
|
71
176
|
values["type"] = "start_stream"
|
|
72
177
|
values["streamUrl"] = streamUrl
|
|
73
178
|
values["streamId"] = streamId
|
|
@@ -90,7 +195,7 @@ data class StreamRequest @JvmOverloads constructor(
|
|
|
90
195
|
?: "",
|
|
91
196
|
streamId = values["streamId"] as? String ?: "",
|
|
92
197
|
keepAlive = values["keepAlive"] as? Boolean ?: true,
|
|
93
|
-
keepAliveIntervalSeconds = (values["keepAliveIntervalSeconds"] as? Number)?.toInt() ?:
|
|
198
|
+
keepAliveIntervalSeconds = (values["keepAliveIntervalSeconds"] as? Number)?.toInt() ?: 5,
|
|
94
199
|
sound = values["sound"] as? Boolean ?: true,
|
|
95
200
|
video = StreamVideoConfig.fromMap(stringMapValue(values["video"])),
|
|
96
201
|
audio = StreamAudioConfig.fromMap(stringMapValue(values["audio"])),
|
|
@@ -99,7 +204,10 @@ data class StreamRequest @JvmOverloads constructor(
|
|
|
99
204
|
}
|
|
100
205
|
}
|
|
101
206
|
|
|
102
|
-
|
|
207
|
+
internal fun StreamRequest.isExternallyManagedKeepAlive(): Boolean =
|
|
208
|
+
stringValue(extraValues, "keepAliveMode") == "external"
|
|
209
|
+
|
|
210
|
+
internal data class StreamKeepAliveRequest @JvmOverloads constructor(
|
|
103
211
|
val streamId: String,
|
|
104
212
|
val ackId: String,
|
|
105
213
|
val extraValues: Map<String, Any> = emptyMap(),
|
|
@@ -162,6 +270,7 @@ sealed interface StreamStatus {
|
|
|
162
270
|
val state: StreamState
|
|
163
271
|
val streamId: String?
|
|
164
272
|
val timestamp: Long?
|
|
273
|
+
val resolvedConfig: StreamResolvedConfig?
|
|
165
274
|
|
|
166
275
|
fun toMap(): Map<String, Any> {
|
|
167
276
|
val values = mutableMapOf<String, Any>(
|
|
@@ -170,6 +279,7 @@ sealed interface StreamStatus {
|
|
|
170
279
|
)
|
|
171
280
|
streamId?.takeIf { it.isNotBlank() }?.let { values["streamId"] = it }
|
|
172
281
|
timestamp?.let { values["timestamp"] = it }
|
|
282
|
+
resolvedConfig?.let { values["resolvedConfig"] = it.toMap() }
|
|
173
283
|
|
|
174
284
|
when (this) {
|
|
175
285
|
is Lifecycle -> Unit
|
|
@@ -197,6 +307,7 @@ sealed interface StreamStatus {
|
|
|
197
307
|
override val state: StreamState,
|
|
198
308
|
override val streamId: String?,
|
|
199
309
|
override val timestamp: Long?,
|
|
310
|
+
override val resolvedConfig: StreamResolvedConfig?,
|
|
200
311
|
) : StreamStatus {
|
|
201
312
|
override val kind: StreamStatusKind = StreamStatusKind.LIFECYCLE
|
|
202
313
|
}
|
|
@@ -207,6 +318,7 @@ sealed interface StreamStatus {
|
|
|
207
318
|
val maxAttempts: Int,
|
|
208
319
|
val reason: String,
|
|
209
320
|
override val timestamp: Long?,
|
|
321
|
+
override val resolvedConfig: StreamResolvedConfig?,
|
|
210
322
|
) : StreamStatus {
|
|
211
323
|
override val kind: StreamStatusKind = StreamStatusKind.RECONNECT
|
|
212
324
|
override val state: StreamState = StreamState.RECONNECTING
|
|
@@ -216,6 +328,7 @@ sealed interface StreamStatus {
|
|
|
216
328
|
override val streamId: String?,
|
|
217
329
|
val attempt: Int,
|
|
218
330
|
override val timestamp: Long?,
|
|
331
|
+
override val resolvedConfig: StreamResolvedConfig?,
|
|
219
332
|
) : StreamStatus {
|
|
220
333
|
override val kind: StreamStatusKind = StreamStatusKind.RECONNECT
|
|
221
334
|
override val state: StreamState = StreamState.RECONNECTED
|
|
@@ -225,6 +338,7 @@ sealed interface StreamStatus {
|
|
|
225
338
|
override val streamId: String?,
|
|
226
339
|
val maxAttempts: Int,
|
|
227
340
|
override val timestamp: Long?,
|
|
341
|
+
override val resolvedConfig: StreamResolvedConfig?,
|
|
228
342
|
) : StreamStatus {
|
|
229
343
|
override val kind: StreamStatusKind = StreamStatusKind.RECONNECT
|
|
230
344
|
override val state: StreamState = StreamState.RECONNECT_FAILED
|
|
@@ -234,6 +348,7 @@ sealed interface StreamStatus {
|
|
|
234
348
|
override val streamId: String?,
|
|
235
349
|
val errorDetails: String,
|
|
236
350
|
override val timestamp: Long?,
|
|
351
|
+
override val resolvedConfig: StreamResolvedConfig?,
|
|
237
352
|
) : StreamStatus {
|
|
238
353
|
override val kind: StreamStatusKind = StreamStatusKind.ERROR
|
|
239
354
|
override val state: StreamState = StreamState.ERROR
|
|
@@ -246,6 +361,7 @@ sealed interface StreamStatus {
|
|
|
246
361
|
override val streamId: String?,
|
|
247
362
|
val attempt: Int?,
|
|
248
363
|
override val timestamp: Long?,
|
|
364
|
+
override val resolvedConfig: StreamResolvedConfig?,
|
|
249
365
|
) : StreamStatus {
|
|
250
366
|
override val kind: StreamStatusKind = StreamStatusKind.SNAPSHOT
|
|
251
367
|
}
|
|
@@ -258,6 +374,7 @@ sealed interface StreamStatus {
|
|
|
258
374
|
val reconnecting = boolValue(values, "reconnecting") ?: false
|
|
259
375
|
val streamId = stringValue(values, "streamId")
|
|
260
376
|
val timestamp = longValue(values, "timestamp")
|
|
377
|
+
val resolvedConfig = StreamResolvedConfig.fromMap(stringMapValue(values["resolvedConfig"]))
|
|
261
378
|
val attempt = numberValue(values, "attempt")
|
|
262
379
|
val maxAttempts = numberValue(values, "maxAttempts") ?: 0
|
|
263
380
|
|
|
@@ -273,6 +390,7 @@ sealed interface StreamStatus {
|
|
|
273
390
|
streamId = streamId,
|
|
274
391
|
attempt = attempt,
|
|
275
392
|
timestamp = timestamp,
|
|
393
|
+
resolvedConfig = resolvedConfig,
|
|
276
394
|
)
|
|
277
395
|
}
|
|
278
396
|
|
|
@@ -281,6 +399,7 @@ sealed interface StreamStatus {
|
|
|
281
399
|
streamId = streamId,
|
|
282
400
|
errorDetails = rawState?.let { "Unknown stream status: $it" } ?: "Missing stream status",
|
|
283
401
|
timestamp = timestamp,
|
|
402
|
+
resolvedConfig = resolvedConfig,
|
|
284
403
|
)
|
|
285
404
|
|
|
286
405
|
return when (state) {
|
|
@@ -290,27 +409,32 @@ sealed interface StreamStatus {
|
|
|
290
409
|
maxAttempts = maxAttempts,
|
|
291
410
|
reason = stringValue(values, "reason") ?: "",
|
|
292
411
|
timestamp = timestamp,
|
|
412
|
+
resolvedConfig = resolvedConfig,
|
|
293
413
|
)
|
|
294
414
|
StreamState.RECONNECTED -> Reconnected(
|
|
295
415
|
streamId = streamId,
|
|
296
416
|
attempt = attempt ?: 0,
|
|
297
417
|
timestamp = timestamp,
|
|
418
|
+
resolvedConfig = resolvedConfig,
|
|
298
419
|
)
|
|
299
420
|
StreamState.RECONNECT_FAILED -> ReconnectFailed(
|
|
300
421
|
streamId = streamId,
|
|
301
422
|
maxAttempts = maxAttempts,
|
|
302
423
|
timestamp = timestamp,
|
|
424
|
+
resolvedConfig = resolvedConfig,
|
|
303
425
|
)
|
|
304
426
|
StreamState.ERROR -> Error(
|
|
305
427
|
streamId = streamId,
|
|
306
428
|
errorDetails = stringValue(values, "errorDetails")
|
|
307
429
|
?: if (rawState == "error_not_streaming") "not_streaming" else "Unknown stream error",
|
|
308
430
|
timestamp = timestamp,
|
|
431
|
+
resolvedConfig = resolvedConfig,
|
|
309
432
|
)
|
|
310
433
|
else -> Lifecycle(
|
|
311
434
|
state = state,
|
|
312
435
|
streamId = streamId,
|
|
313
436
|
timestamp = timestamp,
|
|
437
|
+
resolvedConfig = resolvedConfig,
|
|
314
438
|
)
|
|
315
439
|
}
|
|
316
440
|
}
|
|
@@ -324,6 +448,7 @@ data class StreamStatusEvent(
|
|
|
324
448
|
|
|
325
449
|
val state: StreamState get() = status.state
|
|
326
450
|
val streamId: String? get() = status.streamId
|
|
451
|
+
val resolvedConfig: StreamResolvedConfig? get() = status.resolvedConfig
|
|
327
452
|
val values: Map<String, Any> get() = status.toEventMap()
|
|
328
453
|
}
|
|
329
454
|
|
|
@@ -226,8 +226,7 @@ class SherpaOnnxTranscriber(private val context: Context) {
|
|
|
226
226
|
* Feed PCM audio data (16-bit little endian) into the transcriber. This method should be called
|
|
227
227
|
* continuously with short chunks (e.g., 100-300ms).
|
|
228
228
|
*
|
|
229
|
-
*
|
|
230
|
-
* directly queued for processing without additional VAD checks.
|
|
229
|
+
* Audio is queued directly; microphone VAD gating is not applied in the SDK.
|
|
231
230
|
*/
|
|
232
231
|
fun acceptAudio(pcm16le: ByteArray) {
|
|
233
232
|
if (!running.get()) {
|
|
@@ -235,7 +234,6 @@ class SherpaOnnxTranscriber(private val context: Context) {
|
|
|
235
234
|
return
|
|
236
235
|
}
|
|
237
236
|
|
|
238
|
-
// Directly queue the audio data for processing
|
|
239
237
|
if (!pcmQueue.offer(pcm16le.copyOf())) {
|
|
240
238
|
// Queue is full, drop oldest and try again
|
|
241
239
|
pcmQueue.poll()
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
package com.mentra.bluetoothsdk.utils;
|
|
2
|
+
|
|
3
|
+
import android.util.Log;
|
|
4
|
+
|
|
5
|
+
import java.util.Map;
|
|
6
|
+
import java.util.concurrent.ConcurrentHashMap;
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Reassembles compact JSON chunks from the glasses before normal event dispatch.
|
|
10
|
+
*/
|
|
11
|
+
public class MessageChunkReassembler {
|
|
12
|
+
private static final String TAG = "MessageChunkReassembler";
|
|
13
|
+
private static final long CHUNK_TIMEOUT_MS = 30000;
|
|
14
|
+
private static final int MAX_CONCURRENT_SESSIONS = 10;
|
|
15
|
+
|
|
16
|
+
private final Map<String, ChunkSession> activeSessions = new ConcurrentHashMap<>();
|
|
17
|
+
|
|
18
|
+
public String addChunk(String chunkId, int chunkIndex, int totalChunks, String data) {
|
|
19
|
+
cleanupTimedOutSessions();
|
|
20
|
+
|
|
21
|
+
if (activeSessions.size() >= MAX_CONCURRENT_SESSIONS && !activeSessions.containsKey(chunkId)) {
|
|
22
|
+
Log.w(TAG, "Maximum concurrent chunk sessions reached, dropping oldest");
|
|
23
|
+
removeOldestSession();
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
ChunkSession session = activeSessions.computeIfAbsent(chunkId, ignored -> new ChunkSession(chunkId, totalChunks));
|
|
27
|
+
boolean added = session.addChunk(chunkIndex, data);
|
|
28
|
+
if (!added) {
|
|
29
|
+
Log.w(TAG, "Failed to add chunk " + chunkIndex + " to session " + chunkId);
|
|
30
|
+
return null;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
Log.d(TAG, "Added chunk " + chunkIndex + "/" + (totalChunks - 1) + " for session " + chunkId);
|
|
34
|
+
|
|
35
|
+
if (!session.isComplete()) {
|
|
36
|
+
return null;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
String reassembled = session.reassemble();
|
|
40
|
+
activeSessions.remove(chunkId);
|
|
41
|
+
Log.d(TAG, "Reassembled message of " + reassembled.length() + " bytes from " + totalChunks + " chunks");
|
|
42
|
+
return reassembled;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
public void clear() {
|
|
46
|
+
activeSessions.clear();
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
private void cleanupTimedOutSessions() {
|
|
50
|
+
long now = System.currentTimeMillis();
|
|
51
|
+
activeSessions.entrySet().removeIf(entry -> now - entry.getValue().createdAt > CHUNK_TIMEOUT_MS);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
private void removeOldestSession() {
|
|
55
|
+
String oldestId = null;
|
|
56
|
+
long oldestTime = Long.MAX_VALUE;
|
|
57
|
+
for (Map.Entry<String, ChunkSession> entry : activeSessions.entrySet()) {
|
|
58
|
+
if (entry.getValue().createdAt < oldestTime) {
|
|
59
|
+
oldestTime = entry.getValue().createdAt;
|
|
60
|
+
oldestId = entry.getKey();
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
if (oldestId != null) {
|
|
64
|
+
activeSessions.remove(oldestId);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
private static class ChunkSession {
|
|
69
|
+
final String chunkId;
|
|
70
|
+
final int totalChunks;
|
|
71
|
+
final long createdAt;
|
|
72
|
+
final Map<Integer, String> chunks;
|
|
73
|
+
|
|
74
|
+
ChunkSession(String chunkId, int totalChunks) {
|
|
75
|
+
this.chunkId = chunkId;
|
|
76
|
+
this.totalChunks = totalChunks;
|
|
77
|
+
this.createdAt = System.currentTimeMillis();
|
|
78
|
+
this.chunks = new ConcurrentHashMap<>();
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
boolean addChunk(int index, String data) {
|
|
82
|
+
if (index < 0 || index >= totalChunks) {
|
|
83
|
+
return false;
|
|
84
|
+
}
|
|
85
|
+
chunks.put(index, data);
|
|
86
|
+
return true;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
boolean isComplete() {
|
|
90
|
+
return chunks.size() == totalChunks;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
String reassemble() {
|
|
94
|
+
StringBuilder reassembled = new StringBuilder();
|
|
95
|
+
for (int i = 0; i < totalChunks; i++) {
|
|
96
|
+
String chunk = chunks.get(i);
|
|
97
|
+
if (chunk == null) {
|
|
98
|
+
throw new IllegalStateException("Missing chunk " + i + " for session " + chunkId);
|
|
99
|
+
}
|
|
100
|
+
reassembled.append(chunk);
|
|
101
|
+
}
|
|
102
|
+
return reassembled.toString();
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
}
|