@mentra/bluetooth-sdk 0.1.6 → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/README.md +26 -22
  2. package/android/.project +28 -0
  3. package/android/build.gradle +5 -9
  4. package/android/lc3Lib/.project +28 -0
  5. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +54 -33
  6. package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +48 -10
  7. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +27 -36
  8. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt +8 -4
  9. package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +170 -14
  10. package/android/src/main/java/com/mentra/bluetoothsdk/audio/AudioModels.kt +6 -6
  11. package/android/src/main/java/com/mentra/bluetoothsdk/camera/CameraModels.kt +69 -10
  12. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/ControllerManager.kt +5 -3
  13. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/R1.kt +2 -1
  14. package/android/src/main/java/com/mentra/bluetoothsdk/debug/BleTraceLogger.kt +149 -0
  15. package/android/src/main/java/com/mentra/bluetoothsdk/events/BluetoothEvents.kt +13 -0
  16. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoUploadServer.kt +297 -0
  17. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/MentraPhotoReceiverModule.kt +120 -0
  18. package/android/src/main/java/com/mentra/bluetoothsdk/services/Foreground.kt +13 -0
  19. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G1.java +1 -1
  20. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt +3 -1
  21. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Mach1.java +1 -1
  22. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraLive.java +269 -28
  23. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraNex.kt +1 -1
  24. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt +8 -3
  25. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Simulated.kt +4 -1
  26. package/android/src/main/java/com/mentra/bluetoothsdk/status/DeviceStatus.kt +14 -12
  27. package/android/src/main/java/com/mentra/bluetoothsdk/status/RuntimeState.kt +4 -2
  28. package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +131 -6
  29. package/android/src/main/java/com/mentra/bluetoothsdk/stt/SherpaOnnxTranscriber.kt +1 -3
  30. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunkReassembler.java +105 -0
  31. package/build/BluetoothSdk.types.d.ts +107 -48
  32. package/build/BluetoothSdk.types.d.ts.map +1 -1
  33. package/build/BluetoothSdk.types.js +4 -0
  34. package/build/BluetoothSdk.types.js.map +1 -1
  35. package/build/_private/BluetoothSdkModule.d.ts +8 -6
  36. package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
  37. package/build/_private/BluetoothSdkModule.js +58 -9
  38. package/build/_private/BluetoothSdkModule.js.map +1 -1
  39. package/build/index.d.ts +2 -2
  40. package/build/index.d.ts.map +1 -1
  41. package/build/index.js +6 -5
  42. package/build/index.js.map +1 -1
  43. package/build/photo-receiver/MentraPhotoReceiver.types.d.ts +18 -0
  44. package/build/photo-receiver/MentraPhotoReceiver.types.d.ts.map +1 -0
  45. package/build/photo-receiver/MentraPhotoReceiver.types.js +2 -0
  46. package/build/photo-receiver/MentraPhotoReceiver.types.js.map +1 -0
  47. package/build/photo-receiver/MentraPhotoReceiverModule.d.ts +10 -0
  48. package/build/photo-receiver/MentraPhotoReceiverModule.d.ts.map +1 -0
  49. package/build/photo-receiver/MentraPhotoReceiverModule.js +3 -0
  50. package/build/photo-receiver/MentraPhotoReceiverModule.js.map +1 -0
  51. package/build/photo-receiver/MentraPhotoReceiverModule.web.d.ts +10 -0
  52. package/build/photo-receiver/MentraPhotoReceiverModule.web.d.ts.map +1 -0
  53. package/build/photo-receiver/MentraPhotoReceiverModule.web.js +12 -0
  54. package/build/photo-receiver/MentraPhotoReceiverModule.web.js.map +1 -0
  55. package/build/photo-receiver/index.d.ts +3 -0
  56. package/build/photo-receiver/index.d.ts.map +1 -0
  57. package/build/photo-receiver/index.js +3 -0
  58. package/build/photo-receiver/index.js.map +1 -0
  59. package/build/react/useMentraBluetooth.d.ts +5 -3
  60. package/build/react/useMentraBluetooth.d.ts.map +1 -1
  61. package/build/react/useMentraBluetooth.js +15 -4
  62. package/build/react/useMentraBluetooth.js.map +1 -1
  63. package/expo-module.config.json +5 -2
  64. package/ios/BluetoothSdkModule.swift +59 -26
  65. package/ios/LocalPhotoUploadServer.swift +411 -0
  66. package/ios/MentraBluetoothSDK.podspec +8 -5
  67. package/ios/MentraPhotoReceiverModule.swift +147 -0
  68. package/ios/Source/Audio/AudioModels.swift +10 -10
  69. package/ios/Source/Bridge.swift +42 -12
  70. package/ios/Source/Camera/CameraModels.swift +74 -19
  71. package/ios/Source/DeviceManager.swift +23 -98
  72. package/ios/Source/DeviceStore.swift +12 -4
  73. package/ios/Source/Events/BluetoothEvents.swift +37 -0
  74. package/ios/Source/MentraBluetoothSDK.swift +135 -14
  75. package/ios/Source/Status/DeviceStatus.swift +12 -4
  76. package/ios/Source/Status/RuntimeState.swift +22 -13
  77. package/ios/Source/Streaming/StreamModels.swift +239 -45
  78. package/ios/Source/controllers/ControllerManager.swift +3 -3
  79. package/ios/Source/controllers/R1.swift +9 -3
  80. package/ios/Source/sgcs/G1.swift +5 -4
  81. package/ios/Source/sgcs/G2.swift +5 -6
  82. package/ios/Source/sgcs/Mach1.swift +1 -1
  83. package/ios/Source/sgcs/MentraLive.swift +183 -22
  84. package/ios/Source/sgcs/MentraNex.swift +1 -1
  85. package/ios/Source/sgcs/SGCManager.swift +11 -3
  86. package/ios/Source/sgcs/Simulated.swift +2 -2
  87. package/ios/Source/stt/SherpaOnnxTranscriber.swift +1 -4
  88. package/ios/Source/utils/MessageChunkReassembler.swift +89 -0
  89. package/package.json +6 -1
  90. package/src/BluetoothSdk.types.ts +133 -62
  91. package/src/_private/BluetoothSdkModule.ts +79 -26
  92. package/src/index.ts +15 -8
  93. package/src/photo-receiver/MentraPhotoReceiver.types.ts +20 -0
  94. package/src/photo-receiver/MentraPhotoReceiverModule.ts +14 -0
  95. package/src/photo-receiver/MentraPhotoReceiverModule.web.ts +20 -0
  96. package/src/photo-receiver/index.ts +2 -0
  97. package/src/react/useMentraBluetooth.ts +19 -7
  98. package/android/src/main/java/com/mentra/bluetoothsdk/stt/VadGateSpeechPolicy.kt +0 -221
  99. package/ios/Packages/VAD/Common/VAD/VADQuality.swift +0 -41
  100. package/ios/Packages/VAD/Common/VAD/VADState.swift +0 -26
  101. package/ios/Packages/VAD/Common/VAD/VADStrategy.swift +0 -29
  102. package/ios/Packages/VAD/Common/VAD/VADType.swift +0 -14
  103. package/ios/Packages/VAD/Data/Configuration.swift +0 -68
  104. package/ios/Packages/VAD/Data/FrameSize.swift +0 -39
  105. package/ios/Packages/VAD/Data/Record.swift +0 -13
  106. package/ios/Packages/VAD/Data/Result.swift +0 -22
  107. package/ios/Packages/VAD/Data/SampleRate.swift +0 -48
  108. package/ios/Packages/VAD/Silero/Model/silero_vad.onnx +0 -0
  109. package/ios/Packages/VAD/Silero/SileroVAD.swift +0 -284
  110. package/ios/Packages/VAD/Silero/SileroVADStrategy.swift +0 -64
@@ -32,6 +32,7 @@ internal data class GlassesStatus(
32
32
  val fullyBooted: Boolean,
33
33
  val connected: Boolean,
34
34
  val micEnabled: Boolean,
35
+ val voiceActivityDetectionEnabled: Boolean,
35
36
  val connectionState: GlassesConnectionState,
36
37
  val bluetoothClassicConnected: Boolean,
37
38
  val signalStrength: Int,
@@ -72,6 +73,7 @@ internal data class GlassesStatus(
72
73
  mapOf(
73
74
  "connection" to connectionState.toStatusMap(connected, fullyBooted),
74
75
  "micEnabled" to micEnabled,
76
+ "voiceActivityDetectionEnabled" to voiceActivityDetectionEnabled,
75
77
  "bluetoothClassicConnected" to bluetoothClassicConnected,
76
78
  "signalStrength" to signalStrength,
77
79
  "signalStrengthUpdatedAt" to signalStrengthUpdatedAt,
@@ -114,6 +116,7 @@ internal data class GlassesStatus(
114
116
  fullyBooted = boolValue(values, "fullyBooted") ?: false,
115
117
  connected = boolValue(values, "connected") ?: false,
116
118
  micEnabled = boolValue(values, "micEnabled") ?: false,
119
+ voiceActivityDetectionEnabled = boolValue(values, "voiceActivityDetectionEnabled") ?: true,
117
120
  connectionState = GlassesConnectionState.fromValue(stringValue(values, "connectionState")),
118
121
  bluetoothClassicConnected = boolValue(values, "bluetoothClassicConnected") ?: false,
119
122
  signalStrength = numberValue(values, "signalStrength") ?: -1,
@@ -186,7 +189,7 @@ internal data class BluetoothStatus(
186
189
  val dashboardDepth: Int,
187
190
  val headUpAngle: Int,
188
191
  val contextualDashboard: Boolean,
189
- val galleryModeAuto: Boolean,
192
+ val galleryModeEnabled: Boolean,
190
193
  val buttonPhotoSize: ButtonPhotoSize,
191
194
  val buttonCameraLed: Boolean,
192
195
  val buttonMaxRecordingTime: Int,
@@ -196,7 +199,6 @@ internal data class BluetoothStatus(
196
199
  val shouldSendPcm: Boolean,
197
200
  val shouldSendLc3: Boolean,
198
201
  val shouldSendTranscript: Boolean,
199
- val bypassVad: Boolean,
200
202
  val offlineCaptionsRunning: Boolean,
201
203
  val localSttFallbackActive: Boolean,
202
204
  val shouldSendBootingMessage: Boolean,
@@ -240,7 +242,7 @@ internal data class BluetoothStatus(
240
242
  "dashboard_depth" to dashboardDepth,
241
243
  "head_up_angle" to headUpAngle,
242
244
  "contextual_dashboard" to contextualDashboard,
243
- "galleryModeAuto" to galleryModeAuto,
245
+ "galleryModeEnabled" to galleryModeEnabled,
244
246
  "button_photo_size" to buttonPhotoSize.value,
245
247
  "button_camera_led" to buttonCameraLed,
246
248
  "button_max_recording_time" to buttonMaxRecordingTime,
@@ -250,7 +252,6 @@ internal data class BluetoothStatus(
250
252
  "should_send_pcm" to shouldSendPcm,
251
253
  "should_send_lc3" to shouldSendLc3,
252
254
  "should_send_transcript" to shouldSendTranscript,
253
- "bypass_vad" to bypassVad,
254
255
  "offline_captions_running" to offlineCaptionsRunning,
255
256
  "local_stt_fallback_active" to localSttFallbackActive,
256
257
  "shouldSendBootingMessage" to shouldSendBootingMessage,
@@ -288,7 +289,8 @@ internal data class BluetoothStatus(
288
289
  dashboardDepth = numberValue(values, "dashboard_depth") ?: 2,
289
290
  headUpAngle = numberValue(values, "head_up_angle") ?: 30,
290
291
  contextualDashboard = boolValue(values, "contextual_dashboard") ?: true,
291
- galleryModeAuto = boolValue(values, "galleryModeAuto") ?: true,
292
+ galleryModeEnabled =
293
+ boolValue(values, "gallery_mode") ?: boolValue(values, "galleryModeEnabled") ?: true,
292
294
  buttonPhotoSize = ButtonPhotoSize.fromValue(stringValue(values, "button_photo_size")),
293
295
  buttonCameraLed = boolValue(values, "button_camera_led") ?: true,
294
296
  buttonMaxRecordingTime = numberValue(values, "button_max_recording_time") ?: 10,
@@ -298,7 +300,6 @@ internal data class BluetoothStatus(
298
300
  shouldSendPcm = boolValue(values, "should_send_pcm") ?: false,
299
301
  shouldSendLc3 = boolValue(values, "should_send_lc3") ?: false,
300
302
  shouldSendTranscript = boolValue(values, "should_send_transcript") ?: false,
301
- bypassVad = boolValue(values, "bypass_vad") ?: true,
302
303
  offlineCaptionsRunning = boolValue(values, "offline_captions_running") ?: false,
303
304
  localSttFallbackActive = boolValue(values, "local_stt_fallback_active") ?: false,
304
305
  shouldSendBootingMessage = boolValue(values, "shouldSendBootingMessage") ?: true,
@@ -310,6 +311,7 @@ internal data class GlassesStatusUpdate(
310
311
  val fullyBooted: Boolean? = null,
311
312
  val connected: Boolean? = null,
312
313
  val micEnabled: Boolean? = null,
314
+ val voiceActivityDetectionEnabled: Boolean? = null,
313
315
  val connectionState: GlassesConnectionState? = null,
314
316
  val bluetoothClassicConnected: Boolean? = null,
315
317
  val signalStrength: Int? = null,
@@ -359,6 +361,7 @@ internal data class GlassesStatusUpdate(
359
361
  put("connection", state.toStatusMap(connected == true, fullyBooted == true))
360
362
  }
361
363
  putIfNotNull("micEnabled", micEnabled)
364
+ putIfNotNull("voiceActivityDetectionEnabled", voiceActivityDetectionEnabled)
362
365
  putIfNotNull("bluetoothClassicConnected", bluetoothClassicConnected)
363
366
  putIfNotNull("signalStrength", signalStrength)
364
367
  putIfNotNull("signalStrengthUpdatedAt", signalStrengthUpdatedAt)
@@ -405,6 +408,7 @@ internal data class GlassesStatusUpdate(
405
408
  fullyBooted = optionalBoolValue(values, "fullyBooted"),
406
409
  connected = optionalBoolValue(values, "connected"),
407
410
  micEnabled = optionalBoolValue(values, "micEnabled"),
411
+ voiceActivityDetectionEnabled = optionalBoolValue(values, "voiceActivityDetectionEnabled"),
408
412
  connectionState = GlassesConnectionState.optionalFromValue(optionalStringValue(values, "connectionState")),
409
413
  bluetoothClassicConnected = optionalBoolValue(values, "bluetoothClassicConnected"),
410
414
  signalStrength = optionalNumberValue(values, "signalStrength"),
@@ -491,7 +495,7 @@ internal data class BluetoothStatusUpdate(
491
495
  val dashboardDepth: Int? = null,
492
496
  val headUpAngle: Int? = null,
493
497
  val contextualDashboard: Boolean? = null,
494
- val galleryModeAuto: Boolean? = null,
498
+ val galleryModeEnabled: Boolean? = null,
495
499
  val buttonPhotoSize: ButtonPhotoSize? = null,
496
500
  val buttonCameraLed: Boolean? = null,
497
501
  val buttonMaxRecordingTime: Int? = null,
@@ -501,7 +505,6 @@ internal data class BluetoothStatusUpdate(
501
505
  val shouldSendPcm: Boolean? = null,
502
506
  val shouldSendLc3: Boolean? = null,
503
507
  val shouldSendTranscript: Boolean? = null,
504
- val bypassVad: Boolean? = null,
505
508
  val offlineCaptionsRunning: Boolean? = null,
506
509
  val localSttFallbackActive: Boolean? = null,
507
510
  val shouldSendBootingMessage: Boolean? = null,
@@ -535,7 +538,7 @@ internal data class BluetoothStatusUpdate(
535
538
  putIfNotNull("dashboard_depth", dashboardDepth)
536
539
  putIfNotNull("head_up_angle", headUpAngle)
537
540
  putIfNotNull("contextual_dashboard", contextualDashboard)
538
- putIfNotNull("galleryModeAuto", galleryModeAuto)
541
+ putIfNotNull("galleryModeEnabled", galleryModeEnabled)
539
542
  buttonPhotoSize?.let { put("button_photo_size", it.value) }
540
543
  putIfNotNull("button_camera_led", buttonCameraLed)
541
544
  putIfNotNull("button_max_recording_time", buttonMaxRecordingTime)
@@ -545,7 +548,6 @@ internal data class BluetoothStatusUpdate(
545
548
  putIfNotNull("should_send_pcm", shouldSendPcm)
546
549
  putIfNotNull("should_send_lc3", shouldSendLc3)
547
550
  putIfNotNull("should_send_transcript", shouldSendTranscript)
548
- putIfNotNull("bypass_vad", bypassVad)
549
551
  putIfNotNull("offline_captions_running", offlineCaptionsRunning)
550
552
  putIfNotNull("local_stt_fallback_active", localSttFallbackActive)
551
553
  putIfNotNull("shouldSendBootingMessage", shouldSendBootingMessage)
@@ -585,7 +587,8 @@ internal data class BluetoothStatusUpdate(
585
587
  dashboardDepth = optionalNumberValue(values, "dashboard_depth"),
586
588
  headUpAngle = optionalNumberValue(values, "head_up_angle"),
587
589
  contextualDashboard = optionalBoolValue(values, "contextual_dashboard"),
588
- galleryModeAuto = optionalBoolValue(values, "galleryModeAuto"),
590
+ galleryModeEnabled =
591
+ optionalBoolValue(values, "gallery_mode") ?: optionalBoolValue(values, "galleryModeEnabled"),
589
592
  buttonPhotoSize =
590
593
  optionalStringValue(values, "button_photo_size")?.let(ButtonPhotoSize::fromValue),
591
594
  buttonCameraLed = optionalBoolValue(values, "button_camera_led"),
@@ -596,7 +599,6 @@ internal data class BluetoothStatusUpdate(
596
599
  shouldSendPcm = optionalBoolValue(values, "should_send_pcm"),
597
600
  shouldSendLc3 = optionalBoolValue(values, "should_send_lc3"),
598
601
  shouldSendTranscript = optionalBoolValue(values, "should_send_transcript"),
599
- bypassVad = optionalBoolValue(values, "bypass_vad"),
600
602
  offlineCaptionsRunning = optionalBoolValue(values, "offline_captions_running"),
601
603
  localSttFallbackActive = optionalBoolValue(values, "local_stt_fallback_active"),
602
604
  shouldSendBootingMessage = optionalBoolValue(values, "shouldSendBootingMessage"),
@@ -74,6 +74,7 @@ sealed interface GlassesRuntimeState {
74
74
  val hotspot: HotspotStatus,
75
75
  override val ready: Boolean,
76
76
  val signal: SignalState,
77
+ val voiceActivityDetectionEnabled: Boolean,
77
78
  val wifi: WifiStatus,
78
79
  ) : GlassesRuntimeState {
79
80
  override val connected: Boolean = true
@@ -95,6 +96,7 @@ sealed interface GlassesRuntimeState {
95
96
  strengthDbm = status.signalStrength.takeUnless { it == -1 },
96
97
  updatedAt = status.signalStrengthUpdatedAt.takeUnless { it <= 0L },
97
98
  ),
99
+ voiceActivityDetectionEnabled = status.voiceActivityDetectionEnabled,
98
100
  wifi = status.wifi,
99
101
  )
100
102
  }
@@ -149,7 +151,7 @@ enum class MicMode(val value: String) {
149
151
  }
150
152
 
151
153
  data class GalleryModeState(
152
- val desired: GalleryMode,
154
+ val enabled: Boolean,
153
155
  )
154
156
 
155
157
  data class PhoneSdkRuntimeState(
@@ -169,7 +171,7 @@ data class PhoneSdkRuntimeState(
169
171
  PhoneSdkRuntimeState(
170
172
  currentMic = MicMode.fromValue(status.currentMic),
171
173
  defaultDevice = status.defaultDevice,
172
- galleryMode = GalleryModeState(if (status.galleryModeAuto) GalleryMode.AUTO else GalleryMode.MANUAL),
174
+ galleryMode = GalleryModeState(enabled = status.galleryModeEnabled),
173
175
  lastLog = status.lastLog,
174
176
  micRanking = status.micRanking.mapNotNull { MicMode.fromValue(it) },
175
177
  otherBluetoothConnected = status.otherBtConnected,
@@ -4,14 +4,14 @@ data class StreamVideoConfig @JvmOverloads constructor(
4
4
  val width: Int? = null,
5
5
  val height: Int? = null,
6
6
  val bitrate: Int? = null,
7
- val frameRate: Int? = null,
7
+ val fps: Int? = null,
8
8
  ) {
9
9
  fun toMap(): Map<String, Any> =
10
10
  listOfNotNull(
11
11
  width?.let { "width" to it },
12
12
  height?.let { "height" to it },
13
13
  bitrate?.let { "bitrate" to it },
14
- frameRate?.let { "frameRate" to it },
14
+ fps?.let { "frameRate" to it },
15
15
  ).toMap()
16
16
 
17
17
  companion object {
@@ -22,7 +22,7 @@ data class StreamVideoConfig @JvmOverloads constructor(
22
22
  width = numberValue(values, "width"),
23
23
  height = numberValue(values, "height"),
24
24
  bitrate = numberValue(values, "bitrate"),
25
- frameRate = numberValue(values, "frameRate"),
25
+ fps = numberValue(values, "fps"),
26
26
  )
27
27
  }
28
28
  }
@@ -56,11 +56,115 @@ data class StreamAudioConfig @JvmOverloads constructor(
56
56
  }
57
57
  }
58
58
 
59
+ data class StreamResolvedVideoConfig @JvmOverloads constructor(
60
+ val width: Int,
61
+ val height: Int,
62
+ val captureWidth: Int? = null,
63
+ val captureHeight: Int? = null,
64
+ val bitrate: Int,
65
+ val fps: Int,
66
+ ) {
67
+ fun toMap(): Map<String, Any> =
68
+ buildMap {
69
+ put("width", width)
70
+ put("height", height)
71
+ captureWidth?.let { put("captureWidth", it) }
72
+ captureHeight?.let { put("captureHeight", it) }
73
+ put("bitrate", bitrate)
74
+ put("fps", fps)
75
+ }
76
+
77
+ companion object {
78
+ @JvmStatic
79
+ fun fromMap(values: Map<String, Any>?): StreamResolvedVideoConfig? {
80
+ values ?: return null
81
+ return StreamResolvedVideoConfig(
82
+ width = numberValue(values, "width") ?: return null,
83
+ height = numberValue(values, "height") ?: return null,
84
+ captureWidth = numberValue(values, "captureWidth"),
85
+ captureHeight = numberValue(values, "captureHeight"),
86
+ bitrate = numberValue(values, "bitrate") ?: return null,
87
+ fps = numberValue(values, "fps") ?: return null,
88
+ )
89
+ }
90
+ }
91
+ }
92
+
93
+ data class StreamResolvedAudioConfig @JvmOverloads constructor(
94
+ val bitrate: Int? = null,
95
+ val sampleRate: Int? = null,
96
+ val echoCancellation: Boolean? = null,
97
+ val noiseSuppression: Boolean? = null,
98
+ ) {
99
+ fun toMap(): Map<String, Any> =
100
+ buildMap {
101
+ bitrate?.let { put("bitrate", it) }
102
+ sampleRate?.let { put("sampleRate", it) }
103
+ echoCancellation?.let { put("echoCancellation", it) }
104
+ noiseSuppression?.let { put("noiseSuppression", it) }
105
+ }
106
+
107
+ companion object {
108
+ @JvmStatic
109
+ fun fromMap(values: Map<String, Any>?): StreamResolvedAudioConfig? {
110
+ values ?: return null
111
+ return StreamResolvedAudioConfig(
112
+ bitrate = numberValue(values, "bitrate"),
113
+ sampleRate = numberValue(values, "sampleRate"),
114
+ echoCancellation = boolValue(values, "echoCancellation"),
115
+ noiseSuppression = boolValue(values, "noiseSuppression"),
116
+ )
117
+ }
118
+ }
119
+ }
120
+
121
+ enum class StreamTransport(val value: String) {
122
+ RTMP("rtmp"),
123
+ SRT("srt"),
124
+ WHIP("whip");
125
+
126
+ companion object {
127
+ @JvmStatic
128
+ fun fromValue(value: String?): StreamTransport? =
129
+ when (value?.lowercase()) {
130
+ "rtmp" -> RTMP
131
+ "srt" -> SRT
132
+ "whip" -> WHIP
133
+ else -> null
134
+ }
135
+ }
136
+ }
137
+
138
+ data class StreamResolvedConfig @JvmOverloads constructor(
139
+ val transport: StreamTransport? = null,
140
+ val video: StreamResolvedVideoConfig? = null,
141
+ val audio: StreamResolvedAudioConfig? = null,
142
+ ) {
143
+ fun toMap(): Map<String, Any> =
144
+ buildMap {
145
+ transport?.let { put("transport", it.value) }
146
+ video?.let { put("video", it.toMap()) }
147
+ audio?.let { put("audio", it.toMap()) }
148
+ }
149
+
150
+ companion object {
151
+ @JvmStatic
152
+ fun fromMap(values: Map<String, Any>?): StreamResolvedConfig? {
153
+ values ?: return null
154
+ return StreamResolvedConfig(
155
+ transport = StreamTransport.fromValue(stringValue(values, "transport")),
156
+ video = StreamResolvedVideoConfig.fromMap(stringMapValue(values["video"])),
157
+ audio = StreamResolvedAudioConfig.fromMap(stringMapValue(values["audio"])),
158
+ )
159
+ }
160
+ }
161
+ }
162
+
59
163
  data class StreamRequest @JvmOverloads constructor(
60
164
  val streamUrl: String,
61
165
  val streamId: String = "",
62
166
  val keepAlive: Boolean = true,
63
- val keepAliveIntervalSeconds: Int = 15,
167
+ val keepAliveIntervalSeconds: Int = 5,
64
168
  val sound: Boolean = true,
65
169
  val video: StreamVideoConfig? = null,
66
170
  val audio: StreamAudioConfig? = null,
@@ -68,6 +172,7 @@ data class StreamRequest @JvmOverloads constructor(
68
172
  ) {
69
173
  fun toMap(): Map<String, Any> {
70
174
  val values = extraValues.toMutableMap()
175
+ values.remove("keepAliveMode")
71
176
  values["type"] = "start_stream"
72
177
  values["streamUrl"] = streamUrl
73
178
  values["streamId"] = streamId
@@ -90,7 +195,7 @@ data class StreamRequest @JvmOverloads constructor(
90
195
  ?: "",
91
196
  streamId = values["streamId"] as? String ?: "",
92
197
  keepAlive = values["keepAlive"] as? Boolean ?: true,
93
- keepAliveIntervalSeconds = (values["keepAliveIntervalSeconds"] as? Number)?.toInt() ?: 15,
198
+ keepAliveIntervalSeconds = (values["keepAliveIntervalSeconds"] as? Number)?.toInt() ?: 5,
94
199
  sound = values["sound"] as? Boolean ?: true,
95
200
  video = StreamVideoConfig.fromMap(stringMapValue(values["video"])),
96
201
  audio = StreamAudioConfig.fromMap(stringMapValue(values["audio"])),
@@ -99,7 +204,10 @@ data class StreamRequest @JvmOverloads constructor(
99
204
  }
100
205
  }
101
206
 
102
- data class StreamKeepAliveRequest @JvmOverloads constructor(
207
+ internal fun StreamRequest.isExternallyManagedKeepAlive(): Boolean =
208
+ stringValue(extraValues, "keepAliveMode") == "external"
209
+
210
+ internal data class StreamKeepAliveRequest @JvmOverloads constructor(
103
211
  val streamId: String,
104
212
  val ackId: String,
105
213
  val extraValues: Map<String, Any> = emptyMap(),
@@ -162,6 +270,7 @@ sealed interface StreamStatus {
162
270
  val state: StreamState
163
271
  val streamId: String?
164
272
  val timestamp: Long?
273
+ val resolvedConfig: StreamResolvedConfig?
165
274
 
166
275
  fun toMap(): Map<String, Any> {
167
276
  val values = mutableMapOf<String, Any>(
@@ -170,6 +279,7 @@ sealed interface StreamStatus {
170
279
  )
171
280
  streamId?.takeIf { it.isNotBlank() }?.let { values["streamId"] = it }
172
281
  timestamp?.let { values["timestamp"] = it }
282
+ resolvedConfig?.let { values["resolvedConfig"] = it.toMap() }
173
283
 
174
284
  when (this) {
175
285
  is Lifecycle -> Unit
@@ -197,6 +307,7 @@ sealed interface StreamStatus {
197
307
  override val state: StreamState,
198
308
  override val streamId: String?,
199
309
  override val timestamp: Long?,
310
+ override val resolvedConfig: StreamResolvedConfig?,
200
311
  ) : StreamStatus {
201
312
  override val kind: StreamStatusKind = StreamStatusKind.LIFECYCLE
202
313
  }
@@ -207,6 +318,7 @@ sealed interface StreamStatus {
207
318
  val maxAttempts: Int,
208
319
  val reason: String,
209
320
  override val timestamp: Long?,
321
+ override val resolvedConfig: StreamResolvedConfig?,
210
322
  ) : StreamStatus {
211
323
  override val kind: StreamStatusKind = StreamStatusKind.RECONNECT
212
324
  override val state: StreamState = StreamState.RECONNECTING
@@ -216,6 +328,7 @@ sealed interface StreamStatus {
216
328
  override val streamId: String?,
217
329
  val attempt: Int,
218
330
  override val timestamp: Long?,
331
+ override val resolvedConfig: StreamResolvedConfig?,
219
332
  ) : StreamStatus {
220
333
  override val kind: StreamStatusKind = StreamStatusKind.RECONNECT
221
334
  override val state: StreamState = StreamState.RECONNECTED
@@ -225,6 +338,7 @@ sealed interface StreamStatus {
225
338
  override val streamId: String?,
226
339
  val maxAttempts: Int,
227
340
  override val timestamp: Long?,
341
+ override val resolvedConfig: StreamResolvedConfig?,
228
342
  ) : StreamStatus {
229
343
  override val kind: StreamStatusKind = StreamStatusKind.RECONNECT
230
344
  override val state: StreamState = StreamState.RECONNECT_FAILED
@@ -234,6 +348,7 @@ sealed interface StreamStatus {
234
348
  override val streamId: String?,
235
349
  val errorDetails: String,
236
350
  override val timestamp: Long?,
351
+ override val resolvedConfig: StreamResolvedConfig?,
237
352
  ) : StreamStatus {
238
353
  override val kind: StreamStatusKind = StreamStatusKind.ERROR
239
354
  override val state: StreamState = StreamState.ERROR
@@ -246,6 +361,7 @@ sealed interface StreamStatus {
246
361
  override val streamId: String?,
247
362
  val attempt: Int?,
248
363
  override val timestamp: Long?,
364
+ override val resolvedConfig: StreamResolvedConfig?,
249
365
  ) : StreamStatus {
250
366
  override val kind: StreamStatusKind = StreamStatusKind.SNAPSHOT
251
367
  }
@@ -258,6 +374,7 @@ sealed interface StreamStatus {
258
374
  val reconnecting = boolValue(values, "reconnecting") ?: false
259
375
  val streamId = stringValue(values, "streamId")
260
376
  val timestamp = longValue(values, "timestamp")
377
+ val resolvedConfig = StreamResolvedConfig.fromMap(stringMapValue(values["resolvedConfig"]))
261
378
  val attempt = numberValue(values, "attempt")
262
379
  val maxAttempts = numberValue(values, "maxAttempts") ?: 0
263
380
 
@@ -273,6 +390,7 @@ sealed interface StreamStatus {
273
390
  streamId = streamId,
274
391
  attempt = attempt,
275
392
  timestamp = timestamp,
393
+ resolvedConfig = resolvedConfig,
276
394
  )
277
395
  }
278
396
 
@@ -281,6 +399,7 @@ sealed interface StreamStatus {
281
399
  streamId = streamId,
282
400
  errorDetails = rawState?.let { "Unknown stream status: $it" } ?: "Missing stream status",
283
401
  timestamp = timestamp,
402
+ resolvedConfig = resolvedConfig,
284
403
  )
285
404
 
286
405
  return when (state) {
@@ -290,27 +409,32 @@ sealed interface StreamStatus {
290
409
  maxAttempts = maxAttempts,
291
410
  reason = stringValue(values, "reason") ?: "",
292
411
  timestamp = timestamp,
412
+ resolvedConfig = resolvedConfig,
293
413
  )
294
414
  StreamState.RECONNECTED -> Reconnected(
295
415
  streamId = streamId,
296
416
  attempt = attempt ?: 0,
297
417
  timestamp = timestamp,
418
+ resolvedConfig = resolvedConfig,
298
419
  )
299
420
  StreamState.RECONNECT_FAILED -> ReconnectFailed(
300
421
  streamId = streamId,
301
422
  maxAttempts = maxAttempts,
302
423
  timestamp = timestamp,
424
+ resolvedConfig = resolvedConfig,
303
425
  )
304
426
  StreamState.ERROR -> Error(
305
427
  streamId = streamId,
306
428
  errorDetails = stringValue(values, "errorDetails")
307
429
  ?: if (rawState == "error_not_streaming") "not_streaming" else "Unknown stream error",
308
430
  timestamp = timestamp,
431
+ resolvedConfig = resolvedConfig,
309
432
  )
310
433
  else -> Lifecycle(
311
434
  state = state,
312
435
  streamId = streamId,
313
436
  timestamp = timestamp,
437
+ resolvedConfig = resolvedConfig,
314
438
  )
315
439
  }
316
440
  }
@@ -324,6 +448,7 @@ data class StreamStatusEvent(
324
448
 
325
449
  val state: StreamState get() = status.state
326
450
  val streamId: String? get() = status.streamId
451
+ val resolvedConfig: StreamResolvedConfig? get() = status.resolvedConfig
327
452
  val values: Map<String, Any> get() = status.toEventMap()
328
453
  }
329
454
 
@@ -226,8 +226,7 @@ class SherpaOnnxTranscriber(private val context: Context) {
226
226
  * Feed PCM audio data (16-bit little endian) into the transcriber. This method should be called
227
227
  * continuously with short chunks (e.g., 100-300ms).
228
228
  *
229
- * Note: Audio passed to this method is assumed to have already passed VAD elsewhere, so it's
230
- * directly queued for processing without additional VAD checks.
229
+ * Audio is queued directly; microphone VAD gating is not applied in the SDK.
231
230
  */
232
231
  fun acceptAudio(pcm16le: ByteArray) {
233
232
  if (!running.get()) {
@@ -235,7 +234,6 @@ class SherpaOnnxTranscriber(private val context: Context) {
235
234
  return
236
235
  }
237
236
 
238
- // Directly queue the audio data for processing
239
237
  if (!pcmQueue.offer(pcm16le.copyOf())) {
240
238
  // Queue is full, drop oldest and try again
241
239
  pcmQueue.poll()
@@ -0,0 +1,105 @@
1
+ package com.mentra.bluetoothsdk.utils;
2
+
3
+ import android.util.Log;
4
+
5
+ import java.util.Map;
6
+ import java.util.concurrent.ConcurrentHashMap;
7
+
8
+ /**
9
+ * Reassembles compact JSON chunks from the glasses before normal event dispatch.
10
+ */
11
+ public class MessageChunkReassembler {
12
+ private static final String TAG = "MessageChunkReassembler";
13
+ private static final long CHUNK_TIMEOUT_MS = 30000;
14
+ private static final int MAX_CONCURRENT_SESSIONS = 10;
15
+
16
+ private final Map<String, ChunkSession> activeSessions = new ConcurrentHashMap<>();
17
+
18
+ public String addChunk(String chunkId, int chunkIndex, int totalChunks, String data) {
19
+ cleanupTimedOutSessions();
20
+
21
+ if (activeSessions.size() >= MAX_CONCURRENT_SESSIONS && !activeSessions.containsKey(chunkId)) {
22
+ Log.w(TAG, "Maximum concurrent chunk sessions reached, dropping oldest");
23
+ removeOldestSession();
24
+ }
25
+
26
+ ChunkSession session = activeSessions.computeIfAbsent(chunkId, ignored -> new ChunkSession(chunkId, totalChunks));
27
+ boolean added = session.addChunk(chunkIndex, data);
28
+ if (!added) {
29
+ Log.w(TAG, "Failed to add chunk " + chunkIndex + " to session " + chunkId);
30
+ return null;
31
+ }
32
+
33
+ Log.d(TAG, "Added chunk " + chunkIndex + "/" + (totalChunks - 1) + " for session " + chunkId);
34
+
35
+ if (!session.isComplete()) {
36
+ return null;
37
+ }
38
+
39
+ String reassembled = session.reassemble();
40
+ activeSessions.remove(chunkId);
41
+ Log.d(TAG, "Reassembled message of " + reassembled.length() + " bytes from " + totalChunks + " chunks");
42
+ return reassembled;
43
+ }
44
+
45
+ public void clear() {
46
+ activeSessions.clear();
47
+ }
48
+
49
+ private void cleanupTimedOutSessions() {
50
+ long now = System.currentTimeMillis();
51
+ activeSessions.entrySet().removeIf(entry -> now - entry.getValue().createdAt > CHUNK_TIMEOUT_MS);
52
+ }
53
+
54
+ private void removeOldestSession() {
55
+ String oldestId = null;
56
+ long oldestTime = Long.MAX_VALUE;
57
+ for (Map.Entry<String, ChunkSession> entry : activeSessions.entrySet()) {
58
+ if (entry.getValue().createdAt < oldestTime) {
59
+ oldestTime = entry.getValue().createdAt;
60
+ oldestId = entry.getKey();
61
+ }
62
+ }
63
+ if (oldestId != null) {
64
+ activeSessions.remove(oldestId);
65
+ }
66
+ }
67
+
68
+ private static class ChunkSession {
69
+ final String chunkId;
70
+ final int totalChunks;
71
+ final long createdAt;
72
+ final Map<Integer, String> chunks;
73
+
74
+ ChunkSession(String chunkId, int totalChunks) {
75
+ this.chunkId = chunkId;
76
+ this.totalChunks = totalChunks;
77
+ this.createdAt = System.currentTimeMillis();
78
+ this.chunks = new ConcurrentHashMap<>();
79
+ }
80
+
81
+ boolean addChunk(int index, String data) {
82
+ if (index < 0 || index >= totalChunks) {
83
+ return false;
84
+ }
85
+ chunks.put(index, data);
86
+ return true;
87
+ }
88
+
89
+ boolean isComplete() {
90
+ return chunks.size() == totalChunks;
91
+ }
92
+
93
+ String reassemble() {
94
+ StringBuilder reassembled = new StringBuilder();
95
+ for (int i = 0; i < totalChunks; i++) {
96
+ String chunk = chunks.get(i);
97
+ if (chunk == null) {
98
+ throw new IllegalStateException("Missing chunk " + i + " for session " + chunkId);
99
+ }
100
+ reassembled.append(chunk);
101
+ }
102
+ return reassembled.toString();
103
+ }
104
+ }
105
+ }