@mentra/bluetooth-sdk 0.1.9 → 0.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (147) hide show
  1. package/README.md +116 -18
  2. package/android/build.gradle +289 -16
  3. package/android/gradle.properties.example +9 -0
  4. package/android/lc3Lib/build.gradle +95 -6
  5. package/android/settings.gradle +3 -0
  6. package/android/silero/LICENSE.md +17 -0
  7. package/android/silero/build.gradle +42 -0
  8. package/android/silero/proguard-rules.pro +24 -0
  9. package/android/silero/src/androidTest/assets/hello.wav +0 -0
  10. package/android/silero/src/androidTest/java/com/konovalov/vad/silero/VadSileroTest.kt +79 -0
  11. package/android/silero/src/main/assets/LICENSE +21 -0
  12. package/android/silero/src/main/assets/silero_vad.onnx +0 -0
  13. package/android/silero/src/main/java/com/konovalov/vad/silero/Vad.kt +160 -0
  14. package/android/silero/src/main/java/com/konovalov/vad/silero/VadSilero.kt +449 -0
  15. package/android/silero/src/main/java/com/konovalov/vad/silero/config/FrameSize.kt +15 -0
  16. package/android/silero/src/main/java/com/konovalov/vad/silero/config/Mode.kt +14 -0
  17. package/android/silero/src/main/java/com/konovalov/vad/silero/config/SampleRate.kt +12 -0
  18. package/android/silero/src/main/java/com/konovalov/vad/silero/utils/AudioUtils.kt +47 -0
  19. package/android/silero/src/main/java/com/konovalov/vad/silero/utils/TensorMap.kt +41 -0
  20. package/android/silero/src/test/java/com/konovalov/vad/silero/utils/AudioUtilsTest.kt +44 -0
  21. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkDefaults.kt +6 -0
  22. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +195 -34
  23. package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +103 -21
  24. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +210 -18
  25. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt +30 -2
  26. package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +787 -85
  27. package/android/src/main/java/com/mentra/bluetoothsdk/audio/AudioModels.kt +6 -2
  28. package/android/src/main/java/com/mentra/bluetoothsdk/camera/CameraModels.kt +125 -11
  29. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/ControllerManager.kt +12 -2
  30. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/R1.kt +8 -2
  31. package/android/src/main/java/com/mentra/bluetoothsdk/debug/BleTraceLogger.kt +33 -17
  32. package/android/src/main/java/com/mentra/bluetoothsdk/events/BluetoothEvents.kt +101 -0
  33. package/android/src/main/java/com/mentra/bluetoothsdk/internal/MapParsing.kt +8 -0
  34. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistry.java +108 -0
  35. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoUploadServer.kt +399 -74
  36. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/MentraPhotoReceiverModule.kt +100 -17
  37. package/android/src/main/java/com/mentra/bluetoothsdk/requests/DisplayRequests.kt +17 -0
  38. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G1.java +2 -2
  39. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt +1293 -758
  40. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Mach1.java +2 -2
  41. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraLive.java +271 -66
  42. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraNex.kt +197 -149
  43. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraosBle.java +12753 -27036
  44. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt +64 -2
  45. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Simulated.kt +16 -4
  46. package/android/src/main/java/com/mentra/bluetoothsdk/status/DeviceStatus.kt +60 -3
  47. package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +12 -3
  48. package/android/src/main/java/com/mentra/bluetoothsdk/stt/STTTools.kt +87 -36
  49. package/android/src/main/java/com/mentra/bluetoothsdk/stt/SherpaOnnxTranscriber.kt +13 -4
  50. package/android/src/main/java/com/mentra/bluetoothsdk/stt/VadGateSpeechPolicy.kt +229 -0
  51. package/android/src/main/java/com/mentra/bluetoothsdk/tts/TTSTools.kt +180 -0
  52. package/android/src/main/java/com/mentra/bluetoothsdk/utils/AvifExifStripper.java +573 -0
  53. package/android/src/main/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadService.java +488 -50
  54. package/android/src/main/java/com/mentra/bluetoothsdk/utils/HeifExifTagReader.java +199 -0
  55. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunkReassembler.java +18 -1
  56. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunker.java +64 -18
  57. package/android/src/main/java/com/mentra/bluetoothsdk/utils/NexSGCUtils.kt +48 -39
  58. package/android/src/test/java/com/mentra/bluetoothsdk/camera/PhotoRequestTest.kt +23 -0
  59. package/android/src/test/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistryTest.java +75 -0
  60. package/android/src/test/java/com/mentra/bluetoothsdk/utils/AvifExifStripperTest.java +30 -0
  61. package/android/src/test/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadServiceTest.java +98 -0
  62. package/android/src/test/resources/avif_with_exif.avif +0 -0
  63. package/build/BluetoothSdk.types.d.ts +253 -44
  64. package/build/BluetoothSdk.types.d.ts.map +1 -1
  65. package/build/BluetoothSdk.types.js +11 -10
  66. package/build/BluetoothSdk.types.js.map +1 -1
  67. package/build/_private/BluetoothSdkModule.d.ts +37 -23
  68. package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
  69. package/build/_private/BluetoothSdkModule.js +43 -52
  70. package/build/_private/BluetoothSdkModule.js.map +1 -1
  71. package/build/_private/photoRequestPayload.d.ts +4 -0
  72. package/build/_private/photoRequestPayload.d.ts.map +1 -0
  73. package/build/_private/photoRequestPayload.js +25 -0
  74. package/build/_private/photoRequestPayload.js.map +1 -0
  75. package/build/index.d.ts +1 -1
  76. package/build/index.d.ts.map +1 -1
  77. package/build/index.js +25 -0
  78. package/build/index.js.map +1 -1
  79. package/build/react/useMentraBluetooth.d.ts +2 -2
  80. package/build/react/useMentraBluetooth.d.ts.map +1 -1
  81. package/build/react/useMentraBluetooth.js +3 -3
  82. package/build/react/useMentraBluetooth.js.map +1 -1
  83. package/ios/BluetoothSdkModule.swift +177 -70
  84. package/ios/LocalPhotoUploadServer.swift +370 -105
  85. package/ios/MentraBluetoothSDK.podspec +3 -1
  86. package/ios/MentraPhotoReceiverModule.swift +62 -10
  87. package/ios/Packages/CoreObjC/include/PcmConverter.h +22 -0
  88. package/ios/Packages/SherpaOnnx/SherpaOnnx.swift +668 -26
  89. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Info.plist +13 -9
  90. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
  91. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
  92. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/libsherpa-onnx.a +0 -0
  93. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
  94. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
  95. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/libsherpa-onnx.a +0 -0
  96. package/ios/Source/BluetoothSdkDefaults.swift +6 -0
  97. package/ios/Source/Bridge.swift +68 -19
  98. package/ios/Source/DeviceManager.swift +167 -32
  99. package/ios/Source/DeviceStore.swift +24 -8
  100. package/ios/Source/MentraBluetoothSDK.swift +803 -47
  101. package/ios/Source/ObservableStore.swift +6 -0
  102. package/ios/Source/{Audio → audio}/AudioModels.swift +9 -7
  103. package/ios/Source/{Camera → camera}/CameraModels.swift +226 -16
  104. package/ios/Source/controllers/ControllerManager.swift +4 -3
  105. package/ios/Source/controllers/R1.swift +3 -3
  106. package/ios/Source/{Events → events}/BluetoothEvents.swift +183 -1
  107. package/ios/Source/{Internal → internal}/ValueParsing.swift +10 -1
  108. package/ios/Source/{Requests → requests}/DisplayRequests.swift +28 -4
  109. package/ios/Source/services/PhoneMic.swift +1 -0
  110. package/ios/Source/sgcs/G1.swift +5 -5
  111. package/ios/Source/sgcs/G2.swift +1494 -870
  112. package/ios/Source/sgcs/Mach1.swift +4 -4
  113. package/ios/Source/sgcs/MentraLive.swift +814 -321
  114. package/ios/Source/sgcs/MentraNex.swift +604 -244
  115. package/ios/Source/sgcs/SGCManager.swift +68 -5
  116. package/ios/Source/sgcs/Simulated.swift +9 -5
  117. package/ios/Source/sgcs/mentraos_ble.pb.swift +3073 -3134
  118. package/ios/Source/status/DeviceStatus.swift +974 -0
  119. package/ios/Source/{Status → status}/WifiHotspotStatus.swift +4 -4
  120. package/ios/Source/{Streaming → streaming}/StreamModels.swift +24 -9
  121. package/ios/Source/stt/STTTools.swift +27 -12
  122. package/ios/Source/stt/SherpaOnnxTranscriber.swift +31 -8
  123. package/ios/Source/tts/TTSTools.swift +164 -0
  124. package/ios/Source/utils/JSCExperiment.swift +7 -7
  125. package/ios/Source/utils/MessageChunkReassembler.swift +20 -1
  126. package/ios/Source/utils/MessageChunker.swift +78 -21
  127. package/ios/Source/utils/TarBz2Extractor.swift +53 -2
  128. package/package.json +1 -1
  129. package/plugin/build/withAndroid.js +68 -4
  130. package/src/BluetoothSdk.types.ts +344 -60
  131. package/src/_private/BluetoothSdkModule.ts +132 -102
  132. package/src/_private/photoRequestPayload.ts +28 -0
  133. package/src/index.ts +61 -15
  134. package/src/react/useMentraBluetooth.ts +6 -4
  135. package/android/.project +0 -28
  136. package/android/lc3Lib/.project +0 -28
  137. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/cargs.h +0 -162
  138. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/c-api.h +0 -1852
  139. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/cxx-api.h +0 -674
  140. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/sherpa-onnx.a +0 -0
  141. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/sherpa-onnx.a +0 -0
  142. package/ios/Source/Status/DeviceStatus.swift +0 -471
  143. /package/ios/Source/{Connection → connection}/ScanSession.swift +0 -0
  144. /package/ios/Source/{Errors → errors}/BluetoothError.swift +0 -0
  145. /package/ios/Source/{Internal → internal}/BluetoothAvailability.swift +0 -0
  146. /package/ios/Source/{Status → status}/RuntimeState.swift +0 -0
  147. /package/ios/Source/{Types → types}/DeviceModels.swift +0 -0
@@ -21,7 +21,9 @@ data class MicPcmEvent(
21
21
  bitsPerSample = numberValue(values, "bitsPerSample") ?: BITS_PER_SAMPLE,
22
22
  channels = numberValue(values, "channels") ?: CHANNELS,
23
23
  encoding = stringValue(values, "encoding") ?: ENCODING,
24
- voiceActivityDetectionEnabled = boolValue(values, "voiceActivityDetectionEnabled") ?: true,
24
+ voiceActivityDetectionEnabled =
25
+ boolValue(values, "voiceActivityDetectionEnabled")
26
+ ?: BluetoothSdkDefaults.VOICE_ACTIVITY_DETECTION_ENABLED,
25
27
  )
26
28
 
27
29
  fun toMap(): Map<String, Any> =
@@ -63,7 +65,9 @@ data class MicLc3Event(
63
65
  frameSizeBytes = numberValue(values, "frameSizeBytes") ?: DEFAULT_FRAME_SIZE_BYTES,
64
66
  bitrate = numberValue(values, "bitrate") ?: DEFAULT_BITRATE,
65
67
  packetizedFromGlasses = boolValue(values, "packetizedFromGlasses") ?: false,
66
- voiceActivityDetectionEnabled = boolValue(values, "voiceActivityDetectionEnabled") ?: true,
68
+ voiceActivityDetectionEnabled =
69
+ boolValue(values, "voiceActivityDetectionEnabled")
70
+ ?: BluetoothSdkDefaults.VOICE_ACTIVITY_DETECTION_ENABLED,
67
71
  )
68
72
 
69
73
  fun toMap(): Map<String, Any> =
@@ -16,7 +16,8 @@ enum class PhotoSize(val value: String) {
16
16
  enum class ButtonPhotoSize(val value: String) {
17
17
  SMALL("small"),
18
18
  MEDIUM("medium"),
19
- LARGE("large");
19
+ LARGE("large"),
20
+ MAX("max");
20
21
 
21
22
  companion object {
22
23
  @JvmStatic
@@ -47,20 +48,86 @@ data class ButtonVideoRecordingSettings(
47
48
  val fps: Int,
48
49
  )
49
50
 
51
+ enum class CameraRoiPosition(val value: Int, val label: String) {
52
+ CENTER(0, "center"),
53
+ BOTTOM(1, "bottom"),
54
+ TOP(2, "top");
55
+
56
+ companion object {
57
+ @JvmStatic
58
+ fun fromValue(value: Int?): CameraRoiPosition =
59
+ values().firstOrNull { it.value == value } ?: CENTER
60
+
61
+ @JvmStatic
62
+ fun fromName(value: String?): CameraRoiPosition =
63
+ values().firstOrNull { it.label == value } ?: CENTER
64
+ }
65
+ }
66
+
50
67
  class CameraFov @JvmOverloads constructor(
51
68
  fov: Int = DEFAULT_FOV,
52
- roiPosition: Int = DEFAULT_ROI_POSITION,
69
+ roiPosition: CameraRoiPosition = DEFAULT_ROI_POSITION,
53
70
  ) {
54
71
  val fov: Int = fov.coerceIn(MIN_FOV, MAX_FOV)
55
- val roiPosition: Int = roiPosition.coerceIn(MIN_ROI_POSITION, MAX_ROI_POSITION)
72
+ val roiPosition: CameraRoiPosition = roiPosition
56
73
 
57
74
  companion object {
58
- const val MIN_FOV = 82
75
+ const val MIN_FOV = 62
59
76
  const val MAX_FOV = 118
60
77
  const val DEFAULT_FOV = 102
61
- const val MIN_ROI_POSITION = 0
62
- const val MAX_ROI_POSITION = 2
63
- const val DEFAULT_ROI_POSITION = 0
78
+ const val NARROW_FOV = 82
79
+ @JvmField
80
+ val DEFAULT_ROI_POSITION = CameraRoiPosition.CENTER
81
+ @JvmField
82
+ val NARROW = CameraFov(NARROW_FOV, DEFAULT_ROI_POSITION)
83
+ @JvmField
84
+ val STANDARD = CameraFov(DEFAULT_FOV, DEFAULT_ROI_POSITION)
85
+ @JvmField
86
+ val WIDE = CameraFov(MAX_FOV, DEFAULT_ROI_POSITION)
87
+ }
88
+ }
89
+
90
+ data class CameraFovResult(
91
+ val requestId: String,
92
+ val fov: Int,
93
+ val roiPosition: CameraRoiPosition,
94
+ val timestamp: Long,
95
+ ) {
96
+ val values: Map<String, Any>
97
+ get() =
98
+ mapOf(
99
+ "requestId" to requestId,
100
+ "fov" to fov,
101
+ "roiPosition" to roiPosition.label,
102
+ "timestamp" to timestamp,
103
+ )
104
+
105
+ companion object {
106
+ @JvmStatic
107
+ fun fromAck(
108
+ ack: SettingsAckEvent,
109
+ fallback: CameraFov,
110
+ ): CameraFovResult {
111
+ if (ack.status == "error") {
112
+ throw BluetoothException(
113
+ ack.errorCode ?: "camera_fov_failed",
114
+ ack.errorMessage ?: "Camera FOV request failed.",
115
+ )
116
+ }
117
+ if (!ack.hardwareApplied) {
118
+ throw BluetoothException(
119
+ "camera_fov_not_applied",
120
+ "Camera FOV was saved but not applied to hardware.",
121
+ )
122
+ }
123
+
124
+ return CameraFovResult(
125
+ requestId = ack.requestId,
126
+ fov = ack.fov ?: fallback.fov,
127
+ roiPosition = CameraRoiPosition.fromValue(ack.roiPosition ?: fallback.roiPosition.value),
128
+ timestamp = ack.timestamp,
129
+ )
130
+ }
64
131
  }
65
132
  }
66
133
 
@@ -72,6 +139,7 @@ data class PhotoRequest @JvmOverloads constructor(
72
139
  val authToken: String? = null,
73
140
  val compress: PhotoCompression = PhotoCompression.MEDIUM,
74
141
  val flash: Boolean = true,
142
+ val save: Boolean = false,
75
143
  val sound: Boolean = true,
76
144
  /** Sensor exposure time for this capture only (ns), or null for auto exposure */
77
145
  val exposureTimeNs: Double? = null,
@@ -107,6 +175,7 @@ data class PhotoRequest @JvmOverloads constructor(
107
175
  authToken = stringValue(values, "authToken", "auth_token")?.takeIf { it.isNotBlank() },
108
176
  compress = PhotoCompression.fromValue(stringValue(values, "compress") ?: "none"),
109
177
  flash = boolValue(values, "flash") ?: true,
178
+ save = boolValue(values, "save", "saveToGallery") ?: false,
110
179
  sound = boolValue(values, "sound") ?: true,
111
180
  exposureTimeNs = exposureTimeNs,
112
181
  iso = iso,
@@ -154,8 +223,23 @@ data class VideoRecordingRequest(
154
223
  val requestId: String,
155
224
  val save: Boolean,
156
225
  val sound: Boolean,
226
+ // Optional per-recording overrides; 0 means "use the saved button-video default".
227
+ val width: Int = 0,
228
+ val height: Int = 0,
229
+ val fps: Int = 0,
157
230
  )
158
231
 
232
+ data class VideoRecordingStatusEvent(
233
+ val values: Map<String, Any>,
234
+ ) {
235
+ val requestId: String get() = stringValue(values, "requestId").orEmpty()
236
+ val success: Boolean get() = boolValue(values, "success") ?: false
237
+ val status: String get() = stringValue(values, "status").orEmpty()
238
+ val details: String? get() = stringValue(values, "details")
239
+ val timestamp: Long get() = longValue(values, "timestamp") ?: System.currentTimeMillis()
240
+ val data: Map<String, Any>? get() = stringMapValue(values["data"])
241
+ }
242
+
159
243
  data class GalleryStatusEvent(
160
244
  val values: Map<String, Any>,
161
245
  )
@@ -172,7 +256,7 @@ sealed interface PhotoResponse {
172
256
  "requestId" to requestId,
173
257
  "uploadUrl" to uploadUrl,
174
258
  "timestamp" to timestamp,
175
- )
259
+ ).withOptionalPhotoMetadata(this)
176
260
 
177
261
  is Error -> mutableMapOf<String, Any>(
178
262
  "state" to state,
@@ -191,6 +275,10 @@ sealed interface PhotoResponse {
191
275
  data class Success(
192
276
  override val requestId: String,
193
277
  val uploadUrl: String,
278
+ val photoUrl: String?,
279
+ val statusUrl: String?,
280
+ val contentType: String?,
281
+ val fileSizeBytes: Long?,
194
282
  override val timestamp: Long,
195
283
  ) : PhotoResponse {
196
284
  override val state: String = "success"
@@ -210,14 +298,27 @@ sealed interface PhotoResponse {
210
298
  val requestId = stringValue(values, "requestId").orEmpty()
211
299
  val timestamp = longValue(values, "timestamp") ?: System.currentTimeMillis()
212
300
  val state = stringValue(values, "state")?.lowercase()
213
- return if (state == "success") {
301
+ val success = state == "success" || boolValue(values, "success") == true
302
+ return if (success) {
214
303
  val uploadUrl = stringValue(values, "uploadUrl").orEmpty()
215
- Success(requestId = requestId, uploadUrl = uploadUrl, timestamp = timestamp)
304
+ Success(
305
+ requestId = requestId,
306
+ uploadUrl = uploadUrl,
307
+ photoUrl = stringValue(values, "photoUrl"),
308
+ statusUrl = stringValue(values, "statusUrl"),
309
+ contentType = stringValue(values, "contentType", "mimeType"),
310
+ fileSizeBytes =
311
+ longValue(values, "fileSizeBytes")
312
+ ?: longValue(values, "bytes")
313
+ ?: longValue(values, "size"),
314
+ timestamp = timestamp,
315
+ )
216
316
  } else {
217
317
  Error(
218
318
  requestId = requestId,
219
319
  errorCode = stringValue(values, "errorCode"),
220
- errorMessage = stringValue(values, "errorMessage") ?: "Unknown photo error",
320
+ errorMessage =
321
+ stringValue(values, "errorMessage", "error") ?: "Unknown photo error",
221
322
  timestamp = timestamp,
222
323
  )
223
324
  }
@@ -225,6 +326,16 @@ sealed interface PhotoResponse {
225
326
  }
226
327
  }
227
328
 
329
+ private fun Map<String, Any>.withOptionalPhotoMetadata(
330
+ success: PhotoResponse.Success,
331
+ ): Map<String, Any> =
332
+ toMutableMap().apply {
333
+ success.photoUrl?.takeIf { it.isNotBlank() }?.let { this["photoUrl"] = it }
334
+ success.statusUrl?.takeIf { it.isNotBlank() }?.let { this["statusUrl"] = it }
335
+ success.contentType?.takeIf { it.isNotBlank() }?.let { this["contentType"] = it }
336
+ success.fileSizeBytes?.let { this["fileSizeBytes"] = it }
337
+ }
338
+
228
339
  data class PhotoResponseEvent(
229
340
  val response: PhotoResponse,
230
341
  ) {
@@ -241,6 +352,9 @@ data class PhotoStatusEvent(
241
352
  val status: String get() = stringValue(values, "status").orEmpty()
242
353
  val timestamp: Long get() = longValue(values, "timestamp") ?: System.currentTimeMillis()
243
354
  val resolvedConfig: Map<String, Any>? get() = stringMapValue(values["resolvedConfig"])
355
+ val requestedCaptureConfig: Map<String, Any>? get() = stringMapValue(values["requestedCaptureConfig"])
356
+ val meteredPreview: Map<String, Any>? get() = stringMapValue(values["meteredPreview"])
357
+ val captureMetadata: Map<String, Any>? get() = stringMapValue(values["captureMetadata"])
244
358
  val errorCode: String? get() = stringValue(values, "errorCode")
245
359
  val errorMessage: String? get() = stringValue(values, "errorMessage")
246
360
  }
@@ -1,5 +1,6 @@
1
1
  package com.mentra.bluetoothsdk.controllers
2
2
 
3
+ import com.mentra.bluetoothsdk.BluetoothSdkDefaults
3
4
  import com.mentra.bluetoothsdk.DeviceStore
4
5
 
5
6
  abstract class ControllerManager {
@@ -22,6 +23,7 @@ abstract class ControllerManager {
22
23
  authToken: String?,
23
24
  compress: String?,
24
25
  flash: Boolean,
26
+ save: Boolean,
25
27
  sound: Boolean,
26
28
  exposureTimeNs: Long?,
27
29
  iso: Int?,
@@ -43,7 +45,13 @@ abstract class ControllerManager {
43
45
  abstract fun clearDisplay()
44
46
  abstract fun sendTextWall(text: String)
45
47
  abstract fun sendDoubleTextWall(top: String, bottom: String)
46
- abstract fun displayBitmap(base64ImageData: String): Boolean
48
+ abstract fun displayBitmap(
49
+ base64ImageData: String,
50
+ x: Int? = null,
51
+ y: Int? = null,
52
+ width: Int? = null,
53
+ height: Int? = null
54
+ ): Boolean
47
55
  abstract fun showDashboard()
48
56
  abstract fun setDashboardPosition(height: Int, depth: Int)
49
57
 
@@ -135,7 +143,9 @@ abstract class ControllerManager {
135
143
  get() = DeviceStore.get("glasses", "micEnabled") as? Boolean ?: false
136
144
 
137
145
  val voiceActivityDetectionEnabled: Boolean
138
- get() = DeviceStore.get("glasses", "voiceActivityDetectionEnabled") as? Boolean ?: true
146
+ get() =
147
+ DeviceStore.get("glasses", "voiceActivityDetectionEnabled") as? Boolean
148
+ ?: BluetoothSdkDefaults.VOICE_ACTIVITY_DETECTION_ENABLED
139
149
 
140
150
  val batteryLevel: Int
141
151
  get() = DeviceStore.get("glasses", "batteryLevel") as? Int ?: -1
@@ -863,7 +863,7 @@ class R1 : ControllerManager() {
863
863
  override fun sendJson(jsonOriginal: Map<String, Any>, wakeUp: Boolean, requireAck: Boolean) {}
864
864
  override fun requestPhoto(
865
865
  requestId: String, appId: String, size: String?, webhookUrl: String?,
866
- authToken: String?, compress: String?, flash: Boolean, sound: Boolean,
866
+ authToken: String?, compress: String?, flash: Boolean, save: Boolean, sound: Boolean,
867
867
  exposureTimeNs: Long?, iso: Int?,
868
868
  ) {}
869
869
  override fun startVideoRecording(requestId: String, save: Boolean, flash: Boolean, sound: Boolean) {}
@@ -879,7 +879,13 @@ class R1 : ControllerManager() {
879
879
  override fun clearDisplay() {}
880
880
  override fun sendTextWall(text: String) {}
881
881
  override fun sendDoubleTextWall(top: String, bottom: String) {}
882
- override fun displayBitmap(base64ImageData: String): Boolean = false
882
+ override fun displayBitmap(
883
+ base64ImageData: String,
884
+ x: Int?,
885
+ y: Int?,
886
+ width: Int?,
887
+ height: Int?
888
+ ): Boolean = false
883
889
  override fun showDashboard() {}
884
890
  override fun setDashboardPosition(height: Int, depth: Int) {}
885
891
  override fun setHeadUpAngle(angle: Int) {}
@@ -10,6 +10,7 @@ import org.json.JSONObject
10
10
  object BleTraceLogger {
11
11
  private const val TAG = "MentraBleTrace"
12
12
  private const val MAX_PAYLOAD_CHARS = 3000
13
+ private const val K900_TYPE = "k900"
13
14
  private val sensitiveKeyParts =
14
15
  listOf("password", "pass", "token", "secret", "authorization", "auth", "email")
15
16
 
@@ -26,8 +27,12 @@ object BleTraceLogger {
26
27
 
27
28
  @JvmStatic
28
29
  fun logMap(direction: String, layer: String, type: String?, payload: Map<String, Any>) {
29
- val sanitized = sanitize(JSONObject(payload))
30
- Log.i(TAG, format(direction, layer, caller(), type ?: extractType(sanitized), null, sanitized.toString()))
30
+ try {
31
+ val sanitized = sanitize(JSONObject(payload))
32
+ Log.i(TAG, format(direction, layer, caller(), type ?: extractType(sanitized), null, sanitized.toString()))
33
+ } catch (e: Exception) {
34
+ Log.d(TAG, "BLE trace map logging failed", e)
35
+ }
31
36
  }
32
37
 
33
38
  @JvmStatic
@@ -37,20 +42,24 @@ object BleTraceLogger {
37
42
  event: String,
38
43
  extra: Map<String, Any?> = emptyMap(),
39
44
  ) {
40
- val payload = JSONObject()
41
- payload.put("event", event)
42
- payload.put("component", component)
43
- payload.put("pid", Process.myPid())
44
- payload.put("model", Build.MODEL)
45
- payload.put("sdkInt", Build.VERSION.SDK_INT)
46
-
47
- context?.let {
48
- payload.put("package", it.packageName)
49
- packageVersion(it)?.let { version -> payload.put("version", version) }
50
- }
45
+ try {
46
+ val payload = JSONObject()
47
+ payload.put("event", event)
48
+ payload.put("component", component)
49
+ payload.put("pid", Process.myPid())
50
+ payload.put("model", Build.MODEL)
51
+ payload.put("sdkInt", Build.VERSION.SDK_INT)
52
+
53
+ context?.let {
54
+ payload.put("package", it.packageName)
55
+ packageVersion(it)?.let { version -> payload.put("version", version) }
56
+ }
51
57
 
52
- extra.forEach { (key, value) -> payload.put(key, value ?: JSONObject.NULL) }
53
- Log.i(TAG, format("phone_app", "app_lifecycle", caller(), event, null, sanitize(payload).toString()))
58
+ extra.forEach { (key, value) -> payload.put(key, value ?: JSONObject.NULL) }
59
+ Log.i(TAG, format("phone_app", "app_lifecycle", caller(), event, null, sanitize(payload).toString()))
60
+ } catch (e: Exception) {
61
+ Log.d(TAG, "BLE trace lifecycle logging failed", e)
62
+ }
54
63
  }
55
64
 
56
65
  private fun format(
@@ -72,12 +81,16 @@ object BleTraceLogger {
72
81
  val inner = JSONObject(cValue)
73
82
  inner.optString("type").takeIf { it.isNotBlank() }?.let { return it }
74
83
  } catch (_: Exception) {
75
- return "k900:${cValue.take(40)}"
84
+ return extractK900Type(cValue)
76
85
  }
86
+ return extractK900Type(cValue)
77
87
  }
78
88
  return "unknown"
79
89
  }
80
90
 
91
+ private fun extractK900Type(cValue: String): String =
92
+ if (cValue == "sr_log") "$K900_TYPE:sr_log" else K900_TYPE
93
+
81
94
  private fun sanitize(value: JSONObject): JSONObject {
82
95
  val output = JSONObject()
83
96
  value.keys().forEach { key ->
@@ -102,7 +115,7 @@ object BleTraceLogger {
102
115
  try {
103
116
  return sanitize(JSONObject(value)).toString()
104
117
  } catch (_: Exception) {
105
- return truncate(value)
118
+ return sanitizeK900Command(value)
106
119
  }
107
120
  }
108
121
  return when (value) {
@@ -113,6 +126,9 @@ object BleTraceLogger {
113
126
  }
114
127
  }
115
128
 
129
+ private fun sanitizeK900Command(value: String): String =
130
+ if (value == "sr_log") value else "<non-json C payload>"
131
+
116
132
  private fun caller(): String {
117
133
  val frame =
118
134
  Throwable().stackTrace.firstOrNull {
@@ -39,6 +39,100 @@ data class SpeakingStatusEvent(
39
39
  val values: Map<String, Any>,
40
40
  )
41
41
 
42
+ data class OtaUpdateAvailableEvent(
43
+ val versionCode: Long?,
44
+ val versionName: String?,
45
+ val updates: List<String>,
46
+ val totalSize: Long?,
47
+ val cacheReady: Boolean?,
48
+ val values: Map<String, Any>,
49
+ ) {
50
+ companion object {
51
+ internal fun fromMap(values: Map<String, Any>): OtaUpdateAvailableEvent =
52
+ OtaUpdateAvailableEvent(
53
+ versionCode = longValue(values, "version_code"),
54
+ versionName = stringValue(values, "version_name"),
55
+ updates = (values["updates"] as? List<*>)?.filterIsInstance<String>() ?: emptyList(),
56
+ totalSize = longValue(values, "total_size"),
57
+ cacheReady = boolValue(values, "cache_ready"),
58
+ values = values,
59
+ )
60
+ }
61
+ }
62
+
63
+ data class OtaStartAckEvent(
64
+ val timestamp: Long?,
65
+ val values: Map<String, Any>,
66
+ ) {
67
+ companion object {
68
+ internal fun fromMap(values: Map<String, Any>): OtaStartAckEvent =
69
+ OtaStartAckEvent(
70
+ timestamp = longValue(values, "timestamp"),
71
+ values = values,
72
+ )
73
+ }
74
+ }
75
+
76
+ data class OtaStatusEvent(
77
+ val sessionId: String,
78
+ val totalSteps: Int,
79
+ val currentStep: Int,
80
+ val stepType: String,
81
+ val phase: String,
82
+ val stepPercent: Int,
83
+ val overallPercent: Int,
84
+ val status: String,
85
+ val errorMessage: String?,
86
+ val glassesTimeMs: Long?,
87
+ val values: Map<String, Any>,
88
+ ) {
89
+ companion object {
90
+ internal fun fromMap(values: Map<String, Any>): OtaStatusEvent =
91
+ OtaStatusEvent(
92
+ sessionId = stringValue(values, "session_id") ?: "",
93
+ totalSteps = numberValue(values, "total_steps") ?: 0,
94
+ currentStep = numberValue(values, "current_step") ?: 0,
95
+ stepType = stringValue(values, "step_type") ?: "",
96
+ phase = stringValue(values, "phase") ?: "",
97
+ stepPercent = numberValue(values, "step_percent") ?: 0,
98
+ overallPercent = numberValue(values, "overall_percent") ?: 0,
99
+ status = stringValue(values, "status") ?: "",
100
+ errorMessage = stringValue(values, "error_message"),
101
+ glassesTimeMs = longValue(values, "glasses_time_ms"),
102
+ values = values,
103
+ )
104
+ }
105
+ }
106
+
107
+ data class OtaQueryResult(
108
+ val values: Map<String, Any>,
109
+ ) {
110
+ val type: String get() = stringValue(values, "type").orEmpty()
111
+ val status: String? get() = stringValue(values, "status")
112
+ }
113
+
114
+ data class SettingsAckEvent(
115
+ val values: Map<String, Any>,
116
+ ) {
117
+ val requestId: String get() = stringValue(values, "requestId").orEmpty()
118
+ val setting: String get() = stringValue(values, "setting").orEmpty()
119
+ val status: String get() = stringValue(values, "status") ?: "applied"
120
+ val timestamp: Long get() = longValue(values, "timestamp") ?: System.currentTimeMillis()
121
+ val fov: Int? get() = numberValue(values, "fov")
122
+ val roiPosition: Int? get() = numberValue(values, "roiPosition", "roi_position")
123
+ val hardwareApplied: Boolean get() = boolValue(values, "hardwareApplied", "hardware_applied") ?: false
124
+ val errorCode: String? get() = stringValue(values, "errorCode")
125
+ val errorMessage: String? get() = stringValue(values, "errorMessage")
126
+ }
127
+
128
+ data class RgbLedControlResponseEvent(
129
+ val values: Map<String, Any>,
130
+ ) {
131
+ val requestId: String get() = stringValue(values, "requestId").orEmpty()
132
+ val state: String get() = stringValue(values, "state") ?: "error"
133
+ val errorCode: String? get() = stringValue(values, "errorCode")
134
+ }
135
+
42
136
  interface MentraBluetoothSdkListener {
43
137
  fun onStateChanged(state: MentraBluetoothState) {}
44
138
  fun onGlassesChanged(glasses: GlassesRuntimeState) {}
@@ -59,8 +153,15 @@ interface MentraBluetoothSdkListener {
59
153
  fun onGalleryStatus(event: GalleryStatusEvent) {}
60
154
  fun onPhotoResponse(event: PhotoResponseEvent) {}
61
155
  fun onPhotoStatus(event: PhotoStatusEvent) {}
156
+ fun onVideoRecordingStatus(event: VideoRecordingStatusEvent) {}
157
+ fun onRgbLedControlResponse(event: RgbLedControlResponseEvent) {}
62
158
  fun onStreamStatus(event: StreamStatusEvent) {}
63
159
  fun onKeepAliveAck(event: KeepAliveAckEvent) {}
160
+ fun onOtaUpdateAvailable(event: OtaUpdateAvailableEvent) {}
161
+ fun onOtaStartAck(event: OtaStartAckEvent) {}
162
+ fun onOtaStatus(event: OtaStatusEvent) {}
163
+ fun onSettingsAck(event: SettingsAckEvent) {}
164
+ fun onVersionInfo(event: VersionInfoResult) {}
64
165
  fun onMicPcm(event: MicPcmEvent) {}
65
166
  fun onMicLc3(event: MicLc3Event) {}
66
167
  fun onLocalTranscription(event: LocalTranscriptionEvent) {}
@@ -8,6 +8,14 @@ internal fun numberValue(
8
8
  (values[key] as? Number)?.toInt()
9
9
  }
10
10
 
11
+ internal fun doubleValue(
12
+ values: Map<String, Any>,
13
+ vararg keys: String,
14
+ ): Double? =
15
+ keys.firstNotNullOfOrNull { key ->
16
+ (values[key] as? Number)?.toDouble()
17
+ }
18
+
11
19
  internal fun stringValue(
12
20
  values: Map<String, Any>,
13
21
  vararg keys: String,
@@ -0,0 +1,108 @@
1
+ package com.mentra.bluetoothsdk.photoreceiver;
2
+
3
+ import androidx.annotation.Nullable;
4
+
5
+ import java.net.URI;
6
+ import java.util.Collections;
7
+ import java.util.HashSet;
8
+ import java.util.Objects;
9
+ import java.util.Set;
10
+
11
+ /**
12
+ * Tracks the SDK photo receiver that is hosted inside the phone app process.
13
+ *
14
+ * <p>The glasses need a LAN-reachable URL, but the phone-side BLE fallback relay
15
+ * can upload to the same receiver over loopback. Keeping this tiny registry lets
16
+ * the relay avoid stale Wi-Fi interface addresses after network changes.
17
+ */
18
+ public final class LocalPhotoReceiverRegistry {
19
+ private static volatile Set<URI> activeUploadUris = Collections.emptySet();
20
+
21
+ private LocalPhotoReceiverRegistry() {}
22
+
23
+ public static void register(String uploadUrl) {
24
+ try {
25
+ URI uri = URI.create(uploadUrl);
26
+ if (isSupportedUploadUri(uri)) {
27
+ Set<URI> nextUploadUris = new HashSet<>(activeUploadUris);
28
+ nextUploadUris.add(uri);
29
+ activeUploadUris = Collections.unmodifiableSet(nextUploadUris);
30
+ }
31
+ } catch (Exception ignored) {
32
+ }
33
+ }
34
+
35
+ public static void unregister() {
36
+ activeUploadUris = Collections.emptySet();
37
+ }
38
+
39
+ public static boolean isActive() {
40
+ return !activeUploadUris.isEmpty();
41
+ }
42
+
43
+ @Nullable
44
+ public static String loopbackUploadUrlFor(@Nullable String webhookUrl) {
45
+ Set<URI> registeredUris = activeUploadUris;
46
+ if (registeredUris.isEmpty() || webhookUrl == null || webhookUrl.isEmpty()) {
47
+ return null;
48
+ }
49
+
50
+ try {
51
+ URI uri = URI.create(webhookUrl);
52
+ for (URI registeredUri : registeredUris) {
53
+ if (matchesRegisteredUploadUri(uri, registeredUri)) {
54
+ return new URI(
55
+ uri.getScheme(),
56
+ null,
57
+ "127.0.0.1",
58
+ effectivePort(uri),
59
+ normalizedPath(uri),
60
+ uri.getRawQuery(),
61
+ uri.getRawFragment()
62
+ ).toString();
63
+ }
64
+ }
65
+ return null;
66
+ } catch (Exception ignored) {
67
+ return null;
68
+ }
69
+ }
70
+
71
+ private static boolean matchesRegisteredUploadUri(URI candidate, URI registered) {
72
+ return isSupportedUploadUri(candidate)
73
+ && equalsIgnoreCase(candidate.getScheme(), registered.getScheme())
74
+ && equalsIgnoreCase(candidate.getHost(), registered.getHost())
75
+ && effectivePort(candidate) == effectivePort(registered)
76
+ && Objects.equals(normalizedPath(candidate), normalizedPath(registered))
77
+ && Objects.equals(candidate.getRawQuery(), registered.getRawQuery())
78
+ && Objects.equals(candidate.getRawFragment(), registered.getRawFragment());
79
+ }
80
+
81
+ private static boolean isSupportedUploadUri(URI uri) {
82
+ String scheme = uri.getScheme();
83
+ if (!"http".equalsIgnoreCase(scheme) && !"https".equalsIgnoreCase(scheme)) {
84
+ return false;
85
+ }
86
+ return uri.getHost() != null && effectivePort(uri) > 0 && "/upload".equals(normalizedPath(uri));
87
+ }
88
+
89
+ private static int effectivePort(URI uri) {
90
+ int port = uri.getPort();
91
+ if (port != -1) {
92
+ return port;
93
+ }
94
+ return "https".equalsIgnoreCase(uri.getScheme()) ? 443 : 80;
95
+ }
96
+
97
+ private static String normalizedPath(URI uri) {
98
+ String path = uri.getRawPath();
99
+ return (path == null || path.isEmpty()) ? "/" : path;
100
+ }
101
+
102
+ private static boolean equalsIgnoreCase(@Nullable String first, @Nullable String second) {
103
+ if (first == null || second == null) {
104
+ return first == second;
105
+ }
106
+ return first.equalsIgnoreCase(second);
107
+ }
108
+ }