@mentra/bluetooth-sdk 0.1.9 → 0.1.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (147) hide show
  1. package/README.md +116 -18
  2. package/android/build.gradle +289 -16
  3. package/android/gradle.properties.example +9 -0
  4. package/android/lc3Lib/build.gradle +95 -6
  5. package/android/settings.gradle +3 -0
  6. package/android/silero/LICENSE.md +17 -0
  7. package/android/silero/build.gradle +42 -0
  8. package/android/silero/proguard-rules.pro +24 -0
  9. package/android/silero/src/androidTest/assets/hello.wav +0 -0
  10. package/android/silero/src/androidTest/java/com/konovalov/vad/silero/VadSileroTest.kt +79 -0
  11. package/android/silero/src/main/assets/LICENSE +21 -0
  12. package/android/silero/src/main/assets/silero_vad.onnx +0 -0
  13. package/android/silero/src/main/java/com/konovalov/vad/silero/Vad.kt +160 -0
  14. package/android/silero/src/main/java/com/konovalov/vad/silero/VadSilero.kt +449 -0
  15. package/android/silero/src/main/java/com/konovalov/vad/silero/config/FrameSize.kt +15 -0
  16. package/android/silero/src/main/java/com/konovalov/vad/silero/config/Mode.kt +14 -0
  17. package/android/silero/src/main/java/com/konovalov/vad/silero/config/SampleRate.kt +12 -0
  18. package/android/silero/src/main/java/com/konovalov/vad/silero/utils/AudioUtils.kt +47 -0
  19. package/android/silero/src/main/java/com/konovalov/vad/silero/utils/TensorMap.kt +41 -0
  20. package/android/silero/src/test/java/com/konovalov/vad/silero/utils/AudioUtilsTest.kt +44 -0
  21. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkDefaults.kt +6 -0
  22. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +165 -27
  23. package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +89 -21
  24. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +189 -16
  25. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt +22 -2
  26. package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +787 -85
  27. package/android/src/main/java/com/mentra/bluetoothsdk/audio/AudioModels.kt +6 -2
  28. package/android/src/main/java/com/mentra/bluetoothsdk/camera/CameraModels.kt +123 -10
  29. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/ControllerManager.kt +12 -2
  30. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/R1.kt +8 -2
  31. package/android/src/main/java/com/mentra/bluetoothsdk/debug/BleTraceLogger.kt +33 -17
  32. package/android/src/main/java/com/mentra/bluetoothsdk/events/BluetoothEvents.kt +101 -0
  33. package/android/src/main/java/com/mentra/bluetoothsdk/internal/MapParsing.kt +8 -0
  34. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistry.java +108 -0
  35. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoUploadServer.kt +399 -74
  36. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/MentraPhotoReceiverModule.kt +100 -17
  37. package/android/src/main/java/com/mentra/bluetoothsdk/requests/DisplayRequests.kt +17 -0
  38. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G1.java +2 -2
  39. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt +569 -220
  40. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Mach1.java +2 -2
  41. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraLive.java +271 -66
  42. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraNex.kt +176 -104
  43. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraosBle.java +12753 -27036
  44. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt +54 -2
  45. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Simulated.kt +16 -4
  46. package/android/src/main/java/com/mentra/bluetoothsdk/status/DeviceStatus.kt +60 -3
  47. package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +12 -3
  48. package/android/src/main/java/com/mentra/bluetoothsdk/stt/STTTools.kt +87 -36
  49. package/android/src/main/java/com/mentra/bluetoothsdk/stt/SherpaOnnxTranscriber.kt +13 -4
  50. package/android/src/main/java/com/mentra/bluetoothsdk/stt/VadGateSpeechPolicy.kt +229 -0
  51. package/android/src/main/java/com/mentra/bluetoothsdk/tts/TTSTools.kt +180 -0
  52. package/android/src/main/java/com/mentra/bluetoothsdk/utils/AvifExifStripper.java +573 -0
  53. package/android/src/main/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadService.java +488 -50
  54. package/android/src/main/java/com/mentra/bluetoothsdk/utils/HeifExifTagReader.java +199 -0
  55. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunkReassembler.java +18 -1
  56. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunker.java +64 -18
  57. package/android/src/main/java/com/mentra/bluetoothsdk/utils/NexSGCUtils.kt +48 -39
  58. package/android/src/test/java/com/mentra/bluetoothsdk/camera/PhotoRequestTest.kt +23 -0
  59. package/android/src/test/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistryTest.java +75 -0
  60. package/android/src/test/java/com/mentra/bluetoothsdk/utils/AvifExifStripperTest.java +30 -0
  61. package/android/src/test/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadServiceTest.java +98 -0
  62. package/android/src/test/resources/avif_with_exif.avif +0 -0
  63. package/build/BluetoothSdk.types.d.ts +241 -43
  64. package/build/BluetoothSdk.types.d.ts.map +1 -1
  65. package/build/BluetoothSdk.types.js +11 -10
  66. package/build/BluetoothSdk.types.js.map +1 -1
  67. package/build/_private/BluetoothSdkModule.d.ts +36 -23
  68. package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
  69. package/build/_private/BluetoothSdkModule.js +30 -52
  70. package/build/_private/BluetoothSdkModule.js.map +1 -1
  71. package/build/_private/photoRequestPayload.d.ts +4 -0
  72. package/build/_private/photoRequestPayload.d.ts.map +1 -0
  73. package/build/_private/photoRequestPayload.js +25 -0
  74. package/build/_private/photoRequestPayload.js.map +1 -0
  75. package/build/index.d.ts +1 -1
  76. package/build/index.d.ts.map +1 -1
  77. package/build/index.js +23 -0
  78. package/build/index.js.map +1 -1
  79. package/build/react/useMentraBluetooth.d.ts +2 -2
  80. package/build/react/useMentraBluetooth.d.ts.map +1 -1
  81. package/build/react/useMentraBluetooth.js +3 -3
  82. package/build/react/useMentraBluetooth.js.map +1 -1
  83. package/ios/BluetoothSdkModule.swift +174 -70
  84. package/ios/LocalPhotoUploadServer.swift +370 -105
  85. package/ios/MentraBluetoothSDK.podspec +3 -1
  86. package/ios/MentraPhotoReceiverModule.swift +62 -10
  87. package/ios/Packages/CoreObjC/include/PcmConverter.h +22 -0
  88. package/ios/Packages/SherpaOnnx/SherpaOnnx.swift +668 -26
  89. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Info.plist +13 -9
  90. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
  91. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
  92. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/libsherpa-onnx.a +0 -0
  93. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
  94. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
  95. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/libsherpa-onnx.a +0 -0
  96. package/ios/Source/BluetoothSdkDefaults.swift +6 -0
  97. package/ios/Source/Bridge.swift +58 -19
  98. package/ios/Source/DeviceManager.swift +149 -17
  99. package/ios/Source/DeviceStore.swift +16 -6
  100. package/ios/Source/MentraBluetoothSDK.swift +803 -47
  101. package/ios/Source/ObservableStore.swift +6 -0
  102. package/ios/Source/{Audio → audio}/AudioModels.swift +9 -7
  103. package/ios/Source/{Camera → camera}/CameraModels.swift +225 -16
  104. package/ios/Source/controllers/ControllerManager.swift +4 -3
  105. package/ios/Source/controllers/R1.swift +3 -3
  106. package/ios/Source/{Events → events}/BluetoothEvents.swift +183 -1
  107. package/ios/Source/{Internal → internal}/ValueParsing.swift +10 -1
  108. package/ios/Source/{Requests → requests}/DisplayRequests.swift +28 -4
  109. package/ios/Source/services/PhoneMic.swift +1 -0
  110. package/ios/Source/sgcs/G1.swift +2 -2
  111. package/ios/Source/sgcs/G2.swift +645 -403
  112. package/ios/Source/sgcs/Mach1.swift +2 -2
  113. package/ios/Source/sgcs/MentraLive.swift +812 -319
  114. package/ios/Source/sgcs/MentraNex.swift +412 -182
  115. package/ios/Source/sgcs/SGCManager.swift +57 -3
  116. package/ios/Source/sgcs/Simulated.swift +7 -3
  117. package/ios/Source/sgcs/mentraos_ble.pb.swift +3073 -3134
  118. package/ios/Source/status/DeviceStatus.swift +974 -0
  119. package/ios/Source/{Status → status}/WifiHotspotStatus.swift +4 -4
  120. package/ios/Source/{Streaming → streaming}/StreamModels.swift +24 -9
  121. package/ios/Source/stt/STTTools.swift +27 -12
  122. package/ios/Source/stt/SherpaOnnxTranscriber.swift +31 -8
  123. package/ios/Source/tts/TTSTools.swift +164 -0
  124. package/ios/Source/utils/JSCExperiment.swift +7 -7
  125. package/ios/Source/utils/MessageChunkReassembler.swift +20 -1
  126. package/ios/Source/utils/MessageChunker.swift +78 -21
  127. package/ios/Source/utils/TarBz2Extractor.swift +53 -2
  128. package/package.json +1 -1
  129. package/plugin/build/withAndroid.js +68 -4
  130. package/src/BluetoothSdk.types.ts +331 -59
  131. package/src/_private/BluetoothSdkModule.ts +113 -102
  132. package/src/_private/photoRequestPayload.ts +28 -0
  133. package/src/index.ts +58 -15
  134. package/src/react/useMentraBluetooth.ts +6 -4
  135. package/android/.project +0 -28
  136. package/android/lc3Lib/.project +0 -28
  137. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/cargs.h +0 -162
  138. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/c-api.h +0 -1852
  139. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/cxx-api.h +0 -674
  140. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/sherpa-onnx.a +0 -0
  141. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/sherpa-onnx.a +0 -0
  142. package/ios/Source/Status/DeviceStatus.swift +0 -471
  143. /package/ios/Source/{Connection → connection}/ScanSession.swift +0 -0
  144. /package/ios/Source/{Errors → errors}/BluetoothError.swift +0 -0
  145. /package/ios/Source/{Internal → internal}/BluetoothAvailability.swift +0 -0
  146. /package/ios/Source/{Status → status}/RuntimeState.swift +0 -0
  147. /package/ios/Source/{Types → types}/DeviceModels.swift +0 -0
@@ -21,7 +21,9 @@ data class MicPcmEvent(
21
21
  bitsPerSample = numberValue(values, "bitsPerSample") ?: BITS_PER_SAMPLE,
22
22
  channels = numberValue(values, "channels") ?: CHANNELS,
23
23
  encoding = stringValue(values, "encoding") ?: ENCODING,
24
- voiceActivityDetectionEnabled = boolValue(values, "voiceActivityDetectionEnabled") ?: true,
24
+ voiceActivityDetectionEnabled =
25
+ boolValue(values, "voiceActivityDetectionEnabled")
26
+ ?: BluetoothSdkDefaults.VOICE_ACTIVITY_DETECTION_ENABLED,
25
27
  )
26
28
 
27
29
  fun toMap(): Map<String, Any> =
@@ -63,7 +65,9 @@ data class MicLc3Event(
63
65
  frameSizeBytes = numberValue(values, "frameSizeBytes") ?: DEFAULT_FRAME_SIZE_BYTES,
64
66
  bitrate = numberValue(values, "bitrate") ?: DEFAULT_BITRATE,
65
67
  packetizedFromGlasses = boolValue(values, "packetizedFromGlasses") ?: false,
66
- voiceActivityDetectionEnabled = boolValue(values, "voiceActivityDetectionEnabled") ?: true,
68
+ voiceActivityDetectionEnabled =
69
+ boolValue(values, "voiceActivityDetectionEnabled")
70
+ ?: BluetoothSdkDefaults.VOICE_ACTIVITY_DETECTION_ENABLED,
67
71
  )
68
72
 
69
73
  fun toMap(): Map<String, Any> =
@@ -47,20 +47,86 @@ data class ButtonVideoRecordingSettings(
47
47
  val fps: Int,
48
48
  )
49
49
 
50
+ enum class CameraRoiPosition(val value: Int, val label: String) {
51
+ CENTER(0, "center"),
52
+ BOTTOM(1, "bottom"),
53
+ TOP(2, "top");
54
+
55
+ companion object {
56
+ @JvmStatic
57
+ fun fromValue(value: Int?): CameraRoiPosition =
58
+ values().firstOrNull { it.value == value } ?: CENTER
59
+
60
+ @JvmStatic
61
+ fun fromName(value: String?): CameraRoiPosition =
62
+ values().firstOrNull { it.label == value } ?: CENTER
63
+ }
64
+ }
65
+
50
66
  class CameraFov @JvmOverloads constructor(
51
67
  fov: Int = DEFAULT_FOV,
52
- roiPosition: Int = DEFAULT_ROI_POSITION,
68
+ roiPosition: CameraRoiPosition = DEFAULT_ROI_POSITION,
53
69
  ) {
54
70
  val fov: Int = fov.coerceIn(MIN_FOV, MAX_FOV)
55
- val roiPosition: Int = roiPosition.coerceIn(MIN_ROI_POSITION, MAX_ROI_POSITION)
71
+ val roiPosition: CameraRoiPosition = roiPosition
56
72
 
57
73
  companion object {
58
- const val MIN_FOV = 82
74
+ const val MIN_FOV = 62
59
75
  const val MAX_FOV = 118
60
76
  const val DEFAULT_FOV = 102
61
- const val MIN_ROI_POSITION = 0
62
- const val MAX_ROI_POSITION = 2
63
- const val DEFAULT_ROI_POSITION = 0
77
+ const val NARROW_FOV = 82
78
+ @JvmField
79
+ val DEFAULT_ROI_POSITION = CameraRoiPosition.CENTER
80
+ @JvmField
81
+ val NARROW = CameraFov(NARROW_FOV, DEFAULT_ROI_POSITION)
82
+ @JvmField
83
+ val STANDARD = CameraFov(DEFAULT_FOV, DEFAULT_ROI_POSITION)
84
+ @JvmField
85
+ val WIDE = CameraFov(MAX_FOV, DEFAULT_ROI_POSITION)
86
+ }
87
+ }
88
+
89
+ data class CameraFovResult(
90
+ val requestId: String,
91
+ val fov: Int,
92
+ val roiPosition: CameraRoiPosition,
93
+ val timestamp: Long,
94
+ ) {
95
+ val values: Map<String, Any>
96
+ get() =
97
+ mapOf(
98
+ "requestId" to requestId,
99
+ "fov" to fov,
100
+ "roiPosition" to roiPosition.label,
101
+ "timestamp" to timestamp,
102
+ )
103
+
104
+ companion object {
105
+ @JvmStatic
106
+ fun fromAck(
107
+ ack: SettingsAckEvent,
108
+ fallback: CameraFov,
109
+ ): CameraFovResult {
110
+ if (ack.status == "error") {
111
+ throw BluetoothException(
112
+ ack.errorCode ?: "camera_fov_failed",
113
+ ack.errorMessage ?: "Camera FOV request failed.",
114
+ )
115
+ }
116
+ if (!ack.hardwareApplied) {
117
+ throw BluetoothException(
118
+ "camera_fov_not_applied",
119
+ "Camera FOV was saved but not applied to hardware.",
120
+ )
121
+ }
122
+
123
+ return CameraFovResult(
124
+ requestId = ack.requestId,
125
+ fov = ack.fov ?: fallback.fov,
126
+ roiPosition = CameraRoiPosition.fromValue(ack.roiPosition ?: fallback.roiPosition.value),
127
+ timestamp = ack.timestamp,
128
+ )
129
+ }
64
130
  }
65
131
  }
66
132
 
@@ -72,6 +138,7 @@ data class PhotoRequest @JvmOverloads constructor(
72
138
  val authToken: String? = null,
73
139
  val compress: PhotoCompression = PhotoCompression.MEDIUM,
74
140
  val flash: Boolean = true,
141
+ val save: Boolean = false,
75
142
  val sound: Boolean = true,
76
143
  /** Sensor exposure time for this capture only (ns), or null for auto exposure */
77
144
  val exposureTimeNs: Double? = null,
@@ -107,6 +174,7 @@ data class PhotoRequest @JvmOverloads constructor(
107
174
  authToken = stringValue(values, "authToken", "auth_token")?.takeIf { it.isNotBlank() },
108
175
  compress = PhotoCompression.fromValue(stringValue(values, "compress") ?: "none"),
109
176
  flash = boolValue(values, "flash") ?: true,
177
+ save = boolValue(values, "save", "saveToGallery") ?: false,
110
178
  sound = boolValue(values, "sound") ?: true,
111
179
  exposureTimeNs = exposureTimeNs,
112
180
  iso = iso,
@@ -154,8 +222,23 @@ data class VideoRecordingRequest(
154
222
  val requestId: String,
155
223
  val save: Boolean,
156
224
  val sound: Boolean,
225
+ // Optional per-recording overrides; 0 means "use the saved button-video default".
226
+ val width: Int = 0,
227
+ val height: Int = 0,
228
+ val fps: Int = 0,
157
229
  )
158
230
 
231
+ data class VideoRecordingStatusEvent(
232
+ val values: Map<String, Any>,
233
+ ) {
234
+ val requestId: String get() = stringValue(values, "requestId").orEmpty()
235
+ val success: Boolean get() = boolValue(values, "success") ?: false
236
+ val status: String get() = stringValue(values, "status").orEmpty()
237
+ val details: String? get() = stringValue(values, "details")
238
+ val timestamp: Long get() = longValue(values, "timestamp") ?: System.currentTimeMillis()
239
+ val data: Map<String, Any>? get() = stringMapValue(values["data"])
240
+ }
241
+
159
242
  data class GalleryStatusEvent(
160
243
  val values: Map<String, Any>,
161
244
  )
@@ -172,7 +255,7 @@ sealed interface PhotoResponse {
172
255
  "requestId" to requestId,
173
256
  "uploadUrl" to uploadUrl,
174
257
  "timestamp" to timestamp,
175
- )
258
+ ).withOptionalPhotoMetadata(this)
176
259
 
177
260
  is Error -> mutableMapOf<String, Any>(
178
261
  "state" to state,
@@ -191,6 +274,10 @@ sealed interface PhotoResponse {
191
274
  data class Success(
192
275
  override val requestId: String,
193
276
  val uploadUrl: String,
277
+ val photoUrl: String?,
278
+ val statusUrl: String?,
279
+ val contentType: String?,
280
+ val fileSizeBytes: Long?,
194
281
  override val timestamp: Long,
195
282
  ) : PhotoResponse {
196
283
  override val state: String = "success"
@@ -210,14 +297,27 @@ sealed interface PhotoResponse {
210
297
  val requestId = stringValue(values, "requestId").orEmpty()
211
298
  val timestamp = longValue(values, "timestamp") ?: System.currentTimeMillis()
212
299
  val state = stringValue(values, "state")?.lowercase()
213
- return if (state == "success") {
300
+ val success = state == "success" || boolValue(values, "success") == true
301
+ return if (success) {
214
302
  val uploadUrl = stringValue(values, "uploadUrl").orEmpty()
215
- Success(requestId = requestId, uploadUrl = uploadUrl, timestamp = timestamp)
303
+ Success(
304
+ requestId = requestId,
305
+ uploadUrl = uploadUrl,
306
+ photoUrl = stringValue(values, "photoUrl"),
307
+ statusUrl = stringValue(values, "statusUrl"),
308
+ contentType = stringValue(values, "contentType", "mimeType"),
309
+ fileSizeBytes =
310
+ longValue(values, "fileSizeBytes")
311
+ ?: longValue(values, "bytes")
312
+ ?: longValue(values, "size"),
313
+ timestamp = timestamp,
314
+ )
216
315
  } else {
217
316
  Error(
218
317
  requestId = requestId,
219
318
  errorCode = stringValue(values, "errorCode"),
220
- errorMessage = stringValue(values, "errorMessage") ?: "Unknown photo error",
319
+ errorMessage =
320
+ stringValue(values, "errorMessage", "error") ?: "Unknown photo error",
221
321
  timestamp = timestamp,
222
322
  )
223
323
  }
@@ -225,6 +325,16 @@ sealed interface PhotoResponse {
225
325
  }
226
326
  }
227
327
 
328
+ private fun Map<String, Any>.withOptionalPhotoMetadata(
329
+ success: PhotoResponse.Success,
330
+ ): Map<String, Any> =
331
+ toMutableMap().apply {
332
+ success.photoUrl?.takeIf { it.isNotBlank() }?.let { this["photoUrl"] = it }
333
+ success.statusUrl?.takeIf { it.isNotBlank() }?.let { this["statusUrl"] = it }
334
+ success.contentType?.takeIf { it.isNotBlank() }?.let { this["contentType"] = it }
335
+ success.fileSizeBytes?.let { this["fileSizeBytes"] = it }
336
+ }
337
+
228
338
  data class PhotoResponseEvent(
229
339
  val response: PhotoResponse,
230
340
  ) {
@@ -241,6 +351,9 @@ data class PhotoStatusEvent(
241
351
  val status: String get() = stringValue(values, "status").orEmpty()
242
352
  val timestamp: Long get() = longValue(values, "timestamp") ?: System.currentTimeMillis()
243
353
  val resolvedConfig: Map<String, Any>? get() = stringMapValue(values["resolvedConfig"])
354
+ val requestedCaptureConfig: Map<String, Any>? get() = stringMapValue(values["requestedCaptureConfig"])
355
+ val meteredPreview: Map<String, Any>? get() = stringMapValue(values["meteredPreview"])
356
+ val captureMetadata: Map<String, Any>? get() = stringMapValue(values["captureMetadata"])
244
357
  val errorCode: String? get() = stringValue(values, "errorCode")
245
358
  val errorMessage: String? get() = stringValue(values, "errorMessage")
246
359
  }
@@ -1,5 +1,6 @@
1
1
  package com.mentra.bluetoothsdk.controllers
2
2
 
3
+ import com.mentra.bluetoothsdk.BluetoothSdkDefaults
3
4
  import com.mentra.bluetoothsdk.DeviceStore
4
5
 
5
6
  abstract class ControllerManager {
@@ -22,6 +23,7 @@ abstract class ControllerManager {
22
23
  authToken: String?,
23
24
  compress: String?,
24
25
  flash: Boolean,
26
+ save: Boolean,
25
27
  sound: Boolean,
26
28
  exposureTimeNs: Long?,
27
29
  iso: Int?,
@@ -43,7 +45,13 @@ abstract class ControllerManager {
43
45
  abstract fun clearDisplay()
44
46
  abstract fun sendTextWall(text: String)
45
47
  abstract fun sendDoubleTextWall(top: String, bottom: String)
46
- abstract fun displayBitmap(base64ImageData: String): Boolean
48
+ abstract fun displayBitmap(
49
+ base64ImageData: String,
50
+ x: Int? = null,
51
+ y: Int? = null,
52
+ width: Int? = null,
53
+ height: Int? = null
54
+ ): Boolean
47
55
  abstract fun showDashboard()
48
56
  abstract fun setDashboardPosition(height: Int, depth: Int)
49
57
 
@@ -135,7 +143,9 @@ abstract class ControllerManager {
135
143
  get() = DeviceStore.get("glasses", "micEnabled") as? Boolean ?: false
136
144
 
137
145
  val voiceActivityDetectionEnabled: Boolean
138
- get() = DeviceStore.get("glasses", "voiceActivityDetectionEnabled") as? Boolean ?: true
146
+ get() =
147
+ DeviceStore.get("glasses", "voiceActivityDetectionEnabled") as? Boolean
148
+ ?: BluetoothSdkDefaults.VOICE_ACTIVITY_DETECTION_ENABLED
139
149
 
140
150
  val batteryLevel: Int
141
151
  get() = DeviceStore.get("glasses", "batteryLevel") as? Int ?: -1
@@ -863,7 +863,7 @@ class R1 : ControllerManager() {
863
863
  override fun sendJson(jsonOriginal: Map<String, Any>, wakeUp: Boolean, requireAck: Boolean) {}
864
864
  override fun requestPhoto(
865
865
  requestId: String, appId: String, size: String?, webhookUrl: String?,
866
- authToken: String?, compress: String?, flash: Boolean, sound: Boolean,
866
+ authToken: String?, compress: String?, flash: Boolean, save: Boolean, sound: Boolean,
867
867
  exposureTimeNs: Long?, iso: Int?,
868
868
  ) {}
869
869
  override fun startVideoRecording(requestId: String, save: Boolean, flash: Boolean, sound: Boolean) {}
@@ -879,7 +879,13 @@ class R1 : ControllerManager() {
879
879
  override fun clearDisplay() {}
880
880
  override fun sendTextWall(text: String) {}
881
881
  override fun sendDoubleTextWall(top: String, bottom: String) {}
882
- override fun displayBitmap(base64ImageData: String): Boolean = false
882
+ override fun displayBitmap(
883
+ base64ImageData: String,
884
+ x: Int?,
885
+ y: Int?,
886
+ width: Int?,
887
+ height: Int?
888
+ ): Boolean = false
883
889
  override fun showDashboard() {}
884
890
  override fun setDashboardPosition(height: Int, depth: Int) {}
885
891
  override fun setHeadUpAngle(angle: Int) {}
@@ -10,6 +10,7 @@ import org.json.JSONObject
10
10
  object BleTraceLogger {
11
11
  private const val TAG = "MentraBleTrace"
12
12
  private const val MAX_PAYLOAD_CHARS = 3000
13
+ private const val K900_TYPE = "k900"
13
14
  private val sensitiveKeyParts =
14
15
  listOf("password", "pass", "token", "secret", "authorization", "auth", "email")
15
16
 
@@ -26,8 +27,12 @@ object BleTraceLogger {
26
27
 
27
28
  @JvmStatic
28
29
  fun logMap(direction: String, layer: String, type: String?, payload: Map<String, Any>) {
29
- val sanitized = sanitize(JSONObject(payload))
30
- Log.i(TAG, format(direction, layer, caller(), type ?: extractType(sanitized), null, sanitized.toString()))
30
+ try {
31
+ val sanitized = sanitize(JSONObject(payload))
32
+ Log.i(TAG, format(direction, layer, caller(), type ?: extractType(sanitized), null, sanitized.toString()))
33
+ } catch (e: Exception) {
34
+ Log.d(TAG, "BLE trace map logging failed", e)
35
+ }
31
36
  }
32
37
 
33
38
  @JvmStatic
@@ -37,20 +42,24 @@ object BleTraceLogger {
37
42
  event: String,
38
43
  extra: Map<String, Any?> = emptyMap(),
39
44
  ) {
40
- val payload = JSONObject()
41
- payload.put("event", event)
42
- payload.put("component", component)
43
- payload.put("pid", Process.myPid())
44
- payload.put("model", Build.MODEL)
45
- payload.put("sdkInt", Build.VERSION.SDK_INT)
46
-
47
- context?.let {
48
- payload.put("package", it.packageName)
49
- packageVersion(it)?.let { version -> payload.put("version", version) }
50
- }
45
+ try {
46
+ val payload = JSONObject()
47
+ payload.put("event", event)
48
+ payload.put("component", component)
49
+ payload.put("pid", Process.myPid())
50
+ payload.put("model", Build.MODEL)
51
+ payload.put("sdkInt", Build.VERSION.SDK_INT)
52
+
53
+ context?.let {
54
+ payload.put("package", it.packageName)
55
+ packageVersion(it)?.let { version -> payload.put("version", version) }
56
+ }
51
57
 
52
- extra.forEach { (key, value) -> payload.put(key, value ?: JSONObject.NULL) }
53
- Log.i(TAG, format("phone_app", "app_lifecycle", caller(), event, null, sanitize(payload).toString()))
58
+ extra.forEach { (key, value) -> payload.put(key, value ?: JSONObject.NULL) }
59
+ Log.i(TAG, format("phone_app", "app_lifecycle", caller(), event, null, sanitize(payload).toString()))
60
+ } catch (e: Exception) {
61
+ Log.d(TAG, "BLE trace lifecycle logging failed", e)
62
+ }
54
63
  }
55
64
 
56
65
  private fun format(
@@ -72,12 +81,16 @@ object BleTraceLogger {
72
81
  val inner = JSONObject(cValue)
73
82
  inner.optString("type").takeIf { it.isNotBlank() }?.let { return it }
74
83
  } catch (_: Exception) {
75
- return "k900:${cValue.take(40)}"
84
+ return extractK900Type(cValue)
76
85
  }
86
+ return extractK900Type(cValue)
77
87
  }
78
88
  return "unknown"
79
89
  }
80
90
 
91
+ private fun extractK900Type(cValue: String): String =
92
+ if (cValue == "sr_log") "$K900_TYPE:sr_log" else K900_TYPE
93
+
81
94
  private fun sanitize(value: JSONObject): JSONObject {
82
95
  val output = JSONObject()
83
96
  value.keys().forEach { key ->
@@ -102,7 +115,7 @@ object BleTraceLogger {
102
115
  try {
103
116
  return sanitize(JSONObject(value)).toString()
104
117
  } catch (_: Exception) {
105
- return truncate(value)
118
+ return sanitizeK900Command(value)
106
119
  }
107
120
  }
108
121
  return when (value) {
@@ -113,6 +126,9 @@ object BleTraceLogger {
113
126
  }
114
127
  }
115
128
 
129
+ private fun sanitizeK900Command(value: String): String =
130
+ if (value == "sr_log") value else "<non-json C payload>"
131
+
116
132
  private fun caller(): String {
117
133
  val frame =
118
134
  Throwable().stackTrace.firstOrNull {
@@ -39,6 +39,100 @@ data class SpeakingStatusEvent(
39
39
  val values: Map<String, Any>,
40
40
  )
41
41
 
42
+ data class OtaUpdateAvailableEvent(
43
+ val versionCode: Long?,
44
+ val versionName: String?,
45
+ val updates: List<String>,
46
+ val totalSize: Long?,
47
+ val cacheReady: Boolean?,
48
+ val values: Map<String, Any>,
49
+ ) {
50
+ companion object {
51
+ internal fun fromMap(values: Map<String, Any>): OtaUpdateAvailableEvent =
52
+ OtaUpdateAvailableEvent(
53
+ versionCode = longValue(values, "version_code"),
54
+ versionName = stringValue(values, "version_name"),
55
+ updates = (values["updates"] as? List<*>)?.filterIsInstance<String>() ?: emptyList(),
56
+ totalSize = longValue(values, "total_size"),
57
+ cacheReady = boolValue(values, "cache_ready"),
58
+ values = values,
59
+ )
60
+ }
61
+ }
62
+
63
+ data class OtaStartAckEvent(
64
+ val timestamp: Long?,
65
+ val values: Map<String, Any>,
66
+ ) {
67
+ companion object {
68
+ internal fun fromMap(values: Map<String, Any>): OtaStartAckEvent =
69
+ OtaStartAckEvent(
70
+ timestamp = longValue(values, "timestamp"),
71
+ values = values,
72
+ )
73
+ }
74
+ }
75
+
76
+ data class OtaStatusEvent(
77
+ val sessionId: String,
78
+ val totalSteps: Int,
79
+ val currentStep: Int,
80
+ val stepType: String,
81
+ val phase: String,
82
+ val stepPercent: Int,
83
+ val overallPercent: Int,
84
+ val status: String,
85
+ val errorMessage: String?,
86
+ val glassesTimeMs: Long?,
87
+ val values: Map<String, Any>,
88
+ ) {
89
+ companion object {
90
+ internal fun fromMap(values: Map<String, Any>): OtaStatusEvent =
91
+ OtaStatusEvent(
92
+ sessionId = stringValue(values, "session_id") ?: "",
93
+ totalSteps = numberValue(values, "total_steps") ?: 0,
94
+ currentStep = numberValue(values, "current_step") ?: 0,
95
+ stepType = stringValue(values, "step_type") ?: "",
96
+ phase = stringValue(values, "phase") ?: "",
97
+ stepPercent = numberValue(values, "step_percent") ?: 0,
98
+ overallPercent = numberValue(values, "overall_percent") ?: 0,
99
+ status = stringValue(values, "status") ?: "",
100
+ errorMessage = stringValue(values, "error_message"),
101
+ glassesTimeMs = longValue(values, "glasses_time_ms"),
102
+ values = values,
103
+ )
104
+ }
105
+ }
106
+
107
+ data class OtaQueryResult(
108
+ val values: Map<String, Any>,
109
+ ) {
110
+ val type: String get() = stringValue(values, "type").orEmpty()
111
+ val status: String? get() = stringValue(values, "status")
112
+ }
113
+
114
+ data class SettingsAckEvent(
115
+ val values: Map<String, Any>,
116
+ ) {
117
+ val requestId: String get() = stringValue(values, "requestId").orEmpty()
118
+ val setting: String get() = stringValue(values, "setting").orEmpty()
119
+ val status: String get() = stringValue(values, "status") ?: "applied"
120
+ val timestamp: Long get() = longValue(values, "timestamp") ?: System.currentTimeMillis()
121
+ val fov: Int? get() = numberValue(values, "fov")
122
+ val roiPosition: Int? get() = numberValue(values, "roiPosition", "roi_position")
123
+ val hardwareApplied: Boolean get() = boolValue(values, "hardwareApplied", "hardware_applied") ?: false
124
+ val errorCode: String? get() = stringValue(values, "errorCode")
125
+ val errorMessage: String? get() = stringValue(values, "errorMessage")
126
+ }
127
+
128
+ data class RgbLedControlResponseEvent(
129
+ val values: Map<String, Any>,
130
+ ) {
131
+ val requestId: String get() = stringValue(values, "requestId").orEmpty()
132
+ val state: String get() = stringValue(values, "state") ?: "error"
133
+ val errorCode: String? get() = stringValue(values, "errorCode")
134
+ }
135
+
42
136
  interface MentraBluetoothSdkListener {
43
137
  fun onStateChanged(state: MentraBluetoothState) {}
44
138
  fun onGlassesChanged(glasses: GlassesRuntimeState) {}
@@ -59,8 +153,15 @@ interface MentraBluetoothSdkListener {
59
153
  fun onGalleryStatus(event: GalleryStatusEvent) {}
60
154
  fun onPhotoResponse(event: PhotoResponseEvent) {}
61
155
  fun onPhotoStatus(event: PhotoStatusEvent) {}
156
+ fun onVideoRecordingStatus(event: VideoRecordingStatusEvent) {}
157
+ fun onRgbLedControlResponse(event: RgbLedControlResponseEvent) {}
62
158
  fun onStreamStatus(event: StreamStatusEvent) {}
63
159
  fun onKeepAliveAck(event: KeepAliveAckEvent) {}
160
+ fun onOtaUpdateAvailable(event: OtaUpdateAvailableEvent) {}
161
+ fun onOtaStartAck(event: OtaStartAckEvent) {}
162
+ fun onOtaStatus(event: OtaStatusEvent) {}
163
+ fun onSettingsAck(event: SettingsAckEvent) {}
164
+ fun onVersionInfo(event: VersionInfoResult) {}
64
165
  fun onMicPcm(event: MicPcmEvent) {}
65
166
  fun onMicLc3(event: MicLc3Event) {}
66
167
  fun onLocalTranscription(event: LocalTranscriptionEvent) {}
@@ -8,6 +8,14 @@ internal fun numberValue(
8
8
  (values[key] as? Number)?.toInt()
9
9
  }
10
10
 
11
+ internal fun doubleValue(
12
+ values: Map<String, Any>,
13
+ vararg keys: String,
14
+ ): Double? =
15
+ keys.firstNotNullOfOrNull { key ->
16
+ (values[key] as? Number)?.toDouble()
17
+ }
18
+
11
19
  internal fun stringValue(
12
20
  values: Map<String, Any>,
13
21
  vararg keys: String,
@@ -0,0 +1,108 @@
1
+ package com.mentra.bluetoothsdk.photoreceiver;
2
+
3
+ import androidx.annotation.Nullable;
4
+
5
+ import java.net.URI;
6
+ import java.util.Collections;
7
+ import java.util.HashSet;
8
+ import java.util.Objects;
9
+ import java.util.Set;
10
+
11
+ /**
12
+ * Tracks the SDK photo receiver that is hosted inside the phone app process.
13
+ *
14
+ * <p>The glasses need a LAN-reachable URL, but the phone-side BLE fallback relay
15
+ * can upload to the same receiver over loopback. Keeping this tiny registry lets
16
+ * the relay avoid stale Wi-Fi interface addresses after network changes.
17
+ */
18
+ public final class LocalPhotoReceiverRegistry {
19
+ private static volatile Set<URI> activeUploadUris = Collections.emptySet();
20
+
21
+ private LocalPhotoReceiverRegistry() {}
22
+
23
+ public static void register(String uploadUrl) {
24
+ try {
25
+ URI uri = URI.create(uploadUrl);
26
+ if (isSupportedUploadUri(uri)) {
27
+ Set<URI> nextUploadUris = new HashSet<>(activeUploadUris);
28
+ nextUploadUris.add(uri);
29
+ activeUploadUris = Collections.unmodifiableSet(nextUploadUris);
30
+ }
31
+ } catch (Exception ignored) {
32
+ }
33
+ }
34
+
35
+ public static void unregister() {
36
+ activeUploadUris = Collections.emptySet();
37
+ }
38
+
39
+ public static boolean isActive() {
40
+ return !activeUploadUris.isEmpty();
41
+ }
42
+
43
+ @Nullable
44
+ public static String loopbackUploadUrlFor(@Nullable String webhookUrl) {
45
+ Set<URI> registeredUris = activeUploadUris;
46
+ if (registeredUris.isEmpty() || webhookUrl == null || webhookUrl.isEmpty()) {
47
+ return null;
48
+ }
49
+
50
+ try {
51
+ URI uri = URI.create(webhookUrl);
52
+ for (URI registeredUri : registeredUris) {
53
+ if (matchesRegisteredUploadUri(uri, registeredUri)) {
54
+ return new URI(
55
+ uri.getScheme(),
56
+ null,
57
+ "127.0.0.1",
58
+ effectivePort(uri),
59
+ normalizedPath(uri),
60
+ uri.getRawQuery(),
61
+ uri.getRawFragment()
62
+ ).toString();
63
+ }
64
+ }
65
+ return null;
66
+ } catch (Exception ignored) {
67
+ return null;
68
+ }
69
+ }
70
+
71
+ private static boolean matchesRegisteredUploadUri(URI candidate, URI registered) {
72
+ return isSupportedUploadUri(candidate)
73
+ && equalsIgnoreCase(candidate.getScheme(), registered.getScheme())
74
+ && equalsIgnoreCase(candidate.getHost(), registered.getHost())
75
+ && effectivePort(candidate) == effectivePort(registered)
76
+ && Objects.equals(normalizedPath(candidate), normalizedPath(registered))
77
+ && Objects.equals(candidate.getRawQuery(), registered.getRawQuery())
78
+ && Objects.equals(candidate.getRawFragment(), registered.getRawFragment());
79
+ }
80
+
81
+ private static boolean isSupportedUploadUri(URI uri) {
82
+ String scheme = uri.getScheme();
83
+ if (!"http".equalsIgnoreCase(scheme) && !"https".equalsIgnoreCase(scheme)) {
84
+ return false;
85
+ }
86
+ return uri.getHost() != null && effectivePort(uri) > 0 && "/upload".equals(normalizedPath(uri));
87
+ }
88
+
89
+ private static int effectivePort(URI uri) {
90
+ int port = uri.getPort();
91
+ if (port != -1) {
92
+ return port;
93
+ }
94
+ return "https".equalsIgnoreCase(uri.getScheme()) ? 443 : 80;
95
+ }
96
+
97
+ private static String normalizedPath(URI uri) {
98
+ String path = uri.getRawPath();
99
+ return (path == null || path.isEmpty()) ? "/" : path;
100
+ }
101
+
102
+ private static boolean equalsIgnoreCase(@Nullable String first, @Nullable String second) {
103
+ if (first == null || second == null) {
104
+ return first == second;
105
+ }
106
+ return first.equalsIgnoreCase(second);
107
+ }
108
+ }