@mentra/bluetooth-sdk 0.1.8 → 0.1.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (147) hide show
  1. package/README.md +116 -18
  2. package/android/build.gradle +289 -16
  3. package/android/gradle.properties.example +9 -0
  4. package/android/lc3Lib/build.gradle +95 -6
  5. package/android/settings.gradle +3 -0
  6. package/android/silero/LICENSE.md +17 -0
  7. package/android/silero/build.gradle +42 -0
  8. package/android/silero/proguard-rules.pro +24 -0
  9. package/android/silero/src/androidTest/assets/hello.wav +0 -0
  10. package/android/silero/src/androidTest/java/com/konovalov/vad/silero/VadSileroTest.kt +79 -0
  11. package/android/silero/src/main/assets/LICENSE +21 -0
  12. package/android/silero/src/main/assets/silero_vad.onnx +0 -0
  13. package/android/silero/src/main/java/com/konovalov/vad/silero/Vad.kt +160 -0
  14. package/android/silero/src/main/java/com/konovalov/vad/silero/VadSilero.kt +449 -0
  15. package/android/silero/src/main/java/com/konovalov/vad/silero/config/FrameSize.kt +15 -0
  16. package/android/silero/src/main/java/com/konovalov/vad/silero/config/Mode.kt +14 -0
  17. package/android/silero/src/main/java/com/konovalov/vad/silero/config/SampleRate.kt +12 -0
  18. package/android/silero/src/main/java/com/konovalov/vad/silero/utils/AudioUtils.kt +47 -0
  19. package/android/silero/src/main/java/com/konovalov/vad/silero/utils/TensorMap.kt +41 -0
  20. package/android/silero/src/test/java/com/konovalov/vad/silero/utils/AudioUtilsTest.kt +44 -0
  21. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkDefaults.kt +6 -0
  22. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +165 -27
  23. package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +89 -21
  24. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +189 -16
  25. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt +22 -2
  26. package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +787 -85
  27. package/android/src/main/java/com/mentra/bluetoothsdk/audio/AudioModels.kt +6 -2
  28. package/android/src/main/java/com/mentra/bluetoothsdk/camera/CameraModels.kt +123 -10
  29. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/ControllerManager.kt +12 -2
  30. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/R1.kt +8 -2
  31. package/android/src/main/java/com/mentra/bluetoothsdk/debug/BleTraceLogger.kt +33 -17
  32. package/android/src/main/java/com/mentra/bluetoothsdk/events/BluetoothEvents.kt +101 -0
  33. package/android/src/main/java/com/mentra/bluetoothsdk/internal/MapParsing.kt +8 -0
  34. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistry.java +108 -0
  35. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoUploadServer.kt +399 -74
  36. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/MentraPhotoReceiverModule.kt +100 -17
  37. package/android/src/main/java/com/mentra/bluetoothsdk/requests/DisplayRequests.kt +17 -0
  38. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G1.java +2 -2
  39. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt +569 -220
  40. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Mach1.java +2 -2
  41. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraLive.java +271 -66
  42. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraNex.kt +176 -104
  43. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraosBle.java +12753 -27036
  44. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt +54 -2
  45. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Simulated.kt +16 -4
  46. package/android/src/main/java/com/mentra/bluetoothsdk/status/DeviceStatus.kt +60 -3
  47. package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +12 -3
  48. package/android/src/main/java/com/mentra/bluetoothsdk/stt/STTTools.kt +87 -36
  49. package/android/src/main/java/com/mentra/bluetoothsdk/stt/SherpaOnnxTranscriber.kt +13 -4
  50. package/android/src/main/java/com/mentra/bluetoothsdk/stt/VadGateSpeechPolicy.kt +229 -0
  51. package/android/src/main/java/com/mentra/bluetoothsdk/tts/TTSTools.kt +180 -0
  52. package/android/src/main/java/com/mentra/bluetoothsdk/utils/AvifExifStripper.java +573 -0
  53. package/android/src/main/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadService.java +488 -50
  54. package/android/src/main/java/com/mentra/bluetoothsdk/utils/HeifExifTagReader.java +199 -0
  55. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunkReassembler.java +18 -1
  56. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunker.java +64 -18
  57. package/android/src/main/java/com/mentra/bluetoothsdk/utils/NexSGCUtils.kt +48 -39
  58. package/android/src/test/java/com/mentra/bluetoothsdk/camera/PhotoRequestTest.kt +23 -0
  59. package/android/src/test/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistryTest.java +75 -0
  60. package/android/src/test/java/com/mentra/bluetoothsdk/utils/AvifExifStripperTest.java +30 -0
  61. package/android/src/test/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadServiceTest.java +98 -0
  62. package/android/src/test/resources/avif_with_exif.avif +0 -0
  63. package/build/BluetoothSdk.types.d.ts +241 -43
  64. package/build/BluetoothSdk.types.d.ts.map +1 -1
  65. package/build/BluetoothSdk.types.js +11 -10
  66. package/build/BluetoothSdk.types.js.map +1 -1
  67. package/build/_private/BluetoothSdkModule.d.ts +36 -23
  68. package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
  69. package/build/_private/BluetoothSdkModule.js +30 -52
  70. package/build/_private/BluetoothSdkModule.js.map +1 -1
  71. package/build/_private/photoRequestPayload.d.ts +4 -0
  72. package/build/_private/photoRequestPayload.d.ts.map +1 -0
  73. package/build/_private/photoRequestPayload.js +25 -0
  74. package/build/_private/photoRequestPayload.js.map +1 -0
  75. package/build/index.d.ts +1 -1
  76. package/build/index.d.ts.map +1 -1
  77. package/build/index.js +23 -0
  78. package/build/index.js.map +1 -1
  79. package/build/react/useMentraBluetooth.d.ts +2 -2
  80. package/build/react/useMentraBluetooth.d.ts.map +1 -1
  81. package/build/react/useMentraBluetooth.js +3 -3
  82. package/build/react/useMentraBluetooth.js.map +1 -1
  83. package/ios/BluetoothSdkModule.swift +174 -70
  84. package/ios/LocalPhotoUploadServer.swift +370 -105
  85. package/ios/MentraBluetoothSDK.podspec +3 -1
  86. package/ios/MentraPhotoReceiverModule.swift +62 -10
  87. package/ios/Packages/CoreObjC/include/PcmConverter.h +22 -0
  88. package/ios/Packages/SherpaOnnx/SherpaOnnx.swift +668 -26
  89. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Info.plist +13 -9
  90. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
  91. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
  92. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/libsherpa-onnx.a +0 -0
  93. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
  94. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
  95. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/libsherpa-onnx.a +0 -0
  96. package/ios/Source/BluetoothSdkDefaults.swift +6 -0
  97. package/ios/Source/Bridge.swift +58 -19
  98. package/ios/Source/DeviceManager.swift +149 -17
  99. package/ios/Source/DeviceStore.swift +16 -6
  100. package/ios/Source/MentraBluetoothSDK.swift +803 -47
  101. package/ios/Source/ObservableStore.swift +8 -2
  102. package/ios/Source/{Audio → audio}/AudioModels.swift +9 -7
  103. package/ios/Source/{Camera → camera}/CameraModels.swift +225 -16
  104. package/ios/Source/controllers/ControllerManager.swift +4 -3
  105. package/ios/Source/controllers/R1.swift +3 -3
  106. package/ios/Source/{Events → events}/BluetoothEvents.swift +183 -1
  107. package/ios/Source/{Internal → internal}/ValueParsing.swift +10 -1
  108. package/ios/Source/{Requests → requests}/DisplayRequests.swift +28 -4
  109. package/ios/Source/services/PhoneMic.swift +1 -0
  110. package/ios/Source/sgcs/G1.swift +2 -2
  111. package/ios/Source/sgcs/G2.swift +645 -403
  112. package/ios/Source/sgcs/Mach1.swift +2 -2
  113. package/ios/Source/sgcs/MentraLive.swift +812 -319
  114. package/ios/Source/sgcs/MentraNex.swift +412 -182
  115. package/ios/Source/sgcs/SGCManager.swift +57 -3
  116. package/ios/Source/sgcs/Simulated.swift +7 -3
  117. package/ios/Source/sgcs/mentraos_ble.pb.swift +3073 -3134
  118. package/ios/Source/status/DeviceStatus.swift +974 -0
  119. package/ios/Source/{Status → status}/WifiHotspotStatus.swift +4 -4
  120. package/ios/Source/{Streaming → streaming}/StreamModels.swift +24 -9
  121. package/ios/Source/stt/STTTools.swift +27 -12
  122. package/ios/Source/stt/SherpaOnnxTranscriber.swift +31 -8
  123. package/ios/Source/tts/TTSTools.swift +164 -0
  124. package/ios/Source/utils/JSCExperiment.swift +7 -7
  125. package/ios/Source/utils/MessageChunkReassembler.swift +20 -1
  126. package/ios/Source/utils/MessageChunker.swift +78 -21
  127. package/ios/Source/utils/TarBz2Extractor.swift +53 -2
  128. package/package.json +1 -1
  129. package/plugin/build/withAndroid.js +68 -4
  130. package/src/BluetoothSdk.types.ts +331 -59
  131. package/src/_private/BluetoothSdkModule.ts +113 -102
  132. package/src/_private/photoRequestPayload.ts +28 -0
  133. package/src/index.ts +58 -15
  134. package/src/react/useMentraBluetooth.ts +6 -4
  135. package/android/.project +0 -28
  136. package/android/lc3Lib/.project +0 -28
  137. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/cargs.h +0 -162
  138. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/c-api.h +0 -1852
  139. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/cxx-api.h +0 -674
  140. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/sherpa-onnx.a +0 -0
  141. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/sherpa-onnx.a +0 -0
  142. package/ios/Source/Status/DeviceStatus.swift +0 -471
  143. /package/ios/Source/{Connection → connection}/ScanSession.swift +0 -0
  144. /package/ios/Source/{Errors → errors}/BluetoothError.swift +0 -0
  145. /package/ios/Source/{Internal → internal}/BluetoothAvailability.swift +0 -0
  146. /package/ios/Source/{Status → status}/RuntimeState.swift +0 -0
  147. /package/ios/Source/{Types → types}/DeviceModels.swift +0 -0
@@ -132,7 +132,7 @@ public class Mach1 extends SGCManager {
132
132
  }
133
133
 
134
134
  @Override
135
- public void requestPhoto(@NonNull String requestId, @NonNull String appId, @NonNull String size, @Nullable String webhookUrl, @Nullable String authToken, @Nullable String compress, boolean flash, boolean sound, @Nullable Long exposureTimeNs, @Nullable Integer iso) {
135
+ public void requestPhoto(@NonNull String requestId, @NonNull String appId, @NonNull String size, @Nullable String webhookUrl, @Nullable String authToken, @Nullable String compress, boolean flash, boolean save, boolean sound, @Nullable Long exposureTimeNs, @Nullable Integer iso) {
136
136
 
137
137
  }
138
138
 
@@ -211,7 +211,7 @@ public class Mach1 extends SGCManager {
211
211
  }
212
212
 
213
213
  @Override
214
- public boolean displayBitmap(@NonNull String base64ImageData) {
214
+ public boolean displayBitmap(@NonNull String base64ImageData, Integer x, Integer y, Integer width, Integer height) {
215
215
  try {
216
216
  Bridge.log("Mach1: displayBitmap() - decoding base64");
217
217
  // Decode base64 to byte array
@@ -32,10 +32,11 @@ import androidx.core.app.ActivityCompat;
32
32
  // import androidx.preference.PreferenceManager;
33
33
 
34
34
  // Mentra
35
- import com.mentra.bluetoothsdk.sgcs.SGCManager;
36
- import com.mentra.bluetoothsdk.DeviceManager;
35
+ import com.mentra.bluetoothsdk.BluetoothSdkDefaults;
37
36
  import com.mentra.bluetoothsdk.Bridge;
37
+ import com.mentra.bluetoothsdk.DeviceManager;
38
38
  import com.mentra.bluetoothsdk.debug.BleTraceLogger;
39
+ import com.mentra.bluetoothsdk.sgcs.SGCManager;
39
40
  import com.mentra.bluetoothsdk.utils.DeviceTypes;
40
41
  import com.mentra.bluetoothsdk.utils.ConnTypes;
41
42
  import com.mentra.bluetoothsdk.utils.BitmapJavaUtils;
@@ -684,6 +685,9 @@ public class MentraLive extends SGCManager {
684
685
  private void updateConnectionState(String state) {
685
686
  boolean isEqual = state.equals(getConnectionState());
686
687
  if (isEqual) {
688
+ if (state.equals(ConnTypes.DISCONNECTED)) {
689
+ incomingChunkReassembler.clear();
690
+ }
687
691
  return;
688
692
  }
689
693
 
@@ -710,6 +714,7 @@ public class MentraLive extends SGCManager {
710
714
  DeviceStore.INSTANCE.apply("glasses", "connected", false);
711
715
  DeviceStore.INSTANCE.apply("glasses", "signalStrength", -1);
712
716
  DeviceStore.INSTANCE.apply("glasses", "signalStrengthUpdatedAt", 0L);
717
+ incomingChunkReassembler.clear();
713
718
  // Drop OTA caches when fully disconnected — avoids leaking session/step state
714
719
  // from a previous pairing into the next one.
715
720
  resetOtaCache();
@@ -2262,14 +2267,13 @@ public class MentraLive extends SGCManager {
2262
2267
  if (Log.isLoggable(TAG, Log.DEBUG)) {
2263
2268
  Log.d(TAG, "LIVE: Got some JSON from glasses: " + json.toString());
2264
2269
  }
2270
+ BleTraceLogger.logJson("glasses_to_phone", "sdk_ble_event", json, null);
2265
2271
 
2266
2272
  if (MessageChunker.isChunkedMessage(json)) {
2267
2273
  processChunkedJsonMessage(json);
2268
2274
  return;
2269
2275
  }
2270
2276
 
2271
- BleTraceLogger.logJson("glasses_to_phone", "sdk_ble_event", json, json.toString().length());
2272
-
2273
2277
  // Check if this is an ACK response
2274
2278
  String type = json.optString("type", "");
2275
2279
  if ("msg_ack".equals(type)) {
@@ -2346,9 +2350,15 @@ public class MentraLive extends SGCManager {
2346
2350
  }
2347
2351
  break;
2348
2352
 
2353
+ case "video_recording_status":
2354
+ emitVideoRecordingStatus(json);
2355
+ break;
2356
+
2349
2357
  case "voice_activity_detection_status":
2350
2358
  handleVoiceActivityDetectionStatus(
2351
- json.optBoolean("voiceActivityDetectionEnabled", true));
2359
+ json.optBoolean(
2360
+ "voiceActivityDetectionEnabled",
2361
+ BluetoothSdkDefaults.VOICE_ACTIVITY_DETECTION_ENABLED));
2352
2362
  break;
2353
2363
 
2354
2364
  case "speaking_status":
@@ -2411,6 +2421,12 @@ public class MentraLive extends SGCManager {
2411
2421
  String photoState = json.optString("state", "");
2412
2422
  boolean photoSuccess = "success".equals(photoState) || json.optBoolean("success", false);
2413
2423
 
2424
+ try {
2425
+ Bridge.sendPhotoResponse(jsonObjectToMap(json));
2426
+ } catch (JSONException e) {
2427
+ Log.e(TAG, "Error converting photo response to Map", e);
2428
+ }
2429
+
2414
2430
  if (!photoSuccess) {
2415
2431
  // Handle failed photo response
2416
2432
  String errorMsg = json.optString("errorMessage", json.optString("error", "Unknown error"));
@@ -2469,7 +2485,8 @@ public class MentraLive extends SGCManager {
2469
2485
  }
2470
2486
  }
2471
2487
 
2472
- Bridge.updateWifiScanResults(networks);
2488
+ boolean scanComplete = json.optBoolean("scan_complete", json.optBoolean("scanComplete", false));
2489
+ Bridge.updateWifiScanResults(networks, scanComplete);
2473
2490
  break;
2474
2491
 
2475
2492
  case "token_status":
@@ -2545,8 +2562,10 @@ public class MentraLive extends SGCManager {
2545
2562
  Bridge.log("LIVE: 📱 OTA status - step " + osCurrentStep + "/" + osTotalSteps +
2546
2563
  " " + osPhase + " " + osStatus + " " + osOverallPercent + "%");
2547
2564
 
2565
+ long glassesTimeMs = json.optLong("glasses_time_ms", 0);
2548
2566
  Bridge.sendOtaStatus(osSessionId, osTotalSteps, osCurrentStep, osStepType,
2549
- osPhase, osStepPercent, osOverallPercent, osStatus, osErrorMessage);
2567
+ osPhase, osStepPercent, osOverallPercent, osStatus, osErrorMessage,
2568
+ glassesTimeMs > 0 ? glassesTimeMs : null);
2550
2569
  break;
2551
2570
 
2552
2571
  case "ota_progress":
@@ -2593,12 +2612,29 @@ public class MentraLive extends SGCManager {
2593
2612
  int totalCount = json.optInt("total", 0);
2594
2613
  long totalSize = json.optLong("total_size", 0);
2595
2614
  boolean hasContent = json.optBoolean("has_content", false);
2615
+ Object cameraBusyValue = json.opt("camera_busy");
2616
+ boolean cameraBusy = json.optBoolean("cameraBusy", false);
2617
+ String cameraBusyReason = json.optString("cameraBusyReason", null);
2618
+ if (cameraBusyValue instanceof Boolean) {
2619
+ cameraBusy = cameraBusy || (Boolean) cameraBusyValue;
2620
+ } else if (cameraBusyValue instanceof String) {
2621
+ String busyReason = ((String) cameraBusyValue).trim();
2622
+ if (!busyReason.isEmpty() && !"false".equalsIgnoreCase(busyReason)) {
2623
+ cameraBusy = true;
2624
+ cameraBusyReason = busyReason;
2625
+ }
2626
+ }
2596
2627
 
2597
2628
  Bridge.log("LIVE: 📸 Received gallery status: " + photoCount + " photos, " +
2598
2629
  videoCount + " videos, total size: " + totalSize + " bytes");
2599
2630
 
2600
2631
  // Send gallery status to React Native frontend (matches iOS pattern)
2601
- Bridge.sendGalleryStatus(photoCount, videoCount, totalCount, totalSize, hasContent);
2632
+ Bridge.sendGalleryStatus(photoCount, videoCount, totalCount, totalSize, hasContent,
2633
+ cameraBusy, cameraBusyReason);
2634
+ break;
2635
+
2636
+ case "settings_ack":
2637
+ emitSettingsAck(json);
2602
2638
  break;
2603
2639
 
2604
2640
  // case "touch_event":
@@ -2769,13 +2805,7 @@ public class MentraLive extends SGCManager {
2769
2805
 
2770
2806
  // Forward to websocket system via Bridge (matches iOS emitKeepAliveAck)
2771
2807
  try {
2772
- Map<String, Object> ackMap = new HashMap<>();
2773
- Iterator<String> keys = json.keys();
2774
- while (keys.hasNext()) {
2775
- String key = keys.next();
2776
- ackMap.put(key, json.get(key));
2777
- }
2778
- Bridge.sendKeepAliveAck(ackMap);
2808
+ Bridge.sendKeepAliveAck(jsonObjectToMap(json));
2779
2809
  } catch (JSONException e) {
2780
2810
  Log.e(TAG, "Error converting keep_alive_ack to Map", e);
2781
2811
  }
@@ -2806,6 +2836,15 @@ public class MentraLive extends SGCManager {
2806
2836
  DeviceStore.INSTANCE.apply("glasses", "firmwareVersion", firmwareVersionLegacy);
2807
2837
  DeviceStore.INSTANCE.apply("glasses", "bluetoothMacAddress", btMacAddressLegacy);
2808
2838
 
2839
+ Map<String, Object> versionInfoLegacy = new HashMap<>();
2840
+ versionInfoLegacy.put("appVersion", appVersionLegacy);
2841
+ versionInfoLegacy.put("buildNumber", buildNumberLegacy);
2842
+ versionInfoLegacy.put("deviceModel", deviceModelLegacy);
2843
+ versionInfoLegacy.put("androidVersion", androidVersionLegacy);
2844
+ versionInfoLegacy.put("otaVersionUrl", otaVersionUrlLegacy != null ? otaVersionUrlLegacy : "");
2845
+ versionInfoLegacy.put("firmwareVersion", firmwareVersionLegacy);
2846
+ Bridge.sendVersionInfo(versionInfoLegacy);
2847
+
2809
2848
  // Parse build number as integer for version checks (local field)
2810
2849
  try {
2811
2850
  buildNumberInt = Integer.parseInt(buildNumberLegacy);
@@ -2997,9 +3036,15 @@ public class MentraLive extends SGCManager {
2997
3036
  if (fields.containsKey("bt_mac_address")) {
2998
3037
  DeviceStore.INSTANCE.apply("glasses", "bluetoothMacAddress", (String) fields.get("bt_mac_address"));
2999
3038
  }
3000
-
3039
+ if (fields.containsKey("system_time_ms")) {
3040
+ Object v = fields.get("system_time_ms");
3041
+ if (v instanceof Number) {
3042
+ DeviceStore.INSTANCE.apply("glasses", "systemTimeMs", ((Number) v).longValue());
3043
+ }
3044
+ }
3001
3045
 
3002
3046
  Bridge.log("LIVE: Processed version_info fields and sent to RN");
3047
+ Bridge.sendVersionInfo(fields);
3003
3048
  } else {
3004
3049
  Log.d(TAG, "📦 Unknown message type: " + type);
3005
3050
  }
@@ -3007,6 +3052,58 @@ public class MentraLive extends SGCManager {
3007
3052
  }
3008
3053
  }
3009
3054
 
3055
+ private void emitSettingsAck(JSONObject json) {
3056
+ try {
3057
+ Map<String, Object> body = jsonObjectToMap(json);
3058
+ if (body.containsKey("request_id") && !body.containsKey("requestId")) {
3059
+ body.put("requestId", body.get("request_id"));
3060
+ }
3061
+ if (body.containsKey("roi_position") && !body.containsKey("roiPosition")) {
3062
+ body.put("roiPosition", body.get("roi_position"));
3063
+ }
3064
+ if (body.containsKey("error_code") && !body.containsKey("errorCode")) {
3065
+ body.put("errorCode", body.get("error_code"));
3066
+ }
3067
+ if (body.containsKey("error_message") && !body.containsKey("errorMessage")) {
3068
+ body.put("errorMessage", body.get("error_message"));
3069
+ }
3070
+ if (body.containsKey("hardware_applied") && !body.containsKey("hardwareApplied")) {
3071
+ body.put("hardwareApplied", body.get("hardware_applied"));
3072
+ }
3073
+ body.remove("request_id");
3074
+ body.remove("roi_position");
3075
+ body.remove("error_code");
3076
+ body.remove("error_message");
3077
+ body.remove("hardware_applied");
3078
+ if (!body.containsKey("timestamp")) {
3079
+ body.put("timestamp", System.currentTimeMillis());
3080
+ }
3081
+ Bridge.sendSettingsAck(body);
3082
+ } catch (JSONException e) {
3083
+ Log.e(TAG, "Error converting settings_ack to Map", e);
3084
+ }
3085
+ }
3086
+
3087
+ private void emitVideoRecordingStatus(JSONObject json) {
3088
+ try {
3089
+ Map<String, Object> body = jsonObjectToMap(json);
3090
+ if (body.containsKey("request_id") && !body.containsKey("requestId")) {
3091
+ body.put("requestId", body.get("request_id"));
3092
+ }
3093
+ if (body.containsKey("error_details") && !body.containsKey("details")) {
3094
+ body.put("details", body.get("error_details"));
3095
+ }
3096
+ body.remove("request_id");
3097
+ body.remove("error_details");
3098
+ if (!body.containsKey("timestamp")) {
3099
+ body.put("timestamp", System.currentTimeMillis());
3100
+ }
3101
+ Bridge.sendVideoRecordingStatus(body);
3102
+ } catch (JSONException e) {
3103
+ Log.e(TAG, "Error converting video_recording_status to Map", e);
3104
+ }
3105
+ }
3106
+
3010
3107
  private Map<String, Object> jsonObjectToMap(JSONObject json) throws JSONException {
3011
3108
  Map<String, Object> map = new HashMap<>();
3012
3109
  Iterator<String> keys = json.keys();
@@ -3046,6 +3143,14 @@ public class MentraLive extends SGCManager {
3046
3143
  Log.w(TAG, "LIVE: Received malformed chunked message: " + json);
3047
3144
  return;
3048
3145
  }
3146
+ if (chunkInfo.chunkId == null || chunkInfo.chunkId.isEmpty()
3147
+ || chunkInfo.totalChunks <= 0
3148
+ || chunkInfo.chunkIndex < 0
3149
+ || chunkInfo.chunkIndex >= chunkInfo.totalChunks
3150
+ || chunkInfo.data == null) {
3151
+ Log.w(TAG, "LIVE: Received invalid chunk metadata: " + json);
3152
+ return;
3153
+ }
3049
3154
 
3050
3155
  String reassembled = incomingChunkReassembler.addChunk(
3051
3156
  chunkInfo.chunkId,
@@ -3062,7 +3167,6 @@ public class MentraLive extends SGCManager {
3062
3167
  Log.e(TAG, "Error processing chunked JSON message", e);
3063
3168
  }
3064
3169
  }
3065
-
3066
3170
  /**
3067
3171
  * Process K900 command format JSON messages (messages with "C" field)
3068
3172
  */
@@ -3751,6 +3855,18 @@ public class MentraLive extends SGCManager {
3751
3855
  }
3752
3856
  }
3753
3857
 
3858
+ public void sendOtaRetryVersionCheck() {
3859
+ try {
3860
+ JSONObject json = new JSONObject();
3861
+ json.put("type", "ota_retry_version_check");
3862
+ json.put("timestamp", System.currentTimeMillis());
3863
+ sendJson(json, true);
3864
+ Bridge.log("LIVE: ⏰ Sending ota_retry_version_check command to glasses");
3865
+ } catch (JSONException e) {
3866
+ Log.e(TAG, "⏰ Error creating ota_retry_version_check command", e);
3867
+ }
3868
+ }
3869
+
3754
3870
  /**
3755
3871
  * Request version info from glasses.
3756
3872
  * Glasses will respond with version_info message containing build number, firmware version, etc.
@@ -3770,10 +3886,17 @@ public class MentraLive extends SGCManager {
3770
3886
  @Override
3771
3887
  public void sendGalleryMode() {
3772
3888
  boolean active = (Boolean) DeviceStore.INSTANCE.get("bluetooth", "gallery_mode");
3889
+ sendGalleryMode(null, active);
3890
+ }
3891
+
3892
+ public void sendGalleryMode(String requestId, boolean active) {
3773
3893
  Bridge.log("LIVE: 📸 Sending gallery mode active to glasses: " + active);
3774
3894
  try {
3775
3895
  JSONObject json = new JSONObject();
3776
3896
  json.put("type", "save_in_gallery_mode");
3897
+ if (requestId != null && !requestId.isEmpty()) {
3898
+ json.put("request_id", requestId);
3899
+ }
3777
3900
  json.put("active", active);
3778
3901
  json.put("timestamp", System.currentTimeMillis());
3779
3902
  sendJson(json, true);
@@ -4113,7 +4236,8 @@ public class MentraLive extends SGCManager {
4113
4236
  public void dbg1() {}
4114
4237
  public void dbg2() {}
4115
4238
 
4116
- public boolean displayBitmap(String base64) {
4239
+ @Override
4240
+ public boolean displayBitmap(String base64, Integer x, Integer y, Integer width, Integer height) {
4117
4241
  return false;
4118
4242
  }
4119
4243
 
@@ -4236,9 +4360,9 @@ public class MentraLive extends SGCManager {
4236
4360
  }
4237
4361
  }
4238
4362
 
4239
- public void requestPhoto(String requestId, String appId, String size, String webhookUrl, String authToken, String compress, boolean flash, boolean sound, Long exposureTimeNs, Integer iso) {
4363
+ public void requestPhoto(String requestId, String appId, String size, String webhookUrl, String authToken, String compress, boolean flash, boolean save, boolean sound, Long exposureTimeNs, Integer iso) {
4240
4364
  boolean hasAuthToken = authToken != null && !authToken.isEmpty();
4241
- Bridge.log("LIVE: Requesting photo: " + requestId + " for app: " + appId + " with size: " + size + ", webhookUrl: " + webhookUrl + ", authToken: " + (hasAuthToken ? "***" : "none") + ", compress=" + compress + ", flash=" + flash + ", sound=" + sound + ", exposureTimeNs=" + exposureTimeNs + ", iso=" + iso);
4365
+ Bridge.log("LIVE: Requesting photo: " + requestId + " for app: " + appId + " with size: " + size + ", webhookUrl: " + webhookUrl + ", authToken: " + (hasAuthToken ? "***" : "none") + ", compress=" + compress + ", flash=" + flash + ", save=" + save + ", sound=" + sound + ", exposureTimeNs=" + exposureTimeNs + ", iso=" + iso);
4242
4366
  Bridge.log("LIVE: PHOTO PIPELINE [5/6] requestPhoto() entry — requestId=" + requestId + ", appId=" + appId);
4243
4367
 
4244
4368
  try {
@@ -4261,6 +4385,7 @@ public class MentraLive extends SGCManager {
4261
4385
  json.put("compress", "none");
4262
4386
  }
4263
4387
  json.put("flash", flash);
4388
+ json.put("save", save);
4264
4389
  json.put("sound", sound);
4265
4390
  if (exposureTimeNs != null && exposureTimeNs > 0L) {
4266
4391
  Bridge.log("LIVE: Using manual exposure time for photo request " + requestId + ": " + exposureTimeNs + " ns");
@@ -4836,6 +4961,7 @@ public class MentraLive extends SGCManager {
4836
4961
 
4837
4962
  // Clean up message tracking
4838
4963
  pendingMessages.clear();
4964
+ incomingChunkReassembler.clear();
4839
4965
  Bridge.log("LIVE: Cleared pending message tracking");
4840
4966
 
4841
4967
  // Release RGB LED control authority before disconnecting
@@ -4891,10 +5017,17 @@ public class MentraLive extends SGCManager {
4891
5017
  // }
4892
5018
 
4893
5019
  public void sendButtonPhotoSettings(String size) {
5020
+ sendButtonPhotoSettings(null, size);
5021
+ }
5022
+
5023
+ public void sendButtonPhotoSettings(String requestId, String size) {
4894
5024
  // Send photo size settings to glasses
4895
5025
  JSONObject command = new JSONObject();
4896
5026
  try {
4897
5027
  command.put("type", "button_photo_setting");
5028
+ if (requestId != null && !requestId.isEmpty()) {
5029
+ command.put("request_id", requestId);
5030
+ }
4898
5031
  command.put("size", size);
4899
5032
  sendJson(command, true);
4900
5033
  } catch (Exception e) {
@@ -4929,11 +5062,21 @@ public class MentraLive extends SGCManager {
4929
5062
  videoFps = ((Number) fps).intValue();
4930
5063
  }
4931
5064
  }
4932
-
5065
+ sendButtonVideoRecordingSettings(null, videoWidth, videoHeight, videoFps);
5066
+ } catch (Exception e) {
5067
+ Log.e(TAG, "❌ [SETTINGS_SYNC] Error sending button video recording settings", e);
5068
+ }
5069
+ }
5070
+
5071
+ public void sendButtonVideoRecordingSettings(String requestId, int videoWidth, int videoHeight, int videoFps) {
5072
+ try {
4933
5073
  Bridge.log("LIVE: 🎥 [SETTINGS_SYNC] Sending button video recording settings: " + videoWidth + "x" + videoHeight + "@" + videoFps + "fps");
4934
5074
 
4935
5075
  JSONObject json = new JSONObject();
4936
5076
  json.put("type", "button_video_recording_setting");
5077
+ if (requestId != null && !requestId.isEmpty()) {
5078
+ json.put("request_id", requestId);
5079
+ }
4937
5080
  JSONObject settings = new JSONObject();
4938
5081
  settings.put("width", videoWidth);
4939
5082
  settings.put("height", videoHeight);
@@ -4948,10 +5091,17 @@ public class MentraLive extends SGCManager {
4948
5091
  }
4949
5092
 
4950
5093
  public void sendButtonCameraLedSetting(boolean enabled) {
5094
+ sendButtonCameraLedSetting(null, enabled);
5095
+ }
5096
+
5097
+ public void sendButtonCameraLedSetting(String requestId, boolean enabled) {
4951
5098
  // Send LED setting to glasses
4952
5099
  JSONObject command = new JSONObject();
4953
5100
  try {
4954
5101
  command.put("type", "button_camera_led");
5102
+ if (requestId != null && !requestId.isEmpty()) {
5103
+ command.put("request_id", requestId);
5104
+ }
4955
5105
  command.put("enabled", enabled);
4956
5106
  sendJson(command, true);
4957
5107
  } catch (Exception e) {
@@ -5678,6 +5828,18 @@ public class MentraLive extends SGCManager {
5678
5828
  }
5679
5829
  }
5680
5830
 
5831
+ private double[] jsonArrayToDoubleArray(JSONArray source, int expectedLength) {
5832
+ if (source == null) {
5833
+ return new double[0];
5834
+ }
5835
+ int length = Math.min(expectedLength, source.length());
5836
+ double[] out = new double[length];
5837
+ for (int i = 0; i < length; i++) {
5838
+ out[i] = source.optDouble(i, 0.0);
5839
+ }
5840
+ return out;
5841
+ }
5842
+
5681
5843
  private void handleStreamImuData(JSONObject json) {
5682
5844
  try {
5683
5845
  JSONArray readings = json.getJSONArray("readings");
@@ -5751,7 +5913,7 @@ public class MentraLive extends SGCManager {
5751
5913
  }
5752
5914
 
5753
5915
  // Create chunks
5754
- List<JSONObject> chunks = MessageChunker.createChunks(data, messageId);
5916
+ List<JSONObject> chunks = MessageChunker.createChunks(data, messageId, wakeup);
5755
5917
  Bridge.log("LIVE: Sending " + chunks.size() + " chunks");
5756
5918
  if (isPhotoRequest) {
5757
5919
  Bridge.log("LIVE: PHOTO PIPELINE BLE handoff — created " + chunks.size() + " chunks for transmission");
@@ -5923,6 +6085,19 @@ public class MentraLive extends SGCManager {
5923
6085
  }
5924
6086
  }
5925
6087
 
6088
+ @Override
6089
+ public void sendSetSystemTime(long timestampMs) {
6090
+ Bridge.log("LIVE: ⏰ Sending set_system_time to glasses: " + timestampMs);
6091
+ try {
6092
+ JSONObject command = new JSONObject();
6093
+ command.put("type", "set_system_time");
6094
+ command.put("timestamp_ms", timestampMs);
6095
+ sendJson(command, true);
6096
+ } catch (JSONException e) {
6097
+ Log.e(TAG, "⏰ Error creating set_system_time JSON", e);
6098
+ }
6099
+ }
6100
+
5926
6101
  /**
5927
6102
  * Sends user email to glasses for crash reporting identification
5928
6103
  *
@@ -6506,14 +6681,14 @@ public class MentraLive extends SGCManager {
6506
6681
  transfer.authToken,
6507
6682
  new BlePhotoUploadService.UploadCallback() {
6508
6683
  @Override
6509
- public void onSuccess(String requestId) {
6684
+ public void onSuccess(String requestId, String responseBody) {
6510
6685
  long uploadDuration = System.currentTimeMillis() - uploadStartTime;
6511
6686
  long totalDuration = System.currentTimeMillis() - transfer.phoneStartTime;
6512
6687
 
6513
6688
  Bridge.log("LIVE: ✅ BLE photo uploaded successfully via phone relay for requestId: " + requestId);
6514
6689
  Bridge.log("LIVE: ⏱️ Upload duration: " + uploadDuration + "ms");
6515
6690
  Bridge.log("LIVE: ⏱️ Total end-to-end duration: " + totalDuration + "ms");
6516
- //sendPhotoUploadSuccess(requestId);
6691
+ sendPhotoTerminalSuccessResponse(requestId, transfer.webhookUrl, responseBody);
6517
6692
  }
6518
6693
 
6519
6694
  @Override
@@ -6521,12 +6696,52 @@ public class MentraLive extends SGCManager {
6521
6696
  long uploadDuration = System.currentTimeMillis() - uploadStartTime;
6522
6697
  Log.e(TAG, "❌ BLE photo upload failed for requestId: " + requestId + ", error: " + error);
6523
6698
  Log.e(TAG, "⏱️ Failed after: " + uploadDuration + "ms");
6524
- //sendPhotoUploadError(requestId, error);
6699
+ Bridge.sendPhotoError(
6700
+ requestId,
6701
+ "PHONE_UPLOAD_FAILED",
6702
+ "BLE photo upload failed: " + error);
6525
6703
  }
6526
6704
  }
6527
6705
  );
6528
6706
  }
6529
6707
 
6708
+ private void sendPhotoTerminalSuccessResponse(
6709
+ String requestId, String uploadUrl, String responseBody) {
6710
+ Map<String, Object> event = new HashMap<>();
6711
+ event.put("type", "photo_response");
6712
+ event.put("state", "success");
6713
+ event.put("success", true);
6714
+ event.put("requestId", requestId);
6715
+ event.put("uploadUrl", uploadUrl != null ? uploadUrl : "");
6716
+ event.put("timestamp", System.currentTimeMillis());
6717
+ copyPhotoUploadResponseMetadata(event, responseBody);
6718
+ Bridge.sendPhotoResponse(event);
6719
+ }
6720
+
6721
+ private void copyPhotoUploadResponseMetadata(Map<String, Object> event, String responseBody) {
6722
+ if (responseBody == null || responseBody.trim().isEmpty()) {
6723
+ return;
6724
+ }
6725
+ try {
6726
+ JSONObject response = new JSONObject(responseBody);
6727
+ copyJsonField(event, response, "photoUrl");
6728
+ copyJsonField(event, response, "statusUrl");
6729
+ copyJsonField(event, response, "mimeType");
6730
+ copyJsonField(event, response, "contentType");
6731
+ copyJsonField(event, response, "bytes");
6732
+ copyJsonField(event, response, "size");
6733
+ } catch (JSONException e) {
6734
+ Bridge.log("LIVE: BLE upload response body was not JSON metadata");
6735
+ }
6736
+ }
6737
+
6738
+ private void copyJsonField(Map<String, Object> event, JSONObject response, String key)
6739
+ throws JSONException {
6740
+ if (response.has(key) && !response.isNull(key)) {
6741
+ event.put(key, response.get(key));
6742
+ }
6743
+ }
6744
+
6530
6745
  /**
6531
6746
  * Send photo upload success notification to glasses
6532
6747
  */
@@ -6646,7 +6861,10 @@ public class MentraLive extends SGCManager {
6646
6861
  @Override
6647
6862
  public void sendVoiceActivityDetectionSetting() {
6648
6863
  Object value = DeviceStore.INSTANCE.get("bluetooth", "voice_activity_detection_enabled");
6649
- boolean enabled = value instanceof Boolean ? (Boolean) value : true;
6864
+ boolean enabled =
6865
+ value instanceof Boolean
6866
+ ? (Boolean) value
6867
+ : BluetoothSdkDefaults.VOICE_ACTIVITY_DETECTION_ENABLED;
6650
6868
 
6651
6869
  Bridge.log("LIVE: 🎤 Sending Voice Activity Detection setting to glasses: " + enabled);
6652
6870
 
@@ -6685,22 +6903,7 @@ public class MentraLive extends SGCManager {
6685
6903
  */
6686
6904
  public void sendButtonPhotoSettings() {
6687
6905
  String size = (String) DeviceStore.INSTANCE.get("bluetooth", "button_photo_size");
6688
-
6689
- Bridge.log("LIVE: Sending button photo setting: " + size);
6690
-
6691
- if (!isConnected) {
6692
- Log.w(TAG, "Cannot send button photo settings - not connected");
6693
- return;
6694
- }
6695
-
6696
- try {
6697
- JSONObject json = new JSONObject();
6698
- json.put("type", "button_photo_setting");
6699
- json.put("size", size);
6700
- sendJson(json);
6701
- } catch (JSONException e) {
6702
- Log.e(TAG, "Error creating button photo settings message", e);
6703
- }
6906
+ sendButtonPhotoSettings(null, size);
6704
6907
  }
6705
6908
 
6706
6909
  /**
@@ -6709,22 +6912,7 @@ public class MentraLive extends SGCManager {
6709
6912
  @Override
6710
6913
  public void sendButtonCameraLedSetting() {
6711
6914
  boolean enabled = (Boolean) DeviceStore.INSTANCE.get("bluetooth", "button_camera_led");
6712
-
6713
- Bridge.log("LIVE: Sending button camera LED setting: " + enabled);
6714
-
6715
- if (!isConnected) {
6716
- Log.w(TAG, "Cannot send button camera LED setting - not connected");
6717
- return;
6718
- }
6719
-
6720
- try {
6721
- JSONObject json = new JSONObject();
6722
- json.put("type", "button_camera_led");
6723
- json.put("enabled", enabled);
6724
- sendJson(json, true);
6725
- } catch (JSONException e) {
6726
- Log.e(TAG, "Error creating button camera LED setting message", e);
6727
- }
6915
+ sendButtonCameraLedSetting(null, enabled);
6728
6916
  }
6729
6917
 
6730
6918
  /**
@@ -6748,6 +6936,10 @@ public class MentraLive extends SGCManager {
6748
6936
  Log.w(TAG, "Could not read camera_fov from store, using defaults", e);
6749
6937
  }
6750
6938
 
6939
+ sendCameraFovSetting(null, fov, roiPosition);
6940
+ }
6941
+
6942
+ public void sendCameraFovSetting(String requestId, int fov, int roiPosition) {
6751
6943
  Bridge.log("LIVE: Sending camera FOV setting: fov=" + fov + ", roiPosition=" + roiPosition);
6752
6944
 
6753
6945
  if (!isConnected) {
@@ -6758,6 +6950,9 @@ public class MentraLive extends SGCManager {
6758
6950
  try {
6759
6951
  JSONObject json = new JSONObject();
6760
6952
  json.put("type", "camera_fov_setting");
6953
+ if (requestId != null && !requestId.isEmpty()) {
6954
+ json.put("request_id", requestId);
6955
+ }
6761
6956
  JSONObject params = new JSONObject();
6762
6957
  params.put("fov", fov);
6763
6958
  params.put("roi_position", roiPosition);
@@ -6774,6 +6969,12 @@ public class MentraLive extends SGCManager {
6774
6969
  */
6775
6970
  @Override
6776
6971
  public void sendButtonMaxRecordingTime() {
6972
+ Object rawMinutes = DeviceStore.INSTANCE.get("bluetooth", "button_max_recording_time");
6973
+ int minutes = (rawMinutes instanceof Number) ? ((Number) rawMinutes).intValue() : 10;
6974
+ sendButtonMaxRecordingTime(null, minutes);
6975
+ }
6976
+
6977
+ public void sendButtonMaxRecordingTime(String requestId, int minutes) {
6777
6978
  Bridge.log("LIVE: Sending button max recording time");
6778
6979
 
6779
6980
  if (!isConnected) {
@@ -6781,12 +6982,12 @@ public class MentraLive extends SGCManager {
6781
6982
  return;
6782
6983
  }
6783
6984
 
6784
- Object rawMinutes = DeviceStore.INSTANCE.get("bluetooth", "button_max_recording_time");
6785
- int minutes = (rawMinutes instanceof Number) ? ((Number) rawMinutes).intValue() : 10;
6786
-
6787
6985
  try {
6788
6986
  JSONObject json = new JSONObject();
6789
6987
  json.put("type", "button_max_recording_time");
6988
+ if (requestId != null && !requestId.isEmpty()) {
6989
+ json.put("request_id", requestId);
6990
+ }
6790
6991
  json.put("minutes", minutes);
6791
6992
  sendJson(json, true);
6792
6993
  } catch (JSONException e) {
@@ -6809,6 +7010,7 @@ public class MentraLive extends SGCManager {
6809
7010
  * @param height Video height (0 for default)
6810
7011
  * @param fps Video frame rate (0 for default)
6811
7012
  */
7013
+ @Override
6812
7014
  public void startVideoRecording(String requestId, boolean save, boolean flash, boolean sound, int width, int height, int fps) {
6813
7015
  Bridge.log("LIVE: Starting video recording: requestId=" + requestId + ", save=" + save +
6814
7016
  ", flash=" + flash + ", sound=" + sound + ", resolution=" + width + "x" + height + "@" + fps + "fps");
@@ -6826,12 +7028,15 @@ public class MentraLive extends SGCManager {
6826
7028
  json.put("flash", flash);
6827
7029
  json.put("sound", sound);
6828
7030
 
6829
- // Add video settings if provided
6830
- if (width > 0 && height > 0) {
7031
+ // Add video settings when any field is overridden. Each field is sent
7032
+ // only when > 0; the glasses merge the missing fields onto their saved
7033
+ // button-video defaults, so a partial override (e.g. fps-only) still
7034
+ // takes effect instead of being dropped here.
7035
+ if (width > 0 || height > 0 || fps > 0) {
6831
7036
  JSONObject settings = new JSONObject();
6832
- settings.put("width", width);
6833
- settings.put("height", height);
6834
- settings.put("fps", fps > 0 ? fps : 30);
7037
+ if (width > 0) settings.put("width", width);
7038
+ if (height > 0) settings.put("height", height);
7039
+ if (fps > 0) settings.put("fps", fps);
6835
7040
  json.put("settings", settings);
6836
7041
  }
6837
7042