@mentra/bluetooth-sdk 0.1.9 → 0.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (147) hide show
  1. package/README.md +116 -18
  2. package/android/build.gradle +289 -16
  3. package/android/gradle.properties.example +9 -0
  4. package/android/lc3Lib/build.gradle +95 -6
  5. package/android/settings.gradle +3 -0
  6. package/android/silero/LICENSE.md +17 -0
  7. package/android/silero/build.gradle +42 -0
  8. package/android/silero/proguard-rules.pro +24 -0
  9. package/android/silero/src/androidTest/assets/hello.wav +0 -0
  10. package/android/silero/src/androidTest/java/com/konovalov/vad/silero/VadSileroTest.kt +79 -0
  11. package/android/silero/src/main/assets/LICENSE +21 -0
  12. package/android/silero/src/main/assets/silero_vad.onnx +0 -0
  13. package/android/silero/src/main/java/com/konovalov/vad/silero/Vad.kt +160 -0
  14. package/android/silero/src/main/java/com/konovalov/vad/silero/VadSilero.kt +449 -0
  15. package/android/silero/src/main/java/com/konovalov/vad/silero/config/FrameSize.kt +15 -0
  16. package/android/silero/src/main/java/com/konovalov/vad/silero/config/Mode.kt +14 -0
  17. package/android/silero/src/main/java/com/konovalov/vad/silero/config/SampleRate.kt +12 -0
  18. package/android/silero/src/main/java/com/konovalov/vad/silero/utils/AudioUtils.kt +47 -0
  19. package/android/silero/src/main/java/com/konovalov/vad/silero/utils/TensorMap.kt +41 -0
  20. package/android/silero/src/test/java/com/konovalov/vad/silero/utils/AudioUtilsTest.kt +44 -0
  21. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkDefaults.kt +6 -0
  22. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +195 -34
  23. package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +103 -21
  24. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +210 -18
  25. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt +30 -2
  26. package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +787 -85
  27. package/android/src/main/java/com/mentra/bluetoothsdk/audio/AudioModels.kt +6 -2
  28. package/android/src/main/java/com/mentra/bluetoothsdk/camera/CameraModels.kt +125 -11
  29. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/ControllerManager.kt +12 -2
  30. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/R1.kt +8 -2
  31. package/android/src/main/java/com/mentra/bluetoothsdk/debug/BleTraceLogger.kt +33 -17
  32. package/android/src/main/java/com/mentra/bluetoothsdk/events/BluetoothEvents.kt +101 -0
  33. package/android/src/main/java/com/mentra/bluetoothsdk/internal/MapParsing.kt +8 -0
  34. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistry.java +108 -0
  35. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoUploadServer.kt +399 -74
  36. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/MentraPhotoReceiverModule.kt +100 -17
  37. package/android/src/main/java/com/mentra/bluetoothsdk/requests/DisplayRequests.kt +17 -0
  38. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G1.java +2 -2
  39. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt +1293 -758
  40. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Mach1.java +2 -2
  41. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraLive.java +271 -66
  42. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraNex.kt +197 -149
  43. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraosBle.java +12753 -27036
  44. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt +64 -2
  45. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Simulated.kt +16 -4
  46. package/android/src/main/java/com/mentra/bluetoothsdk/status/DeviceStatus.kt +60 -3
  47. package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +12 -3
  48. package/android/src/main/java/com/mentra/bluetoothsdk/stt/STTTools.kt +87 -36
  49. package/android/src/main/java/com/mentra/bluetoothsdk/stt/SherpaOnnxTranscriber.kt +13 -4
  50. package/android/src/main/java/com/mentra/bluetoothsdk/stt/VadGateSpeechPolicy.kt +229 -0
  51. package/android/src/main/java/com/mentra/bluetoothsdk/tts/TTSTools.kt +180 -0
  52. package/android/src/main/java/com/mentra/bluetoothsdk/utils/AvifExifStripper.java +573 -0
  53. package/android/src/main/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadService.java +488 -50
  54. package/android/src/main/java/com/mentra/bluetoothsdk/utils/HeifExifTagReader.java +199 -0
  55. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunkReassembler.java +18 -1
  56. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunker.java +64 -18
  57. package/android/src/main/java/com/mentra/bluetoothsdk/utils/NexSGCUtils.kt +48 -39
  58. package/android/src/test/java/com/mentra/bluetoothsdk/camera/PhotoRequestTest.kt +23 -0
  59. package/android/src/test/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistryTest.java +75 -0
  60. package/android/src/test/java/com/mentra/bluetoothsdk/utils/AvifExifStripperTest.java +30 -0
  61. package/android/src/test/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadServiceTest.java +98 -0
  62. package/android/src/test/resources/avif_with_exif.avif +0 -0
  63. package/build/BluetoothSdk.types.d.ts +253 -44
  64. package/build/BluetoothSdk.types.d.ts.map +1 -1
  65. package/build/BluetoothSdk.types.js +11 -10
  66. package/build/BluetoothSdk.types.js.map +1 -1
  67. package/build/_private/BluetoothSdkModule.d.ts +37 -23
  68. package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
  69. package/build/_private/BluetoothSdkModule.js +43 -52
  70. package/build/_private/BluetoothSdkModule.js.map +1 -1
  71. package/build/_private/photoRequestPayload.d.ts +4 -0
  72. package/build/_private/photoRequestPayload.d.ts.map +1 -0
  73. package/build/_private/photoRequestPayload.js +25 -0
  74. package/build/_private/photoRequestPayload.js.map +1 -0
  75. package/build/index.d.ts +1 -1
  76. package/build/index.d.ts.map +1 -1
  77. package/build/index.js +25 -0
  78. package/build/index.js.map +1 -1
  79. package/build/react/useMentraBluetooth.d.ts +2 -2
  80. package/build/react/useMentraBluetooth.d.ts.map +1 -1
  81. package/build/react/useMentraBluetooth.js +3 -3
  82. package/build/react/useMentraBluetooth.js.map +1 -1
  83. package/ios/BluetoothSdkModule.swift +177 -70
  84. package/ios/LocalPhotoUploadServer.swift +370 -105
  85. package/ios/MentraBluetoothSDK.podspec +3 -1
  86. package/ios/MentraPhotoReceiverModule.swift +62 -10
  87. package/ios/Packages/CoreObjC/include/PcmConverter.h +22 -0
  88. package/ios/Packages/SherpaOnnx/SherpaOnnx.swift +668 -26
  89. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Info.plist +13 -9
  90. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
  91. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
  92. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/libsherpa-onnx.a +0 -0
  93. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
  94. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
  95. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/libsherpa-onnx.a +0 -0
  96. package/ios/Source/BluetoothSdkDefaults.swift +6 -0
  97. package/ios/Source/Bridge.swift +68 -19
  98. package/ios/Source/DeviceManager.swift +167 -32
  99. package/ios/Source/DeviceStore.swift +24 -8
  100. package/ios/Source/MentraBluetoothSDK.swift +803 -47
  101. package/ios/Source/ObservableStore.swift +6 -0
  102. package/ios/Source/{Audio → audio}/AudioModels.swift +9 -7
  103. package/ios/Source/{Camera → camera}/CameraModels.swift +226 -16
  104. package/ios/Source/controllers/ControllerManager.swift +4 -3
  105. package/ios/Source/controllers/R1.swift +3 -3
  106. package/ios/Source/{Events → events}/BluetoothEvents.swift +183 -1
  107. package/ios/Source/{Internal → internal}/ValueParsing.swift +10 -1
  108. package/ios/Source/{Requests → requests}/DisplayRequests.swift +28 -4
  109. package/ios/Source/services/PhoneMic.swift +1 -0
  110. package/ios/Source/sgcs/G1.swift +5 -5
  111. package/ios/Source/sgcs/G2.swift +1494 -870
  112. package/ios/Source/sgcs/Mach1.swift +4 -4
  113. package/ios/Source/sgcs/MentraLive.swift +814 -321
  114. package/ios/Source/sgcs/MentraNex.swift +604 -244
  115. package/ios/Source/sgcs/SGCManager.swift +68 -5
  116. package/ios/Source/sgcs/Simulated.swift +9 -5
  117. package/ios/Source/sgcs/mentraos_ble.pb.swift +3073 -3134
  118. package/ios/Source/status/DeviceStatus.swift +974 -0
  119. package/ios/Source/{Status → status}/WifiHotspotStatus.swift +4 -4
  120. package/ios/Source/{Streaming → streaming}/StreamModels.swift +24 -9
  121. package/ios/Source/stt/STTTools.swift +27 -12
  122. package/ios/Source/stt/SherpaOnnxTranscriber.swift +31 -8
  123. package/ios/Source/tts/TTSTools.swift +164 -0
  124. package/ios/Source/utils/JSCExperiment.swift +7 -7
  125. package/ios/Source/utils/MessageChunkReassembler.swift +20 -1
  126. package/ios/Source/utils/MessageChunker.swift +78 -21
  127. package/ios/Source/utils/TarBz2Extractor.swift +53 -2
  128. package/package.json +1 -1
  129. package/plugin/build/withAndroid.js +68 -4
  130. package/src/BluetoothSdk.types.ts +344 -60
  131. package/src/_private/BluetoothSdkModule.ts +132 -102
  132. package/src/_private/photoRequestPayload.ts +28 -0
  133. package/src/index.ts +61 -15
  134. package/src/react/useMentraBluetooth.ts +6 -4
  135. package/android/.project +0 -28
  136. package/android/lc3Lib/.project +0 -28
  137. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/cargs.h +0 -162
  138. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/c-api.h +0 -1852
  139. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/cxx-api.h +0 -674
  140. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/sherpa-onnx.a +0 -0
  141. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/sherpa-onnx.a +0 -0
  142. package/ios/Source/Status/DeviceStatus.swift +0 -471
  143. /package/ios/Source/{Connection → connection}/ScanSession.swift +0 -0
  144. /package/ios/Source/{Errors → errors}/BluetoothError.swift +0 -0
  145. /package/ios/Source/{Internal → internal}/BluetoothAvailability.swift +0 -0
  146. /package/ios/Source/{Status → status}/RuntimeState.swift +0 -0
  147. /package/ios/Source/{Types → types}/DeviceModels.swift +0 -0
@@ -6,9 +6,21 @@ import android.os.Build;
6
6
  import android.util.Log;
7
7
 
8
8
  import androidx.annotation.Nullable;
9
+ import androidx.annotation.VisibleForTesting;
10
+ import androidx.exifinterface.media.ExifInterface;
9
11
 
10
- import java.io.ByteArrayOutputStream;
12
+ import com.mentra.bluetoothsdk.debug.BleTraceLogger;
13
+ import com.mentra.bluetoothsdk.photoreceiver.LocalPhotoReceiverRegistry;
14
+ import com.radzivon.bartoshyk.avif.coder.HeifCoder;
15
+ import com.radzivon.bartoshyk.avif.coder.PreferredColorConfig;
16
+
17
+ import org.json.JSONArray;
18
+
19
+ import java.io.File;
20
+ import java.io.FileOutputStream;
11
21
  import java.io.IOException;
22
+ import java.net.URI;
23
+ import java.nio.charset.StandardCharsets;
12
24
  import java.util.concurrent.TimeUnit;
13
25
 
14
26
  import okhttp3.MediaType;
@@ -17,15 +29,18 @@ import okhttp3.OkHttpClient;
17
29
  import okhttp3.Request;
18
30
  import okhttp3.RequestBody;
19
31
  import okhttp3.Response;
32
+ import org.json.JSONObject;
20
33
 
21
34
  /**
22
- * Service to handle BLE photo uploads including AVIF decoding and webhook posting
35
+ * Service to handle BLE photo uploads including AVIF decoding and webhook posting.
36
+ * Preserves IMU metadata embedded in EXIF UserComment when re-encoding to JPEG.
23
37
  */
24
38
  public class BlePhotoUploadService {
25
39
  private static final String TAG = "BlePhotoUploadService";
40
+ private static final int JPEG_QUALITY = 90;
26
41
 
27
42
  public interface UploadCallback {
28
- void onSuccess(String requestId);
43
+ void onSuccess(String requestId, String responseBody);
29
44
  void onError(String requestId, String error);
30
45
  }
31
46
 
@@ -44,27 +59,15 @@ public class BlePhotoUploadService {
44
59
  try {
45
60
  Log.d(TAG, "Processing BLE photo for upload. Image size: " + imageData.length + " bytes");
46
61
 
47
- // 1. Decode image (AVIF or JPEG) to Bitmap
48
- Bitmap bitmap = decodeImage(imageData);
49
- if (bitmap == null) {
50
- throw new Exception("Failed to decode image data");
51
- }
52
-
53
- Log.d(TAG, "Decoded image to bitmap: " + bitmap.getWidth() + "x" + bitmap.getHeight());
54
-
55
- // 2. Convert to JPEG for upload (in case it was AVIF)
56
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
57
- bitmap.compress(Bitmap.CompressFormat.JPEG, 90, baos);
58
- byte[] jpegData = baos.toByteArray();
59
- bitmap.recycle();
60
-
62
+ byte[] jpegData = convertToJpegPreservingExif(imageData);
61
63
  Log.d(TAG, "Converted to JPEG for upload. Size: " + jpegData.length + " bytes");
62
64
 
63
65
  // 3. Upload to webhook
64
- uploadToWebhook(jpegData, requestId, webhookUrl, authToken);
66
+ String responseBody =
67
+ uploadToWebhook(jpegData, imageData.length, requestId, webhookUrl, authToken);
65
68
 
66
69
  Log.d(TAG, "Photo uploaded successfully for requestId: " + requestId);
67
- callback.onSuccess(requestId);
70
+ callback.onSuccess(requestId, responseBody);
68
71
 
69
72
  } catch (Exception e) {
70
73
  Log.e(TAG, "Error processing BLE photo for requestId: " + requestId, e);
@@ -73,6 +76,192 @@ public class BlePhotoUploadService {
73
76
  }).start();
74
77
  }
75
78
 
79
+ /**
80
+ * Decode incoming AVIF/JPEG, re-encode as JPEG, and re-attach IMU EXIF when present.
81
+ */
82
+ @VisibleForTesting
83
+ static byte[] convertToJpegPreservingExif(byte[] imageData) throws Exception {
84
+ File inputFile = File.createTempFile("ble_photo_in_", guessExtension(imageData));
85
+ File outputFile = File.createTempFile("ble_photo_out_", ".jpg");
86
+ try {
87
+ try (FileOutputStream fos = new FileOutputStream(inputFile)) {
88
+ fos.write(imageData);
89
+ }
90
+
91
+ logIncomingImageDiagnostics(imageData, inputFile.getAbsolutePath());
92
+
93
+ String imuJson = readImuJsonFromBleImage(imageData, inputFile.getAbsolutePath());
94
+
95
+ Bitmap bitmap = decodeImage(imageData);
96
+ if (bitmap == null) {
97
+ throw new Exception("Failed to decode image data");
98
+ }
99
+
100
+ Log.d(TAG, "Decoded image to bitmap: " + bitmap.getWidth() + "x" + bitmap.getHeight());
101
+
102
+ try (FileOutputStream fos = new FileOutputStream(outputFile)) {
103
+ bitmap.compress(Bitmap.CompressFormat.JPEG, JPEG_QUALITY, fos);
104
+ } finally {
105
+ bitmap.recycle();
106
+ }
107
+
108
+ if (imuJson != null && !imuJson.isEmpty()) {
109
+ logImuData(imuJson);
110
+ // IMU EXIF is enrichment, not the payload: a write failure must not drop the
111
+ // already-decoded photo. Log and upload the plain JPEG instead.
112
+ try {
113
+ writeImuJsonToJpeg(outputFile.getAbsolutePath(), imuJson);
114
+ Log.d(TAG, "Re-attached IMU EXIF UserComment on output JPEG (" + imuJson.length() + " chars)");
115
+ } catch (Exception e) {
116
+ Log.w(TAG, "Failed to attach IMU EXIF; uploading photo without it", e);
117
+ }
118
+ } else {
119
+ boolean rawHasExif = containsExifMarkerInBytes(imageData);
120
+ Log.w(
121
+ TAG,
122
+ "No IMU from ExifInterface on "
123
+ + inputFile.getName()
124
+ + " (container="
125
+ + describeContainer(imageData)
126
+ + ", rawHasExifMarker="
127
+ + rawHasExif
128
+ + "). If rawHasExifMarker=true, EXIF may be present but unreadable via"
129
+ + " ExifInterface on this container.");
130
+ }
131
+
132
+ return java.nio.file.Files.readAllBytes(outputFile.toPath());
133
+ } finally {
134
+ if (!inputFile.delete()) {
135
+ inputFile.deleteOnExit();
136
+ }
137
+ if (!outputFile.delete()) {
138
+ outputFile.deleteOnExit();
139
+ }
140
+ }
141
+ }
142
+
143
+ @Nullable
144
+ @VisibleForTesting
145
+ static String readImuJsonFromBleImage(byte[] imageData, String imagePath) {
146
+ String fromExif = readImuJsonFromImageFile(imagePath);
147
+ if (fromExif != null && !fromExif.isEmpty()) {
148
+ return fromExif;
149
+ }
150
+ if (containsExifMarkerInBytes(imageData)) {
151
+ String fromTiff = HeifExifTagReader.readImuJson(imageData);
152
+ if (fromTiff != null && !fromTiff.isEmpty()) {
153
+ Log.d(
154
+ TAG,
155
+ "Read IMU UserComment via TIFF scan ("
156
+ + fromTiff.length()
157
+ + " chars), container="
158
+ + describeContainer(imageData));
159
+ return fromTiff;
160
+ }
161
+ }
162
+ return null;
163
+ }
164
+
165
+ @Nullable
166
+ @VisibleForTesting
167
+ static String readImuJsonFromImageFile(String imagePath) {
168
+ try {
169
+ ExifInterface exif = new ExifInterface(imagePath);
170
+ String userComment = exif.getAttribute(ExifInterface.TAG_USER_COMMENT);
171
+ String imageDescription = exif.getAttribute(ExifInterface.TAG_IMAGE_DESCRIPTION);
172
+ Log.d(
173
+ TAG,
174
+ "ExifInterface on "
175
+ + imagePath
176
+ + ": UserComment="
177
+ + describeExifAttribute(userComment)
178
+ + ", ImageDescription="
179
+ + describeExifAttribute(imageDescription));
180
+ if (userComment != null && !userComment.isEmpty()) {
181
+ return userComment;
182
+ }
183
+ return imageDescription;
184
+ } catch (IOException e) {
185
+ Log.w(TAG, "Could not read EXIF from BLE image: " + imagePath, e);
186
+ return null;
187
+ }
188
+ }
189
+
190
+ private static void logIncomingImageDiagnostics(byte[] imageData, String tempPath) {
191
+ Log.d(
192
+ TAG,
193
+ "BLE image diagnostics: size="
194
+ + imageData.length
195
+ + " bytes, container="
196
+ + describeContainer(imageData)
197
+ + ", tempFile="
198
+ + tempPath
199
+ + ", rawHasExifMarker="
200
+ + containsExifMarkerInBytes(imageData));
201
+ }
202
+
203
+ /** ISO BMFF major brand or JPEG; used to correlate with ExifInterface behavior on AVIF. */
204
+ @VisibleForTesting
205
+ static String describeContainer(byte[] data) {
206
+ if (data.length >= 2 && (data[0] & 0xFF) == 0xFF && (data[1] & 0xFF) == 0xD8) {
207
+ return "jpeg";
208
+ }
209
+ if (data.length >= 12
210
+ && data[4] == 'f'
211
+ && data[5] == 't'
212
+ && data[6] == 'y'
213
+ && data[7] == 'p') {
214
+ String brand = new String(data, 8, 4, StandardCharsets.US_ASCII);
215
+ return "iso_bmff/ftyp=" + brand;
216
+ }
217
+ return "unknown";
218
+ }
219
+
220
+ /** Scans file bytes for the TIFF EXIF header (does not parse tags). */
221
+ @VisibleForTesting
222
+ static boolean containsExifMarkerInBytes(byte[] data) {
223
+ byte[] marker = new byte[] {'E', 'x', 'i', 'f', 0, 0};
224
+ outer:
225
+ for (int i = 0; i <= data.length - marker.length; i++) {
226
+ for (int j = 0; j < marker.length; j++) {
227
+ if (data[i + j] != marker[j]) {
228
+ continue outer;
229
+ }
230
+ }
231
+ return true;
232
+ }
233
+ return false;
234
+ }
235
+
236
+ private static String describeExifAttribute(@Nullable String value) {
237
+ if (value == null) {
238
+ return "null";
239
+ }
240
+ if (value.isEmpty()) {
241
+ return "empty";
242
+ }
243
+ String preview = value.length() > 80 ? value.substring(0, 80) + "…" : value;
244
+ return "len=" + value.length() + " preview=\"" + preview + "\"";
245
+ }
246
+
247
+ @VisibleForTesting
248
+ static void writeImuJsonToJpeg(String jpegPath, String imuJson) throws IOException {
249
+ ExifInterface exif = new ExifInterface(jpegPath);
250
+ exif.setAttribute(ExifInterface.TAG_USER_COMMENT, imuJson);
251
+ exif.saveAttributes();
252
+ }
253
+
254
+ private static String guessExtension(byte[] imageData) {
255
+ if (imageData.length > 12
256
+ && imageData[4] == 'f'
257
+ && imageData[5] == 't'
258
+ && imageData[6] == 'y'
259
+ && imageData[7] == 'p') {
260
+ return ".avif";
261
+ }
262
+ return ".jpg";
263
+ }
264
+
76
265
  /**
77
266
  * Decode image data (AVIF or JPEG) to Bitmap
78
267
  * @param imageData Raw image bytes
@@ -80,53 +269,125 @@ public class BlePhotoUploadService {
80
269
  */
81
270
  private static Bitmap decodeImage(byte[] imageData) {
82
271
  try {
83
- // Check if this is AVIF by looking for "ftyp" box
84
- boolean isAvif = imageData.length > 12 &&
85
- imageData[4] == 'f' && imageData[5] == 't' &&
86
- imageData[6] == 'y' && imageData[7] == 'p' &&
87
- (imageData[8] == 'a' && imageData[9] == 'v' && imageData[10] == 'i' && imageData[11] == 'f');
272
+ boolean isAvif = imageData.length > 12
273
+ && imageData[4] == 'f' && imageData[5] == 't'
274
+ && imageData[6] == 'y' && imageData[7] == 'p'
275
+ && imageData[8] == 'a' && imageData[9] == 'v'
276
+ && imageData[10] == 'i' && imageData[11] == 'f';
88
277
 
89
278
  if (isAvif) {
90
279
  Log.d(TAG, "Detected AVIF image format");
91
- // AVIF decoding - requires Android API 31+ for native support
92
- if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
93
- // Android 12+ has native AVIF support
94
- BitmapFactory.Options options = new BitmapFactory.Options();
95
- options.inPreferredConfig = Bitmap.Config.ARGB_8888;
96
- return BitmapFactory.decodeByteArray(imageData, 0, imageData.length, options);
97
- } else {
98
- // For older Android versions, we could add a library or convert on glasses side
99
- Log.e(TAG, "AVIF decoding requires Android 12+ (API 31+). Current API: " + Build.VERSION.SDK_INT);
100
- throw new UnsupportedOperationException("AVIF not supported on Android " + Build.VERSION.SDK_INT);
280
+ byte[] strippedBytes = imageData;
281
+ if (containsExifMarkerInBytes(imageData)) {
282
+ try {
283
+ strippedBytes = AvifExifStripper.stripForDecode(imageData);
284
+ Log.d(
285
+ TAG,
286
+ "Stripped Exif metadata item for decode: "
287
+ + imageData.length
288
+ + " -> "
289
+ + strippedBytes.length
290
+ + " bytes");
291
+ } catch (Exception e) {
292
+ Log.w(TAG, "stripForDecode failed, using raw AVIF: " + e.getMessage());
293
+ strippedBytes = imageData;
294
+ }
101
295
  }
102
- } else {
103
- Log.d(TAG, "Detected JPEG image format");
104
- // Standard JPEG decoding
105
- return BitmapFactory.decodeByteArray(imageData, 0, imageData.length);
296
+ Bitmap bmp = decodeAvifBytes(strippedBytes);
297
+ if (bmp == null && strippedBytes != imageData) {
298
+ Log.w(TAG, "Stripped AVIF decode failed; retrying original BLE bytes");
299
+ bmp = decodeAvifBytes(imageData);
300
+ }
301
+ return bmp;
106
302
  }
303
+ Log.d(TAG, "Detected JPEG image format");
304
+ return BitmapFactory.decodeByteArray(imageData, 0, imageData.length);
107
305
  } catch (Exception e) {
108
306
  Log.e(TAG, "Failed to decode image", e);
109
307
  return null;
110
308
  }
111
309
  }
112
310
 
311
+ @Nullable
312
+ private static Bitmap decodeAvifBytes(byte[] avifBytes) {
313
+ try {
314
+ Bitmap bmp = new HeifCoder().decode(avifBytes, PreferredColorConfig.RGBA_8888);
315
+ if (bmp != null) {
316
+ return bmp;
317
+ }
318
+ } catch (Exception | LinkageError e) {
319
+ Log.w(TAG, "HeifCoder AVIF decode unavailable/failed, trying BitmapFactory: " + e.getMessage());
320
+ }
321
+ if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
322
+ return BitmapFactory.decodeByteArray(avifBytes, 0, avifBytes.length);
323
+ }
324
+ Log.e(TAG, "AVIF decoding requires Android 12+ (API 31+). Current API: " + Build.VERSION.SDK_INT);
325
+ return null;
326
+ }
327
+
113
328
  /**
114
329
  * Upload JPEG data to webhook
115
- * @param jpegData JPEG image bytes
116
- * @param requestId Request ID for tracking
117
- * @param webhookUrl Destination URL
118
- * @param authToken Optional bearer token for auth
119
- * @throws IOException If upload fails
120
330
  */
121
- private static void uploadToWebhook(byte[] jpegData, String requestId,
331
+ private static void logImuData(String imuJson) {
332
+ try {
333
+ JSONObject root = new JSONObject(imuJson);
334
+ int sampleCount = root.optInt("sampleCount", 0);
335
+ double samplingRateHz = root.optDouble("samplingRateHz", 0);
336
+ long durationMs = root.optLong("durationMs", 0);
337
+ long startTimeNs = root.optLong("startTimeNs", 0);
338
+ JSONArray samples = root.optJSONArray("samples");
339
+
340
+ StringBuilder sb = new StringBuilder();
341
+ sb.append("IMU data received: sampleCount=").append(sampleCount)
342
+ .append(" rate=").append(samplingRateHz).append("Hz")
343
+ .append(" duration=").append(durationMs).append("ms")
344
+ .append(" startNs=").append(startTimeNs);
345
+ Log.d(TAG, sb.toString());
346
+
347
+ if (samples != null && samples.length() > 0) {
348
+ // Log first and last sample: [timestampMs, ax, ay, az, gx, gy, gz]
349
+ logSample("first", samples.optJSONArray(0));
350
+ if (samples.length() > 1) {
351
+ logSample("last", samples.optJSONArray(samples.length() - 1));
352
+ }
353
+ }
354
+ } catch (Exception e) {
355
+ Log.w(TAG, "logImuData: failed to parse IMU JSON: " + e.getMessage());
356
+ }
357
+ }
358
+
359
+ private static void logSample(String label, @Nullable JSONArray sample) {
360
+ if (sample == null || sample.length() < 7) return;
361
+ try {
362
+ Log.d(
363
+ TAG,
364
+ "IMU " + label + " sample:"
365
+ + " t=" + sample.optLong(0) + "ms"
366
+ + " accel=[" + String.format("%.3f", sample.optDouble(1))
367
+ + ", " + String.format("%.3f", sample.optDouble(2))
368
+ + ", " + String.format("%.3f", sample.optDouble(3)) + "]m/s²"
369
+ + " gyro=[" + String.format("%.3f", sample.optDouble(4))
370
+ + ", " + String.format("%.3f", sample.optDouble(5))
371
+ + ", " + String.format("%.3f", sample.optDouble(6)) + "]rad/s");
372
+ } catch (Exception e) {
373
+ Log.w(TAG, "logSample: " + e.getMessage());
374
+ }
375
+ }
376
+
377
+ private static String uploadToWebhook(byte[] jpegData, String requestId,
122
378
  String webhookUrl, @Nullable String authToken) throws IOException {
379
+ return uploadToWebhook(jpegData, -1, requestId, webhookUrl, authToken);
380
+ }
381
+
382
+ private static String uploadToWebhook(byte[] jpegData, int sourceImageBytes, String requestId,
383
+ String webhookUrl, @Nullable String authToken) throws IOException {
384
+ String effectiveWebhookUrl = resolvePhoneRelayWebhookUrl(webhookUrl);
123
385
  OkHttpClient client = new OkHttpClient.Builder()
124
386
  .connectTimeout(30, TimeUnit.SECONDS)
125
387
  .writeTimeout(30, TimeUnit.SECONDS)
126
388
  .readTimeout(30, TimeUnit.SECONDS)
127
389
  .build();
128
390
 
129
- // Build multipart request
130
391
  RequestBody requestBody = new MultipartBody.Builder()
131
392
  .setType(MultipartBody.FORM)
132
393
  .addFormDataPart("requestId", requestId)
@@ -135,9 +396,8 @@ public class BlePhotoUploadService {
135
396
  RequestBody.create(MediaType.parse("image/jpeg"), jpegData))
136
397
  .build();
137
398
 
138
- // Build request with auth header
139
399
  Request.Builder requestBuilder = new Request.Builder()
140
- .url(webhookUrl)
400
+ .url(effectiveWebhookUrl)
141
401
  .post(requestBody);
142
402
 
143
403
  if (authToken != null && !authToken.isEmpty()) {
@@ -146,15 +406,193 @@ public class BlePhotoUploadService {
146
406
 
147
407
  Request request = requestBuilder.build();
148
408
 
149
- Log.d(TAG, "Uploading photo to webhook: " + webhookUrl);
409
+ if (!effectiveWebhookUrl.equals(webhookUrl)) {
410
+ Log.d(TAG, "Uploading BLE fallback photo to local receiver via loopback: " + effectiveWebhookUrl);
411
+ } else {
412
+ Log.d(TAG, "Uploading photo to webhook: " + webhookUrl);
413
+ }
414
+ long startMs = System.currentTimeMillis();
415
+ traceRelayUploadStart(requestId, effectiveWebhookUrl, webhookUrl, authToken, sourceImageBytes, jpegData.length, startMs);
150
416
 
417
+ boolean responseTraced = false;
151
418
  try (Response response = client.newCall(request).execute()) {
152
419
  if (!response.isSuccessful()) {
153
420
  String errorBody = response.body() != null ? response.body().string() : "No response body";
421
+ traceRelayUploadEnd(
422
+ requestId,
423
+ effectiveWebhookUrl,
424
+ webhookUrl,
425
+ sourceImageBytes,
426
+ jpegData.length,
427
+ startMs,
428
+ response.code(),
429
+ false,
430
+ "http_error");
431
+ responseTraced = true;
154
432
  throw new IOException("Upload failed with code " + response.code() + ": " + errorBody);
155
433
  }
156
434
 
435
+ String responseBody = response.body() != null ? response.body().string() : "";
436
+ traceRelayUploadEnd(
437
+ requestId,
438
+ effectiveWebhookUrl,
439
+ webhookUrl,
440
+ sourceImageBytes,
441
+ jpegData.length,
442
+ startMs,
443
+ response.code(),
444
+ true,
445
+ "uploaded");
446
+ responseTraced = true;
157
447
  Log.d(TAG, "Upload successful. Response code: " + response.code());
448
+ return responseBody;
449
+ } catch (IOException e) {
450
+ if (!responseTraced) {
451
+ traceRelayUploadError(
452
+ requestId,
453
+ effectiveWebhookUrl,
454
+ webhookUrl,
455
+ sourceImageBytes,
456
+ jpegData.length,
457
+ startMs,
458
+ e,
459
+ "failed");
460
+ }
461
+ throw e;
462
+ }
463
+ }
464
+
465
+ private static String resolvePhoneRelayWebhookUrl(String webhookUrl) {
466
+ String loopbackUrl = LocalPhotoReceiverRegistry.loopbackUploadUrlFor(webhookUrl);
467
+ return loopbackUrl != null ? loopbackUrl : webhookUrl;
468
+ }
469
+
470
+ private static void traceRelayUploadStart(String requestId, String webhookUrl,
471
+ String originalWebhookUrl,
472
+ @Nullable String authToken, int sourceImageBytes,
473
+ int jpegBytes, long startMs) {
474
+ JSONObject payload = createRelayUploadPayload(
475
+ "photo_relay_upload_start",
476
+ requestId,
477
+ webhookUrl,
478
+ sourceImageBytes,
479
+ jpegBytes,
480
+ startMs);
481
+ putRelayRewrite(payload, webhookUrl, originalWebhookUrl);
482
+ putJson(payload, "bearerHeaderPresent", authToken != null && !authToken.isEmpty());
483
+ safeTraceJson("phone_to_wifi", "wifi_http_output", payload);
484
+ }
485
+
486
+ private static void traceRelayUploadEnd(String requestId, String webhookUrl,
487
+ String originalWebhookUrl,
488
+ int sourceImageBytes, int jpegBytes, long startMs,
489
+ int statusCode, boolean success, String outcome) {
490
+ JSONObject payload = createRelayUploadPayload(
491
+ "photo_relay_upload_end",
492
+ requestId,
493
+ webhookUrl,
494
+ sourceImageBytes,
495
+ jpegBytes,
496
+ startMs);
497
+ putRelayRewrite(payload, webhookUrl, originalWebhookUrl);
498
+ long endMs = System.currentTimeMillis();
499
+ putJson(payload, "endMs", endMs);
500
+ putJson(payload, "durationMs", endMs - startMs);
501
+ putJson(payload, "statusCode", statusCode);
502
+ putJson(payload, "success", success);
503
+ putJson(payload, "outcome", outcome);
504
+ safeTraceJson("wifi_to_phone", "wifi_http_input", payload);
505
+ }
506
+
507
+ private static void traceRelayUploadError(String requestId, String webhookUrl,
508
+ String originalWebhookUrl,
509
+ int sourceImageBytes, int jpegBytes, long startMs,
510
+ Exception error, String outcome) {
511
+ JSONObject payload = createRelayUploadPayload(
512
+ "photo_relay_upload_error",
513
+ requestId,
514
+ webhookUrl,
515
+ sourceImageBytes,
516
+ jpegBytes,
517
+ startMs);
518
+ putRelayRewrite(payload, webhookUrl, originalWebhookUrl);
519
+ long endMs = System.currentTimeMillis();
520
+ putJson(payload, "endMs", endMs);
521
+ putJson(payload, "durationMs", endMs - startMs);
522
+ putJson(payload, "success", false);
523
+ putJson(payload, "outcome", outcome);
524
+ putJson(payload, "errorClass", error.getClass().getSimpleName());
525
+ putJson(payload, "errorMessage", error.getMessage());
526
+ safeTraceJson("wifi_to_phone", "wifi_http_input", payload);
527
+ }
528
+
529
+ private static JSONObject createRelayUploadPayload(String type, String requestId,
530
+ String webhookUrl, int sourceImageBytes,
531
+ int jpegBytes, long startMs) {
532
+ JSONObject payload = new JSONObject();
533
+ putJson(payload, "type", type);
534
+ putJson(payload, "requestId", requestId);
535
+ putJson(payload, "source", "ble_fallback");
536
+ if (sourceImageBytes >= 0) {
537
+ putJson(payload, "imageBytes", sourceImageBytes);
538
+ }
539
+ putJson(payload, "jpegBytes", jpegBytes);
540
+ putJson(payload, "startMs", startMs);
541
+ putWebhookSummary(payload, webhookUrl);
542
+ return payload;
543
+ }
544
+
545
+ private static void putRelayRewrite(JSONObject payload, String webhookUrl, String originalWebhookUrl) {
546
+ if (webhookUrl == null || originalWebhookUrl == null || webhookUrl.equals(originalWebhookUrl)) {
547
+ return;
548
+ }
549
+
550
+ putJson(payload, "relayRewrite", "loopback");
551
+ try {
552
+ URI original = URI.create(originalWebhookUrl);
553
+ putJson(payload, "originalUrlHost", original.getHost());
554
+ if (original.getPort() != -1) {
555
+ putJson(payload, "originalUrlPort", original.getPort());
556
+ }
557
+ } catch (Exception e) {
558
+ putJson(payload, "originalUrlParseError", e.getClass().getSimpleName());
559
+ }
560
+ }
561
+
562
+ private static void putWebhookSummary(JSONObject payload, String webhookUrl) {
563
+ if (webhookUrl == null || webhookUrl.isEmpty()) {
564
+ return;
565
+ }
566
+
567
+ try {
568
+ URI uri = URI.create(webhookUrl);
569
+ putJson(payload, "urlScheme", uri.getScheme());
570
+ putJson(payload, "urlHost", uri.getHost());
571
+ if (uri.getPort() != -1) {
572
+ putJson(payload, "urlPort", uri.getPort());
573
+ }
574
+ String path = uri.getRawPath();
575
+ putJson(payload, "urlHasPath", path != null && !path.isEmpty() && !"/".equals(path));
576
+ putJson(payload, "urlHasQuery", uri.getRawQuery() != null && !uri.getRawQuery().isEmpty());
577
+ } catch (Exception e) {
578
+ putJson(payload, "urlParseError", e.getClass().getSimpleName());
579
+ }
580
+ }
581
+
582
+ private static void putJson(JSONObject payload, String key, Object value) {
583
+ if (payload == null || key == null || value == null) {
584
+ return;
585
+ }
586
+ try {
587
+ payload.put(key, value);
588
+ } catch (Exception ignored) {
589
+ }
590
+ }
591
+
592
+ private static void safeTraceJson(String direction, String layer, JSONObject payload) {
593
+ try {
594
+ BleTraceLogger.logJson(direction, layer, payload, null);
595
+ } catch (Throwable ignored) {
158
596
  }
159
597
  }
160
598
 
@@ -168,8 +606,8 @@ public class BlePhotoUploadService {
168
606
  new Thread(() -> {
169
607
  try {
170
608
  Log.d(TAG, "Uploading pre-decoded JPEG. Size: " + jpegData.length + " bytes");
171
- uploadToWebhook(jpegData, requestId, webhookUrl, authToken);
172
- callback.onSuccess(requestId);
609
+ String responseBody = uploadToWebhook(jpegData, requestId, webhookUrl, authToken);
610
+ callback.onSuccess(requestId, responseBody);
173
611
  } catch (Exception e) {
174
612
  Log.e(TAG, "Error uploading JPEG photo", e);
175
613
  callback.onError(requestId, e.getMessage());