@mentra/bluetooth-sdk 0.1.9 → 0.1.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (147) hide show
  1. package/README.md +116 -18
  2. package/android/build.gradle +289 -16
  3. package/android/gradle.properties.example +9 -0
  4. package/android/lc3Lib/build.gradle +95 -6
  5. package/android/settings.gradle +3 -0
  6. package/android/silero/LICENSE.md +17 -0
  7. package/android/silero/build.gradle +42 -0
  8. package/android/silero/proguard-rules.pro +24 -0
  9. package/android/silero/src/androidTest/assets/hello.wav +0 -0
  10. package/android/silero/src/androidTest/java/com/konovalov/vad/silero/VadSileroTest.kt +79 -0
  11. package/android/silero/src/main/assets/LICENSE +21 -0
  12. package/android/silero/src/main/assets/silero_vad.onnx +0 -0
  13. package/android/silero/src/main/java/com/konovalov/vad/silero/Vad.kt +160 -0
  14. package/android/silero/src/main/java/com/konovalov/vad/silero/VadSilero.kt +449 -0
  15. package/android/silero/src/main/java/com/konovalov/vad/silero/config/FrameSize.kt +15 -0
  16. package/android/silero/src/main/java/com/konovalov/vad/silero/config/Mode.kt +14 -0
  17. package/android/silero/src/main/java/com/konovalov/vad/silero/config/SampleRate.kt +12 -0
  18. package/android/silero/src/main/java/com/konovalov/vad/silero/utils/AudioUtils.kt +47 -0
  19. package/android/silero/src/main/java/com/konovalov/vad/silero/utils/TensorMap.kt +41 -0
  20. package/android/silero/src/test/java/com/konovalov/vad/silero/utils/AudioUtilsTest.kt +44 -0
  21. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkDefaults.kt +6 -0
  22. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +165 -27
  23. package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +89 -21
  24. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +189 -16
  25. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt +22 -2
  26. package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +787 -85
  27. package/android/src/main/java/com/mentra/bluetoothsdk/audio/AudioModels.kt +6 -2
  28. package/android/src/main/java/com/mentra/bluetoothsdk/camera/CameraModels.kt +123 -10
  29. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/ControllerManager.kt +12 -2
  30. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/R1.kt +8 -2
  31. package/android/src/main/java/com/mentra/bluetoothsdk/debug/BleTraceLogger.kt +33 -17
  32. package/android/src/main/java/com/mentra/bluetoothsdk/events/BluetoothEvents.kt +101 -0
  33. package/android/src/main/java/com/mentra/bluetoothsdk/internal/MapParsing.kt +8 -0
  34. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistry.java +108 -0
  35. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoUploadServer.kt +399 -74
  36. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/MentraPhotoReceiverModule.kt +100 -17
  37. package/android/src/main/java/com/mentra/bluetoothsdk/requests/DisplayRequests.kt +17 -0
  38. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G1.java +2 -2
  39. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt +569 -220
  40. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Mach1.java +2 -2
  41. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraLive.java +271 -66
  42. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraNex.kt +176 -104
  43. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraosBle.java +12753 -27036
  44. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt +54 -2
  45. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Simulated.kt +16 -4
  46. package/android/src/main/java/com/mentra/bluetoothsdk/status/DeviceStatus.kt +60 -3
  47. package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +12 -3
  48. package/android/src/main/java/com/mentra/bluetoothsdk/stt/STTTools.kt +87 -36
  49. package/android/src/main/java/com/mentra/bluetoothsdk/stt/SherpaOnnxTranscriber.kt +13 -4
  50. package/android/src/main/java/com/mentra/bluetoothsdk/stt/VadGateSpeechPolicy.kt +229 -0
  51. package/android/src/main/java/com/mentra/bluetoothsdk/tts/TTSTools.kt +180 -0
  52. package/android/src/main/java/com/mentra/bluetoothsdk/utils/AvifExifStripper.java +573 -0
  53. package/android/src/main/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadService.java +488 -50
  54. package/android/src/main/java/com/mentra/bluetoothsdk/utils/HeifExifTagReader.java +199 -0
  55. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunkReassembler.java +18 -1
  56. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunker.java +64 -18
  57. package/android/src/main/java/com/mentra/bluetoothsdk/utils/NexSGCUtils.kt +48 -39
  58. package/android/src/test/java/com/mentra/bluetoothsdk/camera/PhotoRequestTest.kt +23 -0
  59. package/android/src/test/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistryTest.java +75 -0
  60. package/android/src/test/java/com/mentra/bluetoothsdk/utils/AvifExifStripperTest.java +30 -0
  61. package/android/src/test/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadServiceTest.java +98 -0
  62. package/android/src/test/resources/avif_with_exif.avif +0 -0
  63. package/build/BluetoothSdk.types.d.ts +241 -43
  64. package/build/BluetoothSdk.types.d.ts.map +1 -1
  65. package/build/BluetoothSdk.types.js +11 -10
  66. package/build/BluetoothSdk.types.js.map +1 -1
  67. package/build/_private/BluetoothSdkModule.d.ts +36 -23
  68. package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
  69. package/build/_private/BluetoothSdkModule.js +30 -52
  70. package/build/_private/BluetoothSdkModule.js.map +1 -1
  71. package/build/_private/photoRequestPayload.d.ts +4 -0
  72. package/build/_private/photoRequestPayload.d.ts.map +1 -0
  73. package/build/_private/photoRequestPayload.js +25 -0
  74. package/build/_private/photoRequestPayload.js.map +1 -0
  75. package/build/index.d.ts +1 -1
  76. package/build/index.d.ts.map +1 -1
  77. package/build/index.js +23 -0
  78. package/build/index.js.map +1 -1
  79. package/build/react/useMentraBluetooth.d.ts +2 -2
  80. package/build/react/useMentraBluetooth.d.ts.map +1 -1
  81. package/build/react/useMentraBluetooth.js +3 -3
  82. package/build/react/useMentraBluetooth.js.map +1 -1
  83. package/ios/BluetoothSdkModule.swift +174 -70
  84. package/ios/LocalPhotoUploadServer.swift +370 -105
  85. package/ios/MentraBluetoothSDK.podspec +3 -1
  86. package/ios/MentraPhotoReceiverModule.swift +62 -10
  87. package/ios/Packages/CoreObjC/include/PcmConverter.h +22 -0
  88. package/ios/Packages/SherpaOnnx/SherpaOnnx.swift +668 -26
  89. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Info.plist +13 -9
  90. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
  91. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
  92. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/libsherpa-onnx.a +0 -0
  93. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
  94. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
  95. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/libsherpa-onnx.a +0 -0
  96. package/ios/Source/BluetoothSdkDefaults.swift +6 -0
  97. package/ios/Source/Bridge.swift +58 -19
  98. package/ios/Source/DeviceManager.swift +149 -17
  99. package/ios/Source/DeviceStore.swift +16 -6
  100. package/ios/Source/MentraBluetoothSDK.swift +803 -47
  101. package/ios/Source/ObservableStore.swift +6 -0
  102. package/ios/Source/{Audio → audio}/AudioModels.swift +9 -7
  103. package/ios/Source/{Camera → camera}/CameraModels.swift +225 -16
  104. package/ios/Source/controllers/ControllerManager.swift +4 -3
  105. package/ios/Source/controllers/R1.swift +3 -3
  106. package/ios/Source/{Events → events}/BluetoothEvents.swift +183 -1
  107. package/ios/Source/{Internal → internal}/ValueParsing.swift +10 -1
  108. package/ios/Source/{Requests → requests}/DisplayRequests.swift +28 -4
  109. package/ios/Source/services/PhoneMic.swift +1 -0
  110. package/ios/Source/sgcs/G1.swift +2 -2
  111. package/ios/Source/sgcs/G2.swift +645 -403
  112. package/ios/Source/sgcs/Mach1.swift +2 -2
  113. package/ios/Source/sgcs/MentraLive.swift +812 -319
  114. package/ios/Source/sgcs/MentraNex.swift +412 -182
  115. package/ios/Source/sgcs/SGCManager.swift +57 -3
  116. package/ios/Source/sgcs/Simulated.swift +7 -3
  117. package/ios/Source/sgcs/mentraos_ble.pb.swift +3073 -3134
  118. package/ios/Source/status/DeviceStatus.swift +974 -0
  119. package/ios/Source/{Status → status}/WifiHotspotStatus.swift +4 -4
  120. package/ios/Source/{Streaming → streaming}/StreamModels.swift +24 -9
  121. package/ios/Source/stt/STTTools.swift +27 -12
  122. package/ios/Source/stt/SherpaOnnxTranscriber.swift +31 -8
  123. package/ios/Source/tts/TTSTools.swift +164 -0
  124. package/ios/Source/utils/JSCExperiment.swift +7 -7
  125. package/ios/Source/utils/MessageChunkReassembler.swift +20 -1
  126. package/ios/Source/utils/MessageChunker.swift +78 -21
  127. package/ios/Source/utils/TarBz2Extractor.swift +53 -2
  128. package/package.json +1 -1
  129. package/plugin/build/withAndroid.js +68 -4
  130. package/src/BluetoothSdk.types.ts +331 -59
  131. package/src/_private/BluetoothSdkModule.ts +113 -102
  132. package/src/_private/photoRequestPayload.ts +28 -0
  133. package/src/index.ts +58 -15
  134. package/src/react/useMentraBluetooth.ts +6 -4
  135. package/android/.project +0 -28
  136. package/android/lc3Lib/.project +0 -28
  137. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/cargs.h +0 -162
  138. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/c-api.h +0 -1852
  139. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/cxx-api.h +0 -674
  140. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/sherpa-onnx.a +0 -0
  141. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/sherpa-onnx.a +0 -0
  142. package/ios/Source/Status/DeviceStatus.swift +0 -471
  143. /package/ios/Source/{Connection → connection}/ScanSession.swift +0 -0
  144. /package/ios/Source/{Errors → errors}/BluetoothError.swift +0 -0
  145. /package/ios/Source/{Internal → internal}/BluetoothAvailability.swift +0 -0
  146. /package/ios/Source/{Status → status}/RuntimeState.swift +0 -0
  147. /package/ios/Source/{Types → types}/DeviceModels.swift +0 -0
@@ -0,0 +1,199 @@
1
+ package com.mentra.bluetoothsdk.utils;
2
+
3
+ import androidx.annotation.Nullable;
4
+
5
+ import java.nio.charset.StandardCharsets;
6
+
7
+ /**
8
+ * Reads EXIF UserComment / ImageDescription from a TIFF block embedded in AVIF/HEIF when
9
+ * {@link androidx.exifinterface.media.ExifInterface} cannot (e.g. some OEMs omit MMR EXIF keys).
10
+ */
11
+ final class HeifExifTagReader {
12
+ private static final int TAG_USER_COMMENT = 0x9286;
13
+ private static final int TAG_IMAGE_DESCRIPTION = 0x010E;
14
+ private static final int TAG_EXIF_IFD_POINTER = 0x8769;
15
+
16
+ private HeifExifTagReader() {}
17
+
18
+ @Nullable
19
+ static String readImuJson(byte[] fileBytes) {
20
+ int searchFrom = 0;
21
+ while (true) {
22
+ int exif = indexOf(fileBytes, new byte[] {'E', 'x', 'i', 'f', 0, 0}, searchFrom);
23
+ if (exif < 0) {
24
+ return null;
25
+ }
26
+ int tiff = exif + 6;
27
+ if (tiff + 8 <= fileBytes.length && isTiffHeader(fileBytes, tiff)) {
28
+ return readFromTiff(fileBytes, tiff);
29
+ }
30
+ searchFrom = exif + 1;
31
+ }
32
+ }
33
+
34
+ private static boolean isTiffHeader(byte[] data, int tiff) {
35
+ boolean littleEndian = data[tiff] == 'I' && data[tiff + 1] == 'I';
36
+ boolean bigEndian = data[tiff] == 'M' && data[tiff + 1] == 'M';
37
+ if (!littleEndian && !bigEndian) {
38
+ return false;
39
+ }
40
+ int magic = readUInt16(data, tiff + 2, littleEndian);
41
+ return magic == 0x002A;
42
+ }
43
+
44
+ @Nullable
45
+ private static String readFromTiff(byte[] fileBytes, int tiff) {
46
+ boolean littleEndian = fileBytes[tiff] == 'I' && fileBytes[tiff + 1] == 'I';
47
+ int ifd0 = readInt(fileBytes, tiff + 4, littleEndian);
48
+ int ifd0Offset = tiff + ifd0;
49
+ if (ifd0 < 0 || ifd0Offset < tiff || (long) ifd0Offset + 2 > fileBytes.length) {
50
+ return null;
51
+ }
52
+ String userComment = readTagString(fileBytes, tiff, ifd0Offset, TAG_USER_COMMENT, littleEndian);
53
+ if (userComment != null && !userComment.isEmpty()) {
54
+ return userComment;
55
+ }
56
+ userComment =
57
+ readTagFromLinkedIfd(fileBytes, tiff, ifd0Offset, TAG_EXIF_IFD_POINTER, TAG_USER_COMMENT, littleEndian);
58
+ if (userComment != null && !userComment.isEmpty()) {
59
+ return userComment;
60
+ }
61
+ String imageDescription =
62
+ readTagString(fileBytes, tiff, ifd0Offset, TAG_IMAGE_DESCRIPTION, littleEndian);
63
+ if (imageDescription != null && !imageDescription.isEmpty()) {
64
+ return imageDescription;
65
+ }
66
+ return readTagFromLinkedIfd(
67
+ fileBytes, tiff, ifd0Offset, TAG_EXIF_IFD_POINTER, TAG_IMAGE_DESCRIPTION, littleEndian);
68
+ }
69
+
70
+ @Nullable
71
+ private static String readTagFromLinkedIfd(
72
+ byte[] data,
73
+ int tiffStart,
74
+ int ifdOffset,
75
+ int pointerTag,
76
+ int targetTag,
77
+ boolean littleEndian) {
78
+ if ((long) ifdOffset + 2 > data.length) {
79
+ return null;
80
+ }
81
+ int count = readUInt16(data, ifdOffset, littleEndian);
82
+ int entryStart = ifdOffset + 2;
83
+ for (int i = 0; i < count; i++) {
84
+ int entry = entryStart + i * 12;
85
+ if ((long) entry + 12 > data.length) {
86
+ return null;
87
+ }
88
+ int tag = readUInt16(data, entry, littleEndian);
89
+ if (tag != pointerTag) {
90
+ continue;
91
+ }
92
+ int subIfdOffset = readInt(data, entry + 8, littleEndian);
93
+ int subIfdAbs = tiffStart + subIfdOffset;
94
+ if (subIfdOffset < 0
95
+ || subIfdAbs < tiffStart
96
+ || (long) subIfdAbs + 2 > data.length) {
97
+ return null;
98
+ }
99
+ return readTagString(data, tiffStart, subIfdAbs, targetTag, littleEndian);
100
+ }
101
+ return null;
102
+ }
103
+
104
+ private static int indexOf(byte[] data, byte[] needle, int fromIndex) {
105
+ outer:
106
+ for (int i = fromIndex; i <= data.length - needle.length; i++) {
107
+ for (int j = 0; j < needle.length; j++) {
108
+ if (data[i + j] != needle[j]) {
109
+ continue outer;
110
+ }
111
+ }
112
+ return i;
113
+ }
114
+ return -1;
115
+ }
116
+
117
+ @Nullable
118
+ private static String readTagString(
119
+ byte[] data, int tiffStart, int ifdOffset, int tagId, boolean littleEndian) {
120
+ if ((long) ifdOffset + 2 > data.length) {
121
+ return null;
122
+ }
123
+ int count = readUInt16(data, ifdOffset, littleEndian);
124
+ int entryStart = ifdOffset + 2;
125
+ for (int i = 0; i < count; i++) {
126
+ int entry = entryStart + i * 12;
127
+ if ((long) entry + 12 > data.length) {
128
+ return null;
129
+ }
130
+ int tag = readUInt16(data, entry, littleEndian);
131
+ if (tag != tagId) {
132
+ continue;
133
+ }
134
+ int type = readUInt16(data, entry + 2, littleEndian);
135
+ int valueCount = readInt(data, entry + 4, littleEndian);
136
+ int valueOffset = readInt(data, entry + 8, littleEndian);
137
+ int valuePos = valueCount > 4 ? tiffStart + valueOffset : entry + 8;
138
+ if (valueCount > 4 && (valueOffset < 0 || valuePos < tiffStart || valuePos > data.length)) {
139
+ continue;
140
+ }
141
+ return decodeExifString(data, valuePos, valueCount, type);
142
+ }
143
+ return null;
144
+ }
145
+
146
+ @Nullable
147
+ private static String decodeExifString(byte[] data, int offset, int count, int type) {
148
+ if (count <= 0 || offset < 0) {
149
+ return null;
150
+ }
151
+ long end = (long) offset + count;
152
+ if (end > data.length) {
153
+ return null;
154
+ }
155
+ if (type == 2) {
156
+ int len = count;
157
+ while (len > 0 && data[offset + len - 1] == 0) {
158
+ len--;
159
+ }
160
+ return new String(data, offset, len, StandardCharsets.UTF_8);
161
+ }
162
+ if (type == 7 || type == 1) {
163
+ if (count >= 8 && data[offset] == 'A' && data[offset + 1] == 'S') {
164
+ int textStart = offset + 8;
165
+ int textLen = count - 8;
166
+ while (textLen > 0 && data[textStart + textLen - 1] == 0) {
167
+ textLen--;
168
+ }
169
+ return new String(data, textStart, textLen, StandardCharsets.UTF_8);
170
+ }
171
+ int len = count;
172
+ while (len > 0 && data[offset + len - 1] == 0) {
173
+ len--;
174
+ }
175
+ return new String(data, offset, len, StandardCharsets.UTF_8);
176
+ }
177
+ return null;
178
+ }
179
+
180
+ private static int readUInt16(byte[] data, int offset, boolean littleEndian) {
181
+ if (littleEndian) {
182
+ return (data[offset] & 0xFF) | ((data[offset + 1] & 0xFF) << 8);
183
+ }
184
+ return ((data[offset] & 0xFF) << 8) | (data[offset + 1] & 0xFF);
185
+ }
186
+
187
+ private static int readInt(byte[] data, int offset, boolean littleEndian) {
188
+ if (littleEndian) {
189
+ return (data[offset] & 0xFF)
190
+ | ((data[offset + 1] & 0xFF) << 8)
191
+ | ((data[offset + 2] & 0xFF) << 16)
192
+ | ((data[offset + 3] & 0xFF) << 24);
193
+ }
194
+ return ((data[offset] & 0xFF) << 24)
195
+ | ((data[offset + 1] & 0xFF) << 16)
196
+ | ((data[offset + 2] & 0xFF) << 8)
197
+ | (data[offset + 3] & 0xFF);
198
+ }
199
+ }
@@ -18,12 +18,29 @@ public class MessageChunkReassembler {
18
18
  public String addChunk(String chunkId, int chunkIndex, int totalChunks, String data) {
19
19
  cleanupTimedOutSessions();
20
20
 
21
+ if (chunkId == null || chunkId.isEmpty() || totalChunks <= 0 || chunkIndex < 0
22
+ || chunkIndex >= totalChunks || data == null) {
23
+ Log.w(TAG, "Dropping invalid chunk metadata: id=" + chunkId + ", index=" + chunkIndex
24
+ + ", total=" + totalChunks);
25
+ return null;
26
+ }
27
+
21
28
  if (activeSessions.size() >= MAX_CONCURRENT_SESSIONS && !activeSessions.containsKey(chunkId)) {
22
29
  Log.w(TAG, "Maximum concurrent chunk sessions reached, dropping oldest");
23
30
  removeOldestSession();
24
31
  }
25
32
 
26
- ChunkSession session = activeSessions.computeIfAbsent(chunkId, ignored -> new ChunkSession(chunkId, totalChunks));
33
+ ChunkSession session = activeSessions.compute(chunkId, (ignored, existingSession) -> {
34
+ if (existingSession == null) {
35
+ return new ChunkSession(chunkId, totalChunks);
36
+ }
37
+ if (existingSession.totalChunks != totalChunks) {
38
+ Log.w(TAG, "totalChunks mismatch for " + chunkId + " (expected "
39
+ + existingSession.totalChunks + ", got " + totalChunks + "), resetting session");
40
+ return new ChunkSession(chunkId, totalChunks);
41
+ }
42
+ return existingSession;
43
+ });
27
44
  boolean added = session.addChunk(chunkIndex, data);
28
45
  if (!added) {
29
46
  Log.w(TAG, "Failed to add chunk " + chunkIndex + " to session " + chunkId);
@@ -5,6 +5,7 @@ import android.util.Log;
5
5
  import org.json.JSONException;
6
6
  import org.json.JSONObject;
7
7
 
8
+ import java.nio.charset.StandardCharsets;
8
9
  import java.util.ArrayList;
9
10
  import java.util.List;
10
11
 
@@ -20,8 +21,8 @@ import java.util.List;
20
21
  * d = chunk data payload
21
22
  *
22
23
  * Each chunk after C-wrapping + K900 framing must fit within the BES2700's
23
- * 253-byte BLE write limit. With compact keys, 80 bytes of raw data produces
24
- * a final packed size of ~245 bytes worst-case (with heavy JSON escaping).
24
+ * 253-byte BLE write limit. Payload size is selected by measuring the final
25
+ * packed chunk so JSON escaping cannot push a chunk over the BLE limit.
25
26
  */
26
27
  public class MessageChunker {
27
28
  private static final String TAG = "MessageChunker";
@@ -30,9 +31,9 @@ public class MessageChunker {
30
31
  // BES2700 limit is 253 bytes; anything over ~200 bytes packed needs chunking.
31
32
  private static final int MESSAGE_SIZE_THRESHOLD = 200;
32
33
 
33
- // Maximum raw bytes per chunk. After double JSON escaping + compact envelope
34
- // + C-wrapper + K900 framing, 80 bytes stays under the 253-byte BLE limit.
35
- private static final int CHUNK_DATA_SIZE = 80;
34
+ private static final int INITIAL_CHUNK_DATA_SIZE = 80;
35
+ private static final int MIN_CHUNK_DATA_SIZE = 4;
36
+ private static final int MAX_PACKED_CHUNK_SIZE = 253;
36
37
 
37
38
  /**
38
39
  * Check if a message needs to be chunked
@@ -44,7 +45,7 @@ public class MessageChunker {
44
45
  return false;
45
46
  }
46
47
 
47
- int messageBytes = message.getBytes().length;
48
+ int messageBytes = message.getBytes(StandardCharsets.UTF_8).length;
48
49
  boolean needsChunking = messageBytes > MESSAGE_SIZE_THRESHOLD;
49
50
 
50
51
  if (needsChunking) {
@@ -62,29 +63,39 @@ public class MessageChunker {
62
63
  * @return List of chunk JSON objects ready to be C-wrapped and sent
63
64
  */
64
65
  public static List<JSONObject> createChunks(String originalJson, long messageId) throws JSONException {
66
+ return createChunks(originalJson, messageId, false);
67
+ }
68
+
69
+ public static List<JSONObject> createChunks(String originalJson, long messageId, boolean wakeup) throws JSONException {
65
70
  if (originalJson == null) {
66
71
  throw new IllegalArgumentException("Cannot chunk null message");
67
72
  }
68
73
 
69
- List<JSONObject> chunks = new ArrayList<>();
70
- byte[] messageBytes = originalJson.getBytes();
74
+ byte[] messageBytes = originalJson.getBytes(StandardCharsets.UTF_8);
71
75
  int totalBytes = messageBytes.length;
72
76
 
73
77
  // Compact chunk session ID: messageId_timestamp (no "chunk_" prefix)
74
78
  String chunkId = messageId + "_" + System.currentTimeMillis();
75
79
 
76
- // Calculate total chunks needed
77
- int totalChunks = (int) Math.ceil((double) totalBytes / CHUNK_DATA_SIZE);
80
+ for (int chunkSize = INITIAL_CHUNK_DATA_SIZE; chunkSize >= MIN_CHUNK_DATA_SIZE; chunkSize--) {
81
+ List<JSONObject> chunks = buildChunks(messageBytes, chunkId, messageId, chunkSize);
82
+ if (allChunksFit(chunks, wakeup)) {
83
+ Log.d(TAG, "Creating " + chunks.size() + " chunks for message of size " + totalBytes
84
+ + " bytes using " + chunkSize + "-byte UTF-8 slices");
85
+ return chunks;
86
+ }
87
+ }
78
88
 
79
- Log.d(TAG, "Creating " + totalChunks + " chunks for message of size " + totalBytes + " bytes");
89
+ throw new JSONException("Unable to create K900 chunks within " + MAX_PACKED_CHUNK_SIZE + " bytes");
90
+ }
80
91
 
81
- for (int i = 0; i < totalChunks; i++) {
82
- int startIndex = i * CHUNK_DATA_SIZE;
83
- int endIndex = Math.min(startIndex + CHUNK_DATA_SIZE, totalBytes);
84
- int chunkLength = endIndex - startIndex;
92
+ private static List<JSONObject> buildChunks(byte[] messageBytes, String chunkId, long messageId, int chunkSize) throws JSONException {
93
+ List<JSONObject> chunks = new ArrayList<>();
94
+ List<String> chunkDataList = splitUtf8(messageBytes, chunkSize);
95
+ int totalChunks = chunkDataList.size();
85
96
 
86
- // Extract chunk data as string
87
- String chunkData = new String(messageBytes, startIndex, chunkLength);
97
+ for (int i = 0; i < totalChunks; i++) {
98
+ String chunkData = chunkDataList.get(i);
88
99
 
89
100
  // Create chunk JSON with compact keys
90
101
  JSONObject chunk = new JSONObject();
@@ -101,12 +112,47 @@ public class MessageChunker {
101
112
 
102
113
  chunks.add(chunk);
103
114
 
104
- Log.d(TAG, "Created chunk " + i + "/" + (totalChunks - 1) + " with " + chunkLength + " bytes");
115
+ Log.d(TAG, "Created chunk " + i + "/" + (totalChunks - 1) + " with " + chunkData.getBytes(StandardCharsets.UTF_8).length + " bytes");
105
116
  }
106
117
 
107
118
  return chunks;
108
119
  }
109
120
 
121
+ private static boolean allChunksFit(List<JSONObject> chunks, boolean wakeup) {
122
+ for (int i = 0; i < chunks.size(); i++) {
123
+ byte[] packed = K900ProtocolUtils.packJsonToK900(chunks.get(i).toString(), wakeup && i == 0);
124
+ if (packed == null || packed.length > MAX_PACKED_CHUNK_SIZE) {
125
+ Log.d(TAG, "Chunk " + i + " packed to " + (packed != null ? packed.length : 0)
126
+ + " bytes, exceeding " + MAX_PACKED_CHUNK_SIZE);
127
+ return false;
128
+ }
129
+ }
130
+ return true;
131
+ }
132
+
133
+ private static List<String> splitUtf8(byte[] messageBytes, int chunkSize) {
134
+ List<String> chunkDataList = new ArrayList<>();
135
+ int offset = 0;
136
+ while (offset < messageBytes.length) {
137
+ int endIndex = findUtf8ChunkEnd(messageBytes, offset, chunkSize);
138
+ chunkDataList.add(new String(messageBytes, offset, endIndex - offset, StandardCharsets.UTF_8));
139
+ offset = endIndex;
140
+ }
141
+ return chunkDataList;
142
+ }
143
+
144
+ private static int findUtf8ChunkEnd(byte[] messageBytes, int startIndex, int chunkSize) {
145
+ int endIndex = Math.min(startIndex + chunkSize, messageBytes.length);
146
+ while (endIndex > startIndex && endIndex < messageBytes.length && isUtf8ContinuationByte(messageBytes[endIndex])) {
147
+ endIndex--;
148
+ }
149
+ return endIndex > startIndex ? endIndex : Math.min(startIndex + chunkSize, messageBytes.length);
150
+ }
151
+
152
+ private static boolean isUtf8ContinuationByte(byte value) {
153
+ return (value & 0xC0) == 0x80;
154
+ }
155
+
110
156
  /**
111
157
  * Check if a received message is a chunked message.
112
158
  * Supports both verbose ("type":"chunked_msg") and compact ("t":"ck") formats.
@@ -8,12 +8,13 @@ import android.content.Context
8
8
  import android.util.Log
9
9
 
10
10
  import com.mentra.bluetoothsdk.Bridge
11
+ import com.mentra.bluetoothsdk.DeviceStore
11
12
 
12
13
  import mentraos.ble.MentraosBle.DisplayText
13
14
  import mentraos.ble.MentraosBle.ClearDisplay
15
+ import mentraos.ble.MentraosBle.DisconnectRequest
14
16
  import mentraos.ble.MentraosBle.PhoneToGlasses
15
17
  import mentraos.ble.MentraosBle.DisplayImage
16
- import mentraos.ble.MentraosBle.PongResponse
17
18
  import mentraos.ble.MentraosBle.BatteryStateRequest
18
19
  import mentraos.ble.MentraosBle.MicStateConfig
19
20
  import mentraos.ble.MentraosBle.BrightnessConfig
@@ -21,7 +22,7 @@ import mentraos.ble.MentraosBle.AutoBrightnessConfig
21
22
  import mentraos.ble.MentraosBle.HeadUpAngleConfig
22
23
  import mentraos.ble.MentraosBle.DisplayDistanceConfig
23
24
  import mentraos.ble.MentraosBle.DisplayHeightConfig
24
- import mentraos.ble.MentraosBle.VersionRequest
25
+ import mentraos.ble.MentraosBle.VadEnabledConfig
25
26
 
26
27
  import org.json.JSONArray
27
28
  import org.json.JSONException
@@ -43,9 +44,9 @@ object NexDisplayConstants {
43
44
  const val FONT_DIVIDER: Float = 2.0f
44
45
  const val LINES_PER_SCREEN: Int = 5 // Lines per screen
45
46
 
46
- /** Matches dashboard depth slider in app settings (1-3); values outside range clamp. */
47
+ /** Matches dashboard depth slider in app settings (1-4); values outside range clamp. */
47
48
  const val DASHBOARD_DEPTH_MIN: Int = 1
48
- const val DASHBOARD_DEPTH_MAX: Int = 3
49
+ const val DASHBOARD_DEPTH_MAX: Int = 4
49
50
  }
50
51
 
51
52
  object NexBluetoothPacketTypes {
@@ -62,7 +63,7 @@ object NexProtobufUtils {
62
63
 
63
64
  /**
64
65
  * Maps dashboard depth to the value Nex firmware expects in [DisplayDistanceConfig.distance_cm].
65
- * The protobuf field is still named `distance_cm`, but Nex treats it as a **tier** 1–3, not centimeters.
66
+ * The protobuf field is still named `distance_cm`, but Nex treats it as a **tier** 1–4, not centimeters.
66
67
  * Keep in sync with iOS `NexDashboardDisplayWire.depthToWireTier`.
67
68
  */
68
69
  fun dashboardDepthToDistanceCm(depth: Int): Int {
@@ -112,16 +113,6 @@ object NexProtobufUtils {
112
113
  return chunks
113
114
  }
114
115
 
115
- fun constructPongResponse(): ByteArray {
116
- Bridge.log("Nex: Constructing pong response to glasses ping")
117
-
118
- // Create the PongResponse message
119
- val pongResponse = PongResponse.newBuilder().build()
120
- // Create the PhoneToGlasses message with the pong response
121
- val phoneToGlasses = PhoneToGlasses.newBuilder().setPong(pongResponse).build()
122
- return generateProtobufCommandBytes(phoneToGlasses)
123
- }
124
-
125
116
  /**
126
117
  * Gets the current protobuf schema version from the compiled protobuf descriptor
127
118
  */
@@ -202,13 +193,15 @@ object NexProtobufUtils {
202
193
  }
203
194
 
204
195
  fun generateVersionRequestCommandBytes(): ByteArray {
205
- val msgId = "ver_req_${System.currentTimeMillis()}"
206
- val versionRequest = VersionRequest.newBuilder()
207
- .setMsgId(msgId)
208
- .build()
196
+ // VersionRequest/VersionResponse removed from the BLE schema; fw_version now comes via DeviceInfo.
197
+ Bridge.log("Nex: generateVersionRequestCommandBytes is a no-op after schema removal")
198
+ return ByteArray(0)
199
+ }
200
+
201
+ fun generateDisconnectRequestCommandBytes(): ByteArray {
202
+ val disconnectRequest = DisconnectRequest.newBuilder().build()
209
203
  val phoneToGlasses = PhoneToGlasses.newBuilder()
210
- .setMsgId(msgId)
211
- .setVersionRequest(versionRequest)
204
+ .setDisconnect(disconnectRequest)
212
205
  .build()
213
206
  return generateProtobufCommandBytes(phoneToGlasses)
214
207
  }
@@ -253,17 +246,21 @@ object NexProtobufUtils {
253
246
  Bridge.log("Nex: Text: \"$text\"")
254
247
  Bridge.log("Nex: Text Length: ${text.length} characters")
255
248
 
256
- // Replace all m-dashes with normal dash
257
- val textWithNormalDash = text.replace("—", "-")
258
-
259
- val sanitizedText = textWithNormalDash.replace(
260
- Regex("""[^A-Za-z0-9 \r\n.,!?;:\-\[\]\(\)\{\}'"+=/]"""),
261
- ""
262
- )
249
+ // When Chinese captions are disabled (default), strip characters the Nex font
250
+ // can't render (CJK etc.): replace m-dashes, then keep only the supported ASCII set.
251
+ val chineseCaptionsEnabled = DeviceStore.get("bluetooth", "nex_chinese_captions") as? Boolean ?: false
252
+ val displayText = if (chineseCaptionsEnabled) {
253
+ text
254
+ } else {
255
+ text.replace("—", "-").replace(
256
+ Regex("""[^A-Za-z0-9 \r\n.,!?;:\-\[\]\(\)\{\}'"+=/]"""),
257
+ ""
258
+ )
259
+ }
263
260
 
264
261
  val textNewBuilder = DisplayText.newBuilder()
265
262
  .setColor(10000)
266
- .setText(sanitizedText)
263
+ .setText(displayText)
267
264
  .setSize(48)
268
265
  .setX(20)
269
266
  .setY(260)
@@ -361,6 +358,18 @@ object NexProtobufUtils {
361
358
  return generateProtobufCommandBytes(phoneToGlasses)
362
359
  }
363
360
 
361
+ fun generateVadEnabledRequestCommandBytes(enable: Boolean): ByteArray {
362
+ Bridge.log("Nex: VAD Enabled: $enable")
363
+ val vadEnabledConfig = VadEnabledConfig.newBuilder()
364
+ .setEnabled(enable)
365
+ .build()
366
+ val phoneToGlasses = PhoneToGlasses.newBuilder()
367
+ .setVadEnabled(vadEnabledConfig)
368
+ .build()
369
+
370
+ return generateProtobufCommandBytes(phoneToGlasses)
371
+ }
372
+
364
373
  fun generateAutoBrightnessConfigCommandBytes(autoLight: Boolean): ByteArray {
365
374
  Bridge.log("Nex: === SENDING AUTO BRIGHTNESS COMMAND TO GLASSES ===")
366
375
  Bridge.log("Nex: Auto Brightness Enabled: $autoLight")
@@ -425,18 +434,18 @@ object NexProtobufUtils {
425
434
  }
426
435
  }
427
436
 
437
+ /**
438
+ * Returns the raw serialized PhoneToGlasses protobuf bytes.
439
+ *
440
+ * Transport framing (the 0x02 packet type and the [seq][totalChunks][chunkIndex]
441
+ * fragmentation header) is applied at send time by MentraNexSGC.sendProtobuf(),
442
+ * which owns the negotiated MTU / chunk size. Keeping framing out of here lets a
443
+ * single chunker handle every control command uniformly.
444
+ */
428
445
  private fun generateProtobufCommandBytes(phoneToGlasses: PhoneToGlasses): ByteArray {
429
446
  val contentBytes = phoneToGlasses.toByteArray()
430
- val chunk = ByteBuffer.allocate(contentBytes.size + 1)
431
-
432
- chunk.put(NexBluetoothPacketTypes.PACKET_TYPE_PROTOBUF)
433
- chunk.put(contentBytes)
434
-
435
- // Enhanced logging for protobuf messages
436
- val result = chunk.array()
437
- logProtobufMessage(phoneToGlasses, result)
438
-
439
- return result
447
+ logProtobufMessage(phoneToGlasses, contentBytes)
448
+ return contentBytes
440
449
  }
441
450
 
442
451
  private fun logProtobufMessage(phoneToGlasses: PhoneToGlasses, fullMessage: ByteArray) {
@@ -0,0 +1,23 @@
1
+ package com.mentra.bluetoothsdk.camera
2
+
3
+ import kotlin.test.Test
4
+ import kotlin.test.assertNull
5
+
6
+ class PhotoRequestTest {
7
+ @Test
8
+ fun `fromMap defaults exposureTimeNs null`() {
9
+ val request =
10
+ PhotoRequest.fromMap(
11
+ mapOf(
12
+ "requestId" to "photo-1",
13
+ "appId" to "com.test.app",
14
+ "size" to "medium",
15
+ "webhookUrl" to "https://example.com/upload",
16
+ "compress" to "none",
17
+ "sound" to true,
18
+ )
19
+ )
20
+
21
+ assertNull(request.exposureTimeNs)
22
+ }
23
+ }
@@ -0,0 +1,75 @@
1
+ package com.mentra.bluetoothsdk.photoreceiver;
2
+
3
+ import static org.assertj.core.api.Assertions.assertThat;
4
+
5
+ import org.junit.After;
6
+ import org.junit.Test;
7
+
8
+ public class LocalPhotoReceiverRegistryTest {
9
+
10
+ @After
11
+ public void tearDown() {
12
+ LocalPhotoReceiverRegistry.unregister();
13
+ }
14
+
15
+ @Test
16
+ public void loopbackUploadUrlFor_rewritesRegisteredReceiverUrl() {
17
+ LocalPhotoReceiverRegistry.register("http://192.168.1.20:8787/upload");
18
+
19
+ assertThat(LocalPhotoReceiverRegistry.loopbackUploadUrlFor("http://192.168.1.20:8787/upload"))
20
+ .isEqualTo("http://127.0.0.1:8787/upload");
21
+ }
22
+
23
+ @Test
24
+ public void loopbackUploadUrlFor_doesNotRewriteDifferentHostOnSamePort() {
25
+ LocalPhotoReceiverRegistry.register("http://192.168.1.20:8787/upload");
26
+
27
+ assertThat(LocalPhotoReceiverRegistry.loopbackUploadUrlFor("https://api.example.com:8787/upload"))
28
+ .isNull();
29
+ assertThat(LocalPhotoReceiverRegistry.loopbackUploadUrlFor("http://devbox:8787/upload"))
30
+ .isNull();
31
+ }
32
+
33
+ @Test
34
+ public void loopbackUploadUrlFor_doesNotRewriteDifferentPath() {
35
+ LocalPhotoReceiverRegistry.register("http://192.168.1.20:8787/upload");
36
+
37
+ assertThat(LocalPhotoReceiverRegistry.loopbackUploadUrlFor("http://192.168.1.20:8787/other"))
38
+ .isNull();
39
+ }
40
+
41
+ @Test
42
+ public void loopbackUploadUrlFor_keepsEarlierReceiverUrlsUntilUnregister() {
43
+ LocalPhotoReceiverRegistry.register("http://192.168.1.20:8787/upload");
44
+ LocalPhotoReceiverRegistry.register("http://10.0.0.12:8787/upload");
45
+
46
+ assertThat(LocalPhotoReceiverRegistry.loopbackUploadUrlFor("http://192.168.1.20:8787/upload"))
47
+ .isEqualTo("http://127.0.0.1:8787/upload");
48
+ assertThat(LocalPhotoReceiverRegistry.loopbackUploadUrlFor("http://10.0.0.12:8787/upload"))
49
+ .isEqualTo("http://127.0.0.1:8787/upload");
50
+
51
+ LocalPhotoReceiverRegistry.unregister();
52
+
53
+ assertThat(LocalPhotoReceiverRegistry.loopbackUploadUrlFor("http://192.168.1.20:8787/upload"))
54
+ .isNull();
55
+ }
56
+
57
+ @Test
58
+ public void register_keepsExistingReceiverUrlsWhenInvalidUrlArrives() {
59
+ LocalPhotoReceiverRegistry.register("http://192.168.1.20:8787/upload");
60
+ LocalPhotoReceiverRegistry.register("not a url");
61
+
62
+ assertThat(LocalPhotoReceiverRegistry.isActive()).isTrue();
63
+ assertThat(LocalPhotoReceiverRegistry.loopbackUploadUrlFor("http://192.168.1.20:8787/upload"))
64
+ .isEqualTo("http://127.0.0.1:8787/upload");
65
+ }
66
+
67
+ @Test
68
+ public void register_rejectsInvalidReceiverUrl() {
69
+ LocalPhotoReceiverRegistry.register("https://api.example.com/photo");
70
+
71
+ assertThat(LocalPhotoReceiverRegistry.isActive()).isFalse();
72
+ assertThat(LocalPhotoReceiverRegistry.loopbackUploadUrlFor("https://api.example.com/photo"))
73
+ .isNull();
74
+ }
75
+ }
@@ -0,0 +1,30 @@
1
+ package com.mentra.bluetoothsdk.utils;
2
+
3
+ import static org.assertj.core.api.Assertions.assertThat;
4
+
5
+ import org.junit.Test;
6
+ import org.junit.runner.RunWith;
7
+ import org.robolectric.RobolectricTestRunner;
8
+ import org.robolectric.annotation.Config;
9
+
10
+ import java.io.File;
11
+
12
+ @RunWith(RobolectricTestRunner.class)
13
+ @Config(sdk = 31)
14
+ public class AvifExifStripperTest {
15
+
16
+ @Test
17
+ public void stripForDecode_removesExifTailAndKeepsPrimaryIlocValid() throws Exception {
18
+ byte[] withExif =
19
+ java.nio.file.Files.readAllBytes(
20
+ new File("src/test/resources/avif_with_exif.avif").toPath());
21
+ assertThat(BlePhotoUploadService.containsExifMarkerInBytes(withExif)).isTrue();
22
+
23
+ byte[] stripped = AvifExifStripper.stripForDecode(withExif);
24
+ assertThat(stripped.length).isLessThan(withExif.length);
25
+ assertThat(BlePhotoUploadService.containsExifMarkerInBytes(stripped)).isFalse();
26
+
27
+ byte[] jpeg = BlePhotoUploadService.convertToJpegPreservingExif(withExif);
28
+ assertThat(jpeg.length).isGreaterThan(100);
29
+ }
30
+ }