@mentra/bluetooth-sdk 0.1.8 → 0.1.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (147) hide show
  1. package/README.md +116 -18
  2. package/android/build.gradle +289 -16
  3. package/android/gradle.properties.example +9 -0
  4. package/android/lc3Lib/build.gradle +95 -6
  5. package/android/settings.gradle +3 -0
  6. package/android/silero/LICENSE.md +17 -0
  7. package/android/silero/build.gradle +42 -0
  8. package/android/silero/proguard-rules.pro +24 -0
  9. package/android/silero/src/androidTest/assets/hello.wav +0 -0
  10. package/android/silero/src/androidTest/java/com/konovalov/vad/silero/VadSileroTest.kt +79 -0
  11. package/android/silero/src/main/assets/LICENSE +21 -0
  12. package/android/silero/src/main/assets/silero_vad.onnx +0 -0
  13. package/android/silero/src/main/java/com/konovalov/vad/silero/Vad.kt +160 -0
  14. package/android/silero/src/main/java/com/konovalov/vad/silero/VadSilero.kt +449 -0
  15. package/android/silero/src/main/java/com/konovalov/vad/silero/config/FrameSize.kt +15 -0
  16. package/android/silero/src/main/java/com/konovalov/vad/silero/config/Mode.kt +14 -0
  17. package/android/silero/src/main/java/com/konovalov/vad/silero/config/SampleRate.kt +12 -0
  18. package/android/silero/src/main/java/com/konovalov/vad/silero/utils/AudioUtils.kt +47 -0
  19. package/android/silero/src/main/java/com/konovalov/vad/silero/utils/TensorMap.kt +41 -0
  20. package/android/silero/src/test/java/com/konovalov/vad/silero/utils/AudioUtilsTest.kt +44 -0
  21. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkDefaults.kt +6 -0
  22. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +165 -27
  23. package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +89 -21
  24. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +189 -16
  25. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt +22 -2
  26. package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +787 -85
  27. package/android/src/main/java/com/mentra/bluetoothsdk/audio/AudioModels.kt +6 -2
  28. package/android/src/main/java/com/mentra/bluetoothsdk/camera/CameraModels.kt +123 -10
  29. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/ControllerManager.kt +12 -2
  30. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/R1.kt +8 -2
  31. package/android/src/main/java/com/mentra/bluetoothsdk/debug/BleTraceLogger.kt +33 -17
  32. package/android/src/main/java/com/mentra/bluetoothsdk/events/BluetoothEvents.kt +101 -0
  33. package/android/src/main/java/com/mentra/bluetoothsdk/internal/MapParsing.kt +8 -0
  34. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistry.java +108 -0
  35. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoUploadServer.kt +399 -74
  36. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/MentraPhotoReceiverModule.kt +100 -17
  37. package/android/src/main/java/com/mentra/bluetoothsdk/requests/DisplayRequests.kt +17 -0
  38. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G1.java +2 -2
  39. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt +569 -220
  40. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Mach1.java +2 -2
  41. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraLive.java +271 -66
  42. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraNex.kt +176 -104
  43. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraosBle.java +12753 -27036
  44. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt +54 -2
  45. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Simulated.kt +16 -4
  46. package/android/src/main/java/com/mentra/bluetoothsdk/status/DeviceStatus.kt +60 -3
  47. package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +12 -3
  48. package/android/src/main/java/com/mentra/bluetoothsdk/stt/STTTools.kt +87 -36
  49. package/android/src/main/java/com/mentra/bluetoothsdk/stt/SherpaOnnxTranscriber.kt +13 -4
  50. package/android/src/main/java/com/mentra/bluetoothsdk/stt/VadGateSpeechPolicy.kt +229 -0
  51. package/android/src/main/java/com/mentra/bluetoothsdk/tts/TTSTools.kt +180 -0
  52. package/android/src/main/java/com/mentra/bluetoothsdk/utils/AvifExifStripper.java +573 -0
  53. package/android/src/main/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadService.java +488 -50
  54. package/android/src/main/java/com/mentra/bluetoothsdk/utils/HeifExifTagReader.java +199 -0
  55. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunkReassembler.java +18 -1
  56. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunker.java +64 -18
  57. package/android/src/main/java/com/mentra/bluetoothsdk/utils/NexSGCUtils.kt +48 -39
  58. package/android/src/test/java/com/mentra/bluetoothsdk/camera/PhotoRequestTest.kt +23 -0
  59. package/android/src/test/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistryTest.java +75 -0
  60. package/android/src/test/java/com/mentra/bluetoothsdk/utils/AvifExifStripperTest.java +30 -0
  61. package/android/src/test/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadServiceTest.java +98 -0
  62. package/android/src/test/resources/avif_with_exif.avif +0 -0
  63. package/build/BluetoothSdk.types.d.ts +241 -43
  64. package/build/BluetoothSdk.types.d.ts.map +1 -1
  65. package/build/BluetoothSdk.types.js +11 -10
  66. package/build/BluetoothSdk.types.js.map +1 -1
  67. package/build/_private/BluetoothSdkModule.d.ts +36 -23
  68. package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
  69. package/build/_private/BluetoothSdkModule.js +30 -52
  70. package/build/_private/BluetoothSdkModule.js.map +1 -1
  71. package/build/_private/photoRequestPayload.d.ts +4 -0
  72. package/build/_private/photoRequestPayload.d.ts.map +1 -0
  73. package/build/_private/photoRequestPayload.js +25 -0
  74. package/build/_private/photoRequestPayload.js.map +1 -0
  75. package/build/index.d.ts +1 -1
  76. package/build/index.d.ts.map +1 -1
  77. package/build/index.js +23 -0
  78. package/build/index.js.map +1 -1
  79. package/build/react/useMentraBluetooth.d.ts +2 -2
  80. package/build/react/useMentraBluetooth.d.ts.map +1 -1
  81. package/build/react/useMentraBluetooth.js +3 -3
  82. package/build/react/useMentraBluetooth.js.map +1 -1
  83. package/ios/BluetoothSdkModule.swift +174 -70
  84. package/ios/LocalPhotoUploadServer.swift +370 -105
  85. package/ios/MentraBluetoothSDK.podspec +3 -1
  86. package/ios/MentraPhotoReceiverModule.swift +62 -10
  87. package/ios/Packages/CoreObjC/include/PcmConverter.h +22 -0
  88. package/ios/Packages/SherpaOnnx/SherpaOnnx.swift +668 -26
  89. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Info.plist +13 -9
  90. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
  91. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
  92. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/libsherpa-onnx.a +0 -0
  93. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
  94. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
  95. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/libsherpa-onnx.a +0 -0
  96. package/ios/Source/BluetoothSdkDefaults.swift +6 -0
  97. package/ios/Source/Bridge.swift +58 -19
  98. package/ios/Source/DeviceManager.swift +149 -17
  99. package/ios/Source/DeviceStore.swift +16 -6
  100. package/ios/Source/MentraBluetoothSDK.swift +803 -47
  101. package/ios/Source/ObservableStore.swift +8 -2
  102. package/ios/Source/{Audio → audio}/AudioModels.swift +9 -7
  103. package/ios/Source/{Camera → camera}/CameraModels.swift +225 -16
  104. package/ios/Source/controllers/ControllerManager.swift +4 -3
  105. package/ios/Source/controllers/R1.swift +3 -3
  106. package/ios/Source/{Events → events}/BluetoothEvents.swift +183 -1
  107. package/ios/Source/{Internal → internal}/ValueParsing.swift +10 -1
  108. package/ios/Source/{Requests → requests}/DisplayRequests.swift +28 -4
  109. package/ios/Source/services/PhoneMic.swift +1 -0
  110. package/ios/Source/sgcs/G1.swift +2 -2
  111. package/ios/Source/sgcs/G2.swift +645 -403
  112. package/ios/Source/sgcs/Mach1.swift +2 -2
  113. package/ios/Source/sgcs/MentraLive.swift +812 -319
  114. package/ios/Source/sgcs/MentraNex.swift +412 -182
  115. package/ios/Source/sgcs/SGCManager.swift +57 -3
  116. package/ios/Source/sgcs/Simulated.swift +7 -3
  117. package/ios/Source/sgcs/mentraos_ble.pb.swift +3073 -3134
  118. package/ios/Source/status/DeviceStatus.swift +974 -0
  119. package/ios/Source/{Status → status}/WifiHotspotStatus.swift +4 -4
  120. package/ios/Source/{Streaming → streaming}/StreamModels.swift +24 -9
  121. package/ios/Source/stt/STTTools.swift +27 -12
  122. package/ios/Source/stt/SherpaOnnxTranscriber.swift +31 -8
  123. package/ios/Source/tts/TTSTools.swift +164 -0
  124. package/ios/Source/utils/JSCExperiment.swift +7 -7
  125. package/ios/Source/utils/MessageChunkReassembler.swift +20 -1
  126. package/ios/Source/utils/MessageChunker.swift +78 -21
  127. package/ios/Source/utils/TarBz2Extractor.swift +53 -2
  128. package/package.json +1 -1
  129. package/plugin/build/withAndroid.js +68 -4
  130. package/src/BluetoothSdk.types.ts +331 -59
  131. package/src/_private/BluetoothSdkModule.ts +113 -102
  132. package/src/_private/photoRequestPayload.ts +28 -0
  133. package/src/index.ts +58 -15
  134. package/src/react/useMentraBluetooth.ts +6 -4
  135. package/android/.project +0 -28
  136. package/android/lc3Lib/.project +0 -28
  137. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/cargs.h +0 -162
  138. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/c-api.h +0 -1852
  139. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/cxx-api.h +0 -674
  140. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/sherpa-onnx.a +0 -0
  141. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/sherpa-onnx.a +0 -0
  142. package/ios/Source/Status/DeviceStatus.swift +0 -471
  143. /package/ios/Source/{Connection → connection}/ScanSession.swift +0 -0
  144. /package/ios/Source/{Errors → errors}/BluetoothError.swift +0 -0
  145. /package/ios/Source/{Internal → internal}/BluetoothAvailability.swift +0 -0
  146. /package/ios/Source/{Status → status}/RuntimeState.swift +0 -0
  147. /package/ios/Source/{Types → types}/DeviceModels.swift +0 -0
@@ -2,6 +2,8 @@ package com.mentra.bluetoothsdk.photoreceiver
2
2
 
3
3
  import android.content.Context
4
4
  import android.net.Uri
5
+ import android.util.Log
6
+ import com.mentra.bluetoothsdk.debug.BleTraceLogger
5
7
  import expo.modules.kotlin.exception.Exceptions
6
8
  import expo.modules.kotlin.modules.Module
7
9
  import expo.modules.kotlin.modules.ModuleDefinition
@@ -29,13 +31,12 @@ class MentraPhotoReceiverModule : Module() {
29
31
  }
30
32
 
31
33
  OnDestroy {
32
- stopPhotoReceiverInternal()
34
+ closePhotoReceiverInternal()
33
35
  }
34
36
  }
35
37
 
38
+ @Synchronized
36
39
  private fun startPhotoReceiver(): Map<String, Any> {
37
- val host = bestLocalIpv4Address()
38
- ?: throw IllegalStateException("No Wi-Fi/LAN IPv4 address found for this phone.")
39
40
  val server = photoUploadServer ?: LocalPhotoUploadServer(
40
41
  context = reactContext(),
41
42
  onLog = { message -> emitStatus(message) },
@@ -44,17 +45,24 @@ class MentraPhotoReceiverModule : Module() {
44
45
  photoUploadServer = it
45
46
  }
46
47
 
48
+ val host = bestLocalIpv4Address()
49
+ ?: throw IllegalStateException("No Wi-Fi/LAN IPv4 address found for this phone.")
50
+
51
+ server.activePort?.let { activePort ->
52
+ val endpoint = receiverEndpoint(host, activePort)
53
+ LocalPhotoReceiverRegistry.register(endpoint.uploadUrl)
54
+ emitStatus("Photo receiver ready at ${endpoint.uploadUrl}")
55
+ return receiverResult(endpoint)
56
+ }
57
+
47
58
  var lastError: Throwable? = null
48
59
  for (port in PHOTO_PORTS) {
49
60
  try {
50
61
  val actualPort = server.start(port)
51
- val uploadUrl = "http://$host:$actualPort/upload"
52
- emitStatus("Photo receiver ready at $uploadUrl")
53
- return mapOf(
54
- "uploadUrl" to uploadUrl,
55
- "host" to host,
56
- "port" to actualPort,
57
- )
62
+ val endpoint = receiverEndpoint(host, actualPort)
63
+ LocalPhotoReceiverRegistry.register(endpoint.uploadUrl)
64
+ emitStatus("Photo receiver ready at ${endpoint.uploadUrl}")
65
+ return receiverResult(endpoint)
58
66
  } catch (error: Throwable) {
59
67
  lastError = error
60
68
  emitStatus("Port $port unavailable: ${error.message ?: error::class.java.simpleName}")
@@ -66,13 +74,35 @@ class MentraPhotoReceiverModule : Module() {
66
74
  )
67
75
  }
68
76
 
77
+ @Synchronized
69
78
  private fun stopPhotoReceiverInternal() {
70
79
  photoUploadServer?.stop()
80
+ LocalPhotoReceiverRegistry.unregister()
71
81
  emitStatus("Photo receiver stopped")
72
82
  }
73
83
 
84
+ @Synchronized
85
+ private fun closePhotoReceiverInternal() {
86
+ photoUploadServer?.close()
87
+ photoUploadServer = null
88
+ LocalPhotoReceiverRegistry.unregister()
89
+ }
90
+
74
91
  private fun handlePhotoUpload(upload: PhotoUpload) {
75
92
  val fileUri = Uri.fromFile(upload.photoFile).toString()
93
+ try {
94
+ BleTraceLogger.logMap(
95
+ "phone_to_app",
96
+ "photo_receiver_event",
97
+ "photo_upload",
98
+ mapOf(
99
+ "requestId" to upload.requestId.orEmpty(),
100
+ "fileName" to upload.photoFile.name,
101
+ "byteCount" to upload.byteCount,
102
+ ),
103
+ )
104
+ } catch (_: Throwable) {
105
+ }
76
106
  sendEvent(
77
107
  "photoUpload",
78
108
  mapOf(
@@ -85,6 +115,7 @@ class MentraPhotoReceiverModule : Module() {
85
115
  }
86
116
 
87
117
  private fun emitStatus(message: String) {
118
+ Log.d(TAG, message)
88
119
  sendEvent(
89
120
  "receiverStatus",
90
121
  mapOf("message" to message),
@@ -98,23 +129,75 @@ class MentraPhotoReceiverModule : Module() {
98
129
  }
99
130
 
100
131
  private fun bestLocalIpv4Address(): String? {
101
- val candidates = mutableListOf<Inet4Address>()
132
+ val wifiPrivateCandidates = mutableListOf<Inet4Address>()
133
+ val wifiCandidates = mutableListOf<Inet4Address>()
134
+ val privateCandidates = mutableListOf<Inet4Address>()
135
+ val otherCandidates = mutableListOf<Inet4Address>()
102
136
  val interfaces = NetworkInterface.getNetworkInterfaces()?.toList().orEmpty()
103
137
  for (networkInterface in interfaces) {
104
138
  if (!networkInterface.isUp || networkInterface.isLoopback) {
105
139
  continue
106
140
  }
107
- val addresses = networkInterface.inetAddresses.toList().filterIsInstance<Inet4Address>()
108
- candidates += addresses.filterNot { it.isLoopbackAddress || it.isLinkLocalAddress }
141
+ val addresses = networkInterface.inetAddresses.toList()
142
+ .filterIsInstance<Inet4Address>()
143
+ .filter { address ->
144
+ !address.isLoopbackAddress &&
145
+ !address.isLinkLocalAddress
146
+ }
147
+ if (isWifiInterface(networkInterface.name)) {
148
+ wifiPrivateCandidates += addresses.filter { isPrivateIpv4(it.hostAddress.orEmpty()) }
149
+ wifiCandidates += addresses
150
+ } else {
151
+ privateCandidates += addresses.filter { isPrivateIpv4(it.hostAddress.orEmpty()) }
152
+ otherCandidates += addresses
153
+ }
109
154
  }
110
155
 
111
- return candidates.firstOrNull { address ->
112
- val host = address.hostAddress.orEmpty()
113
- host.startsWith("192.168.") || host.startsWith("10.") || host.matches(Regex("^172\\.(1[6-9]|2[0-9]|3[0-1])\\..*"))
114
- }?.hostAddress ?: candidates.firstOrNull()?.hostAddress
156
+ return (
157
+ wifiPrivateCandidates.firstOrNull() ?:
158
+ wifiCandidates.firstOrNull() ?:
159
+ privateCandidates.firstOrNull() ?:
160
+ otherCandidates.firstOrNull()
161
+ )?.hostAddress
162
+ }
163
+
164
+ private fun receiverEndpoint(host: String, port: Int): ReceiverEndpoint {
165
+ return ReceiverEndpoint(
166
+ uploadUrl = "http://$host:$port/upload",
167
+ host = host,
168
+ port = port,
169
+ )
170
+ }
171
+
172
+ private fun receiverResult(endpoint: ReceiverEndpoint): Map<String, Any> {
173
+ return mapOf(
174
+ "uploadUrl" to endpoint.uploadUrl,
175
+ "host" to endpoint.host,
176
+ "port" to endpoint.port,
177
+ )
178
+ }
179
+
180
+ private fun isWifiInterface(name: String): Boolean {
181
+ return name.startsWith("wlan") ||
182
+ name.startsWith("p2p") ||
183
+ name.startsWith("ap") ||
184
+ name.startsWith("swlan")
185
+ }
186
+
187
+ private fun isPrivateIpv4(host: String): Boolean {
188
+ return host.startsWith("192.168.") ||
189
+ host.startsWith("10.") ||
190
+ host.matches(Regex("^172\\.(1[6-9]|2[0-9]|3[0-1])\\..*"))
115
191
  }
116
192
 
117
193
  private companion object {
194
+ const val TAG = "MentraPhotoReceiver"
118
195
  val PHOTO_PORTS = listOf(8787, 8788, 8789, 8790)
119
196
  }
197
+
198
+ private data class ReceiverEndpoint(
199
+ val uploadUrl: String,
200
+ val host: String,
201
+ val port: Int,
202
+ )
120
203
  }
@@ -37,3 +37,20 @@ internal data class DashboardMenuItem(
37
37
  "packageName" to packageName,
38
38
  )
39
39
  }
40
+
41
+ // Mirrors the TS `CalendarEvent` shape: { title, location?, time, endDate }
42
+ // where `endDate` is Unix seconds.
43
+ internal data class CalendarEvent(
44
+ val title: String,
45
+ val time: String,
46
+ val endDate: Long,
47
+ val location: String? = null,
48
+ ) {
49
+ fun toMap(): Map<String, Any> =
50
+ buildMap {
51
+ put("title", title)
52
+ put("time", time)
53
+ put("endDate", endDate)
54
+ location?.let { put("location", it) }
55
+ }
56
+ }
@@ -1530,7 +1530,7 @@ public class G1 extends SGCManager {
1530
1530
  }
1531
1531
 
1532
1532
  @Override
1533
- public void requestPhoto(String requestId, String appId, String size, String webhookUrl, String authToken, String compress, boolean flash, boolean sound, Long exposureTimeNs, Integer iso) {
1533
+ public void requestPhoto(String requestId, String appId, String size, String webhookUrl, String authToken, String compress, boolean flash, boolean save, boolean sound, Long exposureTimeNs, Integer iso) {
1534
1534
 
1535
1535
  }
1536
1536
 
@@ -1608,7 +1608,7 @@ public class G1 extends SGCManager {
1608
1608
  }
1609
1609
 
1610
1610
  @Override
1611
- public boolean displayBitmap(String base64ImageData) {
1611
+ public boolean displayBitmap(String base64ImageData, Integer x, Integer y, Integer width, Integer height) {
1612
1612
  try {
1613
1613
  // Decode base64 to byte array
1614
1614
  byte[] bmpData = android.util.Base64.decode(base64ImageData, android.util.Base64.DEFAULT);