@mentra/bluetooth-sdk 0.1.9 → 0.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (147) hide show
  1. package/README.md +116 -18
  2. package/android/build.gradle +289 -16
  3. package/android/gradle.properties.example +9 -0
  4. package/android/lc3Lib/build.gradle +95 -6
  5. package/android/settings.gradle +3 -0
  6. package/android/silero/LICENSE.md +17 -0
  7. package/android/silero/build.gradle +42 -0
  8. package/android/silero/proguard-rules.pro +24 -0
  9. package/android/silero/src/androidTest/assets/hello.wav +0 -0
  10. package/android/silero/src/androidTest/java/com/konovalov/vad/silero/VadSileroTest.kt +79 -0
  11. package/android/silero/src/main/assets/LICENSE +21 -0
  12. package/android/silero/src/main/assets/silero_vad.onnx +0 -0
  13. package/android/silero/src/main/java/com/konovalov/vad/silero/Vad.kt +160 -0
  14. package/android/silero/src/main/java/com/konovalov/vad/silero/VadSilero.kt +449 -0
  15. package/android/silero/src/main/java/com/konovalov/vad/silero/config/FrameSize.kt +15 -0
  16. package/android/silero/src/main/java/com/konovalov/vad/silero/config/Mode.kt +14 -0
  17. package/android/silero/src/main/java/com/konovalov/vad/silero/config/SampleRate.kt +12 -0
  18. package/android/silero/src/main/java/com/konovalov/vad/silero/utils/AudioUtils.kt +47 -0
  19. package/android/silero/src/main/java/com/konovalov/vad/silero/utils/TensorMap.kt +41 -0
  20. package/android/silero/src/test/java/com/konovalov/vad/silero/utils/AudioUtilsTest.kt +44 -0
  21. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkDefaults.kt +6 -0
  22. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +195 -34
  23. package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +103 -21
  24. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +210 -18
  25. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt +30 -2
  26. package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +787 -85
  27. package/android/src/main/java/com/mentra/bluetoothsdk/audio/AudioModels.kt +6 -2
  28. package/android/src/main/java/com/mentra/bluetoothsdk/camera/CameraModels.kt +125 -11
  29. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/ControllerManager.kt +12 -2
  30. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/R1.kt +8 -2
  31. package/android/src/main/java/com/mentra/bluetoothsdk/debug/BleTraceLogger.kt +33 -17
  32. package/android/src/main/java/com/mentra/bluetoothsdk/events/BluetoothEvents.kt +101 -0
  33. package/android/src/main/java/com/mentra/bluetoothsdk/internal/MapParsing.kt +8 -0
  34. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistry.java +108 -0
  35. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoUploadServer.kt +399 -74
  36. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/MentraPhotoReceiverModule.kt +100 -17
  37. package/android/src/main/java/com/mentra/bluetoothsdk/requests/DisplayRequests.kt +17 -0
  38. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G1.java +2 -2
  39. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt +1293 -758
  40. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Mach1.java +2 -2
  41. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraLive.java +271 -66
  42. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraNex.kt +197 -149
  43. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraosBle.java +12753 -27036
  44. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt +64 -2
  45. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Simulated.kt +16 -4
  46. package/android/src/main/java/com/mentra/bluetoothsdk/status/DeviceStatus.kt +60 -3
  47. package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +12 -3
  48. package/android/src/main/java/com/mentra/bluetoothsdk/stt/STTTools.kt +87 -36
  49. package/android/src/main/java/com/mentra/bluetoothsdk/stt/SherpaOnnxTranscriber.kt +13 -4
  50. package/android/src/main/java/com/mentra/bluetoothsdk/stt/VadGateSpeechPolicy.kt +229 -0
  51. package/android/src/main/java/com/mentra/bluetoothsdk/tts/TTSTools.kt +180 -0
  52. package/android/src/main/java/com/mentra/bluetoothsdk/utils/AvifExifStripper.java +573 -0
  53. package/android/src/main/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadService.java +488 -50
  54. package/android/src/main/java/com/mentra/bluetoothsdk/utils/HeifExifTagReader.java +199 -0
  55. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunkReassembler.java +18 -1
  56. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunker.java +64 -18
  57. package/android/src/main/java/com/mentra/bluetoothsdk/utils/NexSGCUtils.kt +48 -39
  58. package/android/src/test/java/com/mentra/bluetoothsdk/camera/PhotoRequestTest.kt +23 -0
  59. package/android/src/test/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistryTest.java +75 -0
  60. package/android/src/test/java/com/mentra/bluetoothsdk/utils/AvifExifStripperTest.java +30 -0
  61. package/android/src/test/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadServiceTest.java +98 -0
  62. package/android/src/test/resources/avif_with_exif.avif +0 -0
  63. package/build/BluetoothSdk.types.d.ts +253 -44
  64. package/build/BluetoothSdk.types.d.ts.map +1 -1
  65. package/build/BluetoothSdk.types.js +11 -10
  66. package/build/BluetoothSdk.types.js.map +1 -1
  67. package/build/_private/BluetoothSdkModule.d.ts +37 -23
  68. package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
  69. package/build/_private/BluetoothSdkModule.js +43 -52
  70. package/build/_private/BluetoothSdkModule.js.map +1 -1
  71. package/build/_private/photoRequestPayload.d.ts +4 -0
  72. package/build/_private/photoRequestPayload.d.ts.map +1 -0
  73. package/build/_private/photoRequestPayload.js +25 -0
  74. package/build/_private/photoRequestPayload.js.map +1 -0
  75. package/build/index.d.ts +1 -1
  76. package/build/index.d.ts.map +1 -1
  77. package/build/index.js +25 -0
  78. package/build/index.js.map +1 -1
  79. package/build/react/useMentraBluetooth.d.ts +2 -2
  80. package/build/react/useMentraBluetooth.d.ts.map +1 -1
  81. package/build/react/useMentraBluetooth.js +3 -3
  82. package/build/react/useMentraBluetooth.js.map +1 -1
  83. package/ios/BluetoothSdkModule.swift +177 -70
  84. package/ios/LocalPhotoUploadServer.swift +370 -105
  85. package/ios/MentraBluetoothSDK.podspec +3 -1
  86. package/ios/MentraPhotoReceiverModule.swift +62 -10
  87. package/ios/Packages/CoreObjC/include/PcmConverter.h +22 -0
  88. package/ios/Packages/SherpaOnnx/SherpaOnnx.swift +668 -26
  89. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Info.plist +13 -9
  90. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
  91. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
  92. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/libsherpa-onnx.a +0 -0
  93. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
  94. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
  95. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/libsherpa-onnx.a +0 -0
  96. package/ios/Source/BluetoothSdkDefaults.swift +6 -0
  97. package/ios/Source/Bridge.swift +68 -19
  98. package/ios/Source/DeviceManager.swift +167 -32
  99. package/ios/Source/DeviceStore.swift +24 -8
  100. package/ios/Source/MentraBluetoothSDK.swift +803 -47
  101. package/ios/Source/ObservableStore.swift +6 -0
  102. package/ios/Source/{Audio → audio}/AudioModels.swift +9 -7
  103. package/ios/Source/{Camera → camera}/CameraModels.swift +226 -16
  104. package/ios/Source/controllers/ControllerManager.swift +4 -3
  105. package/ios/Source/controllers/R1.swift +3 -3
  106. package/ios/Source/{Events → events}/BluetoothEvents.swift +183 -1
  107. package/ios/Source/{Internal → internal}/ValueParsing.swift +10 -1
  108. package/ios/Source/{Requests → requests}/DisplayRequests.swift +28 -4
  109. package/ios/Source/services/PhoneMic.swift +1 -0
  110. package/ios/Source/sgcs/G1.swift +5 -5
  111. package/ios/Source/sgcs/G2.swift +1494 -870
  112. package/ios/Source/sgcs/Mach1.swift +4 -4
  113. package/ios/Source/sgcs/MentraLive.swift +814 -321
  114. package/ios/Source/sgcs/MentraNex.swift +604 -244
  115. package/ios/Source/sgcs/SGCManager.swift +68 -5
  116. package/ios/Source/sgcs/Simulated.swift +9 -5
  117. package/ios/Source/sgcs/mentraos_ble.pb.swift +3073 -3134
  118. package/ios/Source/status/DeviceStatus.swift +974 -0
  119. package/ios/Source/{Status → status}/WifiHotspotStatus.swift +4 -4
  120. package/ios/Source/{Streaming → streaming}/StreamModels.swift +24 -9
  121. package/ios/Source/stt/STTTools.swift +27 -12
  122. package/ios/Source/stt/SherpaOnnxTranscriber.swift +31 -8
  123. package/ios/Source/tts/TTSTools.swift +164 -0
  124. package/ios/Source/utils/JSCExperiment.swift +7 -7
  125. package/ios/Source/utils/MessageChunkReassembler.swift +20 -1
  126. package/ios/Source/utils/MessageChunker.swift +78 -21
  127. package/ios/Source/utils/TarBz2Extractor.swift +53 -2
  128. package/package.json +1 -1
  129. package/plugin/build/withAndroid.js +68 -4
  130. package/src/BluetoothSdk.types.ts +344 -60
  131. package/src/_private/BluetoothSdkModule.ts +132 -102
  132. package/src/_private/photoRequestPayload.ts +28 -0
  133. package/src/index.ts +61 -15
  134. package/src/react/useMentraBluetooth.ts +6 -4
  135. package/android/.project +0 -28
  136. package/android/lc3Lib/.project +0 -28
  137. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/cargs.h +0 -162
  138. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/c-api.h +0 -1852
  139. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/cxx-api.h +0 -674
  140. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/sherpa-onnx.a +0 -0
  141. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/sherpa-onnx.a +0 -0
  142. package/ios/Source/Status/DeviceStatus.swift +0 -471
  143. /package/ios/Source/{Connection → connection}/ScanSession.swift +0 -0
  144. /package/ios/Source/{Errors → errors}/BluetoothError.swift +0 -0
  145. /package/ios/Source/{Internal → internal}/BluetoothAvailability.swift +0 -0
  146. /package/ios/Source/{Status → status}/RuntimeState.swift +0 -0
  147. /package/ios/Source/{Types → types}/DeviceModels.swift +0 -0
@@ -20,12 +20,19 @@ protocol SGCManager {
20
20
 
21
21
  func requestPhoto(
22
22
  _ requestId: String, appId: String, size: String?, webhookUrl: String?, authToken: String?,
23
- compress: String?, flash: Bool, sound: Bool, exposureTimeNs: Double?, iso: Int?
23
+ compress: String?, flash: Bool, save: Bool, sound: Bool, exposureTimeNs: Double?, iso: Int?
24
24
  )
25
25
  func startStream(_ message: [String: Any])
26
26
  func stopStream()
27
27
  func sendStreamKeepAlive(_ message: [String: Any])
28
28
  func startVideoRecording(requestId: String, save: Bool, flash: Bool, sound: Bool)
29
+ /// Start video recording with optional per-recording resolution/fps. A width,
30
+ /// height, or fps of 0 means "use the device's saved button-video default".
31
+ /// Defaulted in an extension to delegate to the basic recording path; devices
32
+ /// that support custom settings (e.g. Mentra Live) override this.
33
+ func startVideoRecording(
34
+ requestId: String, save: Bool, flash: Bool, sound: Bool, width: Int, height: Int, fps: Int
35
+ )
29
36
  func stopVideoRecording(requestId: String)
30
37
 
31
38
  // MARK: - Button Settings
@@ -40,9 +47,11 @@ protocol SGCManager {
40
47
 
41
48
  func setBrightness(_ level: Int, autoMode: Bool)
42
49
  func clearDisplay()
43
- func sendTextWall(_ text: String)
44
- func sendDoubleTextWall(_ top: String, _ bottom: String)
45
- func displayBitmap(base64ImageData: String) async -> Bool
50
+ func sendTextWall(_ text: String) async
51
+ func sendDoubleTextWall(_ top: String, _ bottom: String) async
52
+ /// Display a bitmap. Optional `x`/`y`/`width`/`height` position and size the target
53
+ /// container (used by G2; other SGCs ignore positioning and render the bitmap as before).
54
+ func displayBitmap(base64ImageData: String, x: Int32?, y: Int32?, width: Int32?, height: Int32?) async -> Bool
46
55
  func showDashboard()
47
56
  func setDashboardPosition(_ height: Int, _ depth: Int)
48
57
  /// Default implementation sends both via [setDashboardPosition]; Nex overrides to one protobuf.
@@ -53,9 +62,24 @@ protocol SGCManager {
53
62
 
54
63
  func setDashboardMenu(_ items: [[String: Any]])
55
64
 
65
+ // MARK: - Notification Panel
66
+
67
+ func showNotificationsPanel() async
68
+
69
+ // MARK: - Calendar Events
70
+
71
+ func sendCalendarEvents(_ events: [[String: Any]])
72
+
73
+ // MARK: - Dashboard Display Settings
74
+
75
+ func sendDashboardDisplaySettings()
76
+
56
77
  // MARK: - Device Control
57
78
 
58
79
  func setHeadUpAngle(_ angle: Int)
80
+ /// Enable/disable raw accelerometer (IMU) reporting from the glasses.
81
+ /// Default no-op; only G2 streams IMU data today.
82
+ func setImuEnabled(_ enabled: Bool) async
59
83
  func getBatteryStatus()
60
84
  func setSilentMode(_ enabled: Bool)
61
85
  func exit()
@@ -89,6 +113,8 @@ protocol SGCManager {
89
113
  func sendHotspotState(_ enabled: Bool)
90
114
  func sendOtaStart()
91
115
  func sendOtaQueryStatus()
116
+ func sendSetSystemTime(_ timestampMs: Int64)
117
+ func sendOtaRetryVersionCheck()
92
118
 
93
119
  // MARK: - User Context (for crash reporting)
94
120
 
@@ -115,6 +141,15 @@ protocol SGCManager {
115
141
  /// doesn't seem to work for concurrency reasons :(
116
142
  /// we can make read-only getters for convienence though:
117
143
  extension SGCManager {
144
+ // MARK: - Video recording (default: ignore custom settings, use saved defaults)
145
+
146
+ func startVideoRecording(
147
+ requestId: String, save: Bool, flash: Bool, sound: Bool, width _: Int, height _: Int,
148
+ fps _: Int
149
+ ) {
150
+ startVideoRecording(requestId: requestId, save: save, flash: flash, sound: sound)
151
+ }
152
+
118
153
  // MARK: - Dashboard (default: combined wire format; Nex implements single-field)
119
154
 
120
155
  func setDashboardHeightOnly(_ height: Int) {
@@ -131,10 +166,37 @@ extension SGCManager {
131
166
 
132
167
  func setDashboardMenu(_: [[String: Any]]) {}
133
168
 
169
+ // MARK: - Notification Panel (default no-op — only G2 supports this)
170
+
171
+ func showNotificationsPanel() async {}
172
+
173
+ // MARK: - IMU (default no-op — only G2 streams accelerometer data)
174
+
175
+ func setImuEnabled(_: Bool) async {
176
+ Bridge.log("SGC: setImuEnabled not supported")
177
+ }
178
+
179
+ // MARK: - Calendar Events (default no-op — only G2 supports this)
180
+
181
+ func sendCalendarEvents(_: [[String: Any]]) {}
182
+
183
+ // MARK: - Dashboard Display Settings (default no-op — only G2 supports this)
184
+
185
+ func sendDashboardDisplaySettings() {}
186
+
134
187
  // MARK: - Voice Activity Detection (default no-op — Mentra Live supports this)
135
188
 
136
189
  func sendVoiceActivityDetectionSetting() {}
137
190
 
191
+ /// Default no-op; Mentra Live overrides when phone detects clock skew during gallery sync.
192
+ func sendSetSystemTime(_: Int64) {
193
+ Bridge.log("SGC: sendSetSystemTime not supported")
194
+ }
195
+
196
+ func sendOtaRetryVersionCheck() {
197
+ Bridge.log("SGC: sendOtaRetryVersionCheck not supported")
198
+ }
199
+
138
200
  // MARK: - Default DeviceStore-backed property implementations
139
201
 
140
202
  var fullyBooted: Bool {
@@ -190,7 +252,8 @@ extension SGCManager {
190
252
  }
191
253
 
192
254
  var voiceActivityDetectionEnabled: Bool {
193
- DeviceStore.shared.get("glasses", "voiceActivityDetectionEnabled") as? Bool ?? true
255
+ DeviceStore.shared.get("glasses", "voiceActivityDetectionEnabled") as? Bool
256
+ ?? BluetoothSdkDefaults.voiceActivityDetectionEnabled
194
257
  }
195
258
 
196
259
  var batteryLevel: Int {
@@ -12,7 +12,11 @@ class Simulated: SGCManager {
12
12
  DeviceStore.shared.apply("glasses", "connected", true)
13
13
  DeviceStore.shared.apply("glasses", "connectionState", ConnTypes.CONNECTED)
14
14
  DeviceStore.shared.apply("glasses", "micEnabled", false)
15
- DeviceStore.shared.apply("glasses", "voiceActivityDetectionEnabled", true)
15
+ DeviceStore.shared.apply(
16
+ "glasses",
17
+ "voiceActivityDetectionEnabled",
18
+ BluetoothSdkDefaults.voiceActivityDetectionEnabled
19
+ )
16
20
  DeviceStore.shared.apply("glasses", "bluetoothClassicConnected", false)
17
21
  }
18
22
 
@@ -75,7 +79,7 @@ class Simulated: SGCManager {
75
79
 
76
80
  // MARK: - Camera & Media
77
81
 
78
- func requestPhoto(_: String, appId _: String, size _: String?, webhookUrl _: String?, authToken _: String?, compress _: String?, flash _: Bool, sound _: Bool, exposureTimeNs _: Double?, iso _: Int?) {
82
+ func requestPhoto(_: String, appId _: String, size _: String?, webhookUrl _: String?, authToken _: String?, compress _: String?, flash _: Bool, save _: Bool, sound _: Bool, exposureTimeNs _: Double?, iso _: Int?) {
79
83
  Bridge.log("requestPhoto")
80
84
  }
81
85
 
@@ -129,15 +133,15 @@ class Simulated: SGCManager {
129
133
  Bridge.log("clearDisplay")
130
134
  }
131
135
 
132
- func sendTextWall(_: String) {
136
+ func sendTextWall(_: String) async {
133
137
  Bridge.log("sendTextWall")
134
138
  }
135
139
 
136
- func sendDoubleTextWall(_: String, _: String) {
140
+ func sendDoubleTextWall(_: String, _: String) async {
137
141
  Bridge.log("sendDoubleTextWall")
138
142
  }
139
143
 
140
- func displayBitmap(base64ImageData _: String) async -> Bool {
144
+ func displayBitmap(base64ImageData _: String, x _: Int32?, y _: Int32?, width _: Int32?, height _: Int32?) async -> Bool {
141
145
  Bridge.log("displayBitmap")
142
146
  return false
143
147
  }