@mentra/bluetooth-sdk 0.1.9 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +116 -18
- package/android/build.gradle +289 -16
- package/android/gradle.properties.example +9 -0
- package/android/lc3Lib/build.gradle +95 -6
- package/android/settings.gradle +3 -0
- package/android/silero/LICENSE.md +17 -0
- package/android/silero/build.gradle +42 -0
- package/android/silero/proguard-rules.pro +24 -0
- package/android/silero/src/androidTest/assets/hello.wav +0 -0
- package/android/silero/src/androidTest/java/com/konovalov/vad/silero/VadSileroTest.kt +79 -0
- package/android/silero/src/main/assets/LICENSE +21 -0
- package/android/silero/src/main/assets/silero_vad.onnx +0 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/Vad.kt +160 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/VadSilero.kt +449 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/config/FrameSize.kt +15 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/config/Mode.kt +14 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/config/SampleRate.kt +12 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/utils/AudioUtils.kt +47 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/utils/TensorMap.kt +41 -0
- package/android/silero/src/test/java/com/konovalov/vad/silero/utils/AudioUtilsTest.kt +44 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkDefaults.kt +6 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +195 -34
- package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +103 -21
- package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +210 -18
- package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt +30 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +787 -85
- package/android/src/main/java/com/mentra/bluetoothsdk/audio/AudioModels.kt +6 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/camera/CameraModels.kt +125 -11
- package/android/src/main/java/com/mentra/bluetoothsdk/controllers/ControllerManager.kt +12 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/controllers/R1.kt +8 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/debug/BleTraceLogger.kt +33 -17
- package/android/src/main/java/com/mentra/bluetoothsdk/events/BluetoothEvents.kt +101 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/internal/MapParsing.kt +8 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistry.java +108 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoUploadServer.kt +399 -74
- package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/MentraPhotoReceiverModule.kt +100 -17
- package/android/src/main/java/com/mentra/bluetoothsdk/requests/DisplayRequests.kt +17 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G1.java +2 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt +1293 -758
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Mach1.java +2 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraLive.java +271 -66
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraNex.kt +197 -149
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraosBle.java +12753 -27036
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt +64 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Simulated.kt +16 -4
- package/android/src/main/java/com/mentra/bluetoothsdk/status/DeviceStatus.kt +60 -3
- package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +12 -3
- package/android/src/main/java/com/mentra/bluetoothsdk/stt/STTTools.kt +87 -36
- package/android/src/main/java/com/mentra/bluetoothsdk/stt/SherpaOnnxTranscriber.kt +13 -4
- package/android/src/main/java/com/mentra/bluetoothsdk/stt/VadGateSpeechPolicy.kt +229 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/tts/TTSTools.kt +180 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/AvifExifStripper.java +573 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadService.java +488 -50
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/HeifExifTagReader.java +199 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunkReassembler.java +18 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunker.java +64 -18
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/NexSGCUtils.kt +48 -39
- package/android/src/test/java/com/mentra/bluetoothsdk/camera/PhotoRequestTest.kt +23 -0
- package/android/src/test/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistryTest.java +75 -0
- package/android/src/test/java/com/mentra/bluetoothsdk/utils/AvifExifStripperTest.java +30 -0
- package/android/src/test/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadServiceTest.java +98 -0
- package/android/src/test/resources/avif_with_exif.avif +0 -0
- package/build/BluetoothSdk.types.d.ts +253 -44
- package/build/BluetoothSdk.types.d.ts.map +1 -1
- package/build/BluetoothSdk.types.js +11 -10
- package/build/BluetoothSdk.types.js.map +1 -1
- package/build/_private/BluetoothSdkModule.d.ts +37 -23
- package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
- package/build/_private/BluetoothSdkModule.js +43 -52
- package/build/_private/BluetoothSdkModule.js.map +1 -1
- package/build/_private/photoRequestPayload.d.ts +4 -0
- package/build/_private/photoRequestPayload.d.ts.map +1 -0
- package/build/_private/photoRequestPayload.js +25 -0
- package/build/_private/photoRequestPayload.js.map +1 -0
- package/build/index.d.ts +1 -1
- package/build/index.d.ts.map +1 -1
- package/build/index.js +25 -0
- package/build/index.js.map +1 -1
- package/build/react/useMentraBluetooth.d.ts +2 -2
- package/build/react/useMentraBluetooth.d.ts.map +1 -1
- package/build/react/useMentraBluetooth.js +3 -3
- package/build/react/useMentraBluetooth.js.map +1 -1
- package/ios/BluetoothSdkModule.swift +177 -70
- package/ios/LocalPhotoUploadServer.swift +370 -105
- package/ios/MentraBluetoothSDK.podspec +3 -1
- package/ios/MentraPhotoReceiverModule.swift +62 -10
- package/ios/Packages/CoreObjC/include/PcmConverter.h +22 -0
- package/ios/Packages/SherpaOnnx/SherpaOnnx.swift +668 -26
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Info.plist +13 -9
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/libsherpa-onnx.a +0 -0
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/libsherpa-onnx.a +0 -0
- package/ios/Source/BluetoothSdkDefaults.swift +6 -0
- package/ios/Source/Bridge.swift +68 -19
- package/ios/Source/DeviceManager.swift +167 -32
- package/ios/Source/DeviceStore.swift +24 -8
- package/ios/Source/MentraBluetoothSDK.swift +803 -47
- package/ios/Source/ObservableStore.swift +6 -0
- package/ios/Source/{Audio → audio}/AudioModels.swift +9 -7
- package/ios/Source/{Camera → camera}/CameraModels.swift +226 -16
- package/ios/Source/controllers/ControllerManager.swift +4 -3
- package/ios/Source/controllers/R1.swift +3 -3
- package/ios/Source/{Events → events}/BluetoothEvents.swift +183 -1
- package/ios/Source/{Internal → internal}/ValueParsing.swift +10 -1
- package/ios/Source/{Requests → requests}/DisplayRequests.swift +28 -4
- package/ios/Source/services/PhoneMic.swift +1 -0
- package/ios/Source/sgcs/G1.swift +5 -5
- package/ios/Source/sgcs/G2.swift +1494 -870
- package/ios/Source/sgcs/Mach1.swift +4 -4
- package/ios/Source/sgcs/MentraLive.swift +814 -321
- package/ios/Source/sgcs/MentraNex.swift +604 -244
- package/ios/Source/sgcs/SGCManager.swift +68 -5
- package/ios/Source/sgcs/Simulated.swift +9 -5
- package/ios/Source/sgcs/mentraos_ble.pb.swift +3073 -3134
- package/ios/Source/status/DeviceStatus.swift +974 -0
- package/ios/Source/{Status → status}/WifiHotspotStatus.swift +4 -4
- package/ios/Source/{Streaming → streaming}/StreamModels.swift +24 -9
- package/ios/Source/stt/STTTools.swift +27 -12
- package/ios/Source/stt/SherpaOnnxTranscriber.swift +31 -8
- package/ios/Source/tts/TTSTools.swift +164 -0
- package/ios/Source/utils/JSCExperiment.swift +7 -7
- package/ios/Source/utils/MessageChunkReassembler.swift +20 -1
- package/ios/Source/utils/MessageChunker.swift +78 -21
- package/ios/Source/utils/TarBz2Extractor.swift +53 -2
- package/package.json +1 -1
- package/plugin/build/withAndroid.js +68 -4
- package/src/BluetoothSdk.types.ts +344 -60
- package/src/_private/BluetoothSdkModule.ts +132 -102
- package/src/_private/photoRequestPayload.ts +28 -0
- package/src/index.ts +61 -15
- package/src/react/useMentraBluetooth.ts +6 -4
- package/android/.project +0 -28
- package/android/lc3Lib/.project +0 -28
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/cargs.h +0 -162
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/c-api.h +0 -1852
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/cxx-api.h +0 -674
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/sherpa-onnx.a +0 -0
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/sherpa-onnx.a +0 -0
- package/ios/Source/Status/DeviceStatus.swift +0 -471
- /package/ios/Source/{Connection → connection}/ScanSession.swift +0 -0
- /package/ios/Source/{Errors → errors}/BluetoothError.swift +0 -0
- /package/ios/Source/{Internal → internal}/BluetoothAvailability.swift +0 -0
- /package/ios/Source/{Status → status}/RuntimeState.swift +0 -0
- /package/ios/Source/{Types → types}/DeviceModels.swift +0 -0
|
@@ -20,12 +20,19 @@ protocol SGCManager {
|
|
|
20
20
|
|
|
21
21
|
func requestPhoto(
|
|
22
22
|
_ requestId: String, appId: String, size: String?, webhookUrl: String?, authToken: String?,
|
|
23
|
-
compress: String?, flash: Bool, sound: Bool, exposureTimeNs: Double?, iso: Int?
|
|
23
|
+
compress: String?, flash: Bool, save: Bool, sound: Bool, exposureTimeNs: Double?, iso: Int?
|
|
24
24
|
)
|
|
25
25
|
func startStream(_ message: [String: Any])
|
|
26
26
|
func stopStream()
|
|
27
27
|
func sendStreamKeepAlive(_ message: [String: Any])
|
|
28
28
|
func startVideoRecording(requestId: String, save: Bool, flash: Bool, sound: Bool)
|
|
29
|
+
/// Start video recording with optional per-recording resolution/fps. A width,
|
|
30
|
+
/// height, or fps of 0 means "use the device's saved button-video default".
|
|
31
|
+
/// Defaulted in an extension to delegate to the basic recording path; devices
|
|
32
|
+
/// that support custom settings (e.g. Mentra Live) override this.
|
|
33
|
+
func startVideoRecording(
|
|
34
|
+
requestId: String, save: Bool, flash: Bool, sound: Bool, width: Int, height: Int, fps: Int
|
|
35
|
+
)
|
|
29
36
|
func stopVideoRecording(requestId: String)
|
|
30
37
|
|
|
31
38
|
// MARK: - Button Settings
|
|
@@ -40,9 +47,11 @@ protocol SGCManager {
|
|
|
40
47
|
|
|
41
48
|
func setBrightness(_ level: Int, autoMode: Bool)
|
|
42
49
|
func clearDisplay()
|
|
43
|
-
func sendTextWall(_ text: String)
|
|
44
|
-
func sendDoubleTextWall(_ top: String, _ bottom: String)
|
|
45
|
-
|
|
50
|
+
func sendTextWall(_ text: String) async
|
|
51
|
+
func sendDoubleTextWall(_ top: String, _ bottom: String) async
|
|
52
|
+
/// Display a bitmap. Optional `x`/`y`/`width`/`height` position and size the target
|
|
53
|
+
/// container (used by G2; other SGCs ignore positioning and render the bitmap as before).
|
|
54
|
+
func displayBitmap(base64ImageData: String, x: Int32?, y: Int32?, width: Int32?, height: Int32?) async -> Bool
|
|
46
55
|
func showDashboard()
|
|
47
56
|
func setDashboardPosition(_ height: Int, _ depth: Int)
|
|
48
57
|
/// Default implementation sends both via [setDashboardPosition]; Nex overrides to one protobuf.
|
|
@@ -53,9 +62,24 @@ protocol SGCManager {
|
|
|
53
62
|
|
|
54
63
|
func setDashboardMenu(_ items: [[String: Any]])
|
|
55
64
|
|
|
65
|
+
// MARK: - Notification Panel
|
|
66
|
+
|
|
67
|
+
func showNotificationsPanel() async
|
|
68
|
+
|
|
69
|
+
// MARK: - Calendar Events
|
|
70
|
+
|
|
71
|
+
func sendCalendarEvents(_ events: [[String: Any]])
|
|
72
|
+
|
|
73
|
+
// MARK: - Dashboard Display Settings
|
|
74
|
+
|
|
75
|
+
func sendDashboardDisplaySettings()
|
|
76
|
+
|
|
56
77
|
// MARK: - Device Control
|
|
57
78
|
|
|
58
79
|
func setHeadUpAngle(_ angle: Int)
|
|
80
|
+
/// Enable/disable raw accelerometer (IMU) reporting from the glasses.
|
|
81
|
+
/// Default no-op; only G2 streams IMU data today.
|
|
82
|
+
func setImuEnabled(_ enabled: Bool) async
|
|
59
83
|
func getBatteryStatus()
|
|
60
84
|
func setSilentMode(_ enabled: Bool)
|
|
61
85
|
func exit()
|
|
@@ -89,6 +113,8 @@ protocol SGCManager {
|
|
|
89
113
|
func sendHotspotState(_ enabled: Bool)
|
|
90
114
|
func sendOtaStart()
|
|
91
115
|
func sendOtaQueryStatus()
|
|
116
|
+
func sendSetSystemTime(_ timestampMs: Int64)
|
|
117
|
+
func sendOtaRetryVersionCheck()
|
|
92
118
|
|
|
93
119
|
// MARK: - User Context (for crash reporting)
|
|
94
120
|
|
|
@@ -115,6 +141,15 @@ protocol SGCManager {
|
|
|
115
141
|
/// doesn't seem to work for concurrency reasons :(
|
|
116
142
|
/// we can make read-only getters for convienence though:
|
|
117
143
|
extension SGCManager {
|
|
144
|
+
// MARK: - Video recording (default: ignore custom settings, use saved defaults)
|
|
145
|
+
|
|
146
|
+
func startVideoRecording(
|
|
147
|
+
requestId: String, save: Bool, flash: Bool, sound: Bool, width _: Int, height _: Int,
|
|
148
|
+
fps _: Int
|
|
149
|
+
) {
|
|
150
|
+
startVideoRecording(requestId: requestId, save: save, flash: flash, sound: sound)
|
|
151
|
+
}
|
|
152
|
+
|
|
118
153
|
// MARK: - Dashboard (default: combined wire format; Nex implements single-field)
|
|
119
154
|
|
|
120
155
|
func setDashboardHeightOnly(_ height: Int) {
|
|
@@ -131,10 +166,37 @@ extension SGCManager {
|
|
|
131
166
|
|
|
132
167
|
func setDashboardMenu(_: [[String: Any]]) {}
|
|
133
168
|
|
|
169
|
+
// MARK: - Notification Panel (default no-op — only G2 supports this)
|
|
170
|
+
|
|
171
|
+
func showNotificationsPanel() async {}
|
|
172
|
+
|
|
173
|
+
// MARK: - IMU (default no-op — only G2 streams accelerometer data)
|
|
174
|
+
|
|
175
|
+
func setImuEnabled(_: Bool) async {
|
|
176
|
+
Bridge.log("SGC: setImuEnabled not supported")
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
// MARK: - Calendar Events (default no-op — only G2 supports this)
|
|
180
|
+
|
|
181
|
+
func sendCalendarEvents(_: [[String: Any]]) {}
|
|
182
|
+
|
|
183
|
+
// MARK: - Dashboard Display Settings (default no-op — only G2 supports this)
|
|
184
|
+
|
|
185
|
+
func sendDashboardDisplaySettings() {}
|
|
186
|
+
|
|
134
187
|
// MARK: - Voice Activity Detection (default no-op — Mentra Live supports this)
|
|
135
188
|
|
|
136
189
|
func sendVoiceActivityDetectionSetting() {}
|
|
137
190
|
|
|
191
|
+
/// Default no-op; Mentra Live overrides when phone detects clock skew during gallery sync.
|
|
192
|
+
func sendSetSystemTime(_: Int64) {
|
|
193
|
+
Bridge.log("SGC: sendSetSystemTime not supported")
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
func sendOtaRetryVersionCheck() {
|
|
197
|
+
Bridge.log("SGC: sendOtaRetryVersionCheck not supported")
|
|
198
|
+
}
|
|
199
|
+
|
|
138
200
|
// MARK: - Default DeviceStore-backed property implementations
|
|
139
201
|
|
|
140
202
|
var fullyBooted: Bool {
|
|
@@ -190,7 +252,8 @@ extension SGCManager {
|
|
|
190
252
|
}
|
|
191
253
|
|
|
192
254
|
var voiceActivityDetectionEnabled: Bool {
|
|
193
|
-
DeviceStore.shared.get("glasses", "voiceActivityDetectionEnabled") as? Bool
|
|
255
|
+
DeviceStore.shared.get("glasses", "voiceActivityDetectionEnabled") as? Bool
|
|
256
|
+
?? BluetoothSdkDefaults.voiceActivityDetectionEnabled
|
|
194
257
|
}
|
|
195
258
|
|
|
196
259
|
var batteryLevel: Int {
|
|
@@ -12,7 +12,11 @@ class Simulated: SGCManager {
|
|
|
12
12
|
DeviceStore.shared.apply("glasses", "connected", true)
|
|
13
13
|
DeviceStore.shared.apply("glasses", "connectionState", ConnTypes.CONNECTED)
|
|
14
14
|
DeviceStore.shared.apply("glasses", "micEnabled", false)
|
|
15
|
-
DeviceStore.shared.apply(
|
|
15
|
+
DeviceStore.shared.apply(
|
|
16
|
+
"glasses",
|
|
17
|
+
"voiceActivityDetectionEnabled",
|
|
18
|
+
BluetoothSdkDefaults.voiceActivityDetectionEnabled
|
|
19
|
+
)
|
|
16
20
|
DeviceStore.shared.apply("glasses", "bluetoothClassicConnected", false)
|
|
17
21
|
}
|
|
18
22
|
|
|
@@ -75,7 +79,7 @@ class Simulated: SGCManager {
|
|
|
75
79
|
|
|
76
80
|
// MARK: - Camera & Media
|
|
77
81
|
|
|
78
|
-
func requestPhoto(_: String, appId _: String, size _: String?, webhookUrl _: String?, authToken _: String?, compress _: String?, flash _: Bool, sound _: Bool, exposureTimeNs _: Double?, iso _: Int?) {
|
|
82
|
+
func requestPhoto(_: String, appId _: String, size _: String?, webhookUrl _: String?, authToken _: String?, compress _: String?, flash _: Bool, save _: Bool, sound _: Bool, exposureTimeNs _: Double?, iso _: Int?) {
|
|
79
83
|
Bridge.log("requestPhoto")
|
|
80
84
|
}
|
|
81
85
|
|
|
@@ -129,15 +133,15 @@ class Simulated: SGCManager {
|
|
|
129
133
|
Bridge.log("clearDisplay")
|
|
130
134
|
}
|
|
131
135
|
|
|
132
|
-
func sendTextWall(_: String) {
|
|
136
|
+
func sendTextWall(_: String) async {
|
|
133
137
|
Bridge.log("sendTextWall")
|
|
134
138
|
}
|
|
135
139
|
|
|
136
|
-
func sendDoubleTextWall(_: String, _: String) {
|
|
140
|
+
func sendDoubleTextWall(_: String, _: String) async {
|
|
137
141
|
Bridge.log("sendDoubleTextWall")
|
|
138
142
|
}
|
|
139
143
|
|
|
140
|
-
func displayBitmap(base64ImageData _: String) async -> Bool {
|
|
144
|
+
func displayBitmap(base64ImageData _: String, x _: Int32?, y _: Int32?, width _: Int32?, height _: Int32?) async -> Bool {
|
|
141
145
|
Bridge.log("displayBitmap")
|
|
142
146
|
return false
|
|
143
147
|
}
|