@mentra/bluetooth-sdk 0.1.8 → 0.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +116 -18
- package/android/build.gradle +289 -16
- package/android/gradle.properties.example +9 -0
- package/android/lc3Lib/build.gradle +95 -6
- package/android/settings.gradle +3 -0
- package/android/silero/LICENSE.md +17 -0
- package/android/silero/build.gradle +42 -0
- package/android/silero/proguard-rules.pro +24 -0
- package/android/silero/src/androidTest/assets/hello.wav +0 -0
- package/android/silero/src/androidTest/java/com/konovalov/vad/silero/VadSileroTest.kt +79 -0
- package/android/silero/src/main/assets/LICENSE +21 -0
- package/android/silero/src/main/assets/silero_vad.onnx +0 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/Vad.kt +160 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/VadSilero.kt +449 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/config/FrameSize.kt +15 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/config/Mode.kt +14 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/config/SampleRate.kt +12 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/utils/AudioUtils.kt +47 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/utils/TensorMap.kt +41 -0
- package/android/silero/src/test/java/com/konovalov/vad/silero/utils/AudioUtilsTest.kt +44 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkDefaults.kt +6 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +165 -27
- package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +89 -21
- package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +189 -16
- package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt +22 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +787 -85
- package/android/src/main/java/com/mentra/bluetoothsdk/audio/AudioModels.kt +6 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/camera/CameraModels.kt +123 -10
- package/android/src/main/java/com/mentra/bluetoothsdk/controllers/ControllerManager.kt +12 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/controllers/R1.kt +8 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/debug/BleTraceLogger.kt +33 -17
- package/android/src/main/java/com/mentra/bluetoothsdk/events/BluetoothEvents.kt +101 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/internal/MapParsing.kt +8 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistry.java +108 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoUploadServer.kt +399 -74
- package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/MentraPhotoReceiverModule.kt +100 -17
- package/android/src/main/java/com/mentra/bluetoothsdk/requests/DisplayRequests.kt +17 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G1.java +2 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt +569 -220
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Mach1.java +2 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraLive.java +271 -66
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraNex.kt +176 -104
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraosBle.java +12753 -27036
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt +54 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Simulated.kt +16 -4
- package/android/src/main/java/com/mentra/bluetoothsdk/status/DeviceStatus.kt +60 -3
- package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +12 -3
- package/android/src/main/java/com/mentra/bluetoothsdk/stt/STTTools.kt +87 -36
- package/android/src/main/java/com/mentra/bluetoothsdk/stt/SherpaOnnxTranscriber.kt +13 -4
- package/android/src/main/java/com/mentra/bluetoothsdk/stt/VadGateSpeechPolicy.kt +229 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/tts/TTSTools.kt +180 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/AvifExifStripper.java +573 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadService.java +488 -50
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/HeifExifTagReader.java +199 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunkReassembler.java +18 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunker.java +64 -18
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/NexSGCUtils.kt +48 -39
- package/android/src/test/java/com/mentra/bluetoothsdk/camera/PhotoRequestTest.kt +23 -0
- package/android/src/test/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistryTest.java +75 -0
- package/android/src/test/java/com/mentra/bluetoothsdk/utils/AvifExifStripperTest.java +30 -0
- package/android/src/test/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadServiceTest.java +98 -0
- package/android/src/test/resources/avif_with_exif.avif +0 -0
- package/build/BluetoothSdk.types.d.ts +241 -43
- package/build/BluetoothSdk.types.d.ts.map +1 -1
- package/build/BluetoothSdk.types.js +11 -10
- package/build/BluetoothSdk.types.js.map +1 -1
- package/build/_private/BluetoothSdkModule.d.ts +36 -23
- package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
- package/build/_private/BluetoothSdkModule.js +30 -52
- package/build/_private/BluetoothSdkModule.js.map +1 -1
- package/build/_private/photoRequestPayload.d.ts +4 -0
- package/build/_private/photoRequestPayload.d.ts.map +1 -0
- package/build/_private/photoRequestPayload.js +25 -0
- package/build/_private/photoRequestPayload.js.map +1 -0
- package/build/index.d.ts +1 -1
- package/build/index.d.ts.map +1 -1
- package/build/index.js +23 -0
- package/build/index.js.map +1 -1
- package/build/react/useMentraBluetooth.d.ts +2 -2
- package/build/react/useMentraBluetooth.d.ts.map +1 -1
- package/build/react/useMentraBluetooth.js +3 -3
- package/build/react/useMentraBluetooth.js.map +1 -1
- package/ios/BluetoothSdkModule.swift +174 -70
- package/ios/LocalPhotoUploadServer.swift +370 -105
- package/ios/MentraBluetoothSDK.podspec +3 -1
- package/ios/MentraPhotoReceiverModule.swift +62 -10
- package/ios/Packages/CoreObjC/include/PcmConverter.h +22 -0
- package/ios/Packages/SherpaOnnx/SherpaOnnx.swift +668 -26
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Info.plist +13 -9
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/libsherpa-onnx.a +0 -0
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/libsherpa-onnx.a +0 -0
- package/ios/Source/BluetoothSdkDefaults.swift +6 -0
- package/ios/Source/Bridge.swift +58 -19
- package/ios/Source/DeviceManager.swift +149 -17
- package/ios/Source/DeviceStore.swift +16 -6
- package/ios/Source/MentraBluetoothSDK.swift +803 -47
- package/ios/Source/ObservableStore.swift +8 -2
- package/ios/Source/{Audio → audio}/AudioModels.swift +9 -7
- package/ios/Source/{Camera → camera}/CameraModels.swift +225 -16
- package/ios/Source/controllers/ControllerManager.swift +4 -3
- package/ios/Source/controllers/R1.swift +3 -3
- package/ios/Source/{Events → events}/BluetoothEvents.swift +183 -1
- package/ios/Source/{Internal → internal}/ValueParsing.swift +10 -1
- package/ios/Source/{Requests → requests}/DisplayRequests.swift +28 -4
- package/ios/Source/services/PhoneMic.swift +1 -0
- package/ios/Source/sgcs/G1.swift +2 -2
- package/ios/Source/sgcs/G2.swift +645 -403
- package/ios/Source/sgcs/Mach1.swift +2 -2
- package/ios/Source/sgcs/MentraLive.swift +812 -319
- package/ios/Source/sgcs/MentraNex.swift +412 -182
- package/ios/Source/sgcs/SGCManager.swift +57 -3
- package/ios/Source/sgcs/Simulated.swift +7 -3
- package/ios/Source/sgcs/mentraos_ble.pb.swift +3073 -3134
- package/ios/Source/status/DeviceStatus.swift +974 -0
- package/ios/Source/{Status → status}/WifiHotspotStatus.swift +4 -4
- package/ios/Source/{Streaming → streaming}/StreamModels.swift +24 -9
- package/ios/Source/stt/STTTools.swift +27 -12
- package/ios/Source/stt/SherpaOnnxTranscriber.swift +31 -8
- package/ios/Source/tts/TTSTools.swift +164 -0
- package/ios/Source/utils/JSCExperiment.swift +7 -7
- package/ios/Source/utils/MessageChunkReassembler.swift +20 -1
- package/ios/Source/utils/MessageChunker.swift +78 -21
- package/ios/Source/utils/TarBz2Extractor.swift +53 -2
- package/package.json +1 -1
- package/plugin/build/withAndroid.js +68 -4
- package/src/BluetoothSdk.types.ts +331 -59
- package/src/_private/BluetoothSdkModule.ts +113 -102
- package/src/_private/photoRequestPayload.ts +28 -0
- package/src/index.ts +58 -15
- package/src/react/useMentraBluetooth.ts +6 -4
- package/android/.project +0 -28
- package/android/lc3Lib/.project +0 -28
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/cargs.h +0 -162
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/c-api.h +0 -1852
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/cxx-api.h +0 -674
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/sherpa-onnx.a +0 -0
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/sherpa-onnx.a +0 -0
- package/ios/Source/Status/DeviceStatus.swift +0 -471
- /package/ios/Source/{Connection → connection}/ScanSession.swift +0 -0
- /package/ios/Source/{Errors → errors}/BluetoothError.swift +0 -0
- /package/ios/Source/{Internal → internal}/BluetoothAvailability.swift +0 -0
- /package/ios/Source/{Status → status}/RuntimeState.swift +0 -0
- /package/ios/Source/{Types → types}/DeviceModels.swift +0 -0
|
@@ -20,12 +20,19 @@ protocol SGCManager {
|
|
|
20
20
|
|
|
21
21
|
func requestPhoto(
|
|
22
22
|
_ requestId: String, appId: String, size: String?, webhookUrl: String?, authToken: String?,
|
|
23
|
-
compress: String?, flash: Bool, sound: Bool, exposureTimeNs: Double?, iso: Int?
|
|
23
|
+
compress: String?, flash: Bool, save: Bool, sound: Bool, exposureTimeNs: Double?, iso: Int?
|
|
24
24
|
)
|
|
25
25
|
func startStream(_ message: [String: Any])
|
|
26
26
|
func stopStream()
|
|
27
27
|
func sendStreamKeepAlive(_ message: [String: Any])
|
|
28
28
|
func startVideoRecording(requestId: String, save: Bool, flash: Bool, sound: Bool)
|
|
29
|
+
/// Start video recording with optional per-recording resolution/fps. A width,
|
|
30
|
+
/// height, or fps of 0 means "use the device's saved button-video default".
|
|
31
|
+
/// Defaulted in an extension to delegate to the basic recording path; devices
|
|
32
|
+
/// that support custom settings (e.g. Mentra Live) override this.
|
|
33
|
+
func startVideoRecording(
|
|
34
|
+
requestId: String, save: Bool, flash: Bool, sound: Bool, width: Int, height: Int, fps: Int
|
|
35
|
+
)
|
|
29
36
|
func stopVideoRecording(requestId: String)
|
|
30
37
|
|
|
31
38
|
// MARK: - Button Settings
|
|
@@ -42,7 +49,9 @@ protocol SGCManager {
|
|
|
42
49
|
func clearDisplay()
|
|
43
50
|
func sendTextWall(_ text: String)
|
|
44
51
|
func sendDoubleTextWall(_ top: String, _ bottom: String)
|
|
45
|
-
|
|
52
|
+
/// Display a bitmap. Optional `x`/`y`/`width`/`height` position and size the target
|
|
53
|
+
/// container (used by G2; other SGCs ignore positioning and render the bitmap as before).
|
|
54
|
+
func displayBitmap(base64ImageData: String, x: Int32?, y: Int32?, width: Int32?, height: Int32?) async -> Bool
|
|
46
55
|
func showDashboard()
|
|
47
56
|
func setDashboardPosition(_ height: Int, _ depth: Int)
|
|
48
57
|
/// Default implementation sends both via [setDashboardPosition]; Nex overrides to one protobuf.
|
|
@@ -53,6 +62,18 @@ protocol SGCManager {
|
|
|
53
62
|
|
|
54
63
|
func setDashboardMenu(_ items: [[String: Any]])
|
|
55
64
|
|
|
65
|
+
// MARK: - Notification Panel
|
|
66
|
+
|
|
67
|
+
func showNotificationsPanel()
|
|
68
|
+
|
|
69
|
+
// MARK: - Calendar Events
|
|
70
|
+
|
|
71
|
+
func sendCalendarEvents(_ events: [[String: Any]])
|
|
72
|
+
|
|
73
|
+
// MARK: - Dashboard Display Settings
|
|
74
|
+
|
|
75
|
+
func sendDashboardDisplaySettings()
|
|
76
|
+
|
|
56
77
|
// MARK: - Device Control
|
|
57
78
|
|
|
58
79
|
func setHeadUpAngle(_ angle: Int)
|
|
@@ -89,6 +110,8 @@ protocol SGCManager {
|
|
|
89
110
|
func sendHotspotState(_ enabled: Bool)
|
|
90
111
|
func sendOtaStart()
|
|
91
112
|
func sendOtaQueryStatus()
|
|
113
|
+
func sendSetSystemTime(_ timestampMs: Int64)
|
|
114
|
+
func sendOtaRetryVersionCheck()
|
|
92
115
|
|
|
93
116
|
// MARK: - User Context (for crash reporting)
|
|
94
117
|
|
|
@@ -115,6 +138,15 @@ protocol SGCManager {
|
|
|
115
138
|
/// doesn't seem to work for concurrency reasons :(
|
|
116
139
|
/// we can make read-only getters for convienence though:
|
|
117
140
|
extension SGCManager {
|
|
141
|
+
// MARK: - Video recording (default: ignore custom settings, use saved defaults)
|
|
142
|
+
|
|
143
|
+
func startVideoRecording(
|
|
144
|
+
requestId: String, save: Bool, flash: Bool, sound: Bool, width _: Int, height _: Int,
|
|
145
|
+
fps _: Int
|
|
146
|
+
) {
|
|
147
|
+
startVideoRecording(requestId: requestId, save: save, flash: flash, sound: sound)
|
|
148
|
+
}
|
|
149
|
+
|
|
118
150
|
// MARK: - Dashboard (default: combined wire format; Nex implements single-field)
|
|
119
151
|
|
|
120
152
|
func setDashboardHeightOnly(_ height: Int) {
|
|
@@ -131,10 +163,31 @@ extension SGCManager {
|
|
|
131
163
|
|
|
132
164
|
func setDashboardMenu(_: [[String: Any]]) {}
|
|
133
165
|
|
|
166
|
+
// MARK: - Notification Panel (default no-op — only G2 supports this)
|
|
167
|
+
|
|
168
|
+
func showNotificationsPanel() {}
|
|
169
|
+
|
|
170
|
+
// MARK: - Calendar Events (default no-op — only G2 supports this)
|
|
171
|
+
|
|
172
|
+
func sendCalendarEvents(_: [[String: Any]]) {}
|
|
173
|
+
|
|
174
|
+
// MARK: - Dashboard Display Settings (default no-op — only G2 supports this)
|
|
175
|
+
|
|
176
|
+
func sendDashboardDisplaySettings() {}
|
|
177
|
+
|
|
134
178
|
// MARK: - Voice Activity Detection (default no-op — Mentra Live supports this)
|
|
135
179
|
|
|
136
180
|
func sendVoiceActivityDetectionSetting() {}
|
|
137
181
|
|
|
182
|
+
/// Default no-op; Mentra Live overrides when phone detects clock skew during gallery sync.
|
|
183
|
+
func sendSetSystemTime(_: Int64) {
|
|
184
|
+
Bridge.log("SGC: sendSetSystemTime not supported")
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
func sendOtaRetryVersionCheck() {
|
|
188
|
+
Bridge.log("SGC: sendOtaRetryVersionCheck not supported")
|
|
189
|
+
}
|
|
190
|
+
|
|
138
191
|
// MARK: - Default DeviceStore-backed property implementations
|
|
139
192
|
|
|
140
193
|
var fullyBooted: Bool {
|
|
@@ -190,7 +243,8 @@ extension SGCManager {
|
|
|
190
243
|
}
|
|
191
244
|
|
|
192
245
|
var voiceActivityDetectionEnabled: Bool {
|
|
193
|
-
DeviceStore.shared.get("glasses", "voiceActivityDetectionEnabled") as? Bool
|
|
246
|
+
DeviceStore.shared.get("glasses", "voiceActivityDetectionEnabled") as? Bool
|
|
247
|
+
?? BluetoothSdkDefaults.voiceActivityDetectionEnabled
|
|
194
248
|
}
|
|
195
249
|
|
|
196
250
|
var batteryLevel: Int {
|
|
@@ -12,7 +12,11 @@ class Simulated: SGCManager {
|
|
|
12
12
|
DeviceStore.shared.apply("glasses", "connected", true)
|
|
13
13
|
DeviceStore.shared.apply("glasses", "connectionState", ConnTypes.CONNECTED)
|
|
14
14
|
DeviceStore.shared.apply("glasses", "micEnabled", false)
|
|
15
|
-
DeviceStore.shared.apply(
|
|
15
|
+
DeviceStore.shared.apply(
|
|
16
|
+
"glasses",
|
|
17
|
+
"voiceActivityDetectionEnabled",
|
|
18
|
+
BluetoothSdkDefaults.voiceActivityDetectionEnabled
|
|
19
|
+
)
|
|
16
20
|
DeviceStore.shared.apply("glasses", "bluetoothClassicConnected", false)
|
|
17
21
|
}
|
|
18
22
|
|
|
@@ -75,7 +79,7 @@ class Simulated: SGCManager {
|
|
|
75
79
|
|
|
76
80
|
// MARK: - Camera & Media
|
|
77
81
|
|
|
78
|
-
func requestPhoto(_: String, appId _: String, size _: String?, webhookUrl _: String?, authToken _: String?, compress _: String?, flash _: Bool, sound _: Bool, exposureTimeNs _: Double?, iso _: Int?) {
|
|
82
|
+
func requestPhoto(_: String, appId _: String, size _: String?, webhookUrl _: String?, authToken _: String?, compress _: String?, flash _: Bool, save _: Bool, sound _: Bool, exposureTimeNs _: Double?, iso _: Int?) {
|
|
79
83
|
Bridge.log("requestPhoto")
|
|
80
84
|
}
|
|
81
85
|
|
|
@@ -137,7 +141,7 @@ class Simulated: SGCManager {
|
|
|
137
141
|
Bridge.log("sendDoubleTextWall")
|
|
138
142
|
}
|
|
139
143
|
|
|
140
|
-
func displayBitmap(base64ImageData _: String) async -> Bool {
|
|
144
|
+
func displayBitmap(base64ImageData _: String, x _: Int32?, y _: Int32?, width _: Int32?, height _: Int32?) async -> Bool {
|
|
141
145
|
Bridge.log("displayBitmap")
|
|
142
146
|
return false
|
|
143
147
|
}
|