@mentra/bluetooth-sdk 0.1.8 → 0.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +116 -18
- package/android/build.gradle +289 -16
- package/android/gradle.properties.example +9 -0
- package/android/lc3Lib/build.gradle +95 -6
- package/android/settings.gradle +3 -0
- package/android/silero/LICENSE.md +17 -0
- package/android/silero/build.gradle +42 -0
- package/android/silero/proguard-rules.pro +24 -0
- package/android/silero/src/androidTest/assets/hello.wav +0 -0
- package/android/silero/src/androidTest/java/com/konovalov/vad/silero/VadSileroTest.kt +79 -0
- package/android/silero/src/main/assets/LICENSE +21 -0
- package/android/silero/src/main/assets/silero_vad.onnx +0 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/Vad.kt +160 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/VadSilero.kt +449 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/config/FrameSize.kt +15 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/config/Mode.kt +14 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/config/SampleRate.kt +12 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/utils/AudioUtils.kt +47 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/utils/TensorMap.kt +41 -0
- package/android/silero/src/test/java/com/konovalov/vad/silero/utils/AudioUtilsTest.kt +44 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkDefaults.kt +6 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +165 -27
- package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +89 -21
- package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +189 -16
- package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt +22 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +787 -85
- package/android/src/main/java/com/mentra/bluetoothsdk/audio/AudioModels.kt +6 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/camera/CameraModels.kt +123 -10
- package/android/src/main/java/com/mentra/bluetoothsdk/controllers/ControllerManager.kt +12 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/controllers/R1.kt +8 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/debug/BleTraceLogger.kt +33 -17
- package/android/src/main/java/com/mentra/bluetoothsdk/events/BluetoothEvents.kt +101 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/internal/MapParsing.kt +8 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistry.java +108 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoUploadServer.kt +399 -74
- package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/MentraPhotoReceiverModule.kt +100 -17
- package/android/src/main/java/com/mentra/bluetoothsdk/requests/DisplayRequests.kt +17 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G1.java +2 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt +569 -220
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Mach1.java +2 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraLive.java +271 -66
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraNex.kt +176 -104
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraosBle.java +12753 -27036
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt +54 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Simulated.kt +16 -4
- package/android/src/main/java/com/mentra/bluetoothsdk/status/DeviceStatus.kt +60 -3
- package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +12 -3
- package/android/src/main/java/com/mentra/bluetoothsdk/stt/STTTools.kt +87 -36
- package/android/src/main/java/com/mentra/bluetoothsdk/stt/SherpaOnnxTranscriber.kt +13 -4
- package/android/src/main/java/com/mentra/bluetoothsdk/stt/VadGateSpeechPolicy.kt +229 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/tts/TTSTools.kt +180 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/AvifExifStripper.java +573 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadService.java +488 -50
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/HeifExifTagReader.java +199 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunkReassembler.java +18 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunker.java +64 -18
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/NexSGCUtils.kt +48 -39
- package/android/src/test/java/com/mentra/bluetoothsdk/camera/PhotoRequestTest.kt +23 -0
- package/android/src/test/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistryTest.java +75 -0
- package/android/src/test/java/com/mentra/bluetoothsdk/utils/AvifExifStripperTest.java +30 -0
- package/android/src/test/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadServiceTest.java +98 -0
- package/android/src/test/resources/avif_with_exif.avif +0 -0
- package/build/BluetoothSdk.types.d.ts +241 -43
- package/build/BluetoothSdk.types.d.ts.map +1 -1
- package/build/BluetoothSdk.types.js +11 -10
- package/build/BluetoothSdk.types.js.map +1 -1
- package/build/_private/BluetoothSdkModule.d.ts +36 -23
- package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
- package/build/_private/BluetoothSdkModule.js +30 -52
- package/build/_private/BluetoothSdkModule.js.map +1 -1
- package/build/_private/photoRequestPayload.d.ts +4 -0
- package/build/_private/photoRequestPayload.d.ts.map +1 -0
- package/build/_private/photoRequestPayload.js +25 -0
- package/build/_private/photoRequestPayload.js.map +1 -0
- package/build/index.d.ts +1 -1
- package/build/index.d.ts.map +1 -1
- package/build/index.js +23 -0
- package/build/index.js.map +1 -1
- package/build/react/useMentraBluetooth.d.ts +2 -2
- package/build/react/useMentraBluetooth.d.ts.map +1 -1
- package/build/react/useMentraBluetooth.js +3 -3
- package/build/react/useMentraBluetooth.js.map +1 -1
- package/ios/BluetoothSdkModule.swift +174 -70
- package/ios/LocalPhotoUploadServer.swift +370 -105
- package/ios/MentraBluetoothSDK.podspec +3 -1
- package/ios/MentraPhotoReceiverModule.swift +62 -10
- package/ios/Packages/CoreObjC/include/PcmConverter.h +22 -0
- package/ios/Packages/SherpaOnnx/SherpaOnnx.swift +668 -26
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Info.plist +13 -9
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/libsherpa-onnx.a +0 -0
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/libsherpa-onnx.a +0 -0
- package/ios/Source/BluetoothSdkDefaults.swift +6 -0
- package/ios/Source/Bridge.swift +58 -19
- package/ios/Source/DeviceManager.swift +149 -17
- package/ios/Source/DeviceStore.swift +16 -6
- package/ios/Source/MentraBluetoothSDK.swift +803 -47
- package/ios/Source/ObservableStore.swift +8 -2
- package/ios/Source/{Audio → audio}/AudioModels.swift +9 -7
- package/ios/Source/{Camera → camera}/CameraModels.swift +225 -16
- package/ios/Source/controllers/ControllerManager.swift +4 -3
- package/ios/Source/controllers/R1.swift +3 -3
- package/ios/Source/{Events → events}/BluetoothEvents.swift +183 -1
- package/ios/Source/{Internal → internal}/ValueParsing.swift +10 -1
- package/ios/Source/{Requests → requests}/DisplayRequests.swift +28 -4
- package/ios/Source/services/PhoneMic.swift +1 -0
- package/ios/Source/sgcs/G1.swift +2 -2
- package/ios/Source/sgcs/G2.swift +645 -403
- package/ios/Source/sgcs/Mach1.swift +2 -2
- package/ios/Source/sgcs/MentraLive.swift +812 -319
- package/ios/Source/sgcs/MentraNex.swift +412 -182
- package/ios/Source/sgcs/SGCManager.swift +57 -3
- package/ios/Source/sgcs/Simulated.swift +7 -3
- package/ios/Source/sgcs/mentraos_ble.pb.swift +3073 -3134
- package/ios/Source/status/DeviceStatus.swift +974 -0
- package/ios/Source/{Status → status}/WifiHotspotStatus.swift +4 -4
- package/ios/Source/{Streaming → streaming}/StreamModels.swift +24 -9
- package/ios/Source/stt/STTTools.swift +27 -12
- package/ios/Source/stt/SherpaOnnxTranscriber.swift +31 -8
- package/ios/Source/tts/TTSTools.swift +164 -0
- package/ios/Source/utils/JSCExperiment.swift +7 -7
- package/ios/Source/utils/MessageChunkReassembler.swift +20 -1
- package/ios/Source/utils/MessageChunker.swift +78 -21
- package/ios/Source/utils/TarBz2Extractor.swift +53 -2
- package/package.json +1 -1
- package/plugin/build/withAndroid.js +68 -4
- package/src/BluetoothSdk.types.ts +331 -59
- package/src/_private/BluetoothSdkModule.ts +113 -102
- package/src/_private/photoRequestPayload.ts +28 -0
- package/src/index.ts +58 -15
- package/src/react/useMentraBluetooth.ts +6 -4
- package/android/.project +0 -28
- package/android/lc3Lib/.project +0 -28
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/cargs.h +0 -162
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/c-api.h +0 -1852
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/cxx-api.h +0 -674
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/sherpa-onnx.a +0 -0
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/sherpa-onnx.a +0 -0
- package/ios/Source/Status/DeviceStatus.swift +0 -471
- /package/ios/Source/{Connection → connection}/ScanSession.swift +0 -0
- /package/ios/Source/{Errors → errors}/BluetoothError.swift +0 -0
- /package/ios/Source/{Internal → internal}/BluetoothAvailability.swift +0 -0
- /package/ios/Source/{Status → status}/RuntimeState.swift +0 -0
- /package/ios/Source/{Types → types}/DeviceModels.swift +0 -0
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
package com.mentra.bluetoothsdk.sgcs
|
|
2
2
|
|
|
3
|
+
import com.mentra.bluetoothsdk.BluetoothSdkDefaults
|
|
4
|
+
import com.mentra.bluetoothsdk.Bridge
|
|
3
5
|
import com.mentra.bluetoothsdk.DeviceStore
|
|
4
6
|
import com.mentra.bluetoothsdk.utils.ConnTypes
|
|
5
7
|
|
|
@@ -21,6 +23,7 @@ abstract class SGCManager {
|
|
|
21
23
|
authToken: String?,
|
|
22
24
|
compress: String?,
|
|
23
25
|
flash: Boolean,
|
|
26
|
+
save: Boolean,
|
|
24
27
|
sound: Boolean,
|
|
25
28
|
exposureTimeNs: Long?,
|
|
26
29
|
iso: Int?,
|
|
@@ -29,6 +32,26 @@ abstract class SGCManager {
|
|
|
29
32
|
abstract fun stopStream()
|
|
30
33
|
abstract fun sendStreamKeepAlive(message: MutableMap<String, Any>)
|
|
31
34
|
abstract fun startVideoRecording(requestId: String, save: Boolean, flash: Boolean, sound: Boolean)
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Start video recording with optional per-recording resolution/fps. A width,
|
|
38
|
+
* height, or fps of 0 means "use the device's saved button-video default".
|
|
39
|
+
* The base implementation ignores the settings and delegates to the default
|
|
40
|
+
* recording path; devices that support custom settings (e.g. Mentra Live)
|
|
41
|
+
* override this.
|
|
42
|
+
*/
|
|
43
|
+
open fun startVideoRecording(
|
|
44
|
+
requestId: String,
|
|
45
|
+
save: Boolean,
|
|
46
|
+
flash: Boolean,
|
|
47
|
+
sound: Boolean,
|
|
48
|
+
width: Int,
|
|
49
|
+
height: Int,
|
|
50
|
+
fps: Int,
|
|
51
|
+
) {
|
|
52
|
+
startVideoRecording(requestId, save, flash, sound)
|
|
53
|
+
}
|
|
54
|
+
|
|
32
55
|
abstract fun stopVideoRecording(requestId: String)
|
|
33
56
|
|
|
34
57
|
// Button Settings
|
|
@@ -43,7 +66,17 @@ abstract class SGCManager {
|
|
|
43
66
|
abstract fun clearDisplay()
|
|
44
67
|
abstract fun sendTextWall(text: String)
|
|
45
68
|
abstract fun sendDoubleTextWall(top: String, bottom: String)
|
|
46
|
-
|
|
69
|
+
/**
|
|
70
|
+
* Display a bitmap. Optional [x]/[y]/[width]/[height] position and size the target
|
|
71
|
+
* container (used by G2; other SGCs ignore positioning and render the bitmap as before).
|
|
72
|
+
*/
|
|
73
|
+
abstract fun displayBitmap(
|
|
74
|
+
base64ImageData: String,
|
|
75
|
+
x: Int? = null,
|
|
76
|
+
y: Int? = null,
|
|
77
|
+
width: Int? = null,
|
|
78
|
+
height: Int? = null
|
|
79
|
+
): Boolean
|
|
47
80
|
abstract fun showDashboard()
|
|
48
81
|
abstract fun setDashboardPosition(height: Int, depth: Int)
|
|
49
82
|
|
|
@@ -62,6 +95,15 @@ abstract class SGCManager {
|
|
|
62
95
|
// Dashboard Menu (default no-op — only G2 supports this)
|
|
63
96
|
open fun setDashboardMenu(items: List<Map<String, Any>>) {}
|
|
64
97
|
|
|
98
|
+
// Calendar Events (default no-op — only G2 supports this)
|
|
99
|
+
open fun sendCalendarEvents(events: List<Map<String, Any>>) {}
|
|
100
|
+
|
|
101
|
+
// Dashboard display settings (default no-op — only G2 supports this)
|
|
102
|
+
open fun sendDashboardDisplaySettings() {}
|
|
103
|
+
|
|
104
|
+
// Notification Panel (default no-op — only G2 supports this)
|
|
105
|
+
open fun showNotificationsPanel() {}
|
|
106
|
+
|
|
65
107
|
// Controller bridging (default no-op — only G2 supports pairing with a ring controller)
|
|
66
108
|
open fun connectController() {}
|
|
67
109
|
open fun disconnectController() {}
|
|
@@ -101,6 +143,11 @@ abstract class SGCManager {
|
|
|
101
143
|
abstract fun forgetWifiNetwork(ssid: String)
|
|
102
144
|
abstract fun sendHotspotState(enabled: Boolean)
|
|
103
145
|
|
|
146
|
+
/** Set glasses system clock (Mentra Live only; no-op on other devices). */
|
|
147
|
+
open fun sendSetSystemTime(timestampMs: Long) {
|
|
148
|
+
Bridge.log("SGC: sendSetSystemTime not supported on $type")
|
|
149
|
+
}
|
|
150
|
+
|
|
104
151
|
// User Context (for crash reporting)
|
|
105
152
|
abstract fun sendUserEmailToGlasses(email: String)
|
|
106
153
|
|
|
@@ -114,6 +161,9 @@ abstract class SGCManager {
|
|
|
114
161
|
// Voice Activity Detection
|
|
115
162
|
open fun sendVoiceActivityDetectionSetting() {}
|
|
116
163
|
|
|
164
|
+
// Start/stop LC3 audio playback from glasses based on the nex_audio_playback flag.
|
|
165
|
+
open fun applyNexAudioPlaybackSetting() {}
|
|
166
|
+
|
|
117
167
|
// Version info
|
|
118
168
|
abstract fun requestVersionInfo()
|
|
119
169
|
|
|
@@ -161,7 +211,9 @@ abstract class SGCManager {
|
|
|
161
211
|
get() = DeviceStore.get("glasses", "micEnabled") as? Boolean ?: false
|
|
162
212
|
|
|
163
213
|
val voiceActivityDetectionEnabled: Boolean
|
|
164
|
-
get() =
|
|
214
|
+
get() =
|
|
215
|
+
DeviceStore.get("glasses", "voiceActivityDetectionEnabled") as? Boolean
|
|
216
|
+
?: BluetoothSdkDefaults.VOICE_ACTIVITY_DETECTION_ENABLED
|
|
165
217
|
|
|
166
218
|
val batteryLevel: Int
|
|
167
219
|
get() = DeviceStore.get("glasses", "batteryLevel") as? Int ?: -1
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
package com.mentra.bluetoothsdk.sgcs
|
|
2
2
|
|
|
3
|
+
import com.mentra.bluetoothsdk.BluetoothSdkDefaults
|
|
3
4
|
import com.mentra.bluetoothsdk.Bridge
|
|
4
5
|
import com.mentra.bluetoothsdk.DeviceManager
|
|
6
|
+
import com.mentra.bluetoothsdk.DeviceStore
|
|
5
7
|
import com.mentra.bluetoothsdk.utils.ConnTypes
|
|
6
8
|
import com.mentra.bluetoothsdk.utils.DeviceTypes
|
|
7
|
-
import com.mentra.bluetoothsdk.DeviceStore
|
|
8
9
|
|
|
9
10
|
class Simulated : SGCManager() {
|
|
10
11
|
|
|
@@ -14,7 +15,11 @@ class Simulated : SGCManager() {
|
|
|
14
15
|
DeviceStore.apply("glasses", "connected", true)
|
|
15
16
|
DeviceStore.apply("glasses", "connectionState", ConnTypes.CONNECTED)
|
|
16
17
|
DeviceStore.apply("glasses", "micEnabled", false)
|
|
17
|
-
DeviceStore.apply(
|
|
18
|
+
DeviceStore.apply(
|
|
19
|
+
"glasses",
|
|
20
|
+
"voiceActivityDetectionEnabled",
|
|
21
|
+
BluetoothSdkDefaults.VOICE_ACTIVITY_DETECTION_ENABLED
|
|
22
|
+
)
|
|
18
23
|
}
|
|
19
24
|
|
|
20
25
|
// Audio Control
|
|
@@ -35,11 +40,12 @@ class Simulated : SGCManager() {
|
|
|
35
40
|
authToken: String?,
|
|
36
41
|
compress: String?,
|
|
37
42
|
flash: Boolean,
|
|
43
|
+
save: Boolean,
|
|
38
44
|
sound: Boolean,
|
|
39
45
|
exposureTimeNs: Long?,
|
|
40
46
|
iso: Int?,
|
|
41
47
|
) {
|
|
42
|
-
Bridge.log("requestPhoto flash=$flash, sound=$sound")
|
|
48
|
+
Bridge.log("requestPhoto flash=$flash, save=$save, sound=$sound")
|
|
43
49
|
}
|
|
44
50
|
|
|
45
51
|
override fun startStream(message: MutableMap<String, Any>) {
|
|
@@ -100,7 +106,13 @@ class Simulated : SGCManager() {
|
|
|
100
106
|
Bridge.log("sendDoubleTextWall")
|
|
101
107
|
}
|
|
102
108
|
|
|
103
|
-
override fun displayBitmap(
|
|
109
|
+
override fun displayBitmap(
|
|
110
|
+
base64ImageData: String,
|
|
111
|
+
x: Int?,
|
|
112
|
+
y: Int?,
|
|
113
|
+
width: Int?,
|
|
114
|
+
height: Int?
|
|
115
|
+
): Boolean {
|
|
104
116
|
Bridge.log("displayBitmap")
|
|
105
117
|
return false
|
|
106
118
|
}
|
|
@@ -47,6 +47,7 @@ internal data class GlassesStatus(
|
|
|
47
47
|
val rightMacAddress: String,
|
|
48
48
|
val macAddress: String,
|
|
49
49
|
val buildNumber: String,
|
|
50
|
+
val systemTimeMs: Long?,
|
|
50
51
|
val otaVersionUrl: String,
|
|
51
52
|
val appVersion: String,
|
|
52
53
|
val bluetoothName: String,
|
|
@@ -69,8 +70,9 @@ internal data class GlassesStatus(
|
|
|
69
70
|
val controllerSignalStrength: Int,
|
|
70
71
|
val ringSignalStrength: Int,
|
|
71
72
|
) {
|
|
72
|
-
internal fun toMap(): Map<String, Any>
|
|
73
|
-
|
|
73
|
+
internal fun toMap(): Map<String, Any> {
|
|
74
|
+
val values =
|
|
75
|
+
mutableMapOf<String, Any>(
|
|
74
76
|
"connection" to connectionState.toStatusMap(connected, fullyBooted),
|
|
75
77
|
"micEnabled" to micEnabled,
|
|
76
78
|
"voiceActivityDetectionEnabled" to voiceActivityDetectionEnabled,
|
|
@@ -109,6 +111,9 @@ internal data class GlassesStatus(
|
|
|
109
111
|
"controllerSignalStrength" to controllerSignalStrength,
|
|
110
112
|
"ringSignalStrength" to ringSignalStrength,
|
|
111
113
|
)
|
|
114
|
+
systemTimeMs?.let { values["systemTimeMs"] = it }
|
|
115
|
+
return values
|
|
116
|
+
}
|
|
112
117
|
|
|
113
118
|
companion object {
|
|
114
119
|
internal fun fromMap(values: Map<String, Any>): GlassesStatus =
|
|
@@ -116,7 +121,9 @@ internal data class GlassesStatus(
|
|
|
116
121
|
fullyBooted = boolValue(values, "fullyBooted") ?: false,
|
|
117
122
|
connected = boolValue(values, "connected") ?: false,
|
|
118
123
|
micEnabled = boolValue(values, "micEnabled") ?: false,
|
|
119
|
-
voiceActivityDetectionEnabled =
|
|
124
|
+
voiceActivityDetectionEnabled =
|
|
125
|
+
boolValue(values, "voiceActivityDetectionEnabled")
|
|
126
|
+
?: BluetoothSdkDefaults.VOICE_ACTIVITY_DETECTION_ENABLED,
|
|
120
127
|
connectionState = GlassesConnectionState.fromValue(stringValue(values, "connectionState")),
|
|
121
128
|
bluetoothClassicConnected = boolValue(values, "bluetoothClassicConnected") ?: false,
|
|
122
129
|
signalStrength = numberValue(values, "signalStrength") ?: -1,
|
|
@@ -131,6 +138,7 @@ internal data class GlassesStatus(
|
|
|
131
138
|
rightMacAddress = stringValue(values, "rightMacAddress") ?: "",
|
|
132
139
|
macAddress = stringValue(values, "macAddress") ?: "",
|
|
133
140
|
buildNumber = stringValue(values, "buildNumber") ?: "",
|
|
141
|
+
systemTimeMs = longValue(values, "systemTimeMs"),
|
|
134
142
|
otaVersionUrl = stringValue(values, "otaVersionUrl") ?: "",
|
|
135
143
|
appVersion = stringValue(values, "appVersion") ?: "",
|
|
136
144
|
bluetoothName = stringValue(values, "bluetoothName") ?: "",
|
|
@@ -156,6 +164,55 @@ internal data class GlassesStatus(
|
|
|
156
164
|
}
|
|
157
165
|
}
|
|
158
166
|
|
|
167
|
+
data class VersionInfoResult(
|
|
168
|
+
val androidVersion: String,
|
|
169
|
+
val firmwareVersion: String,
|
|
170
|
+
val besFirmwareVersion: String,
|
|
171
|
+
val mtkFirmwareVersion: String,
|
|
172
|
+
val buildNumber: String,
|
|
173
|
+
val systemTimeMs: Long?,
|
|
174
|
+
val otaVersionUrl: String,
|
|
175
|
+
val appVersion: String,
|
|
176
|
+
) {
|
|
177
|
+
internal fun toMap(): Map<String, Any> =
|
|
178
|
+
buildMap {
|
|
179
|
+
put("androidVersion", androidVersion)
|
|
180
|
+
put("firmwareVersion", firmwareVersion)
|
|
181
|
+
put("besFirmwareVersion", besFirmwareVersion)
|
|
182
|
+
put("mtkFirmwareVersion", mtkFirmwareVersion)
|
|
183
|
+
put("buildNumber", buildNumber)
|
|
184
|
+
systemTimeMs?.let { put("systemTimeMs", it) }
|
|
185
|
+
put("otaVersionUrl", otaVersionUrl)
|
|
186
|
+
put("appVersion", appVersion)
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
companion object {
|
|
190
|
+
internal fun from(status: GlassesStatus): VersionInfoResult =
|
|
191
|
+
VersionInfoResult(
|
|
192
|
+
androidVersion = status.androidVersion,
|
|
193
|
+
firmwareVersion = status.firmwareVersion,
|
|
194
|
+
besFirmwareVersion = status.besFirmwareVersion,
|
|
195
|
+
mtkFirmwareVersion = status.mtkFirmwareVersion,
|
|
196
|
+
buildNumber = status.buildNumber,
|
|
197
|
+
systemTimeMs = status.systemTimeMs,
|
|
198
|
+
otaVersionUrl = status.otaVersionUrl,
|
|
199
|
+
appVersion = status.appVersion,
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
internal fun fromMap(values: Map<String, Any>): VersionInfoResult =
|
|
203
|
+
VersionInfoResult(
|
|
204
|
+
androidVersion = stringValue(values, "androidVersion", "android_version") ?: "",
|
|
205
|
+
firmwareVersion = stringValue(values, "firmwareVersion", "firmware_version") ?: "",
|
|
206
|
+
besFirmwareVersion = stringValue(values, "besFirmwareVersion", "bes_fw_version") ?: "",
|
|
207
|
+
mtkFirmwareVersion = stringValue(values, "mtkFirmwareVersion", "mtk_fw_version") ?: "",
|
|
208
|
+
buildNumber = stringValue(values, "buildNumber", "build_number") ?: "",
|
|
209
|
+
systemTimeMs = longValue(values, "systemTimeMs", "system_time_ms"),
|
|
210
|
+
otaVersionUrl = stringValue(values, "otaVersionUrl", "ota_version_url") ?: "",
|
|
211
|
+
appVersion = stringValue(values, "appVersion", "app_version") ?: "",
|
|
212
|
+
)
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
|
|
159
216
|
internal data class BluetoothStatus(
|
|
160
217
|
val searching: Boolean,
|
|
161
218
|
val searchingController: Boolean,
|
|
@@ -56,13 +56,20 @@ data class StreamAudioConfig @JvmOverloads constructor(
|
|
|
56
56
|
}
|
|
57
57
|
}
|
|
58
58
|
|
|
59
|
+
/** Effective video settings reported by the glasses after defaults and clamps. */
|
|
59
60
|
data class StreamResolvedVideoConfig @JvmOverloads constructor(
|
|
61
|
+
/** Encoded output width sent to the stream endpoint. */
|
|
60
62
|
val width: Int,
|
|
63
|
+
/** Encoded output height sent to the stream endpoint. */
|
|
61
64
|
val height: Int,
|
|
65
|
+
/** Native camera buffer width selected before crop/downscale. */
|
|
62
66
|
val captureWidth: Int? = null,
|
|
67
|
+
/** Native camera buffer height selected before crop/downscale. */
|
|
63
68
|
val captureHeight: Int? = null,
|
|
69
|
+
/** Encoded video bitrate in bits per second. */
|
|
64
70
|
val bitrate: Int,
|
|
65
|
-
|
|
71
|
+
/** Resolved capture/encode frame rate. */
|
|
72
|
+
val fps: Double,
|
|
66
73
|
) {
|
|
67
74
|
fun toMap(): Map<String, Any> =
|
|
68
75
|
buildMap {
|
|
@@ -84,7 +91,7 @@ data class StreamResolvedVideoConfig @JvmOverloads constructor(
|
|
|
84
91
|
captureWidth = numberValue(values, "captureWidth"),
|
|
85
92
|
captureHeight = numberValue(values, "captureHeight"),
|
|
86
93
|
bitrate = numberValue(values, "bitrate") ?: return null,
|
|
87
|
-
fps =
|
|
94
|
+
fps = doubleValue(values, "fps") ?: return null,
|
|
88
95
|
)
|
|
89
96
|
}
|
|
90
97
|
}
|
|
@@ -377,12 +384,14 @@ sealed interface StreamStatus {
|
|
|
377
384
|
val resolvedConfig = StreamResolvedConfig.fromMap(stringMapValue(values["resolvedConfig"]))
|
|
378
385
|
val attempt = numberValue(values, "attempt")
|
|
379
386
|
val maxAttempts = numberValue(values, "maxAttempts") ?: 0
|
|
387
|
+
val parsedState = StreamState.fromValue(rawState)
|
|
380
388
|
|
|
381
389
|
if (streaming != null || hasAnyKey(values, "reconnecting")) {
|
|
382
390
|
return Snapshot(
|
|
383
391
|
state = when {
|
|
384
392
|
reconnecting -> StreamState.RECONNECTING
|
|
385
393
|
streaming == true -> StreamState.STREAMING
|
|
394
|
+
parsedState != null -> parsedState
|
|
386
395
|
else -> StreamState.STOPPED
|
|
387
396
|
},
|
|
388
397
|
streaming = streaming == true,
|
|
@@ -394,7 +403,7 @@ sealed interface StreamStatus {
|
|
|
394
403
|
)
|
|
395
404
|
}
|
|
396
405
|
|
|
397
|
-
val state =
|
|
406
|
+
val state = parsedState
|
|
398
407
|
?: return Error(
|
|
399
408
|
streamId = streamId,
|
|
400
409
|
errorDetails = rawState?.let { "Unknown stream status: $it" } ?: "Missing stream status",
|
|
@@ -8,9 +8,56 @@ import java.io.BufferedOutputStream
|
|
|
8
8
|
import java.io.File
|
|
9
9
|
import java.io.FileInputStream
|
|
10
10
|
import java.io.FileOutputStream
|
|
11
|
+
import java.io.InputStream
|
|
11
12
|
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
|
|
12
13
|
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream
|
|
13
14
|
|
|
15
|
+
/**
|
|
16
|
+
* Wraps an InputStream so we can observe how many compressed bytes have been
|
|
17
|
+
* consumed from the source .tar.bz2. Used to report extraction progress to JS
|
|
18
|
+
* — bz2 reads sequentially, so bytes-consumed/total is a reasonable proxy for
|
|
19
|
+
* "% of work done".
|
|
20
|
+
*/
|
|
21
|
+
private class ProgressInputStream(
|
|
22
|
+
private val wrapped: InputStream,
|
|
23
|
+
private val totalBytes: Long,
|
|
24
|
+
private val onProgress: (Long, Long) -> Unit,
|
|
25
|
+
) : InputStream() {
|
|
26
|
+
private var bytesRead: Long = 0
|
|
27
|
+
private var lastEmittedAt: Long = 0
|
|
28
|
+
private val emitIntervalMs = 200L
|
|
29
|
+
|
|
30
|
+
override fun read(): Int {
|
|
31
|
+
val byte = wrapped.read()
|
|
32
|
+
if (byte != -1) {
|
|
33
|
+
bytesRead++
|
|
34
|
+
maybeEmit()
|
|
35
|
+
}
|
|
36
|
+
return byte
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
override fun read(b: ByteArray, off: Int, len: Int): Int {
|
|
40
|
+
val n = wrapped.read(b, off, len)
|
|
41
|
+
if (n > 0) {
|
|
42
|
+
bytesRead += n
|
|
43
|
+
maybeEmit()
|
|
44
|
+
}
|
|
45
|
+
return n
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
override fun close() {
|
|
49
|
+
wrapped.close()
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
private fun maybeEmit() {
|
|
53
|
+
val now = System.currentTimeMillis()
|
|
54
|
+
if (now - lastEmittedAt >= emitIntervalMs) {
|
|
55
|
+
lastEmittedAt = now
|
|
56
|
+
onProgress(bytesRead, totalBytes)
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
14
61
|
/**
|
|
15
62
|
* STTTools provides utilities for STT model management.
|
|
16
63
|
*
|
|
@@ -85,21 +132,26 @@ object STTTools {
|
|
|
85
132
|
}
|
|
86
133
|
|
|
87
134
|
// Check for CTC model
|
|
88
|
-
val ctcModelFile =
|
|
135
|
+
val ctcModelFile = findReadableFile(modelDir, listOf("model.int8.onnx", "model.onnx"))
|
|
89
136
|
if (ctcModelFile.exists() && ctcModelFile.canRead() && ctcModelFile.length() > 0) {
|
|
90
137
|
Bridge.log("STT CTC model found at: $path (size: ${ctcModelFile.length()} bytes)")
|
|
91
138
|
return true
|
|
92
139
|
}
|
|
93
140
|
|
|
94
141
|
// Check for transducer model
|
|
95
|
-
val transducerFiles =
|
|
142
|
+
val transducerFiles =
|
|
143
|
+
listOf(
|
|
144
|
+
"encoder" to listOf("encoder.onnx", "encoder.int8.onnx"),
|
|
145
|
+
"decoder" to listOf("decoder.onnx", "decoder.int8.onnx"),
|
|
146
|
+
"joiner" to listOf("joiner.onnx", "joiner.int8.onnx")
|
|
147
|
+
)
|
|
96
148
|
val allTransducerFilesPresent =
|
|
97
|
-
transducerFiles.all {
|
|
98
|
-
val file =
|
|
149
|
+
transducerFiles.all { (label, candidates) ->
|
|
150
|
+
val file = findReadableFile(modelDir, candidates)
|
|
99
151
|
val exists = file.exists() && file.canRead() && file.length() > 0
|
|
100
152
|
if (!exists) {
|
|
101
153
|
Bridge.log(
|
|
102
|
-
"STT model missing or invalid transducer file: $
|
|
154
|
+
"STT model missing or invalid transducer file: $label at $path"
|
|
103
155
|
)
|
|
104
156
|
}
|
|
105
157
|
exists
|
|
@@ -153,9 +205,17 @@ object STTTools {
|
|
|
153
205
|
var fileCount = 0
|
|
154
206
|
var bytesExtracted = 0L
|
|
155
207
|
|
|
208
|
+
val totalCompressedBytes = sourceFile.length()
|
|
209
|
+
Bridge.sendExtractionProgress(0, 0, totalCompressedBytes)
|
|
210
|
+
|
|
156
211
|
// Open the tar.bz2 file with buffered streams for better performance
|
|
157
212
|
FileInputStream(sourceFile).use { fis ->
|
|
158
|
-
|
|
213
|
+
ProgressInputStream(fis, totalCompressedBytes) { read, total ->
|
|
214
|
+
val pct =
|
|
215
|
+
if (total > 0) ((read * 99L) / total).coerceIn(0L, 99L).toInt() else 0
|
|
216
|
+
Bridge.sendExtractionProgress(pct, read, total)
|
|
217
|
+
}.use { progress ->
|
|
218
|
+
BufferedInputStream(progress, 65536).use { bis ->
|
|
159
219
|
Bridge.log("Opening bz2 decompression stream...")
|
|
160
220
|
BZip2CompressorInputStream(bis).use { bzIn ->
|
|
161
221
|
Bridge.log("Opening tar archive stream...")
|
|
@@ -166,31 +226,25 @@ object STTTools {
|
|
|
166
226
|
while (entry != null) {
|
|
167
227
|
try {
|
|
168
228
|
val outputFile = File(destDir, entry.name)
|
|
169
|
-
Bridge.log(
|
|
170
|
-
"Processing entry: ${entry.name} (${entry.size} bytes, isDir=${entry.isDirectory})"
|
|
171
|
-
)
|
|
172
229
|
|
|
173
230
|
if (entry.isDirectory) {
|
|
174
|
-
// Create directory
|
|
175
231
|
if (!outputFile.exists()) {
|
|
176
232
|
outputFile.mkdirs()
|
|
177
233
|
}
|
|
178
234
|
} else {
|
|
179
|
-
// Create parent directories if needed
|
|
180
235
|
outputFile.parentFile?.let { parent ->
|
|
181
236
|
if (!parent.exists()) {
|
|
182
237
|
parent.mkdirs()
|
|
183
238
|
}
|
|
184
239
|
}
|
|
185
240
|
|
|
186
|
-
// Extract file with buffered output for better performance
|
|
187
241
|
FileOutputStream(outputFile).use { fos ->
|
|
188
|
-
BufferedOutputStream(fos).use { bos ->
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
242
|
+
BufferedOutputStream(fos, 65536).use { bos ->
|
|
243
|
+
// 64KB buffer matches iOS path and cuts JNI/stream
|
|
244
|
+
// overhead ~16x vs. the prior 4KB. The per-read
|
|
245
|
+
// progress hook is already 200ms-throttled by
|
|
246
|
+
// ProgressInputStream, so we don't log per-chunk.
|
|
247
|
+
val buffer = ByteArray(65536)
|
|
194
248
|
var len: Int
|
|
195
249
|
var fileBytes = 0L
|
|
196
250
|
val fileSizeMB = entry.size / 1024 / 1024
|
|
@@ -201,16 +255,13 @@ object STTTools {
|
|
|
201
255
|
fileBytes += len
|
|
202
256
|
bytesExtracted += len
|
|
203
257
|
|
|
204
|
-
// For large files (>10MB), log progress every
|
|
205
|
-
// 10MB
|
|
206
258
|
if (fileSizeMB > 10) {
|
|
207
259
|
val currentMB = fileBytes / 1024 / 1024
|
|
208
260
|
if (currentMB >= lastProgressMB + 10) {
|
|
209
261
|
lastProgressMB = currentMB
|
|
210
262
|
val percent =
|
|
211
263
|
if (entry.size > 0)
|
|
212
|
-
(fileBytes * 100 /
|
|
213
|
-
entry.size)
|
|
264
|
+
(fileBytes * 100 / entry.size)
|
|
214
265
|
else 0
|
|
215
266
|
Bridge.log(
|
|
216
267
|
" Extracting ${entry.name}: ${currentMB}MB / ${fileSizeMB}MB (${percent}%)"
|
|
@@ -220,21 +271,11 @@ object STTTools {
|
|
|
220
271
|
}
|
|
221
272
|
|
|
222
273
|
fileCount++
|
|
223
|
-
|
|
224
|
-
// Log progress every file for debugging
|
|
225
|
-
val mbExtracted = bytesExtracted / 1024 / 1024
|
|
226
|
-
Bridge.log(
|
|
227
|
-
"Extracted file $fileCount (${mbExtracted}MB total) - ${entry.name}"
|
|
228
|
-
)
|
|
229
274
|
}
|
|
230
275
|
}
|
|
231
276
|
}
|
|
232
277
|
|
|
233
|
-
Bridge.log("Getting next entry...")
|
|
234
278
|
entry = tarIn.nextEntry
|
|
235
|
-
Bridge.log(
|
|
236
|
-
"Next entry received: ${entry?.name ?: "null (end of archive)"}"
|
|
237
|
-
)
|
|
238
279
|
} catch (e: Exception) {
|
|
239
280
|
Bridge.log(
|
|
240
281
|
"ERROR extracting entry ${entry?.name}: ${e.javaClass.simpleName}: ${e.message}"
|
|
@@ -248,8 +289,11 @@ object STTTools {
|
|
|
248
289
|
}
|
|
249
290
|
}
|
|
250
291
|
}
|
|
292
|
+
}
|
|
251
293
|
}
|
|
252
294
|
|
|
295
|
+
Bridge.sendExtractionProgress(100, totalCompressedBytes, totalCompressedBytes)
|
|
296
|
+
|
|
253
297
|
val duration = (System.currentTimeMillis() - startTime) / 1000
|
|
254
298
|
val mbExtracted = bytesExtracted / 1024 / 1024
|
|
255
299
|
Bridge.log(
|
|
@@ -290,11 +334,12 @@ object STTTools {
|
|
|
290
334
|
if (actualModelDir != destDir) {
|
|
291
335
|
Bridge.log("Moving files from subdirectory to parent directory")
|
|
292
336
|
actualModelDir.listFiles()?.forEach { file ->
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
file.renameTo(targetFile)
|
|
337
|
+
val targetFile = File(destDir, file.name)
|
|
338
|
+
if (targetFile.exists()) {
|
|
339
|
+
targetFile.deleteRecursively()
|
|
297
340
|
}
|
|
341
|
+
Bridge.log("Moving ${file.name}")
|
|
342
|
+
file.renameTo(targetFile)
|
|
298
343
|
}
|
|
299
344
|
// Delete the now-empty subdirectory
|
|
300
345
|
actualModelDir.delete()
|
|
@@ -313,6 +358,12 @@ object STTTools {
|
|
|
313
358
|
return context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE)
|
|
314
359
|
}
|
|
315
360
|
|
|
361
|
+
private fun findReadableFile(modelDir: File, candidates: List<String>): File {
|
|
362
|
+
return candidates.map { File(modelDir, it) }.firstOrNull { file ->
|
|
363
|
+
file.exists() && file.canRead() && file.length() > 0
|
|
364
|
+
} ?: File(modelDir, candidates.first())
|
|
365
|
+
}
|
|
366
|
+
|
|
316
367
|
/** Clear all STT model settings */
|
|
317
368
|
fun clearSttModelSettings(context: Context) {
|
|
318
369
|
val prefs = getPrefs(context)
|
|
@@ -122,8 +122,9 @@ class SherpaOnnxTranscriber(private val context: Context) {
|
|
|
122
122
|
val modelConfig = OnlineModelConfig()
|
|
123
123
|
|
|
124
124
|
// Detect model type based on available files
|
|
125
|
-
val ctcModelFile =
|
|
126
|
-
val transducerEncoderFile =
|
|
125
|
+
val ctcModelFile = findReadableFile(modelDir, listOf("model.int8.onnx", "model.onnx"))
|
|
126
|
+
val transducerEncoderFile =
|
|
127
|
+
findReadableFile(modelDir, listOf("encoder.onnx", "encoder.int8.onnx"))
|
|
127
128
|
|
|
128
129
|
when {
|
|
129
130
|
ctcModelFile.exists() -> {
|
|
@@ -143,8 +144,10 @@ class SherpaOnnxTranscriber(private val context: Context) {
|
|
|
143
144
|
modelType = "transducer"
|
|
144
145
|
Bridge.log("Detected transducer model at $modelPath")
|
|
145
146
|
|
|
146
|
-
val decoderFile =
|
|
147
|
-
|
|
147
|
+
val decoderFile =
|
|
148
|
+
findReadableFile(modelDir, listOf("decoder.onnx", "decoder.int8.onnx"))
|
|
149
|
+
val joinerFile =
|
|
150
|
+
findReadableFile(modelDir, listOf("joiner.onnx", "joiner.int8.onnx"))
|
|
148
151
|
|
|
149
152
|
if (!decoderFile.exists() || !joinerFile.exists()) {
|
|
150
153
|
throw IllegalStateException(
|
|
@@ -222,6 +225,12 @@ class SherpaOnnxTranscriber(private val context: Context) {
|
|
|
222
225
|
}
|
|
223
226
|
}
|
|
224
227
|
|
|
228
|
+
private fun findReadableFile(modelDir: File, candidates: List<String>): File {
|
|
229
|
+
return candidates.map { File(modelDir, it) }.firstOrNull { file ->
|
|
230
|
+
file.exists() && file.canRead() && file.length() > 0
|
|
231
|
+
} ?: File(modelDir, candidates.first())
|
|
232
|
+
}
|
|
233
|
+
|
|
225
234
|
/**
|
|
226
235
|
* Feed PCM audio data (16-bit little endian) into the transcriber. This method should be called
|
|
227
236
|
* continuously with short chunks (e.g., 100-300ms).
|