@mentra/bluetooth-sdk 0.1.8 → 0.1.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (147) hide show
  1. package/README.md +116 -18
  2. package/android/build.gradle +289 -16
  3. package/android/gradle.properties.example +9 -0
  4. package/android/lc3Lib/build.gradle +95 -6
  5. package/android/settings.gradle +3 -0
  6. package/android/silero/LICENSE.md +17 -0
  7. package/android/silero/build.gradle +42 -0
  8. package/android/silero/proguard-rules.pro +24 -0
  9. package/android/silero/src/androidTest/assets/hello.wav +0 -0
  10. package/android/silero/src/androidTest/java/com/konovalov/vad/silero/VadSileroTest.kt +79 -0
  11. package/android/silero/src/main/assets/LICENSE +21 -0
  12. package/android/silero/src/main/assets/silero_vad.onnx +0 -0
  13. package/android/silero/src/main/java/com/konovalov/vad/silero/Vad.kt +160 -0
  14. package/android/silero/src/main/java/com/konovalov/vad/silero/VadSilero.kt +449 -0
  15. package/android/silero/src/main/java/com/konovalov/vad/silero/config/FrameSize.kt +15 -0
  16. package/android/silero/src/main/java/com/konovalov/vad/silero/config/Mode.kt +14 -0
  17. package/android/silero/src/main/java/com/konovalov/vad/silero/config/SampleRate.kt +12 -0
  18. package/android/silero/src/main/java/com/konovalov/vad/silero/utils/AudioUtils.kt +47 -0
  19. package/android/silero/src/main/java/com/konovalov/vad/silero/utils/TensorMap.kt +41 -0
  20. package/android/silero/src/test/java/com/konovalov/vad/silero/utils/AudioUtilsTest.kt +44 -0
  21. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkDefaults.kt +6 -0
  22. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +165 -27
  23. package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +89 -21
  24. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +189 -16
  25. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt +22 -2
  26. package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +787 -85
  27. package/android/src/main/java/com/mentra/bluetoothsdk/audio/AudioModels.kt +6 -2
  28. package/android/src/main/java/com/mentra/bluetoothsdk/camera/CameraModels.kt +123 -10
  29. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/ControllerManager.kt +12 -2
  30. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/R1.kt +8 -2
  31. package/android/src/main/java/com/mentra/bluetoothsdk/debug/BleTraceLogger.kt +33 -17
  32. package/android/src/main/java/com/mentra/bluetoothsdk/events/BluetoothEvents.kt +101 -0
  33. package/android/src/main/java/com/mentra/bluetoothsdk/internal/MapParsing.kt +8 -0
  34. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistry.java +108 -0
  35. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoUploadServer.kt +399 -74
  36. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/MentraPhotoReceiverModule.kt +100 -17
  37. package/android/src/main/java/com/mentra/bluetoothsdk/requests/DisplayRequests.kt +17 -0
  38. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G1.java +2 -2
  39. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt +569 -220
  40. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Mach1.java +2 -2
  41. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraLive.java +271 -66
  42. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraNex.kt +176 -104
  43. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraosBle.java +12753 -27036
  44. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt +54 -2
  45. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Simulated.kt +16 -4
  46. package/android/src/main/java/com/mentra/bluetoothsdk/status/DeviceStatus.kt +60 -3
  47. package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +12 -3
  48. package/android/src/main/java/com/mentra/bluetoothsdk/stt/STTTools.kt +87 -36
  49. package/android/src/main/java/com/mentra/bluetoothsdk/stt/SherpaOnnxTranscriber.kt +13 -4
  50. package/android/src/main/java/com/mentra/bluetoothsdk/stt/VadGateSpeechPolicy.kt +229 -0
  51. package/android/src/main/java/com/mentra/bluetoothsdk/tts/TTSTools.kt +180 -0
  52. package/android/src/main/java/com/mentra/bluetoothsdk/utils/AvifExifStripper.java +573 -0
  53. package/android/src/main/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadService.java +488 -50
  54. package/android/src/main/java/com/mentra/bluetoothsdk/utils/HeifExifTagReader.java +199 -0
  55. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunkReassembler.java +18 -1
  56. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunker.java +64 -18
  57. package/android/src/main/java/com/mentra/bluetoothsdk/utils/NexSGCUtils.kt +48 -39
  58. package/android/src/test/java/com/mentra/bluetoothsdk/camera/PhotoRequestTest.kt +23 -0
  59. package/android/src/test/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistryTest.java +75 -0
  60. package/android/src/test/java/com/mentra/bluetoothsdk/utils/AvifExifStripperTest.java +30 -0
  61. package/android/src/test/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadServiceTest.java +98 -0
  62. package/android/src/test/resources/avif_with_exif.avif +0 -0
  63. package/build/BluetoothSdk.types.d.ts +241 -43
  64. package/build/BluetoothSdk.types.d.ts.map +1 -1
  65. package/build/BluetoothSdk.types.js +11 -10
  66. package/build/BluetoothSdk.types.js.map +1 -1
  67. package/build/_private/BluetoothSdkModule.d.ts +36 -23
  68. package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
  69. package/build/_private/BluetoothSdkModule.js +30 -52
  70. package/build/_private/BluetoothSdkModule.js.map +1 -1
  71. package/build/_private/photoRequestPayload.d.ts +4 -0
  72. package/build/_private/photoRequestPayload.d.ts.map +1 -0
  73. package/build/_private/photoRequestPayload.js +25 -0
  74. package/build/_private/photoRequestPayload.js.map +1 -0
  75. package/build/index.d.ts +1 -1
  76. package/build/index.d.ts.map +1 -1
  77. package/build/index.js +23 -0
  78. package/build/index.js.map +1 -1
  79. package/build/react/useMentraBluetooth.d.ts +2 -2
  80. package/build/react/useMentraBluetooth.d.ts.map +1 -1
  81. package/build/react/useMentraBluetooth.js +3 -3
  82. package/build/react/useMentraBluetooth.js.map +1 -1
  83. package/ios/BluetoothSdkModule.swift +174 -70
  84. package/ios/LocalPhotoUploadServer.swift +370 -105
  85. package/ios/MentraBluetoothSDK.podspec +3 -1
  86. package/ios/MentraPhotoReceiverModule.swift +62 -10
  87. package/ios/Packages/CoreObjC/include/PcmConverter.h +22 -0
  88. package/ios/Packages/SherpaOnnx/SherpaOnnx.swift +668 -26
  89. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Info.plist +13 -9
  90. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
  91. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
  92. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/libsherpa-onnx.a +0 -0
  93. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
  94. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
  95. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/libsherpa-onnx.a +0 -0
  96. package/ios/Source/BluetoothSdkDefaults.swift +6 -0
  97. package/ios/Source/Bridge.swift +58 -19
  98. package/ios/Source/DeviceManager.swift +149 -17
  99. package/ios/Source/DeviceStore.swift +16 -6
  100. package/ios/Source/MentraBluetoothSDK.swift +803 -47
  101. package/ios/Source/ObservableStore.swift +8 -2
  102. package/ios/Source/{Audio → audio}/AudioModels.swift +9 -7
  103. package/ios/Source/{Camera → camera}/CameraModels.swift +225 -16
  104. package/ios/Source/controllers/ControllerManager.swift +4 -3
  105. package/ios/Source/controllers/R1.swift +3 -3
  106. package/ios/Source/{Events → events}/BluetoothEvents.swift +183 -1
  107. package/ios/Source/{Internal → internal}/ValueParsing.swift +10 -1
  108. package/ios/Source/{Requests → requests}/DisplayRequests.swift +28 -4
  109. package/ios/Source/services/PhoneMic.swift +1 -0
  110. package/ios/Source/sgcs/G1.swift +2 -2
  111. package/ios/Source/sgcs/G2.swift +645 -403
  112. package/ios/Source/sgcs/Mach1.swift +2 -2
  113. package/ios/Source/sgcs/MentraLive.swift +812 -319
  114. package/ios/Source/sgcs/MentraNex.swift +412 -182
  115. package/ios/Source/sgcs/SGCManager.swift +57 -3
  116. package/ios/Source/sgcs/Simulated.swift +7 -3
  117. package/ios/Source/sgcs/mentraos_ble.pb.swift +3073 -3134
  118. package/ios/Source/status/DeviceStatus.swift +974 -0
  119. package/ios/Source/{Status → status}/WifiHotspotStatus.swift +4 -4
  120. package/ios/Source/{Streaming → streaming}/StreamModels.swift +24 -9
  121. package/ios/Source/stt/STTTools.swift +27 -12
  122. package/ios/Source/stt/SherpaOnnxTranscriber.swift +31 -8
  123. package/ios/Source/tts/TTSTools.swift +164 -0
  124. package/ios/Source/utils/JSCExperiment.swift +7 -7
  125. package/ios/Source/utils/MessageChunkReassembler.swift +20 -1
  126. package/ios/Source/utils/MessageChunker.swift +78 -21
  127. package/ios/Source/utils/TarBz2Extractor.swift +53 -2
  128. package/package.json +1 -1
  129. package/plugin/build/withAndroid.js +68 -4
  130. package/src/BluetoothSdk.types.ts +331 -59
  131. package/src/_private/BluetoothSdkModule.ts +113 -102
  132. package/src/_private/photoRequestPayload.ts +28 -0
  133. package/src/index.ts +58 -15
  134. package/src/react/useMentraBluetooth.ts +6 -4
  135. package/android/.project +0 -28
  136. package/android/lc3Lib/.project +0 -28
  137. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/cargs.h +0 -162
  138. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/c-api.h +0 -1852
  139. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/cxx-api.h +0 -674
  140. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/sherpa-onnx.a +0 -0
  141. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/sherpa-onnx.a +0 -0
  142. package/ios/Source/Status/DeviceStatus.swift +0 -471
  143. /package/ios/Source/{Connection → connection}/ScanSession.swift +0 -0
  144. /package/ios/Source/{Errors → errors}/BluetoothError.swift +0 -0
  145. /package/ios/Source/{Internal → internal}/BluetoothAvailability.swift +0 -0
  146. /package/ios/Source/{Status → status}/RuntimeState.swift +0 -0
  147. /package/ios/Source/{Types → types}/DeviceModels.swift +0 -0
@@ -1,5 +1,7 @@
1
1
  package com.mentra.bluetoothsdk.sgcs
2
2
 
3
+ import com.mentra.bluetoothsdk.BluetoothSdkDefaults
4
+ import com.mentra.bluetoothsdk.Bridge
3
5
  import com.mentra.bluetoothsdk.DeviceStore
4
6
  import com.mentra.bluetoothsdk.utils.ConnTypes
5
7
 
@@ -21,6 +23,7 @@ abstract class SGCManager {
21
23
  authToken: String?,
22
24
  compress: String?,
23
25
  flash: Boolean,
26
+ save: Boolean,
24
27
  sound: Boolean,
25
28
  exposureTimeNs: Long?,
26
29
  iso: Int?,
@@ -29,6 +32,26 @@ abstract class SGCManager {
29
32
  abstract fun stopStream()
30
33
  abstract fun sendStreamKeepAlive(message: MutableMap<String, Any>)
31
34
  abstract fun startVideoRecording(requestId: String, save: Boolean, flash: Boolean, sound: Boolean)
35
+
36
+ /**
37
+ * Start video recording with optional per-recording resolution/fps. A width,
38
+ * height, or fps of 0 means "use the device's saved button-video default".
39
+ * The base implementation ignores the settings and delegates to the default
40
+ * recording path; devices that support custom settings (e.g. Mentra Live)
41
+ * override this.
42
+ */
43
+ open fun startVideoRecording(
44
+ requestId: String,
45
+ save: Boolean,
46
+ flash: Boolean,
47
+ sound: Boolean,
48
+ width: Int,
49
+ height: Int,
50
+ fps: Int,
51
+ ) {
52
+ startVideoRecording(requestId, save, flash, sound)
53
+ }
54
+
32
55
  abstract fun stopVideoRecording(requestId: String)
33
56
 
34
57
  // Button Settings
@@ -43,7 +66,17 @@ abstract class SGCManager {
43
66
  abstract fun clearDisplay()
44
67
  abstract fun sendTextWall(text: String)
45
68
  abstract fun sendDoubleTextWall(top: String, bottom: String)
46
- abstract fun displayBitmap(base64ImageData: String): Boolean
69
+ /**
70
+ * Display a bitmap. Optional [x]/[y]/[width]/[height] position and size the target
71
+ * container (used by G2; other SGCs ignore positioning and render the bitmap as before).
72
+ */
73
+ abstract fun displayBitmap(
74
+ base64ImageData: String,
75
+ x: Int? = null,
76
+ y: Int? = null,
77
+ width: Int? = null,
78
+ height: Int? = null
79
+ ): Boolean
47
80
  abstract fun showDashboard()
48
81
  abstract fun setDashboardPosition(height: Int, depth: Int)
49
82
 
@@ -62,6 +95,15 @@ abstract class SGCManager {
62
95
  // Dashboard Menu (default no-op — only G2 supports this)
63
96
  open fun setDashboardMenu(items: List<Map<String, Any>>) {}
64
97
 
98
+ // Calendar Events (default no-op — only G2 supports this)
99
+ open fun sendCalendarEvents(events: List<Map<String, Any>>) {}
100
+
101
+ // Dashboard display settings (default no-op — only G2 supports this)
102
+ open fun sendDashboardDisplaySettings() {}
103
+
104
+ // Notification Panel (default no-op — only G2 supports this)
105
+ open fun showNotificationsPanel() {}
106
+
65
107
  // Controller bridging (default no-op — only G2 supports pairing with a ring controller)
66
108
  open fun connectController() {}
67
109
  open fun disconnectController() {}
@@ -101,6 +143,11 @@ abstract class SGCManager {
101
143
  abstract fun forgetWifiNetwork(ssid: String)
102
144
  abstract fun sendHotspotState(enabled: Boolean)
103
145
 
146
+ /** Set glasses system clock (Mentra Live only; no-op on other devices). */
147
+ open fun sendSetSystemTime(timestampMs: Long) {
148
+ Bridge.log("SGC: sendSetSystemTime not supported on $type")
149
+ }
150
+
104
151
  // User Context (for crash reporting)
105
152
  abstract fun sendUserEmailToGlasses(email: String)
106
153
 
@@ -114,6 +161,9 @@ abstract class SGCManager {
114
161
  // Voice Activity Detection
115
162
  open fun sendVoiceActivityDetectionSetting() {}
116
163
 
164
+ // Start/stop LC3 audio playback from glasses based on the nex_audio_playback flag.
165
+ open fun applyNexAudioPlaybackSetting() {}
166
+
117
167
  // Version info
118
168
  abstract fun requestVersionInfo()
119
169
 
@@ -161,7 +211,9 @@ abstract class SGCManager {
161
211
  get() = DeviceStore.get("glasses", "micEnabled") as? Boolean ?: false
162
212
 
163
213
  val voiceActivityDetectionEnabled: Boolean
164
- get() = DeviceStore.get("glasses", "voiceActivityDetectionEnabled") as? Boolean ?: true
214
+ get() =
215
+ DeviceStore.get("glasses", "voiceActivityDetectionEnabled") as? Boolean
216
+ ?: BluetoothSdkDefaults.VOICE_ACTIVITY_DETECTION_ENABLED
165
217
 
166
218
  val batteryLevel: Int
167
219
  get() = DeviceStore.get("glasses", "batteryLevel") as? Int ?: -1
@@ -1,10 +1,11 @@
1
1
  package com.mentra.bluetoothsdk.sgcs
2
2
 
3
+ import com.mentra.bluetoothsdk.BluetoothSdkDefaults
3
4
  import com.mentra.bluetoothsdk.Bridge
4
5
  import com.mentra.bluetoothsdk.DeviceManager
6
+ import com.mentra.bluetoothsdk.DeviceStore
5
7
  import com.mentra.bluetoothsdk.utils.ConnTypes
6
8
  import com.mentra.bluetoothsdk.utils.DeviceTypes
7
- import com.mentra.bluetoothsdk.DeviceStore
8
9
 
9
10
  class Simulated : SGCManager() {
10
11
 
@@ -14,7 +15,11 @@ class Simulated : SGCManager() {
14
15
  DeviceStore.apply("glasses", "connected", true)
15
16
  DeviceStore.apply("glasses", "connectionState", ConnTypes.CONNECTED)
16
17
  DeviceStore.apply("glasses", "micEnabled", false)
17
- DeviceStore.apply("glasses", "voiceActivityDetectionEnabled", true)
18
+ DeviceStore.apply(
19
+ "glasses",
20
+ "voiceActivityDetectionEnabled",
21
+ BluetoothSdkDefaults.VOICE_ACTIVITY_DETECTION_ENABLED
22
+ )
18
23
  }
19
24
 
20
25
  // Audio Control
@@ -35,11 +40,12 @@ class Simulated : SGCManager() {
35
40
  authToken: String?,
36
41
  compress: String?,
37
42
  flash: Boolean,
43
+ save: Boolean,
38
44
  sound: Boolean,
39
45
  exposureTimeNs: Long?,
40
46
  iso: Int?,
41
47
  ) {
42
- Bridge.log("requestPhoto flash=$flash, sound=$sound")
48
+ Bridge.log("requestPhoto flash=$flash, save=$save, sound=$sound")
43
49
  }
44
50
 
45
51
  override fun startStream(message: MutableMap<String, Any>) {
@@ -100,7 +106,13 @@ class Simulated : SGCManager() {
100
106
  Bridge.log("sendDoubleTextWall")
101
107
  }
102
108
 
103
- override fun displayBitmap(base64ImageData: String): Boolean {
109
+ override fun displayBitmap(
110
+ base64ImageData: String,
111
+ x: Int?,
112
+ y: Int?,
113
+ width: Int?,
114
+ height: Int?
115
+ ): Boolean {
104
116
  Bridge.log("displayBitmap")
105
117
  return false
106
118
  }
@@ -47,6 +47,7 @@ internal data class GlassesStatus(
47
47
  val rightMacAddress: String,
48
48
  val macAddress: String,
49
49
  val buildNumber: String,
50
+ val systemTimeMs: Long?,
50
51
  val otaVersionUrl: String,
51
52
  val appVersion: String,
52
53
  val bluetoothName: String,
@@ -69,8 +70,9 @@ internal data class GlassesStatus(
69
70
  val controllerSignalStrength: Int,
70
71
  val ringSignalStrength: Int,
71
72
  ) {
72
- internal fun toMap(): Map<String, Any> =
73
- mapOf(
73
+ internal fun toMap(): Map<String, Any> {
74
+ val values =
75
+ mutableMapOf<String, Any>(
74
76
  "connection" to connectionState.toStatusMap(connected, fullyBooted),
75
77
  "micEnabled" to micEnabled,
76
78
  "voiceActivityDetectionEnabled" to voiceActivityDetectionEnabled,
@@ -109,6 +111,9 @@ internal data class GlassesStatus(
109
111
  "controllerSignalStrength" to controllerSignalStrength,
110
112
  "ringSignalStrength" to ringSignalStrength,
111
113
  )
114
+ systemTimeMs?.let { values["systemTimeMs"] = it }
115
+ return values
116
+ }
112
117
 
113
118
  companion object {
114
119
  internal fun fromMap(values: Map<String, Any>): GlassesStatus =
@@ -116,7 +121,9 @@ internal data class GlassesStatus(
116
121
  fullyBooted = boolValue(values, "fullyBooted") ?: false,
117
122
  connected = boolValue(values, "connected") ?: false,
118
123
  micEnabled = boolValue(values, "micEnabled") ?: false,
119
- voiceActivityDetectionEnabled = boolValue(values, "voiceActivityDetectionEnabled") ?: true,
124
+ voiceActivityDetectionEnabled =
125
+ boolValue(values, "voiceActivityDetectionEnabled")
126
+ ?: BluetoothSdkDefaults.VOICE_ACTIVITY_DETECTION_ENABLED,
120
127
  connectionState = GlassesConnectionState.fromValue(stringValue(values, "connectionState")),
121
128
  bluetoothClassicConnected = boolValue(values, "bluetoothClassicConnected") ?: false,
122
129
  signalStrength = numberValue(values, "signalStrength") ?: -1,
@@ -131,6 +138,7 @@ internal data class GlassesStatus(
131
138
  rightMacAddress = stringValue(values, "rightMacAddress") ?: "",
132
139
  macAddress = stringValue(values, "macAddress") ?: "",
133
140
  buildNumber = stringValue(values, "buildNumber") ?: "",
141
+ systemTimeMs = longValue(values, "systemTimeMs"),
134
142
  otaVersionUrl = stringValue(values, "otaVersionUrl") ?: "",
135
143
  appVersion = stringValue(values, "appVersion") ?: "",
136
144
  bluetoothName = stringValue(values, "bluetoothName") ?: "",
@@ -156,6 +164,55 @@ internal data class GlassesStatus(
156
164
  }
157
165
  }
158
166
 
167
+ data class VersionInfoResult(
168
+ val androidVersion: String,
169
+ val firmwareVersion: String,
170
+ val besFirmwareVersion: String,
171
+ val mtkFirmwareVersion: String,
172
+ val buildNumber: String,
173
+ val systemTimeMs: Long?,
174
+ val otaVersionUrl: String,
175
+ val appVersion: String,
176
+ ) {
177
+ internal fun toMap(): Map<String, Any> =
178
+ buildMap {
179
+ put("androidVersion", androidVersion)
180
+ put("firmwareVersion", firmwareVersion)
181
+ put("besFirmwareVersion", besFirmwareVersion)
182
+ put("mtkFirmwareVersion", mtkFirmwareVersion)
183
+ put("buildNumber", buildNumber)
184
+ systemTimeMs?.let { put("systemTimeMs", it) }
185
+ put("otaVersionUrl", otaVersionUrl)
186
+ put("appVersion", appVersion)
187
+ }
188
+
189
+ companion object {
190
+ internal fun from(status: GlassesStatus): VersionInfoResult =
191
+ VersionInfoResult(
192
+ androidVersion = status.androidVersion,
193
+ firmwareVersion = status.firmwareVersion,
194
+ besFirmwareVersion = status.besFirmwareVersion,
195
+ mtkFirmwareVersion = status.mtkFirmwareVersion,
196
+ buildNumber = status.buildNumber,
197
+ systemTimeMs = status.systemTimeMs,
198
+ otaVersionUrl = status.otaVersionUrl,
199
+ appVersion = status.appVersion,
200
+ )
201
+
202
+ internal fun fromMap(values: Map<String, Any>): VersionInfoResult =
203
+ VersionInfoResult(
204
+ androidVersion = stringValue(values, "androidVersion", "android_version") ?: "",
205
+ firmwareVersion = stringValue(values, "firmwareVersion", "firmware_version") ?: "",
206
+ besFirmwareVersion = stringValue(values, "besFirmwareVersion", "bes_fw_version") ?: "",
207
+ mtkFirmwareVersion = stringValue(values, "mtkFirmwareVersion", "mtk_fw_version") ?: "",
208
+ buildNumber = stringValue(values, "buildNumber", "build_number") ?: "",
209
+ systemTimeMs = longValue(values, "systemTimeMs", "system_time_ms"),
210
+ otaVersionUrl = stringValue(values, "otaVersionUrl", "ota_version_url") ?: "",
211
+ appVersion = stringValue(values, "appVersion", "app_version") ?: "",
212
+ )
213
+ }
214
+ }
215
+
159
216
  internal data class BluetoothStatus(
160
217
  val searching: Boolean,
161
218
  val searchingController: Boolean,
@@ -56,13 +56,20 @@ data class StreamAudioConfig @JvmOverloads constructor(
56
56
  }
57
57
  }
58
58
 
59
+ /** Effective video settings reported by the glasses after defaults and clamps. */
59
60
  data class StreamResolvedVideoConfig @JvmOverloads constructor(
61
+ /** Encoded output width sent to the stream endpoint. */
60
62
  val width: Int,
63
+ /** Encoded output height sent to the stream endpoint. */
61
64
  val height: Int,
65
+ /** Native camera buffer width selected before crop/downscale. */
62
66
  val captureWidth: Int? = null,
67
+ /** Native camera buffer height selected before crop/downscale. */
63
68
  val captureHeight: Int? = null,
69
+ /** Encoded video bitrate in bits per second. */
64
70
  val bitrate: Int,
65
- val fps: Int,
71
+ /** Resolved capture/encode frame rate. */
72
+ val fps: Double,
66
73
  ) {
67
74
  fun toMap(): Map<String, Any> =
68
75
  buildMap {
@@ -84,7 +91,7 @@ data class StreamResolvedVideoConfig @JvmOverloads constructor(
84
91
  captureWidth = numberValue(values, "captureWidth"),
85
92
  captureHeight = numberValue(values, "captureHeight"),
86
93
  bitrate = numberValue(values, "bitrate") ?: return null,
87
- fps = numberValue(values, "fps") ?: return null,
94
+ fps = doubleValue(values, "fps") ?: return null,
88
95
  )
89
96
  }
90
97
  }
@@ -377,12 +384,14 @@ sealed interface StreamStatus {
377
384
  val resolvedConfig = StreamResolvedConfig.fromMap(stringMapValue(values["resolvedConfig"]))
378
385
  val attempt = numberValue(values, "attempt")
379
386
  val maxAttempts = numberValue(values, "maxAttempts") ?: 0
387
+ val parsedState = StreamState.fromValue(rawState)
380
388
 
381
389
  if (streaming != null || hasAnyKey(values, "reconnecting")) {
382
390
  return Snapshot(
383
391
  state = when {
384
392
  reconnecting -> StreamState.RECONNECTING
385
393
  streaming == true -> StreamState.STREAMING
394
+ parsedState != null -> parsedState
386
395
  else -> StreamState.STOPPED
387
396
  },
388
397
  streaming = streaming == true,
@@ -394,7 +403,7 @@ sealed interface StreamStatus {
394
403
  )
395
404
  }
396
405
 
397
- val state = StreamState.fromValue(rawState)
406
+ val state = parsedState
398
407
  ?: return Error(
399
408
  streamId = streamId,
400
409
  errorDetails = rawState?.let { "Unknown stream status: $it" } ?: "Missing stream status",
@@ -8,9 +8,56 @@ import java.io.BufferedOutputStream
8
8
  import java.io.File
9
9
  import java.io.FileInputStream
10
10
  import java.io.FileOutputStream
11
+ import java.io.InputStream
11
12
  import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
12
13
  import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream
13
14
 
15
+ /**
16
+ * Wraps an InputStream so we can observe how many compressed bytes have been
17
+ * consumed from the source .tar.bz2. Used to report extraction progress to JS
18
+ * — bz2 reads sequentially, so bytes-consumed/total is a reasonable proxy for
19
+ * "% of work done".
20
+ */
21
+ private class ProgressInputStream(
22
+ private val wrapped: InputStream,
23
+ private val totalBytes: Long,
24
+ private val onProgress: (Long, Long) -> Unit,
25
+ ) : InputStream() {
26
+ private var bytesRead: Long = 0
27
+ private var lastEmittedAt: Long = 0
28
+ private val emitIntervalMs = 200L
29
+
30
+ override fun read(): Int {
31
+ val byte = wrapped.read()
32
+ if (byte != -1) {
33
+ bytesRead++
34
+ maybeEmit()
35
+ }
36
+ return byte
37
+ }
38
+
39
+ override fun read(b: ByteArray, off: Int, len: Int): Int {
40
+ val n = wrapped.read(b, off, len)
41
+ if (n > 0) {
42
+ bytesRead += n
43
+ maybeEmit()
44
+ }
45
+ return n
46
+ }
47
+
48
+ override fun close() {
49
+ wrapped.close()
50
+ }
51
+
52
+ private fun maybeEmit() {
53
+ val now = System.currentTimeMillis()
54
+ if (now - lastEmittedAt >= emitIntervalMs) {
55
+ lastEmittedAt = now
56
+ onProgress(bytesRead, totalBytes)
57
+ }
58
+ }
59
+ }
60
+
14
61
  /**
15
62
  * STTTools provides utilities for STT model management.
16
63
  *
@@ -85,21 +132,26 @@ object STTTools {
85
132
  }
86
133
 
87
134
  // Check for CTC model
88
- val ctcModelFile = File(modelDir, "model.int8.onnx")
135
+ val ctcModelFile = findReadableFile(modelDir, listOf("model.int8.onnx", "model.onnx"))
89
136
  if (ctcModelFile.exists() && ctcModelFile.canRead() && ctcModelFile.length() > 0) {
90
137
  Bridge.log("STT CTC model found at: $path (size: ${ctcModelFile.length()} bytes)")
91
138
  return true
92
139
  }
93
140
 
94
141
  // Check for transducer model
95
- val transducerFiles = listOf("encoder.onnx", "decoder.onnx", "joiner.onnx")
142
+ val transducerFiles =
143
+ listOf(
144
+ "encoder" to listOf("encoder.onnx", "encoder.int8.onnx"),
145
+ "decoder" to listOf("decoder.onnx", "decoder.int8.onnx"),
146
+ "joiner" to listOf("joiner.onnx", "joiner.int8.onnx")
147
+ )
96
148
  val allTransducerFilesPresent =
97
- transducerFiles.all { fileName ->
98
- val file = File(modelDir, fileName)
149
+ transducerFiles.all { (label, candidates) ->
150
+ val file = findReadableFile(modelDir, candidates)
99
151
  val exists = file.exists() && file.canRead() && file.length() > 0
100
152
  if (!exists) {
101
153
  Bridge.log(
102
- "STT model missing or invalid transducer file: $fileName at $path"
154
+ "STT model missing or invalid transducer file: $label at $path"
103
155
  )
104
156
  }
105
157
  exists
@@ -153,9 +205,17 @@ object STTTools {
153
205
  var fileCount = 0
154
206
  var bytesExtracted = 0L
155
207
 
208
+ val totalCompressedBytes = sourceFile.length()
209
+ Bridge.sendExtractionProgress(0, 0, totalCompressedBytes)
210
+
156
211
  // Open the tar.bz2 file with buffered streams for better performance
157
212
  FileInputStream(sourceFile).use { fis ->
158
- BufferedInputStream(fis).use { bis ->
213
+ ProgressInputStream(fis, totalCompressedBytes) { read, total ->
214
+ val pct =
215
+ if (total > 0) ((read * 99L) / total).coerceIn(0L, 99L).toInt() else 0
216
+ Bridge.sendExtractionProgress(pct, read, total)
217
+ }.use { progress ->
218
+ BufferedInputStream(progress, 65536).use { bis ->
159
219
  Bridge.log("Opening bz2 decompression stream...")
160
220
  BZip2CompressorInputStream(bis).use { bzIn ->
161
221
  Bridge.log("Opening tar archive stream...")
@@ -166,31 +226,25 @@ object STTTools {
166
226
  while (entry != null) {
167
227
  try {
168
228
  val outputFile = File(destDir, entry.name)
169
- Bridge.log(
170
- "Processing entry: ${entry.name} (${entry.size} bytes, isDir=${entry.isDirectory})"
171
- )
172
229
 
173
230
  if (entry.isDirectory) {
174
- // Create directory
175
231
  if (!outputFile.exists()) {
176
232
  outputFile.mkdirs()
177
233
  }
178
234
  } else {
179
- // Create parent directories if needed
180
235
  outputFile.parentFile?.let { parent ->
181
236
  if (!parent.exists()) {
182
237
  parent.mkdirs()
183
238
  }
184
239
  }
185
240
 
186
- // Extract file with buffered output for better performance
187
241
  FileOutputStream(outputFile).use { fos ->
188
- BufferedOutputStream(fos).use { bos ->
189
- val buffer =
190
- ByteArray(
191
- 4096
192
- ) // Use 4KB buffer like original
193
- // implementation
242
+ BufferedOutputStream(fos, 65536).use { bos ->
243
+ // 64KB buffer matches iOS path and cuts JNI/stream
244
+ // overhead ~16x vs. the prior 4KB. The per-read
245
+ // progress hook is already 200ms-throttled by
246
+ // ProgressInputStream, so we don't log per-chunk.
247
+ val buffer = ByteArray(65536)
194
248
  var len: Int
195
249
  var fileBytes = 0L
196
250
  val fileSizeMB = entry.size / 1024 / 1024
@@ -201,16 +255,13 @@ object STTTools {
201
255
  fileBytes += len
202
256
  bytesExtracted += len
203
257
 
204
- // For large files (>10MB), log progress every
205
- // 10MB
206
258
  if (fileSizeMB > 10) {
207
259
  val currentMB = fileBytes / 1024 / 1024
208
260
  if (currentMB >= lastProgressMB + 10) {
209
261
  lastProgressMB = currentMB
210
262
  val percent =
211
263
  if (entry.size > 0)
212
- (fileBytes * 100 /
213
- entry.size)
264
+ (fileBytes * 100 / entry.size)
214
265
  else 0
215
266
  Bridge.log(
216
267
  " Extracting ${entry.name}: ${currentMB}MB / ${fileSizeMB}MB (${percent}%)"
@@ -220,21 +271,11 @@ object STTTools {
220
271
  }
221
272
 
222
273
  fileCount++
223
-
224
- // Log progress every file for debugging
225
- val mbExtracted = bytesExtracted / 1024 / 1024
226
- Bridge.log(
227
- "Extracted file $fileCount (${mbExtracted}MB total) - ${entry.name}"
228
- )
229
274
  }
230
275
  }
231
276
  }
232
277
 
233
- Bridge.log("Getting next entry...")
234
278
  entry = tarIn.nextEntry
235
- Bridge.log(
236
- "Next entry received: ${entry?.name ?: "null (end of archive)"}"
237
- )
238
279
  } catch (e: Exception) {
239
280
  Bridge.log(
240
281
  "ERROR extracting entry ${entry?.name}: ${e.javaClass.simpleName}: ${e.message}"
@@ -248,8 +289,11 @@ object STTTools {
248
289
  }
249
290
  }
250
291
  }
292
+ }
251
293
  }
252
294
 
295
+ Bridge.sendExtractionProgress(100, totalCompressedBytes, totalCompressedBytes)
296
+
253
297
  val duration = (System.currentTimeMillis() - startTime) / 1000
254
298
  val mbExtracted = bytesExtracted / 1024 / 1024
255
299
  Bridge.log(
@@ -290,11 +334,12 @@ object STTTools {
290
334
  if (actualModelDir != destDir) {
291
335
  Bridge.log("Moving files from subdirectory to parent directory")
292
336
  actualModelDir.listFiles()?.forEach { file ->
293
- if (!file.isDirectory) {
294
- val targetFile = File(destDir, file.name)
295
- Bridge.log("Moving ${file.name}")
296
- file.renameTo(targetFile)
337
+ val targetFile = File(destDir, file.name)
338
+ if (targetFile.exists()) {
339
+ targetFile.deleteRecursively()
297
340
  }
341
+ Bridge.log("Moving ${file.name}")
342
+ file.renameTo(targetFile)
298
343
  }
299
344
  // Delete the now-empty subdirectory
300
345
  actualModelDir.delete()
@@ -313,6 +358,12 @@ object STTTools {
313
358
  return context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE)
314
359
  }
315
360
 
361
+ private fun findReadableFile(modelDir: File, candidates: List<String>): File {
362
+ return candidates.map { File(modelDir, it) }.firstOrNull { file ->
363
+ file.exists() && file.canRead() && file.length() > 0
364
+ } ?: File(modelDir, candidates.first())
365
+ }
366
+
316
367
  /** Clear all STT model settings */
317
368
  fun clearSttModelSettings(context: Context) {
318
369
  val prefs = getPrefs(context)
@@ -122,8 +122,9 @@ class SherpaOnnxTranscriber(private val context: Context) {
122
122
  val modelConfig = OnlineModelConfig()
123
123
 
124
124
  // Detect model type based on available files
125
- val ctcModelFile = File(modelDir, "model.int8.onnx")
126
- val transducerEncoderFile = File(modelDir, "encoder.onnx")
125
+ val ctcModelFile = findReadableFile(modelDir, listOf("model.int8.onnx", "model.onnx"))
126
+ val transducerEncoderFile =
127
+ findReadableFile(modelDir, listOf("encoder.onnx", "encoder.int8.onnx"))
127
128
 
128
129
  when {
129
130
  ctcModelFile.exists() -> {
@@ -143,8 +144,10 @@ class SherpaOnnxTranscriber(private val context: Context) {
143
144
  modelType = "transducer"
144
145
  Bridge.log("Detected transducer model at $modelPath")
145
146
 
146
- val decoderFile = File(modelDir, "decoder.onnx")
147
- val joinerFile = File(modelDir, "joiner.onnx")
147
+ val decoderFile =
148
+ findReadableFile(modelDir, listOf("decoder.onnx", "decoder.int8.onnx"))
149
+ val joinerFile =
150
+ findReadableFile(modelDir, listOf("joiner.onnx", "joiner.int8.onnx"))
148
151
 
149
152
  if (!decoderFile.exists() || !joinerFile.exists()) {
150
153
  throw IllegalStateException(
@@ -222,6 +225,12 @@ class SherpaOnnxTranscriber(private val context: Context) {
222
225
  }
223
226
  }
224
227
 
228
+ private fun findReadableFile(modelDir: File, candidates: List<String>): File {
229
+ return candidates.map { File(modelDir, it) }.firstOrNull { file ->
230
+ file.exists() && file.canRead() && file.length() > 0
231
+ } ?: File(modelDir, candidates.first())
232
+ }
233
+
225
234
  /**
226
235
  * Feed PCM audio data (16-bit little endian) into the transcriber. This method should be called
227
236
  * continuously with short chunks (e.g., 100-300ms).