@mentra/bluetooth-sdk 0.1.9 → 0.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (147) hide show
  1. package/README.md +116 -18
  2. package/android/build.gradle +289 -16
  3. package/android/gradle.properties.example +9 -0
  4. package/android/lc3Lib/build.gradle +95 -6
  5. package/android/settings.gradle +3 -0
  6. package/android/silero/LICENSE.md +17 -0
  7. package/android/silero/build.gradle +42 -0
  8. package/android/silero/proguard-rules.pro +24 -0
  9. package/android/silero/src/androidTest/assets/hello.wav +0 -0
  10. package/android/silero/src/androidTest/java/com/konovalov/vad/silero/VadSileroTest.kt +79 -0
  11. package/android/silero/src/main/assets/LICENSE +21 -0
  12. package/android/silero/src/main/assets/silero_vad.onnx +0 -0
  13. package/android/silero/src/main/java/com/konovalov/vad/silero/Vad.kt +160 -0
  14. package/android/silero/src/main/java/com/konovalov/vad/silero/VadSilero.kt +449 -0
  15. package/android/silero/src/main/java/com/konovalov/vad/silero/config/FrameSize.kt +15 -0
  16. package/android/silero/src/main/java/com/konovalov/vad/silero/config/Mode.kt +14 -0
  17. package/android/silero/src/main/java/com/konovalov/vad/silero/config/SampleRate.kt +12 -0
  18. package/android/silero/src/main/java/com/konovalov/vad/silero/utils/AudioUtils.kt +47 -0
  19. package/android/silero/src/main/java/com/konovalov/vad/silero/utils/TensorMap.kt +41 -0
  20. package/android/silero/src/test/java/com/konovalov/vad/silero/utils/AudioUtilsTest.kt +44 -0
  21. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkDefaults.kt +6 -0
  22. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +195 -34
  23. package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +103 -21
  24. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +210 -18
  25. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt +30 -2
  26. package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +787 -85
  27. package/android/src/main/java/com/mentra/bluetoothsdk/audio/AudioModels.kt +6 -2
  28. package/android/src/main/java/com/mentra/bluetoothsdk/camera/CameraModels.kt +125 -11
  29. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/ControllerManager.kt +12 -2
  30. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/R1.kt +8 -2
  31. package/android/src/main/java/com/mentra/bluetoothsdk/debug/BleTraceLogger.kt +33 -17
  32. package/android/src/main/java/com/mentra/bluetoothsdk/events/BluetoothEvents.kt +101 -0
  33. package/android/src/main/java/com/mentra/bluetoothsdk/internal/MapParsing.kt +8 -0
  34. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistry.java +108 -0
  35. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoUploadServer.kt +399 -74
  36. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/MentraPhotoReceiverModule.kt +100 -17
  37. package/android/src/main/java/com/mentra/bluetoothsdk/requests/DisplayRequests.kt +17 -0
  38. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G1.java +2 -2
  39. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt +1293 -758
  40. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Mach1.java +2 -2
  41. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraLive.java +271 -66
  42. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraNex.kt +197 -149
  43. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraosBle.java +12753 -27036
  44. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt +64 -2
  45. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Simulated.kt +16 -4
  46. package/android/src/main/java/com/mentra/bluetoothsdk/status/DeviceStatus.kt +60 -3
  47. package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +12 -3
  48. package/android/src/main/java/com/mentra/bluetoothsdk/stt/STTTools.kt +87 -36
  49. package/android/src/main/java/com/mentra/bluetoothsdk/stt/SherpaOnnxTranscriber.kt +13 -4
  50. package/android/src/main/java/com/mentra/bluetoothsdk/stt/VadGateSpeechPolicy.kt +229 -0
  51. package/android/src/main/java/com/mentra/bluetoothsdk/tts/TTSTools.kt +180 -0
  52. package/android/src/main/java/com/mentra/bluetoothsdk/utils/AvifExifStripper.java +573 -0
  53. package/android/src/main/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadService.java +488 -50
  54. package/android/src/main/java/com/mentra/bluetoothsdk/utils/HeifExifTagReader.java +199 -0
  55. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunkReassembler.java +18 -1
  56. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunker.java +64 -18
  57. package/android/src/main/java/com/mentra/bluetoothsdk/utils/NexSGCUtils.kt +48 -39
  58. package/android/src/test/java/com/mentra/bluetoothsdk/camera/PhotoRequestTest.kt +23 -0
  59. package/android/src/test/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistryTest.java +75 -0
  60. package/android/src/test/java/com/mentra/bluetoothsdk/utils/AvifExifStripperTest.java +30 -0
  61. package/android/src/test/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadServiceTest.java +98 -0
  62. package/android/src/test/resources/avif_with_exif.avif +0 -0
  63. package/build/BluetoothSdk.types.d.ts +253 -44
  64. package/build/BluetoothSdk.types.d.ts.map +1 -1
  65. package/build/BluetoothSdk.types.js +11 -10
  66. package/build/BluetoothSdk.types.js.map +1 -1
  67. package/build/_private/BluetoothSdkModule.d.ts +37 -23
  68. package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
  69. package/build/_private/BluetoothSdkModule.js +43 -52
  70. package/build/_private/BluetoothSdkModule.js.map +1 -1
  71. package/build/_private/photoRequestPayload.d.ts +4 -0
  72. package/build/_private/photoRequestPayload.d.ts.map +1 -0
  73. package/build/_private/photoRequestPayload.js +25 -0
  74. package/build/_private/photoRequestPayload.js.map +1 -0
  75. package/build/index.d.ts +1 -1
  76. package/build/index.d.ts.map +1 -1
  77. package/build/index.js +25 -0
  78. package/build/index.js.map +1 -1
  79. package/build/react/useMentraBluetooth.d.ts +2 -2
  80. package/build/react/useMentraBluetooth.d.ts.map +1 -1
  81. package/build/react/useMentraBluetooth.js +3 -3
  82. package/build/react/useMentraBluetooth.js.map +1 -1
  83. package/ios/BluetoothSdkModule.swift +177 -70
  84. package/ios/LocalPhotoUploadServer.swift +370 -105
  85. package/ios/MentraBluetoothSDK.podspec +3 -1
  86. package/ios/MentraPhotoReceiverModule.swift +62 -10
  87. package/ios/Packages/CoreObjC/include/PcmConverter.h +22 -0
  88. package/ios/Packages/SherpaOnnx/SherpaOnnx.swift +668 -26
  89. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Info.plist +13 -9
  90. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
  91. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
  92. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/libsherpa-onnx.a +0 -0
  93. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
  94. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
  95. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/libsherpa-onnx.a +0 -0
  96. package/ios/Source/BluetoothSdkDefaults.swift +6 -0
  97. package/ios/Source/Bridge.swift +68 -19
  98. package/ios/Source/DeviceManager.swift +167 -32
  99. package/ios/Source/DeviceStore.swift +24 -8
  100. package/ios/Source/MentraBluetoothSDK.swift +803 -47
  101. package/ios/Source/ObservableStore.swift +6 -0
  102. package/ios/Source/{Audio → audio}/AudioModels.swift +9 -7
  103. package/ios/Source/{Camera → camera}/CameraModels.swift +226 -16
  104. package/ios/Source/controllers/ControllerManager.swift +4 -3
  105. package/ios/Source/controllers/R1.swift +3 -3
  106. package/ios/Source/{Events → events}/BluetoothEvents.swift +183 -1
  107. package/ios/Source/{Internal → internal}/ValueParsing.swift +10 -1
  108. package/ios/Source/{Requests → requests}/DisplayRequests.swift +28 -4
  109. package/ios/Source/services/PhoneMic.swift +1 -0
  110. package/ios/Source/sgcs/G1.swift +5 -5
  111. package/ios/Source/sgcs/G2.swift +1494 -870
  112. package/ios/Source/sgcs/Mach1.swift +4 -4
  113. package/ios/Source/sgcs/MentraLive.swift +814 -321
  114. package/ios/Source/sgcs/MentraNex.swift +604 -244
  115. package/ios/Source/sgcs/SGCManager.swift +68 -5
  116. package/ios/Source/sgcs/Simulated.swift +9 -5
  117. package/ios/Source/sgcs/mentraos_ble.pb.swift +3073 -3134
  118. package/ios/Source/status/DeviceStatus.swift +974 -0
  119. package/ios/Source/{Status → status}/WifiHotspotStatus.swift +4 -4
  120. package/ios/Source/{Streaming → streaming}/StreamModels.swift +24 -9
  121. package/ios/Source/stt/STTTools.swift +27 -12
  122. package/ios/Source/stt/SherpaOnnxTranscriber.swift +31 -8
  123. package/ios/Source/tts/TTSTools.swift +164 -0
  124. package/ios/Source/utils/JSCExperiment.swift +7 -7
  125. package/ios/Source/utils/MessageChunkReassembler.swift +20 -1
  126. package/ios/Source/utils/MessageChunker.swift +78 -21
  127. package/ios/Source/utils/TarBz2Extractor.swift +53 -2
  128. package/package.json +1 -1
  129. package/plugin/build/withAndroid.js +68 -4
  130. package/src/BluetoothSdk.types.ts +344 -60
  131. package/src/_private/BluetoothSdkModule.ts +132 -102
  132. package/src/_private/photoRequestPayload.ts +28 -0
  133. package/src/index.ts +61 -15
  134. package/src/react/useMentraBluetooth.ts +6 -4
  135. package/android/.project +0 -28
  136. package/android/lc3Lib/.project +0 -28
  137. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/cargs.h +0 -162
  138. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/c-api.h +0 -1852
  139. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/cxx-api.h +0 -674
  140. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/sherpa-onnx.a +0 -0
  141. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/sherpa-onnx.a +0 -0
  142. package/ios/Source/Status/DeviceStatus.swift +0 -471
  143. /package/ios/Source/{Connection → connection}/ScanSession.swift +0 -0
  144. /package/ios/Source/{Errors → errors}/BluetoothError.swift +0 -0
  145. /package/ios/Source/{Internal → internal}/BluetoothAvailability.swift +0 -0
  146. /package/ios/Source/{Status → status}/RuntimeState.swift +0 -0
  147. /package/ios/Source/{Types → types}/DeviceModels.swift +0 -0
@@ -0,0 +1,6 @@
1
+ import Foundation
2
+
3
+ /// Defaults for the public Bluetooth SDK surface.
4
+ enum BluetoothSdkDefaults {
5
+ static let voiceActivityDetectionEnabled = false
6
+ }
@@ -10,7 +10,7 @@ import Foundation
10
10
  /// Bridge for Bluetooth SDK communication between Expo modules and native iOS code
11
11
  /// Has commands for the Bluetooth SDK to use to send messages to JavaScript
12
12
  class Bridge {
13
- private static let micSampleRate = 16_000
13
+ private static let micSampleRate = 16000
14
14
  private static let pcmBitsPerSample = 16
15
15
  private static let micChannels = 1
16
16
  private static let lc3FrameDurationMs = 10
@@ -67,6 +67,16 @@ class Bridge {
67
67
  Bridge.sendTypedMessage("log", body: data)
68
68
  }
69
69
 
70
+ /// Report tar.bz2 extraction progress to JavaScript.
71
+ static func sendExtractionProgress(percentage: Int, bytesRead: Int64, totalBytes: Int64) {
72
+ let body: [String: Any] = [
73
+ "percentage": percentage,
74
+ "bytesRead": bytesRead,
75
+ "totalBytes": totalBytes,
76
+ ]
77
+ Bridge.sendTypedMessage("extraction_progress", body: body)
78
+ }
79
+
70
80
  static func sendHeadUp(_ isUp: Bool) {
71
81
  let data = ["up": isUp]
72
82
  Bridge.sendTypedMessage("head_up", body: data)
@@ -90,7 +100,8 @@ class Bridge {
90
100
  @MainActor
91
101
  private static func micPcmEventBody(_ data: Data) -> [String: Any] {
92
102
  let voiceActivityDetectionEnabled =
93
- DeviceStore.shared.get("glasses", "voiceActivityDetectionEnabled") as? Bool ?? true
103
+ DeviceStore.shared.get("glasses", "voiceActivityDetectionEnabled") as? Bool
104
+ ?? BluetoothSdkDefaults.voiceActivityDetectionEnabled
94
105
  return [
95
106
  "pcm": data,
96
107
  "sampleRate": micSampleRate,
@@ -104,7 +115,8 @@ class Bridge {
104
115
  @MainActor
105
116
  private static func micLc3EventBody(_ data: Data) -> [String: Any] {
106
117
  let voiceActivityDetectionEnabled =
107
- DeviceStore.shared.get("glasses", "voiceActivityDetectionEnabled") as? Bool ?? true
118
+ DeviceStore.shared.get("glasses", "voiceActivityDetectionEnabled") as? Bool
119
+ ?? BluetoothSdkDefaults.voiceActivityDetectionEnabled
108
120
  let frameSizeBytes = DeviceStore.shared.get("bluetooth", "lc3_frame_size") as? Int ?? defaultLc3FrameSizeBytes
109
121
  return [
110
122
  "lc3": data,
@@ -239,6 +251,16 @@ class Bridge {
239
251
  Bridge.sendTypedMessage("touch_event", body: body)
240
252
  }
241
253
 
254
+ static func sendAccelEvent(x: Float, y: Float, z: Float, timestamp: Int64) {
255
+ let body: [String: Any] = [
256
+ "x": x,
257
+ "y": y,
258
+ "z": z,
259
+ "timestamp": timestamp,
260
+ ]
261
+ Bridge.sendTypedMessage("accel_event", body: body)
262
+ }
263
+
242
264
  static func sendSwipeVolumeStatus(enabled: Bool, timestamp: Int64) {
243
265
  let body: [String: Any] = [
244
266
  "enabled": enabled,
@@ -269,6 +291,35 @@ class Bridge {
269
291
  Bridge.sendTypedMessage("rgb_led_control_response", body: body)
270
292
  }
271
293
 
294
+ static func sendSettingsAck(_ values: [String: Any]) {
295
+ var body = values
296
+ body["type"] = "settings_ack"
297
+ Bridge.sendTypedMessage("settings_ack", body: body)
298
+ }
299
+
300
+ static func sendVideoRecordingStatus(_ values: [String: Any]) {
301
+ var body = values
302
+ body["type"] = "video_recording_status"
303
+ Bridge.sendTypedMessage("video_recording_status", body: body)
304
+ }
305
+
306
+ static func sendVersionInfo(_ values: [String: Any]) {
307
+ var body: [String: Any] = [
308
+ "type": "version_info",
309
+ "androidVersion": stringValue(values, "androidVersion", "android_version") ?? "",
310
+ "firmwareVersion": stringValue(values, "firmwareVersion", "firmware_version") ?? "",
311
+ "besFirmwareVersion": stringValue(values, "besFirmwareVersion", "bes_fw_version") ?? "",
312
+ "mtkFirmwareVersion": stringValue(values, "mtkFirmwareVersion", "mtk_fw_version") ?? "",
313
+ "buildNumber": stringValue(values, "buildNumber", "build_number") ?? "",
314
+ "otaVersionUrl": stringValue(values, "otaVersionUrl", "ota_version_url") ?? "",
315
+ "appVersion": stringValue(values, "appVersion", "app_version") ?? "",
316
+ ]
317
+ if let systemTimeMs = intValue(values["systemTimeMs"]) ?? intValue(values["system_time_ms"]) {
318
+ body["systemTimeMs"] = systemTimeMs
319
+ }
320
+ Bridge.sendTypedMessage("version_info", body: body)
321
+ }
322
+
272
323
  static func sendPhotoError(requestId: String, errorCode: String, errorMessage: String) {
273
324
  let timestamp = Int(Date().timeIntervalSince1970 * 1000)
274
325
  var event: [String: Any] = [
@@ -284,26 +335,16 @@ class Bridge {
284
335
  event["errorMessage"] = errorMessage
285
336
  }
286
337
  Bridge.sendTypedMessage("photo_response", body: event)
287
-
288
- var status: [String: Any] = [
289
- "type": "photo_status",
290
- "status": "failed",
291
- "requestId": requestId,
292
- "timestamp": timestamp,
293
- ]
294
- if !errorCode.isEmpty {
295
- status["errorCode"] = errorCode
296
- }
297
- if !errorMessage.isEmpty {
298
- status["errorMessage"] = errorMessage
299
- }
300
- Bridge.sendTypedMessage("photo_status", body: status)
301
338
  }
302
339
 
303
340
  static func sendPhotoStatus(_ status: [String: Any]) {
304
341
  Bridge.sendTypedMessage("photo_status", body: status)
305
342
  }
306
343
 
344
+ static func sendPhotoResponse(_ response: [String: Any]) {
345
+ Bridge.sendTypedMessage("photo_response", body: response)
346
+ }
347
+
307
348
  static func sendMiniappSelected(packageName: String) {
308
349
  let event: [String: Any] = [
309
350
  "packageName": packageName,
@@ -349,7 +390,7 @@ class Bridge {
349
390
  Bridge.sendTypedMessage("wifi_status_change", body: status.values)
350
391
  }
351
392
 
352
- static func updateWifiScanResults(_ networks: [[String: Any]]) {
393
+ static func updateWifiScanResults(_ networks: [[String: Any]], scanComplete: Bool) {
353
394
  Task {
354
395
  await MainActor.run {
355
396
  var storedNetworks: [[String: Any]] =
@@ -363,6 +404,10 @@ class Bridge {
363
404
  }
364
405
  }
365
406
  DeviceStore.shared.apply("bluetooth", "wifiScanResults", storedNetworks)
407
+ Bridge.sendTypedMessage(
408
+ "wifi_scan_result",
409
+ body: ["networks": storedNetworks, "scanComplete": scanComplete]
410
+ )
366
411
  }
367
412
  }
368
413
  }
@@ -408,7 +453,8 @@ class Bridge {
408
453
  stepPercent: Int,
409
454
  overallPercent: Int,
410
455
  status: String,
411
- errorMessage: String?
456
+ errorMessage: String?,
457
+ glassesTimeMs: Int64? = nil
412
458
  ) {
413
459
  var eventBody: [String: Any] = [
414
460
  "session_id": sessionId,
@@ -423,6 +469,9 @@ class Bridge {
423
469
  if let error = errorMessage {
424
470
  eventBody["error_message"] = error
425
471
  }
472
+ if let glassesTimeMs, glassesTimeMs > 0 {
473
+ eventBody["glasses_time_ms"] = glassesTimeMs
474
+ }
426
475
  Bridge.sendTypedMessage("ota_status", body: eventBody)
427
476
  }
428
477
 
@@ -10,6 +10,9 @@ import Combine
10
10
  import CoreBluetooth
11
11
  import Foundation
12
12
  import UIKit
13
+ #if SWIFT_PACKAGE
14
+ import MentraBluetoothSDKCoreObjC
15
+ #endif
13
16
 
14
17
  struct ViewState {
15
18
  var topText: String
@@ -19,6 +22,11 @@ struct ViewState {
19
22
  var text: String
20
23
  var data: String?
21
24
  var animationData: [String: Any]?
25
+ // Optional bitmap_view container position/size (used by G2; ignored by others)
26
+ var bmpX: Int32?
27
+ var bmpY: Int32?
28
+ var bmpWidth: Int32?
29
+ var bmpHeight: Int32?
22
30
  }
23
31
 
24
32
  @MainActor
@@ -167,6 +175,14 @@ struct ViewState {
167
175
  set { DeviceStore.shared.apply("bluetooth", "sensing_enabled", newValue) }
168
176
  }
169
177
 
178
+ /// Phone-side VAD gating switch. Default is OFF (VAD runs) so that the
179
+ /// coordinator can drive per-utterance offline/online STT switching from
180
+ /// `vad_status` events. Set to `true` only as an emergency kill-switch.
181
+ private var bypassVad: Bool {
182
+ get { DeviceStore.shared.get("bluetooth", "bypass_vad") as? Bool ?? false }
183
+ set { DeviceStore.shared.apply("bluetooth", "bypass_vad", newValue) }
184
+ }
185
+
170
186
  private var offlineCaptionsRunning: Bool {
171
187
  get { DeviceStore.shared.get("bluetooth", "offline_captions_running") as? Bool ?? false }
172
188
  set { DeviceStore.shared.apply("bluetooth", "offline_captions_running", newValue) }
@@ -226,6 +242,8 @@ struct ViewState {
226
242
  set { DeviceStore.shared.apply("bluetooth", "shouldSendBootingMessage", newValue) }
227
243
  }
228
244
 
245
+ private var lastSystemTimeSyncConnectionKey = ""
246
+
229
247
  private var systemMicUnavailable: Bool {
230
248
  get { DeviceStore.shared.get("bluetooth", "systemMicUnavailable") as? Bool ?? false }
231
249
  set { DeviceStore.shared.apply("bluetooth", "systemMicUnavailable", newValue) }
@@ -281,7 +299,9 @@ struct ViewState {
281
299
  private var micReinitTimer: Timer?
282
300
 
283
301
  /// STT:
302
+ #if !SWIFT_PACKAGE || MENTRA_FEATURE_LOCAL_STT
284
303
  private var transcriber: SherpaOnnxTranscriber?
304
+ #endif
285
305
 
286
306
  var viewStates: [ViewState] = [
287
307
  ViewState(
@@ -310,6 +330,7 @@ struct ViewState {
310
330
  // MemoryMonitor.start()
311
331
 
312
332
  // Initialize SherpaOnnx Transcriber
333
+ #if !SWIFT_PACKAGE || MENTRA_FEATURE_LOCAL_STT
313
334
  if let windowScene = UIApplication.shared.connectedScenes.first as? UIWindowScene,
314
335
  let window = windowScene.windows.first,
315
336
  let rootViewController = window.rootViewController
@@ -324,6 +345,7 @@ struct ViewState {
324
345
  transcriber.initialize()
325
346
  Bridge.log("SherpaOnnxTranscriber fully initialized")
326
347
  }
348
+ #endif
327
349
 
328
350
  // Initialize persistent LC3 converter for unified audio encoding
329
351
  lc3Converter = PcmConverter()
@@ -395,9 +417,11 @@ struct ViewState {
395
417
  handleSendingPcm(pcmData)
396
418
 
397
419
  // Send PCM to local transcriber.
420
+ #if !SWIFT_PACKAGE || MENTRA_FEATURE_LOCAL_STT
398
421
  if shouldSendTranscript || offlineCaptionsRunning || localSttFallbackActive {
399
422
  transcriber?.acceptAudio(pcm16le: pcmData)
400
423
  }
424
+ #endif
401
425
  }
402
426
 
403
427
  func updateMicState() {
@@ -541,7 +565,7 @@ struct ViewState {
541
565
  // Check if we've completed all cycles
542
566
  if cycles >= totalCycles {
543
567
  // End animation with final message
544
- sgc?.sendTextWall(" /// MentraOS Connected \\\\\\")
568
+ Task { await sgc?.sendTextWall(" /// MentraOS Connected \\\\\\") }
545
569
  animationQueue.asyncAfter(deadline: .now() + 1.0) {
546
570
  self.sgc?.clearDisplay()
547
571
  }
@@ -551,7 +575,7 @@ struct ViewState {
551
575
  // Display current animation frame
552
576
  let frameText =
553
577
  " \(arrowFrames[frameIndex]) MentraOS Booting \(arrowFrames[frameIndex])"
554
- sgc?.sendTextWall(frameText)
578
+ Task { await sgc?.sendTextWall(frameText) }
555
579
 
556
580
  // Move to next frame
557
581
  frameIndex = (frameIndex + 1) % arrowFrames.count
@@ -581,6 +605,7 @@ struct ViewState {
581
605
  Bridge.log("MAN: Manager already initialized, cleaning up previous sgc")
582
606
  sgc?.cleanup()
583
607
  sgc = nil
608
+ lastSystemTimeSyncConnectionKey = ""
584
609
  }
585
610
 
586
611
  if sgc != nil {
@@ -595,14 +620,24 @@ struct ViewState {
595
620
  sgc = G2()
596
621
  } else if wearable.contains(DeviceTypes.LIVE) {
597
622
  sgc = MentraLive()
598
- } else if wearable.contains(DeviceTypes.MACH1) {
599
- sgc = Mach1()
600
- } else if wearable.contains(DeviceTypes.Z100) {
601
- sgc = Mach1() // Z100 uses same hardware/SDK as Mach1
602
- sgc?.type = DeviceTypes.Z100 // Override type to Z100
603
623
  } else if wearable.contains(DeviceTypes.FRAME) {
604
624
  // sgc = FrameManager()
605
625
  }
626
+ #if !SWIFT_PACKAGE || MENTRA_FEATURE_NEX
627
+ if sgc == nil && wearable.contains(DeviceTypes.NEX) {
628
+ sgc = MentraNexSGC.getInstance()
629
+ }
630
+ #endif
631
+ #if !SWIFT_PACKAGE || MENTRA_FEATURE_VUZIX
632
+ if sgc == nil {
633
+ if wearable.contains(DeviceTypes.MACH1) {
634
+ sgc = Mach1()
635
+ } else if wearable.contains(DeviceTypes.Z100) {
636
+ sgc = Mach1() // Z100 uses same hardware/SDK as Mach1
637
+ sgc?.type = DeviceTypes.Z100 // Override type to Z100
638
+ }
639
+ }
640
+ #endif
606
641
  // update device model:
607
642
  DeviceStore.shared.apply("glasses", "deviceModel", sgc?.type ?? "")
608
643
  }
@@ -658,21 +693,27 @@ struct ViewState {
658
693
  switch layoutType {
659
694
  case "text_wall":
660
695
  let text = currentViewState.text
661
- sgc?.sendTextWall(text)
696
+ await sgc?.sendTextWall(text)
662
697
  case "double_text_wall":
663
698
  let topText = currentViewState.topText
664
699
  let bottomText = currentViewState.bottomText
665
- sgc?.sendDoubleTextWall(topText, bottomText)
700
+ await sgc?.sendDoubleTextWall(topText, bottomText)
666
701
  case "reference_card":
667
- sgc?.sendTextWall(currentViewState.title + "\n\n" + currentViewState.text)
702
+ await sgc?.sendTextWall(currentViewState.title + "\n\n" + currentViewState.text)
668
703
  case "bitmap_view":
669
- Bridge.log("MAN: Processing bitmap_view layout")
704
+ // Bridge.log("MAN: Processing bitmap_view layout")
670
705
  guard let data = currentViewState.data else {
671
706
  Bridge.log("MAN: ERROR: bitmap_view missing data field")
672
707
  return
673
708
  }
674
- Bridge.log("MAN: Processing bitmap_view with base64 data, length: \(data.count)")
675
- await sgc?.displayBitmap(base64ImageData: data)
709
+ // Bridge.log("MAN: Processing bitmap_view with base64 data, length: \(data.count)")
710
+ await sgc?.displayBitmap(
711
+ base64ImageData: data,
712
+ x: currentViewState.bmpX,
713
+ y: currentViewState.bmpY,
714
+ width: currentViewState.bmpWidth,
715
+ height: currentViewState.bmpHeight
716
+ )
676
717
  case "clear_view":
677
718
  sgc?.clearDisplay()
678
719
  default:
@@ -827,8 +868,12 @@ struct ViewState {
827
868
  }
828
869
 
829
870
  func restartTranscriber() {
871
+ #if !SWIFT_PACKAGE || MENTRA_FEATURE_LOCAL_STT
830
872
  Bridge.log("MAN: Restarting SherpaOnnxTranscriber via command")
831
873
  transcriber?.restart()
874
+ #else
875
+ Bridge.log("MAN: Local STT is not included in this SwiftPM build")
876
+ #endif
832
877
  }
833
878
 
834
879
  // MARK: - connection state management
@@ -844,10 +889,26 @@ struct ViewState {
844
889
  defaultWearable = sgc.type
845
890
  searching = false
846
891
 
892
+ let connectionKey = "\(sgc.type):\(deviceName)"
893
+ syncSystemTimeOnceForConnection(sgc, connectionKey: connectionKey)
894
+
895
+ // re-apply display height/depth after reconnection
896
+ DispatchQueue.main.asyncAfter(deadline: .now() + 2) { [weak self] in
897
+ // Re-read the current sgc rather than capturing the connect-time instance: the user may
898
+ // have disconnected or switched glasses during the 2s window, and we must not push to a
899
+ // stale/torn-down connection.
900
+ guard let sgc = self?.sgc else { return }
901
+ let h = DeviceStore.shared.get("bluetooth", "dashboard_height") as? Int ?? 4
902
+ // Fall back to the canonical default (2), matching DeviceStore — not 1.
903
+ let rawDepth = DeviceStore.shared.get("bluetooth", "dashboard_depth") as? Int ?? 2
904
+ let d = min(max(rawDepth, 1), 4)
905
+ sgc.setDashboardPosition(h, d)
906
+ }
907
+
847
908
  // Show welcome message on first connect for all display glasses
848
909
  if shouldSendBootingMessage {
849
910
  Task {
850
- sgc.sendTextWall("// MentraOS Connected")
911
+ await sgc.sendTextWall("// MentraOS Connected")
851
912
  try? await Task.sleep(nanoseconds: 3_000_000_000) // 1 second
852
913
  sgc.clearDisplay()
853
914
  }
@@ -873,12 +934,20 @@ struct ViewState {
873
934
  Bridge.saveSetting("device_name", deviceName)
874
935
  Bridge.saveSetting("device_address", deviceAddress)
875
936
 
876
- // Re-apply display height after reconnection
877
- let h = DeviceStore.shared.get("bluetooth", "dashboard_height") as? Int ?? 4
878
- let d = NexDashboardDisplayWire.clampDepthFromStore(
879
- DeviceStore.shared.get("bluetooth", "dashboard_depth")
880
- )
881
- sgc.setDashboardPosition(h, d)
937
+ }
938
+
939
+ private func syncSystemTimeOnceForConnection(_ sgc: SGCManager, connectionKey: String) {
940
+ if sgc.type.contains(DeviceTypes.SIMULATED) {
941
+ return
942
+ }
943
+ if connectionKey == lastSystemTimeSyncConnectionKey {
944
+ return
945
+ }
946
+
947
+ lastSystemTimeSyncConnectionKey = connectionKey
948
+ let timestampMs = Int64(Date().timeIntervalSince1970 * 1000)
949
+ Bridge.log("MAN: Syncing glasses system time once for connection: \(timestampMs)")
950
+ sgc.sendSetSystemTime(timestampMs)
882
951
  }
883
952
 
884
953
  func handleControllerReady() {
@@ -925,8 +994,9 @@ struct ViewState {
925
994
 
926
995
  func handleDeviceDisconnected() {
927
996
  Bridge.log("MAN: Device disconnected")
997
+ lastSystemTimeSyncConnectionKey = ""
928
998
  DeviceStore.shared.apply("glasses", "headUp", false)
929
- DeviceStore.shared.apply("glasses", "voiceActivityDetectionEnabled", true)
999
+ DeviceStore.shared.apply("glasses", "voiceActivityDetectionEnabled", BluetoothSdkDefaults.voiceActivityDetectionEnabled)
930
1000
  // shouldSendBootingMessage = true // Reset for next first connect
931
1001
  }
932
1002
 
@@ -939,7 +1009,7 @@ struct ViewState {
939
1009
  }
940
1010
 
941
1011
  Bridge.log("MAN: Displaying text: \(text)")
942
- sgc?.sendTextWall(text)
1012
+ Task { await sgc?.sendTextWall(text) }
943
1013
  }
944
1014
 
945
1015
  func displayEvent(_ event: [String: Any]) {
@@ -964,6 +1034,12 @@ struct ViewState {
964
1034
  var title = layout["title"] as? String ?? " "
965
1035
  var data = layout["data"] as? String ?? ""
966
1036
 
1037
+ // Optional bitmap_view container position/size (forwarded to the SGC; used by G2).
1038
+ let bmpX = (layout["x"] as? NSNumber).map { $0.int32Value }
1039
+ let bmpY = (layout["y"] as? NSNumber).map { $0.int32Value }
1040
+ let bmpWidth = (layout["width"] as? NSNumber).map { $0.int32Value }
1041
+ let bmpHeight = (layout["height"] as? NSNumber).map { $0.int32Value }
1042
+
967
1043
  text = parsePlaceholders(text)
968
1044
  topText = parsePlaceholders(topText)
969
1045
  bottomText = parsePlaceholders(bottomText)
@@ -971,7 +1047,8 @@ struct ViewState {
971
1047
 
972
1048
  var newViewState = ViewState(
973
1049
  topText: topText, bottomText: bottomText, title: title, layoutType: layoutType,
974
- text: text, data: data, animationData: nil
1050
+ text: text, data: data, animationData: nil,
1051
+ bmpX: bmpX, bmpY: bmpY, bmpWidth: bmpWidth, bmpHeight: bmpHeight
975
1052
  )
976
1053
 
977
1054
  if layoutType == "bitmap_animation" {
@@ -1021,13 +1098,17 @@ struct ViewState {
1021
1098
  sgc?.showDashboard()
1022
1099
  }
1023
1100
 
1101
+ func showNotificationsPanel() {
1102
+ Task { await sgc?.showNotificationsPanel() }
1103
+ }
1104
+
1024
1105
  func ping() {
1025
1106
  sgc?.ping()
1026
1107
  }
1027
1108
 
1028
1109
  func dbg1() {
1029
- sgc?.disconnectController()
1030
- connectDefaultController()
1110
+ // sgc?.disconnectController()
1111
+ // connectDefaultController()
1031
1112
  }
1032
1113
 
1033
1114
  func dbg2() {}
@@ -1075,6 +1156,11 @@ struct ViewState {
1075
1156
  sgc?.sendHotspotState(enabled)
1076
1157
  }
1077
1158
 
1159
+ func setSystemTime(_ timestampMs: Int64) {
1160
+ Bridge.log("MAN: Setting glasses system time: \(timestampMs)")
1161
+ sgc?.sendSetSystemTime(timestampMs)
1162
+ }
1163
+
1078
1164
  func queryGalleryStatus() {
1079
1165
  Bridge.log("MAN: 📸 Querying gallery status from glasses")
1080
1166
  sgc?.queryGalleryStatus()
@@ -1093,6 +1179,42 @@ struct ViewState {
1093
1179
  (sgc as? MentraLive)?.sendOtaQueryStatus()
1094
1180
  }
1095
1181
 
1182
+ private func liveSgc() throws -> MentraLive {
1183
+ guard let live = sgc as? MentraLive else {
1184
+ throw BluetoothError(code: "unsupported_device", message: "This command requires Mentra Live glasses.")
1185
+ }
1186
+ return live
1187
+ }
1188
+
1189
+ func sendGalleryMode(requestId: String, enabled: Bool) throws {
1190
+ try liveSgc().sendGalleryMode(requestId: requestId, active: enabled)
1191
+ }
1192
+
1193
+ func sendButtonPhotoSettings(requestId: String, size: String) throws {
1194
+ try liveSgc().sendButtonPhotoSettings(requestId: requestId, size: size)
1195
+ }
1196
+
1197
+ func sendButtonVideoRecordingSettings(requestId: String, width: Int, height: Int, fps: Int) throws {
1198
+ try liveSgc().sendButtonVideoRecordingSettings(requestId: requestId, width: width, height: height, fps: fps)
1199
+ }
1200
+
1201
+ func sendButtonCameraLedSetting(requestId: String, enabled: Bool) throws {
1202
+ try liveSgc().sendButtonCameraLedSetting(requestId: requestId, enabled: enabled)
1203
+ }
1204
+
1205
+ func sendButtonMaxRecordingTime(requestId: String, minutes: Int) throws {
1206
+ try liveSgc().sendButtonMaxRecordingTime(requestId: requestId, minutes: minutes)
1207
+ }
1208
+
1209
+ func sendCameraFovSetting(requestId: String, fov: Int, roiPosition: Int) throws {
1210
+ try liveSgc().sendCameraFovSetting(requestId: requestId, fov: fov, roiPosition: roiPosition)
1211
+ }
1212
+
1213
+ func retryOtaVersionCheck() {
1214
+ Bridge.log("MAN: ⏰ Retrying glasses OTA version check after clock sync")
1215
+ (sgc as? MentraLive)?.sendOtaRetryVersionCheck()
1216
+ }
1217
+
1096
1218
  /// Request version info from glasses.
1097
1219
  /// Glasses will respond with version_info message containing build number, firmware version, etc.
1098
1220
  func requestVersionInfo() {
@@ -1114,11 +1236,17 @@ struct ViewState {
1114
1236
  sgc?.sendReboot()
1115
1237
  }
1116
1238
 
1117
- func startVideoRecording(_ requestId: String, _ save: Bool, _ sound: Bool) {
1239
+ func startVideoRecording(
1240
+ _ requestId: String, _ save: Bool, _ sound: Bool, _ width: Int = 0, _ height: Int = 0,
1241
+ _ fps: Int = 0
1242
+ ) {
1118
1243
  Bridge.log(
1119
- "MAN: onStartVideoRecording: requestId=\(requestId), save=\(save), flash=true, sound=\(sound)"
1244
+ "MAN: onStartVideoRecording: requestId=\(requestId), save=\(save), flash=true, sound=\(sound), resolution=\(width)x\(height)@\(fps)fps"
1245
+ )
1246
+ sgc?.startVideoRecording(
1247
+ requestId: requestId, save: save, flash: true, sound: sound, width: width, height: height,
1248
+ fps: fps
1120
1249
  )
1121
- sgc?.startVideoRecording(requestId: requestId, save: save, flash: true, sound: sound)
1122
1250
  }
1123
1251
 
1124
1252
  func stopVideoRecording(_ requestId: String) {
@@ -1195,13 +1323,17 @@ struct ViewState {
1195
1323
  _ authToken: String?,
1196
1324
  _ compress: String?,
1197
1325
  _ flash: Bool,
1326
+ _ save: Bool,
1198
1327
  _ sound: Bool,
1199
1328
  exposureTimeNs: Double? = nil,
1200
1329
  iso: Int? = nil
1201
1330
  ) {
1202
- let manualIso = exposureTimeNs != nil ? iso.flatMap { $0 > 0 ? $0 : nil } : nil
1331
+ // Only honor manual exposure when it is a usable value; manual ISO is a one-shot
1332
+ // companion to manual exposure and must be dropped when exposure is invalid.
1333
+ let manualExposureNs = exposureTimeNs.flatMap { $0.isFinite && $0 > 0 ? $0 : nil }
1334
+ let manualIso = manualExposureNs != nil ? iso.flatMap { $0 > 0 ? $0 : nil } : nil
1203
1335
  Bridge.log(
1204
- "MAN: PHOTO PIPELINE [4/6] DeviceManager.requestPhoto requestId=\(requestId) appId=\(appId) webhookUrl=\(webhookUrl ?? "nil") size=\(size) compress=\(compress ?? "none") flash=\(flash) sound=\(sound) exposureTimeNs=\(exposureTimeNs.map { String($0) } ?? "nil") iso=\(manualIso.map { String($0) } ?? "auto") sgc=\(sgc != nil ? String(describing: type(of: sgc!)) : "null")"
1336
+ "MAN: PHOTO PIPELINE [4/6] DeviceManager.requestPhoto requestId=\(requestId) appId=\(appId) webhookUrl=\(webhookUrl ?? "nil") size=\(size) compress=\(compress ?? "none") flash=\(flash) save=\(save) sound=\(sound) exposureTimeNs=\(manualExposureNs.map { String($0) } ?? "nil") iso=\(manualIso.map { String($0) } ?? "auto") sgc=\(sgc != nil ? String(describing: type(of: sgc!)) : "null")"
1205
1337
  )
1206
1338
  guard let sgc else {
1207
1339
  Bridge.log(
@@ -1211,7 +1343,7 @@ struct ViewState {
1211
1343
  }
1212
1344
  sgc.requestPhoto(
1213
1345
  requestId, appId: appId, size: size, webhookUrl: webhookUrl, authToken: authToken,
1214
- compress: compress, flash: flash, sound: sound, exposureTimeNs: exposureTimeNs, iso: manualIso
1346
+ compress: compress, flash: flash, save: save, sound: sound, exposureTimeNs: manualExposureNs, iso: manualIso
1215
1347
  )
1216
1348
  }
1217
1349
 
@@ -1309,6 +1441,7 @@ struct ViewState {
1309
1441
  sgc?.clearDisplay() // clear the screen
1310
1442
  sgc?.disconnect()
1311
1443
  sgc = nil // Clear the SGC reference after disconnect
1444
+ lastSystemTimeSyncConnectionKey = ""
1312
1445
  searching = false
1313
1446
  micEnabled = false
1314
1447
  updateMicState()
@@ -1317,7 +1450,7 @@ struct ViewState {
1317
1450
  DeviceStore.shared.apply("glasses", "deviceModel", "")
1318
1451
  DeviceStore.shared.apply("glasses", "fullyBooted", false)
1319
1452
  DeviceStore.shared.apply("glasses", "connected", false)
1320
- DeviceStore.shared.apply("glasses", "voiceActivityDetectionEnabled", true)
1453
+ DeviceStore.shared.apply("glasses", "voiceActivityDetectionEnabled", BluetoothSdkDefaults.voiceActivityDetectionEnabled)
1321
1454
  // disconnect the controller as well:
1322
1455
  searchingController = false
1323
1456
  DeviceStore.shared.apply("glasses", "controllerConnected", false)
@@ -1392,8 +1525,10 @@ struct ViewState {
1392
1525
 
1393
1526
  func cleanup() {
1394
1527
  // Clean up transcriber resources
1528
+ #if !SWIFT_PACKAGE || MENTRA_FEATURE_LOCAL_STT
1395
1529
  transcriber?.shutdown()
1396
1530
  transcriber = nil
1531
+ #endif
1397
1532
 
1398
1533
  // Clean up LC3 converter
1399
1534
  lc3Converter = nil