@mentra/bluetooth-sdk 0.1.8 → 0.1.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (147) hide show
  1. package/README.md +116 -18
  2. package/android/build.gradle +289 -16
  3. package/android/gradle.properties.example +9 -0
  4. package/android/lc3Lib/build.gradle +95 -6
  5. package/android/settings.gradle +3 -0
  6. package/android/silero/LICENSE.md +17 -0
  7. package/android/silero/build.gradle +42 -0
  8. package/android/silero/proguard-rules.pro +24 -0
  9. package/android/silero/src/androidTest/assets/hello.wav +0 -0
  10. package/android/silero/src/androidTest/java/com/konovalov/vad/silero/VadSileroTest.kt +79 -0
  11. package/android/silero/src/main/assets/LICENSE +21 -0
  12. package/android/silero/src/main/assets/silero_vad.onnx +0 -0
  13. package/android/silero/src/main/java/com/konovalov/vad/silero/Vad.kt +160 -0
  14. package/android/silero/src/main/java/com/konovalov/vad/silero/VadSilero.kt +449 -0
  15. package/android/silero/src/main/java/com/konovalov/vad/silero/config/FrameSize.kt +15 -0
  16. package/android/silero/src/main/java/com/konovalov/vad/silero/config/Mode.kt +14 -0
  17. package/android/silero/src/main/java/com/konovalov/vad/silero/config/SampleRate.kt +12 -0
  18. package/android/silero/src/main/java/com/konovalov/vad/silero/utils/AudioUtils.kt +47 -0
  19. package/android/silero/src/main/java/com/konovalov/vad/silero/utils/TensorMap.kt +41 -0
  20. package/android/silero/src/test/java/com/konovalov/vad/silero/utils/AudioUtilsTest.kt +44 -0
  21. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkDefaults.kt +6 -0
  22. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +165 -27
  23. package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +89 -21
  24. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +189 -16
  25. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt +22 -2
  26. package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +787 -85
  27. package/android/src/main/java/com/mentra/bluetoothsdk/audio/AudioModels.kt +6 -2
  28. package/android/src/main/java/com/mentra/bluetoothsdk/camera/CameraModels.kt +123 -10
  29. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/ControllerManager.kt +12 -2
  30. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/R1.kt +8 -2
  31. package/android/src/main/java/com/mentra/bluetoothsdk/debug/BleTraceLogger.kt +33 -17
  32. package/android/src/main/java/com/mentra/bluetoothsdk/events/BluetoothEvents.kt +101 -0
  33. package/android/src/main/java/com/mentra/bluetoothsdk/internal/MapParsing.kt +8 -0
  34. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistry.java +108 -0
  35. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoUploadServer.kt +399 -74
  36. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/MentraPhotoReceiverModule.kt +100 -17
  37. package/android/src/main/java/com/mentra/bluetoothsdk/requests/DisplayRequests.kt +17 -0
  38. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G1.java +2 -2
  39. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt +569 -220
  40. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Mach1.java +2 -2
  41. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraLive.java +271 -66
  42. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraNex.kt +176 -104
  43. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraosBle.java +12753 -27036
  44. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt +54 -2
  45. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Simulated.kt +16 -4
  46. package/android/src/main/java/com/mentra/bluetoothsdk/status/DeviceStatus.kt +60 -3
  47. package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +12 -3
  48. package/android/src/main/java/com/mentra/bluetoothsdk/stt/STTTools.kt +87 -36
  49. package/android/src/main/java/com/mentra/bluetoothsdk/stt/SherpaOnnxTranscriber.kt +13 -4
  50. package/android/src/main/java/com/mentra/bluetoothsdk/stt/VadGateSpeechPolicy.kt +229 -0
  51. package/android/src/main/java/com/mentra/bluetoothsdk/tts/TTSTools.kt +180 -0
  52. package/android/src/main/java/com/mentra/bluetoothsdk/utils/AvifExifStripper.java +573 -0
  53. package/android/src/main/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadService.java +488 -50
  54. package/android/src/main/java/com/mentra/bluetoothsdk/utils/HeifExifTagReader.java +199 -0
  55. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunkReassembler.java +18 -1
  56. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunker.java +64 -18
  57. package/android/src/main/java/com/mentra/bluetoothsdk/utils/NexSGCUtils.kt +48 -39
  58. package/android/src/test/java/com/mentra/bluetoothsdk/camera/PhotoRequestTest.kt +23 -0
  59. package/android/src/test/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistryTest.java +75 -0
  60. package/android/src/test/java/com/mentra/bluetoothsdk/utils/AvifExifStripperTest.java +30 -0
  61. package/android/src/test/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadServiceTest.java +98 -0
  62. package/android/src/test/resources/avif_with_exif.avif +0 -0
  63. package/build/BluetoothSdk.types.d.ts +241 -43
  64. package/build/BluetoothSdk.types.d.ts.map +1 -1
  65. package/build/BluetoothSdk.types.js +11 -10
  66. package/build/BluetoothSdk.types.js.map +1 -1
  67. package/build/_private/BluetoothSdkModule.d.ts +36 -23
  68. package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
  69. package/build/_private/BluetoothSdkModule.js +30 -52
  70. package/build/_private/BluetoothSdkModule.js.map +1 -1
  71. package/build/_private/photoRequestPayload.d.ts +4 -0
  72. package/build/_private/photoRequestPayload.d.ts.map +1 -0
  73. package/build/_private/photoRequestPayload.js +25 -0
  74. package/build/_private/photoRequestPayload.js.map +1 -0
  75. package/build/index.d.ts +1 -1
  76. package/build/index.d.ts.map +1 -1
  77. package/build/index.js +23 -0
  78. package/build/index.js.map +1 -1
  79. package/build/react/useMentraBluetooth.d.ts +2 -2
  80. package/build/react/useMentraBluetooth.d.ts.map +1 -1
  81. package/build/react/useMentraBluetooth.js +3 -3
  82. package/build/react/useMentraBluetooth.js.map +1 -1
  83. package/ios/BluetoothSdkModule.swift +174 -70
  84. package/ios/LocalPhotoUploadServer.swift +370 -105
  85. package/ios/MentraBluetoothSDK.podspec +3 -1
  86. package/ios/MentraPhotoReceiverModule.swift +62 -10
  87. package/ios/Packages/CoreObjC/include/PcmConverter.h +22 -0
  88. package/ios/Packages/SherpaOnnx/SherpaOnnx.swift +668 -26
  89. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Info.plist +13 -9
  90. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
  91. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
  92. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/libsherpa-onnx.a +0 -0
  93. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
  94. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
  95. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/libsherpa-onnx.a +0 -0
  96. package/ios/Source/BluetoothSdkDefaults.swift +6 -0
  97. package/ios/Source/Bridge.swift +58 -19
  98. package/ios/Source/DeviceManager.swift +149 -17
  99. package/ios/Source/DeviceStore.swift +16 -6
  100. package/ios/Source/MentraBluetoothSDK.swift +803 -47
  101. package/ios/Source/ObservableStore.swift +8 -2
  102. package/ios/Source/{Audio → audio}/AudioModels.swift +9 -7
  103. package/ios/Source/{Camera → camera}/CameraModels.swift +225 -16
  104. package/ios/Source/controllers/ControllerManager.swift +4 -3
  105. package/ios/Source/controllers/R1.swift +3 -3
  106. package/ios/Source/{Events → events}/BluetoothEvents.swift +183 -1
  107. package/ios/Source/{Internal → internal}/ValueParsing.swift +10 -1
  108. package/ios/Source/{Requests → requests}/DisplayRequests.swift +28 -4
  109. package/ios/Source/services/PhoneMic.swift +1 -0
  110. package/ios/Source/sgcs/G1.swift +2 -2
  111. package/ios/Source/sgcs/G2.swift +645 -403
  112. package/ios/Source/sgcs/Mach1.swift +2 -2
  113. package/ios/Source/sgcs/MentraLive.swift +812 -319
  114. package/ios/Source/sgcs/MentraNex.swift +412 -182
  115. package/ios/Source/sgcs/SGCManager.swift +57 -3
  116. package/ios/Source/sgcs/Simulated.swift +7 -3
  117. package/ios/Source/sgcs/mentraos_ble.pb.swift +3073 -3134
  118. package/ios/Source/status/DeviceStatus.swift +974 -0
  119. package/ios/Source/{Status → status}/WifiHotspotStatus.swift +4 -4
  120. package/ios/Source/{Streaming → streaming}/StreamModels.swift +24 -9
  121. package/ios/Source/stt/STTTools.swift +27 -12
  122. package/ios/Source/stt/SherpaOnnxTranscriber.swift +31 -8
  123. package/ios/Source/tts/TTSTools.swift +164 -0
  124. package/ios/Source/utils/JSCExperiment.swift +7 -7
  125. package/ios/Source/utils/MessageChunkReassembler.swift +20 -1
  126. package/ios/Source/utils/MessageChunker.swift +78 -21
  127. package/ios/Source/utils/TarBz2Extractor.swift +53 -2
  128. package/package.json +1 -1
  129. package/plugin/build/withAndroid.js +68 -4
  130. package/src/BluetoothSdk.types.ts +331 -59
  131. package/src/_private/BluetoothSdkModule.ts +113 -102
  132. package/src/_private/photoRequestPayload.ts +28 -0
  133. package/src/index.ts +58 -15
  134. package/src/react/useMentraBluetooth.ts +6 -4
  135. package/android/.project +0 -28
  136. package/android/lc3Lib/.project +0 -28
  137. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/cargs.h +0 -162
  138. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/c-api.h +0 -1852
  139. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/cxx-api.h +0 -674
  140. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/sherpa-onnx.a +0 -0
  141. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/sherpa-onnx.a +0 -0
  142. package/ios/Source/Status/DeviceStatus.swift +0 -471
  143. /package/ios/Source/{Connection → connection}/ScanSession.swift +0 -0
  144. /package/ios/Source/{Errors → errors}/BluetoothError.swift +0 -0
  145. /package/ios/Source/{Internal → internal}/BluetoothAvailability.swift +0 -0
  146. /package/ios/Source/{Status → status}/RuntimeState.swift +0 -0
  147. /package/ios/Source/{Types → types}/DeviceModels.swift +0 -0
@@ -0,0 +1,6 @@
1
+ import Foundation
2
+
3
+ /// Defaults for the public Bluetooth SDK surface.
4
+ enum BluetoothSdkDefaults {
5
+ static let voiceActivityDetectionEnabled = false
6
+ }
@@ -10,7 +10,7 @@ import Foundation
10
10
  /// Bridge for Bluetooth SDK communication between Expo modules and native iOS code
11
11
  /// Has commands for the Bluetooth SDK to use to send messages to JavaScript
12
12
  class Bridge {
13
- private static let micSampleRate = 16_000
13
+ private static let micSampleRate = 16000
14
14
  private static let pcmBitsPerSample = 16
15
15
  private static let micChannels = 1
16
16
  private static let lc3FrameDurationMs = 10
@@ -67,6 +67,16 @@ class Bridge {
67
67
  Bridge.sendTypedMessage("log", body: data)
68
68
  }
69
69
 
70
+ /// Report tar.bz2 extraction progress to JavaScript.
71
+ static func sendExtractionProgress(percentage: Int, bytesRead: Int64, totalBytes: Int64) {
72
+ let body: [String: Any] = [
73
+ "percentage": percentage,
74
+ "bytesRead": bytesRead,
75
+ "totalBytes": totalBytes,
76
+ ]
77
+ Bridge.sendTypedMessage("extraction_progress", body: body)
78
+ }
79
+
70
80
  static func sendHeadUp(_ isUp: Bool) {
71
81
  let data = ["up": isUp]
72
82
  Bridge.sendTypedMessage("head_up", body: data)
@@ -90,7 +100,8 @@ class Bridge {
90
100
  @MainActor
91
101
  private static func micPcmEventBody(_ data: Data) -> [String: Any] {
92
102
  let voiceActivityDetectionEnabled =
93
- DeviceStore.shared.get("glasses", "voiceActivityDetectionEnabled") as? Bool ?? true
103
+ DeviceStore.shared.get("glasses", "voiceActivityDetectionEnabled") as? Bool
104
+ ?? BluetoothSdkDefaults.voiceActivityDetectionEnabled
94
105
  return [
95
106
  "pcm": data,
96
107
  "sampleRate": micSampleRate,
@@ -104,7 +115,8 @@ class Bridge {
104
115
  @MainActor
105
116
  private static func micLc3EventBody(_ data: Data) -> [String: Any] {
106
117
  let voiceActivityDetectionEnabled =
107
- DeviceStore.shared.get("glasses", "voiceActivityDetectionEnabled") as? Bool ?? true
118
+ DeviceStore.shared.get("glasses", "voiceActivityDetectionEnabled") as? Bool
119
+ ?? BluetoothSdkDefaults.voiceActivityDetectionEnabled
108
120
  let frameSizeBytes = DeviceStore.shared.get("bluetooth", "lc3_frame_size") as? Int ?? defaultLc3FrameSizeBytes
109
121
  return [
110
122
  "lc3": data,
@@ -269,6 +281,35 @@ class Bridge {
269
281
  Bridge.sendTypedMessage("rgb_led_control_response", body: body)
270
282
  }
271
283
 
284
+ static func sendSettingsAck(_ values: [String: Any]) {
285
+ var body = values
286
+ body["type"] = "settings_ack"
287
+ Bridge.sendTypedMessage("settings_ack", body: body)
288
+ }
289
+
290
+ static func sendVideoRecordingStatus(_ values: [String: Any]) {
291
+ var body = values
292
+ body["type"] = "video_recording_status"
293
+ Bridge.sendTypedMessage("video_recording_status", body: body)
294
+ }
295
+
296
+ static func sendVersionInfo(_ values: [String: Any]) {
297
+ var body: [String: Any] = [
298
+ "type": "version_info",
299
+ "androidVersion": stringValue(values, "androidVersion", "android_version") ?? "",
300
+ "firmwareVersion": stringValue(values, "firmwareVersion", "firmware_version") ?? "",
301
+ "besFirmwareVersion": stringValue(values, "besFirmwareVersion", "bes_fw_version") ?? "",
302
+ "mtkFirmwareVersion": stringValue(values, "mtkFirmwareVersion", "mtk_fw_version") ?? "",
303
+ "buildNumber": stringValue(values, "buildNumber", "build_number") ?? "",
304
+ "otaVersionUrl": stringValue(values, "otaVersionUrl", "ota_version_url") ?? "",
305
+ "appVersion": stringValue(values, "appVersion", "app_version") ?? "",
306
+ ]
307
+ if let systemTimeMs = intValue(values["systemTimeMs"]) ?? intValue(values["system_time_ms"]) {
308
+ body["systemTimeMs"] = systemTimeMs
309
+ }
310
+ Bridge.sendTypedMessage("version_info", body: body)
311
+ }
312
+
272
313
  static func sendPhotoError(requestId: String, errorCode: String, errorMessage: String) {
273
314
  let timestamp = Int(Date().timeIntervalSince1970 * 1000)
274
315
  var event: [String: Any] = [
@@ -284,26 +325,16 @@ class Bridge {
284
325
  event["errorMessage"] = errorMessage
285
326
  }
286
327
  Bridge.sendTypedMessage("photo_response", body: event)
287
-
288
- var status: [String: Any] = [
289
- "type": "photo_status",
290
- "status": "failed",
291
- "requestId": requestId,
292
- "timestamp": timestamp,
293
- ]
294
- if !errorCode.isEmpty {
295
- status["errorCode"] = errorCode
296
- }
297
- if !errorMessage.isEmpty {
298
- status["errorMessage"] = errorMessage
299
- }
300
- Bridge.sendTypedMessage("photo_status", body: status)
301
328
  }
302
329
 
303
330
  static func sendPhotoStatus(_ status: [String: Any]) {
304
331
  Bridge.sendTypedMessage("photo_status", body: status)
305
332
  }
306
333
 
334
+ static func sendPhotoResponse(_ response: [String: Any]) {
335
+ Bridge.sendTypedMessage("photo_response", body: response)
336
+ }
337
+
307
338
  static func sendMiniappSelected(packageName: String) {
308
339
  let event: [String: Any] = [
309
340
  "packageName": packageName,
@@ -349,7 +380,7 @@ class Bridge {
349
380
  Bridge.sendTypedMessage("wifi_status_change", body: status.values)
350
381
  }
351
382
 
352
- static func updateWifiScanResults(_ networks: [[String: Any]]) {
383
+ static func updateWifiScanResults(_ networks: [[String: Any]], scanComplete: Bool) {
353
384
  Task {
354
385
  await MainActor.run {
355
386
  var storedNetworks: [[String: Any]] =
@@ -363,6 +394,10 @@ class Bridge {
363
394
  }
364
395
  }
365
396
  DeviceStore.shared.apply("bluetooth", "wifiScanResults", storedNetworks)
397
+ Bridge.sendTypedMessage(
398
+ "wifi_scan_result",
399
+ body: ["networks": storedNetworks, "scanComplete": scanComplete]
400
+ )
366
401
  }
367
402
  }
368
403
  }
@@ -408,7 +443,8 @@ class Bridge {
408
443
  stepPercent: Int,
409
444
  overallPercent: Int,
410
445
  status: String,
411
- errorMessage: String?
446
+ errorMessage: String?,
447
+ glassesTimeMs: Int64? = nil
412
448
  ) {
413
449
  var eventBody: [String: Any] = [
414
450
  "session_id": sessionId,
@@ -423,6 +459,9 @@ class Bridge {
423
459
  if let error = errorMessage {
424
460
  eventBody["error_message"] = error
425
461
  }
462
+ if let glassesTimeMs, glassesTimeMs > 0 {
463
+ eventBody["glasses_time_ms"] = glassesTimeMs
464
+ }
426
465
  Bridge.sendTypedMessage("ota_status", body: eventBody)
427
466
  }
428
467
 
@@ -10,6 +10,9 @@ import Combine
10
10
  import CoreBluetooth
11
11
  import Foundation
12
12
  import UIKit
13
+ #if SWIFT_PACKAGE
14
+ import MentraBluetoothSDKCoreObjC
15
+ #endif
13
16
 
14
17
  struct ViewState {
15
18
  var topText: String
@@ -19,6 +22,11 @@ struct ViewState {
19
22
  var text: String
20
23
  var data: String?
21
24
  var animationData: [String: Any]?
25
+ // Optional bitmap_view container position/size (used by G2; ignored by others)
26
+ var bmpX: Int32?
27
+ var bmpY: Int32?
28
+ var bmpWidth: Int32?
29
+ var bmpHeight: Int32?
22
30
  }
23
31
 
24
32
  @MainActor
@@ -167,6 +175,14 @@ struct ViewState {
167
175
  set { DeviceStore.shared.apply("bluetooth", "sensing_enabled", newValue) }
168
176
  }
169
177
 
178
+ /// Phone-side VAD gating switch. Default is OFF (VAD runs) so that the
179
+ /// coordinator can drive per-utterance offline/online STT switching from
180
+ /// `vad_status` events. Set to `true` only as an emergency kill-switch.
181
+ private var bypassVad: Bool {
182
+ get { DeviceStore.shared.get("bluetooth", "bypass_vad") as? Bool ?? false }
183
+ set { DeviceStore.shared.apply("bluetooth", "bypass_vad", newValue) }
184
+ }
185
+
170
186
  private var offlineCaptionsRunning: Bool {
171
187
  get { DeviceStore.shared.get("bluetooth", "offline_captions_running") as? Bool ?? false }
172
188
  set { DeviceStore.shared.apply("bluetooth", "offline_captions_running", newValue) }
@@ -225,6 +241,7 @@ struct ViewState {
225
241
  get { DeviceStore.shared.get("bluetooth", "shouldSendBootingMessage") as? Bool ?? true }
226
242
  set { DeviceStore.shared.apply("bluetooth", "shouldSendBootingMessage", newValue) }
227
243
  }
244
+ private var lastSystemTimeSyncConnectionKey = ""
228
245
 
229
246
  private var systemMicUnavailable: Bool {
230
247
  get { DeviceStore.shared.get("bluetooth", "systemMicUnavailable") as? Bool ?? false }
@@ -281,7 +298,9 @@ struct ViewState {
281
298
  private var micReinitTimer: Timer?
282
299
 
283
300
  /// STT:
301
+ #if !SWIFT_PACKAGE || MENTRA_FEATURE_LOCAL_STT
284
302
  private var transcriber: SherpaOnnxTranscriber?
303
+ #endif
285
304
 
286
305
  var viewStates: [ViewState] = [
287
306
  ViewState(
@@ -310,6 +329,7 @@ struct ViewState {
310
329
  // MemoryMonitor.start()
311
330
 
312
331
  // Initialize SherpaOnnx Transcriber
332
+ #if !SWIFT_PACKAGE || MENTRA_FEATURE_LOCAL_STT
313
333
  if let windowScene = UIApplication.shared.connectedScenes.first as? UIWindowScene,
314
334
  let window = windowScene.windows.first,
315
335
  let rootViewController = window.rootViewController
@@ -324,6 +344,7 @@ struct ViewState {
324
344
  transcriber.initialize()
325
345
  Bridge.log("SherpaOnnxTranscriber fully initialized")
326
346
  }
347
+ #endif
327
348
 
328
349
  // Initialize persistent LC3 converter for unified audio encoding
329
350
  lc3Converter = PcmConverter()
@@ -395,9 +416,11 @@ struct ViewState {
395
416
  handleSendingPcm(pcmData)
396
417
 
397
418
  // Send PCM to local transcriber.
419
+ #if !SWIFT_PACKAGE || MENTRA_FEATURE_LOCAL_STT
398
420
  if shouldSendTranscript || offlineCaptionsRunning || localSttFallbackActive {
399
421
  transcriber?.acceptAudio(pcm16le: pcmData)
400
422
  }
423
+ #endif
401
424
  }
402
425
 
403
426
  func updateMicState() {
@@ -581,6 +604,7 @@ struct ViewState {
581
604
  Bridge.log("MAN: Manager already initialized, cleaning up previous sgc")
582
605
  sgc?.cleanup()
583
606
  sgc = nil
607
+ lastSystemTimeSyncConnectionKey = ""
584
608
  }
585
609
 
586
610
  if sgc != nil {
@@ -595,14 +619,24 @@ struct ViewState {
595
619
  sgc = G2()
596
620
  } else if wearable.contains(DeviceTypes.LIVE) {
597
621
  sgc = MentraLive()
598
- } else if wearable.contains(DeviceTypes.MACH1) {
599
- sgc = Mach1()
600
- } else if wearable.contains(DeviceTypes.Z100) {
601
- sgc = Mach1() // Z100 uses same hardware/SDK as Mach1
602
- sgc?.type = DeviceTypes.Z100 // Override type to Z100
603
622
  } else if wearable.contains(DeviceTypes.FRAME) {
604
623
  // sgc = FrameManager()
605
624
  }
625
+ #if !SWIFT_PACKAGE || MENTRA_FEATURE_NEX
626
+ if sgc == nil && wearable.contains(DeviceTypes.NEX) {
627
+ sgc = MentraNexSGC.getInstance()
628
+ }
629
+ #endif
630
+ #if !SWIFT_PACKAGE || MENTRA_FEATURE_VUZIX
631
+ if sgc == nil {
632
+ if wearable.contains(DeviceTypes.MACH1) {
633
+ sgc = Mach1()
634
+ } else if wearable.contains(DeviceTypes.Z100) {
635
+ sgc = Mach1() // Z100 uses same hardware/SDK as Mach1
636
+ sgc?.type = DeviceTypes.Z100 // Override type to Z100
637
+ }
638
+ }
639
+ #endif
606
640
  // update device model:
607
641
  DeviceStore.shared.apply("glasses", "deviceModel", sgc?.type ?? "")
608
642
  }
@@ -672,7 +706,13 @@ struct ViewState {
672
706
  return
673
707
  }
674
708
  Bridge.log("MAN: Processing bitmap_view with base64 data, length: \(data.count)")
675
- await sgc?.displayBitmap(base64ImageData: data)
709
+ await sgc?.displayBitmap(
710
+ base64ImageData: data,
711
+ x: currentViewState.bmpX,
712
+ y: currentViewState.bmpY,
713
+ width: currentViewState.bmpWidth,
714
+ height: currentViewState.bmpHeight
715
+ )
676
716
  case "clear_view":
677
717
  sgc?.clearDisplay()
678
718
  default:
@@ -827,8 +867,12 @@ struct ViewState {
827
867
  }
828
868
 
829
869
  func restartTranscriber() {
870
+ #if !SWIFT_PACKAGE || MENTRA_FEATURE_LOCAL_STT
830
871
  Bridge.log("MAN: Restarting SherpaOnnxTranscriber via command")
831
872
  transcriber?.restart()
873
+ #else
874
+ Bridge.log("MAN: Local STT is not included in this SwiftPM build")
875
+ #endif
832
876
  }
833
877
 
834
878
  // MARK: - connection state management
@@ -844,6 +888,9 @@ struct ViewState {
844
888
  defaultWearable = sgc.type
845
889
  searching = false
846
890
 
891
+ let connectionKey = "\(sgc.type):\(deviceName)"
892
+ syncSystemTimeOnceForConnection(sgc, connectionKey: connectionKey)
893
+
847
894
  // Show welcome message on first connect for all display glasses
848
895
  if shouldSendBootingMessage {
849
896
  Task {
@@ -875,12 +922,31 @@ struct ViewState {
875
922
 
876
923
  // Re-apply display height after reconnection
877
924
  let h = DeviceStore.shared.get("bluetooth", "dashboard_height") as? Int ?? 4
925
+ #if !SWIFT_PACKAGE || MENTRA_FEATURE_NEX
878
926
  let d = NexDashboardDisplayWire.clampDepthFromStore(
879
927
  DeviceStore.shared.get("bluetooth", "dashboard_depth")
880
928
  )
929
+ #else
930
+ let rawDepth = DeviceStore.shared.get("bluetooth", "dashboard_depth") as? Int ?? 1
931
+ let d = min(max(rawDepth, 1), 4)
932
+ #endif
881
933
  sgc.setDashboardPosition(h, d)
882
934
  }
883
935
 
936
+ private func syncSystemTimeOnceForConnection(_ sgc: SGCManager, connectionKey: String) {
937
+ if sgc.type.contains(DeviceTypes.SIMULATED) {
938
+ return
939
+ }
940
+ if connectionKey == lastSystemTimeSyncConnectionKey {
941
+ return
942
+ }
943
+
944
+ lastSystemTimeSyncConnectionKey = connectionKey
945
+ let timestampMs = Int64(Date().timeIntervalSince1970 * 1000)
946
+ Bridge.log("MAN: Syncing glasses system time once for connection: \(timestampMs)")
947
+ sgc.sendSetSystemTime(timestampMs)
948
+ }
949
+
884
950
  func handleControllerReady() {
885
951
  guard let controller else {
886
952
  Bridge.log("MAN: Controller is nil, returning")
@@ -925,8 +991,9 @@ struct ViewState {
925
991
 
926
992
  func handleDeviceDisconnected() {
927
993
  Bridge.log("MAN: Device disconnected")
994
+ lastSystemTimeSyncConnectionKey = ""
928
995
  DeviceStore.shared.apply("glasses", "headUp", false)
929
- DeviceStore.shared.apply("glasses", "voiceActivityDetectionEnabled", true)
996
+ DeviceStore.shared.apply("glasses", "voiceActivityDetectionEnabled", BluetoothSdkDefaults.voiceActivityDetectionEnabled)
930
997
  // shouldSendBootingMessage = true // Reset for next first connect
931
998
  }
932
999
 
@@ -964,6 +1031,12 @@ struct ViewState {
964
1031
  var title = layout["title"] as? String ?? " "
965
1032
  var data = layout["data"] as? String ?? ""
966
1033
 
1034
+ // Optional bitmap_view container position/size (forwarded to the SGC; used by G2).
1035
+ let bmpX = (layout["x"] as? NSNumber).map { $0.int32Value }
1036
+ let bmpY = (layout["y"] as? NSNumber).map { $0.int32Value }
1037
+ let bmpWidth = (layout["width"] as? NSNumber).map { $0.int32Value }
1038
+ let bmpHeight = (layout["height"] as? NSNumber).map { $0.int32Value }
1039
+
967
1040
  text = parsePlaceholders(text)
968
1041
  topText = parsePlaceholders(topText)
969
1042
  bottomText = parsePlaceholders(bottomText)
@@ -971,7 +1044,8 @@ struct ViewState {
971
1044
 
972
1045
  var newViewState = ViewState(
973
1046
  topText: topText, bottomText: bottomText, title: title, layoutType: layoutType,
974
- text: text, data: data, animationData: nil
1047
+ text: text, data: data, animationData: nil,
1048
+ bmpX: bmpX, bmpY: bmpY, bmpWidth: bmpWidth, bmpHeight: bmpHeight
975
1049
  )
976
1050
 
977
1051
  if layoutType == "bitmap_animation" {
@@ -1021,13 +1095,17 @@ struct ViewState {
1021
1095
  sgc?.showDashboard()
1022
1096
  }
1023
1097
 
1098
+ func showNotificationsPanel() {
1099
+ sgc?.showNotificationsPanel()
1100
+ }
1101
+
1024
1102
  func ping() {
1025
1103
  sgc?.ping()
1026
1104
  }
1027
1105
 
1028
1106
  func dbg1() {
1029
- sgc?.disconnectController()
1030
- connectDefaultController()
1107
+ // sgc?.disconnectController()
1108
+ // connectDefaultController()
1031
1109
  }
1032
1110
 
1033
1111
  func dbg2() {}
@@ -1075,6 +1153,11 @@ struct ViewState {
1075
1153
  sgc?.sendHotspotState(enabled)
1076
1154
  }
1077
1155
 
1156
+ func setSystemTime(_ timestampMs: Int64) {
1157
+ Bridge.log("MAN: Setting glasses system time: \(timestampMs)")
1158
+ sgc?.sendSetSystemTime(timestampMs)
1159
+ }
1160
+
1078
1161
  func queryGalleryStatus() {
1079
1162
  Bridge.log("MAN: 📸 Querying gallery status from glasses")
1080
1163
  sgc?.queryGalleryStatus()
@@ -1093,6 +1176,42 @@ struct ViewState {
1093
1176
  (sgc as? MentraLive)?.sendOtaQueryStatus()
1094
1177
  }
1095
1178
 
1179
+ private func liveSgc() throws -> MentraLive {
1180
+ guard let live = sgc as? MentraLive else {
1181
+ throw BluetoothError(code: "unsupported_device", message: "This command requires Mentra Live glasses.")
1182
+ }
1183
+ return live
1184
+ }
1185
+
1186
+ func sendGalleryMode(requestId: String, enabled: Bool) throws {
1187
+ try liveSgc().sendGalleryMode(requestId: requestId, active: enabled)
1188
+ }
1189
+
1190
+ func sendButtonPhotoSettings(requestId: String, size: String) throws {
1191
+ try liveSgc().sendButtonPhotoSettings(requestId: requestId, size: size)
1192
+ }
1193
+
1194
+ func sendButtonVideoRecordingSettings(requestId: String, width: Int, height: Int, fps: Int) throws {
1195
+ try liveSgc().sendButtonVideoRecordingSettings(requestId: requestId, width: width, height: height, fps: fps)
1196
+ }
1197
+
1198
+ func sendButtonCameraLedSetting(requestId: String, enabled: Bool) throws {
1199
+ try liveSgc().sendButtonCameraLedSetting(requestId: requestId, enabled: enabled)
1200
+ }
1201
+
1202
+ func sendButtonMaxRecordingTime(requestId: String, minutes: Int) throws {
1203
+ try liveSgc().sendButtonMaxRecordingTime(requestId: requestId, minutes: minutes)
1204
+ }
1205
+
1206
+ func sendCameraFovSetting(requestId: String, fov: Int, roiPosition: Int) throws {
1207
+ try liveSgc().sendCameraFovSetting(requestId: requestId, fov: fov, roiPosition: roiPosition)
1208
+ }
1209
+
1210
+ func retryOtaVersionCheck() {
1211
+ Bridge.log("MAN: ⏰ Retrying glasses OTA version check after clock sync")
1212
+ (sgc as? MentraLive)?.sendOtaRetryVersionCheck()
1213
+ }
1214
+
1096
1215
  /// Request version info from glasses.
1097
1216
  /// Glasses will respond with version_info message containing build number, firmware version, etc.
1098
1217
  func requestVersionInfo() {
@@ -1114,11 +1233,17 @@ struct ViewState {
1114
1233
  sgc?.sendReboot()
1115
1234
  }
1116
1235
 
1117
- func startVideoRecording(_ requestId: String, _ save: Bool, _ sound: Bool) {
1236
+ func startVideoRecording(
1237
+ _ requestId: String, _ save: Bool, _ sound: Bool, _ width: Int = 0, _ height: Int = 0,
1238
+ _ fps: Int = 0
1239
+ ) {
1118
1240
  Bridge.log(
1119
- "MAN: onStartVideoRecording: requestId=\(requestId), save=\(save), flash=true, sound=\(sound)"
1241
+ "MAN: onStartVideoRecording: requestId=\(requestId), save=\(save), flash=true, sound=\(sound), resolution=\(width)x\(height)@\(fps)fps"
1242
+ )
1243
+ sgc?.startVideoRecording(
1244
+ requestId: requestId, save: save, flash: true, sound: sound, width: width, height: height,
1245
+ fps: fps
1120
1246
  )
1121
- sgc?.startVideoRecording(requestId: requestId, save: save, flash: true, sound: sound)
1122
1247
  }
1123
1248
 
1124
1249
  func stopVideoRecording(_ requestId: String) {
@@ -1195,13 +1320,17 @@ struct ViewState {
1195
1320
  _ authToken: String?,
1196
1321
  _ compress: String?,
1197
1322
  _ flash: Bool,
1323
+ _ save: Bool,
1198
1324
  _ sound: Bool,
1199
1325
  exposureTimeNs: Double? = nil,
1200
1326
  iso: Int? = nil
1201
1327
  ) {
1202
- let manualIso = exposureTimeNs != nil ? iso.flatMap { $0 > 0 ? $0 : nil } : nil
1328
+ // Only honor manual exposure when it is a usable value; manual ISO is a one-shot
1329
+ // companion to manual exposure and must be dropped when exposure is invalid.
1330
+ let manualExposureNs = exposureTimeNs.flatMap { $0.isFinite && $0 > 0 ? $0 : nil }
1331
+ let manualIso = manualExposureNs != nil ? iso.flatMap { $0 > 0 ? $0 : nil } : nil
1203
1332
  Bridge.log(
1204
- "MAN: PHOTO PIPELINE [4/6] DeviceManager.requestPhoto requestId=\(requestId) appId=\(appId) webhookUrl=\(webhookUrl ?? "nil") size=\(size) compress=\(compress ?? "none") flash=\(flash) sound=\(sound) exposureTimeNs=\(exposureTimeNs.map { String($0) } ?? "nil") iso=\(manualIso.map { String($0) } ?? "auto") sgc=\(sgc != nil ? String(describing: type(of: sgc!)) : "null")"
1333
+ "MAN: PHOTO PIPELINE [4/6] DeviceManager.requestPhoto requestId=\(requestId) appId=\(appId) webhookUrl=\(webhookUrl ?? "nil") size=\(size) compress=\(compress ?? "none") flash=\(flash) save=\(save) sound=\(sound) exposureTimeNs=\(manualExposureNs.map { String($0) } ?? "nil") iso=\(manualIso.map { String($0) } ?? "auto") sgc=\(sgc != nil ? String(describing: type(of: sgc!)) : "null")"
1205
1334
  )
1206
1335
  guard let sgc else {
1207
1336
  Bridge.log(
@@ -1211,7 +1340,7 @@ struct ViewState {
1211
1340
  }
1212
1341
  sgc.requestPhoto(
1213
1342
  requestId, appId: appId, size: size, webhookUrl: webhookUrl, authToken: authToken,
1214
- compress: compress, flash: flash, sound: sound, exposureTimeNs: exposureTimeNs, iso: manualIso
1343
+ compress: compress, flash: flash, save: save, sound: sound, exposureTimeNs: manualExposureNs, iso: manualIso
1215
1344
  )
1216
1345
  }
1217
1346
 
@@ -1309,6 +1438,7 @@ struct ViewState {
1309
1438
  sgc?.clearDisplay() // clear the screen
1310
1439
  sgc?.disconnect()
1311
1440
  sgc = nil // Clear the SGC reference after disconnect
1441
+ lastSystemTimeSyncConnectionKey = ""
1312
1442
  searching = false
1313
1443
  micEnabled = false
1314
1444
  updateMicState()
@@ -1317,7 +1447,7 @@ struct ViewState {
1317
1447
  DeviceStore.shared.apply("glasses", "deviceModel", "")
1318
1448
  DeviceStore.shared.apply("glasses", "fullyBooted", false)
1319
1449
  DeviceStore.shared.apply("glasses", "connected", false)
1320
- DeviceStore.shared.apply("glasses", "voiceActivityDetectionEnabled", true)
1450
+ DeviceStore.shared.apply("glasses", "voiceActivityDetectionEnabled", BluetoothSdkDefaults.voiceActivityDetectionEnabled)
1321
1451
  // disconnect the controller as well:
1322
1452
  searchingController = false
1323
1453
  DeviceStore.shared.apply("glasses", "controllerConnected", false)
@@ -1392,8 +1522,10 @@ struct ViewState {
1392
1522
 
1393
1523
  func cleanup() {
1394
1524
  // Clean up transcriber resources
1525
+ #if !SWIFT_PACKAGE || MENTRA_FEATURE_LOCAL_STT
1395
1526
  transcriber?.shutdown()
1396
1527
  transcriber = nil
1528
+ #endif
1397
1529
 
1398
1530
  // Clean up LC3 converter
1399
1531
  lc3Converter = nil
@@ -28,7 +28,7 @@ class DeviceStore {
28
28
  store.set("glasses", "deviceModel", "")
29
29
  store.set("glasses", "firmwareVersion", "")
30
30
  store.set("glasses", "micEnabled", false)
31
- store.set("glasses", "voiceActivityDetectionEnabled", true)
31
+ store.set("glasses", "voiceActivityDetectionEnabled", BluetoothSdkDefaults.voiceActivityDetectionEnabled)
32
32
  store.set("glasses", "bluetoothClassicConnected", false)
33
33
  store.set("glasses", "caseRemoved", true)
34
34
  store.set("glasses", "caseOpen", true)
@@ -86,7 +86,9 @@ class DeviceStore {
86
86
  store.set("bluetooth", "head_up_angle", 30)
87
87
  store.set("bluetooth", "contextual_dashboard", true)
88
88
  store.set("bluetooth", "gallery_mode", true)
89
- store.set("bluetooth", "voice_activity_detection_enabled", true)
89
+ store.set("bluetooth", "voice_activity_detection_enabled", BluetoothSdkDefaults.voiceActivityDetectionEnabled)
90
+ // Mentra Nex feature flag (off by default; toggled from Nex Developer Settings):
91
+ store.set("bluetooth", "nex_chinese_captions", false)
90
92
  store.set("bluetooth", "screen_disabled", false)
91
93
  store.set("bluetooth", "button_photo_size", "medium")
92
94
  store.set("bluetooth", "button_camera_led", true)
@@ -134,16 +136,16 @@ class DeviceStore {
134
136
  /// Apply changes with side effects
135
137
  func apply(_ category: String, _ key: String, _ value: Any) {
136
138
  let oldValue = store.get(category, key)
139
+ let storeWouldSkipSet = store.wouldSkipSet(category, key, value)
137
140
  store.set(category, key, value)
141
+ if storeWouldSkipSet {
142
+ return
143
+ }
138
144
 
139
145
  // Trigger hardware updates based on setting changes
140
146
  switch (category, key) {
141
147
  case ("glasses", "fullyBooted"):
142
148
  Bridge.log("STORE: Glasses fullyBooted changed to \(value)")
143
- // skip if the value is the same as the old value:
144
- if let ready = value as? Bool, ready == oldValue as? Bool {
145
- return
146
- }
147
149
  if let ready = value as? Bool {
148
150
  if ready {
149
151
  DeviceManager.shared.handleDeviceReady()
@@ -220,6 +222,14 @@ class DeviceStore {
220
222
  DeviceManager.shared.sgc?.setDashboardMenu(items)
221
223
  }
222
224
 
225
+ case ("bluetooth", "calendar_events"), ("core", "calendar_events"):
226
+ if let items = value as? [[String: Any]] {
227
+ DeviceManager.shared.sgc?.sendCalendarEvents(items)
228
+ }
229
+
230
+ case ("bluetooth", "metric_system"), ("bluetooth", "twelve_hour_time"):
231
+ DeviceManager.shared.sgc?.sendDashboardDisplaySettings()
232
+
223
233
  case ("bluetooth", "gallery_mode"):
224
234
  DeviceManager.shared.sgc?.sendGalleryMode()
225
235