@mentra/bluetooth-sdk 0.1.5 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -27
- package/android/build.gradle +2 -9
- package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +53 -36
- package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +22 -9
- package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +25 -36
- package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt +9 -4
- package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +68 -56
- package/android/src/main/java/com/mentra/bluetoothsdk/audio/AudioModels.kt +125 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/camera/CameraModels.kt +209 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/connection/ConnectionModels.kt +78 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/controllers/ControllerManager.kt +4 -3
- package/android/src/main/java/com/mentra/bluetoothsdk/controllers/R1.kt +2 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/events/BluetoothEvents.kt +72 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/internal/MapParsing.kt +132 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/requests/DisplayRequests.kt +39 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G1.java +1 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt +2 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Mach1.java +1 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraLive.java +168 -10
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraNex.kt +1 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt +7 -3
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Simulated.kt +3 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/status/DeviceStatus.kt +607 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/status/RuntimeState.kt +201 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/status/WifiHotspotStatus.kt +173 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +348 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/stt/SherpaOnnxTranscriber.kt +1 -3
- package/android/src/main/java/com/mentra/bluetoothsdk/types/DeviceModels.kt +75 -0
- package/build/BluetoothSdk.types.d.ts +46 -20
- package/build/BluetoothSdk.types.d.ts.map +1 -1
- package/build/BluetoothSdk.types.js +1 -0
- package/build/BluetoothSdk.types.js.map +1 -1
- package/build/_private/BluetoothSdkModule.d.ts +6 -5
- package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
- package/build/_private/BluetoothSdkModule.js +40 -5
- package/build/_private/BluetoothSdkModule.js.map +1 -1
- package/build/index.d.ts +1 -1
- package/build/index.d.ts.map +1 -1
- package/build/index.js +4 -2
- package/build/index.js.map +1 -1
- package/build/react/useMentraBluetooth.d.ts +5 -3
- package/build/react/useMentraBluetooth.d.ts.map +1 -1
- package/build/react/useMentraBluetooth.js +15 -4
- package/build/react/useMentraBluetooth.js.map +1 -1
- package/ios/BluetoothSdkModule.swift +45 -28
- package/ios/MentraBluetoothSDK.podspec +0 -3
- package/ios/Source/Audio/AudioModels.swift +159 -0
- package/ios/Source/Bridge.swift +22 -11
- package/ios/Source/Camera/CameraModels.swift +248 -0
- package/ios/Source/Connection/ScanSession.swift +27 -0
- package/ios/Source/DeviceManager.swift +20 -97
- package/ios/Source/DeviceStore.swift +13 -4
- package/ios/Source/Errors/BluetoothError.swift +19 -0
- package/ios/Source/Events/BluetoothEvents.swift +149 -0
- package/ios/Source/Internal/BluetoothAvailability.swift +58 -0
- package/ios/Source/Internal/ValueParsing.swift +90 -0
- package/ios/Source/MentraBluetoothSDK.swift +50 -2150
- package/ios/Source/Requests/DisplayRequests.swift +58 -0
- package/ios/Source/Status/DeviceStatus.swift +471 -0
- package/ios/Source/Status/RuntimeState.swift +359 -0
- package/ios/Source/Status/WifiHotspotStatus.swift +326 -0
- package/ios/Source/Streaming/StreamModels.swift +436 -0
- package/ios/Source/Types/DeviceModels.swift +134 -0
- package/ios/Source/controllers/ControllerManager.swift +3 -3
- package/ios/Source/controllers/R1.swift +1 -1
- package/ios/Source/sgcs/G1.swift +1 -1
- package/ios/Source/sgcs/G2.swift +99 -39
- package/ios/Source/sgcs/Mach1.swift +1 -1
- package/ios/Source/sgcs/MentraLive.swift +109 -12
- package/ios/Source/sgcs/MentraNex.swift +1 -1
- package/ios/Source/sgcs/SGCManager.swift +11 -3
- package/ios/Source/sgcs/Simulated.swift +2 -2
- package/ios/Source/stt/SherpaOnnxTranscriber.swift +1 -4
- package/package.json +1 -1
- package/src/BluetoothSdk.types.ts +48 -28
- package/src/_private/BluetoothSdkModule.ts +47 -21
- package/src/index.ts +6 -4
- package/src/react/useMentraBluetooth.ts +19 -7
- package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothModels.kt +0 -1810
- package/android/src/main/java/com/mentra/bluetoothsdk/stt/VadGateSpeechPolicy.kt +0 -221
- package/ios/Packages/VAD/Common/VAD/VADQuality.swift +0 -41
- package/ios/Packages/VAD/Common/VAD/VADState.swift +0 -26
- package/ios/Packages/VAD/Common/VAD/VADStrategy.swift +0 -29
- package/ios/Packages/VAD/Common/VAD/VADType.swift +0 -14
- package/ios/Packages/VAD/Data/Configuration.swift +0 -68
- package/ios/Packages/VAD/Data/FrameSize.swift +0 -39
- package/ios/Packages/VAD/Data/Record.swift +0 -13
- package/ios/Packages/VAD/Data/Result.swift +0 -22
- package/ios/Packages/VAD/Data/SampleRate.swift +0 -48
- package/ios/Packages/VAD/Silero/Model/silero_vad.onnx +0 -0
- package/ios/Packages/VAD/Silero/SileroVAD.swift +0 -284
- package/ios/Packages/VAD/Silero/SileroVADStrategy.swift +0 -64
package/ios/Source/sgcs/G2.swift
CHANGED
|
@@ -1062,6 +1062,8 @@ class G2: NSObject, SGCManager {
|
|
|
1062
1062
|
private var rightAuthenticated: Bool = false
|
|
1063
1063
|
private var isDisconnecting = false
|
|
1064
1064
|
private var pairingTimeoutTimer: DispatchWorkItem?
|
|
1065
|
+
private var useEvenDashboard = true
|
|
1066
|
+
private var dashboardShowing = 0
|
|
1065
1067
|
|
|
1066
1068
|
/// Device search
|
|
1067
1069
|
var DEVICE_SEARCH_ID = "NOT_SET"
|
|
@@ -1452,8 +1454,7 @@ class G2: NSObject, SGCManager {
|
|
|
1452
1454
|
DeviceStore.shared.apply("glasses", "bluetoothName", btName)
|
|
1453
1455
|
DeviceStore.shared.apply("glasses", "deviceModel", DeviceTypes.G2)
|
|
1454
1456
|
|
|
1455
|
-
|
|
1456
|
-
DeviceStore.shared.apply("glasses", "fullyBooted", true)
|
|
1457
|
+
self.setFullyConnected()
|
|
1457
1458
|
|
|
1458
1459
|
// connnect a controller if we have one:
|
|
1459
1460
|
self.connectController()
|
|
@@ -1598,11 +1599,11 @@ class G2: NSObject, SGCManager {
|
|
|
1598
1599
|
func sendTextWall(_ text: String) {
|
|
1599
1600
|
// Bridge.log("G2: sendTextWall(\(text.prefix(50))...)")
|
|
1600
1601
|
|
|
1601
|
-
// ignore events while the dashboard is open:
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1602
|
+
// ignore events while the ER dashboard is open:
|
|
1603
|
+
let useNativeDashboard = DeviceStore.shared.get("bluetooth", "use_native_dashboard") as? Bool ?? false
|
|
1604
|
+
if useNativeDashboard && dashboardShowing > 0 {
|
|
1605
|
+
return
|
|
1606
|
+
}
|
|
1606
1607
|
|
|
1607
1608
|
if text.isEmpty {
|
|
1608
1609
|
clearDisplay()
|
|
@@ -2191,8 +2192,24 @@ class G2: NSObject, SGCManager {
|
|
|
2191
2192
|
return bmp
|
|
2192
2193
|
}
|
|
2193
2194
|
|
|
2195
|
+
/// Bring the Even Realities dashboard (the OS-level home/idle screen) to
|
|
2196
|
+
/// the foreground by tearing down whatever EvenHub page we currently own.
|
|
2197
|
+
/// The glasses fall back to the dashboard automatically when no page is up.
|
|
2194
2198
|
func showDashboard() {
|
|
2195
|
-
|
|
2199
|
+
Bridge.log("G2: showDashboard")
|
|
2200
|
+
dashboardShowing += 2
|
|
2201
|
+
let msg = EvenHubProto.shutdownMessage()
|
|
2202
|
+
sendEvenHubCommand(msg)
|
|
2203
|
+
pageCreated = false
|
|
2204
|
+
pageHasTextContainer = false
|
|
2205
|
+
currentTextContent = ""
|
|
2206
|
+
currentBitmapBase64 = ""
|
|
2207
|
+
DispatchQueue.main.asyncAfter(deadline: .now() + 0.5) { [weak self] in
|
|
2208
|
+
guard let self = self else { return }
|
|
2209
|
+
// activate the dashboard by setting dept to the current setting:
|
|
2210
|
+
let currentDepth = DeviceStore.shared.get("bluetooth", "dashboard_depth") as? Int ?? 0
|
|
2211
|
+
self.setDashboardDepthOnly(currentDepth)
|
|
2212
|
+
}
|
|
2196
2213
|
}
|
|
2197
2214
|
|
|
2198
2215
|
func setDashboardPosition(_ height: Int, _ depth: Int) {
|
|
@@ -2302,21 +2319,33 @@ class G2: NSObject, SGCManager {
|
|
|
2302
2319
|
sendEvenHubCommand(toSend)
|
|
2303
2320
|
}
|
|
2304
2321
|
|
|
2322
|
+
func restartMic() {
|
|
2323
|
+
// if already enabled, set to disabled, then send enabled after 500ms:
|
|
2324
|
+
DeviceStore.shared.apply("glasses", "micEnabled", true)
|
|
2325
|
+
let msg = EvenHubProto.audioControlMessage(enable: false)
|
|
2326
|
+
sendEvenHubCommand(msg)
|
|
2327
|
+
DispatchQueue.main.asyncAfter(deadline: .now() + 0.5) { [weak self] in
|
|
2328
|
+
guard let self = self else { return }
|
|
2329
|
+
let useNativeDashboard = DeviceStore.shared.get("bluetooth", "use_native_dashboard") as? Bool ?? false
|
|
2330
|
+
Bridge.log("G2: setMicEnabled - useNativeDashboard=\(useNativeDashboard), dashboardShowing=\(dashboardShowing)")
|
|
2331
|
+
if useNativeDashboard && dashboardShowing > 0 {
|
|
2332
|
+
return
|
|
2333
|
+
}
|
|
2334
|
+
if (!pageCreated || !pageHasTextContainer) {
|
|
2335
|
+
DeviceManager.shared.sendCurrentState()// should re-create the page if needed
|
|
2336
|
+
}
|
|
2337
|
+
let msg = EvenHubProto.audioControlMessage(enable: true)
|
|
2338
|
+
self.sendEvenHubCommand(msg)
|
|
2339
|
+
}
|
|
2340
|
+
}
|
|
2341
|
+
|
|
2305
2342
|
// MARK: - SGCManager: Audio Control
|
|
2306
2343
|
|
|
2307
2344
|
func setMicEnabled(_ enabled: Bool) {
|
|
2308
2345
|
Bridge.log("G2: setMicEnabled(\(enabled))")
|
|
2309
2346
|
let currentEnabled = DeviceStore.shared.get("glasses", "micEnabled") as? Bool ?? false
|
|
2310
2347
|
if currentEnabled && enabled {
|
|
2311
|
-
|
|
2312
|
-
DeviceStore.shared.apply("glasses", "micEnabled", true)
|
|
2313
|
-
let msg = EvenHubProto.audioControlMessage(enable: false)
|
|
2314
|
-
sendEvenHubCommand(msg)
|
|
2315
|
-
DispatchQueue.main.asyncAfter(deadline: .now() + 0.5) { [weak self] in
|
|
2316
|
-
guard let self = self else { return }
|
|
2317
|
-
let msg = EvenHubProto.audioControlMessage(enable: true)
|
|
2318
|
-
self.sendEvenHubCommand(msg)
|
|
2319
|
-
}
|
|
2348
|
+
restartMic()
|
|
2320
2349
|
return
|
|
2321
2350
|
}
|
|
2322
2351
|
|
|
@@ -2505,7 +2534,8 @@ class G2: NSObject, SGCManager {
|
|
|
2505
2534
|
// }
|
|
2506
2535
|
|
|
2507
2536
|
// connectController("1B:08:26:8E:0E:E6")
|
|
2508
|
-
connectController()
|
|
2537
|
+
// connectController()
|
|
2538
|
+
showDashboard()
|
|
2509
2539
|
}
|
|
2510
2540
|
|
|
2511
2541
|
func dbg2() {
|
|
@@ -2532,6 +2562,8 @@ class G2: NSObject, SGCManager {
|
|
|
2532
2562
|
// // update the text
|
|
2533
2563
|
// Bridge.log("G2: sendTextWall() - updating text container")
|
|
2534
2564
|
// updateText("test2")
|
|
2565
|
+
let currentDepth = DeviceStore.shared.get("bluetooth", "dashboard_depth") as? Int ?? 0
|
|
2566
|
+
setDashboardDepthOnly(currentDepth)
|
|
2535
2567
|
}
|
|
2536
2568
|
|
|
2537
2569
|
// MARK: - SGCManager: Device Control
|
|
@@ -2615,7 +2647,7 @@ class G2: NSObject, SGCManager {
|
|
|
2615
2647
|
|
|
2616
2648
|
func requestPhoto(
|
|
2617
2649
|
_: String, appId _: String, size _: String?, webhookUrl _: String?, authToken _: String?,
|
|
2618
|
-
compress _: String?, flash _: Bool, sound _: Bool
|
|
2650
|
+
compress _: String?, flash _: Bool, sound _: Bool, exposureTimeNs _: Double?
|
|
2619
2651
|
) {}
|
|
2620
2652
|
func startVideoRecording(requestId _: String, save _: Bool, flash _: Bool, sound _: Bool) {}
|
|
2621
2653
|
func startStream(_: [String: Any]) {}
|
|
@@ -3021,15 +3053,22 @@ class G2: NSObject, SGCManager {
|
|
|
3021
3053
|
if eventType == .doubleClick {
|
|
3022
3054
|
// trigger dashboard:
|
|
3023
3055
|
let isHeadUp = DeviceStore.shared.get("glasses", "headUp") as? Bool ?? false
|
|
3024
|
-
|
|
3025
|
-
DeviceStore.shared.
|
|
3026
|
-
if
|
|
3027
|
-
|
|
3028
|
-
|
|
3029
|
-
|
|
3030
|
-
|
|
3031
|
-
}
|
|
3056
|
+
|
|
3057
|
+
let useNativeDashboard = DeviceStore.shared.get("bluetooth", "use_native_dashboard") as? Bool ?? false
|
|
3058
|
+
if useNativeDashboard {
|
|
3059
|
+
showDashboard()
|
|
3060
|
+
} else {
|
|
3061
|
+
// toggle head up:
|
|
3062
|
+
DeviceStore.shared.apply("glasses", "headUp", !isHeadUp)
|
|
3032
3063
|
}
|
|
3064
|
+
|
|
3065
|
+
// if isHeadUp {
|
|
3066
|
+
// // Bridge.log("G2: going back to home, clearing display")
|
|
3067
|
+
// // clear the display after a delay:
|
|
3068
|
+
// DispatchQueue.main.asyncAfter(deadline: .now() + 0.5) {
|
|
3069
|
+
// self.clearDisplay()
|
|
3070
|
+
// }
|
|
3071
|
+
// }
|
|
3033
3072
|
// sendDashboardCommand(DashboardCommand.trigger)
|
|
3034
3073
|
|
|
3035
3074
|
// toggle head up:
|
|
@@ -3373,18 +3412,39 @@ class G2: NSObject, SGCManager {
|
|
|
3373
3412
|
// if we got 08011A00 that means we closed the dashboard, which means the mic is probably dead,
|
|
3374
3413
|
// so we need to revive it:
|
|
3375
3414
|
if data == Data([0x08, 0x01, 0x1A, 0x00]) {
|
|
3376
|
-
Bridge.log("G2:
|
|
3377
|
-
|
|
3378
|
-
|
|
3379
|
-
|
|
3380
|
-
|
|
3381
|
-
|
|
3382
|
-
|
|
3383
|
-
|
|
3384
|
-
|
|
3385
|
-
|
|
3386
|
-
|
|
3387
|
-
|
|
3415
|
+
Bridge.log("G2: dashboard closed / shutdown - dashboardShowing=\(dashboardShowing)")
|
|
3416
|
+
let useNativeDashboard = DeviceStore.shared.get("bluetooth", "use_native_dashboard") as? Bool ?? false
|
|
3417
|
+
if !useNativeDashboard {
|
|
3418
|
+
// make sure the container exists:
|
|
3419
|
+
DeviceManager.shared.sendCurrentState()
|
|
3420
|
+
// re-send mic on / if it's enabled:
|
|
3421
|
+
let micEnabled = DeviceStore.shared.get("glasses", "micEnabled") as? Bool ?? false
|
|
3422
|
+
if micEnabled {
|
|
3423
|
+
restartMic()
|
|
3424
|
+
}
|
|
3425
|
+
// reset the text container (different from clearDisplay())
|
|
3426
|
+
// sendTextWall(" ")
|
|
3427
|
+
// createPageWithText(" ")
|
|
3428
|
+
} else {
|
|
3429
|
+
// if we aren't trying to show the dashboard
|
|
3430
|
+
// then we need to turn the mic back on and display the mentra main page:
|
|
3431
|
+
if dashboardShowing <= 1 {
|
|
3432
|
+
dashboardShowing = 0
|
|
3433
|
+
// make sure the container exists:
|
|
3434
|
+
DeviceManager.shared.sendCurrentState()
|
|
3435
|
+
// set the mic back on if it should be on
|
|
3436
|
+
let micEnabled = DeviceStore.shared.get("glasses", "micEnabled") as? Bool ?? false
|
|
3437
|
+
if micEnabled {
|
|
3438
|
+
restartMic()
|
|
3439
|
+
}
|
|
3440
|
+
return
|
|
3441
|
+
}
|
|
3442
|
+
// do nothing this time since we just closed the dashboard
|
|
3443
|
+
dashboardShowing -= 1
|
|
3444
|
+
if (dashboardShowing < 0) {
|
|
3445
|
+
dashboardShowing = 0
|
|
3446
|
+
}
|
|
3447
|
+
}
|
|
3388
3448
|
}
|
|
3389
3449
|
|
|
3390
3450
|
// if we got 08011097012200 that means we selected a menu item:
|
|
@@ -17,7 +17,7 @@ class Mach1: UltraliteBaseViewController, SGCManager {
|
|
|
17
17
|
|
|
18
18
|
func requestPhoto(
|
|
19
19
|
_: String, appId _: String, size _: String?, webhookUrl _: String?, authToken _: String?,
|
|
20
|
-
compress _: String?, flash _: Bool, sound _: Bool
|
|
20
|
+
compress _: String?, flash _: Bool, sound _: Bool, exposureTimeNs _: Double?
|
|
21
21
|
) {}
|
|
22
22
|
|
|
23
23
|
func sendGalleryMode() {}
|
|
@@ -903,6 +903,7 @@ class MentraLive: NSObject, SGCManager {
|
|
|
903
903
|
// BLOCK_AUDIO_DUPLEX: When true, suspends LC3 mic while phone is playing audio via A2DP
|
|
904
904
|
// to avoid overloading the MCU. Set to false to allow simultaneous A2DP + LC3 mic.
|
|
905
905
|
private let BLOCK_AUDIO_DUPLEX = false
|
|
906
|
+
private static let voiceActivityDetectionSwitchType = 8
|
|
906
907
|
|
|
907
908
|
var connectionState: String = ConnTypes.DISCONNECTED
|
|
908
909
|
|
|
@@ -1311,9 +1312,11 @@ class MentraLive: NSObject, SGCManager {
|
|
|
1311
1312
|
|
|
1312
1313
|
func requestPhoto(
|
|
1313
1314
|
_ requestId: String, appId: String, size: String?, webhookUrl: String?, authToken: String?,
|
|
1314
|
-
compress: String?, flash: Bool, sound: Bool
|
|
1315
|
+
compress: String?, flash: Bool, sound: Bool, exposureTimeNs: Double?
|
|
1315
1316
|
) {
|
|
1316
|
-
Bridge.log(
|
|
1317
|
+
Bridge.log(
|
|
1318
|
+
"LIVE: PHOTO PIPELINE [5/6] requestPhoto() entry requestId=\(requestId) appId=\(appId) flash=\(flash) sound=\(sound)"
|
|
1319
|
+
)
|
|
1317
1320
|
|
|
1318
1321
|
var json: [String: Any] = [
|
|
1319
1322
|
"type": "take_photo",
|
|
@@ -1360,7 +1363,13 @@ class MentraLive: NSObject, SGCManager {
|
|
|
1360
1363
|
json["flash"] = flash
|
|
1361
1364
|
json["sound"] = sound
|
|
1362
1365
|
|
|
1363
|
-
|
|
1366
|
+
if let e = exposureTimeNs, e.isFinite, e > 0, e <= Double(Int64.max) {
|
|
1367
|
+
Bridge.log("LIVE: Using manual exposure time for photo request \(requestId): \(Int64(e)) ns")
|
|
1368
|
+
json["exposureTimeNs"] = Int64(e)
|
|
1369
|
+
}
|
|
1370
|
+
|
|
1371
|
+
Bridge.log("LIVE: PHOTO PIPELINE [5b/6] take_photo JSON ready bleImgId=\(bleImgId) transferMethod=auto")
|
|
1372
|
+
Bridge.log("LIVE: PHOTO PIPELINE [6/6] Dispatching take_photo to sendJson()")
|
|
1364
1373
|
|
|
1365
1374
|
sendJson(json, wakeUp: true)
|
|
1366
1375
|
}
|
|
@@ -1814,6 +1823,14 @@ class MentraLive: NSObject, SGCManager {
|
|
|
1814
1823
|
let isCharging = json["charging"] as? Bool ?? charging
|
|
1815
1824
|
updateBatteryStatus(level: level, isCharging: isCharging)
|
|
1816
1825
|
|
|
1826
|
+
case "voice_activity_detection_status":
|
|
1827
|
+
let enabled = json["voiceActivityDetectionEnabled"] as? Bool ?? true
|
|
1828
|
+
handleVoiceActivityDetectionStatus(enabled: enabled)
|
|
1829
|
+
|
|
1830
|
+
case "speaking_status":
|
|
1831
|
+
let speaking = json["speaking"] as? Bool ?? false
|
|
1832
|
+
handleSpeakingStatus(speaking: speaking)
|
|
1833
|
+
|
|
1817
1834
|
case "wifi_status":
|
|
1818
1835
|
let connected = json["connected"] as? Bool ?? false
|
|
1819
1836
|
let ssid = json["ssid"] as? String ?? ""
|
|
@@ -1893,9 +1910,7 @@ class MentraLive: NSObject, SGCManager {
|
|
|
1893
1910
|
let switchType = (json["switch_type"] as? Int) ?? (json["switchType"] as? Int) ?? -1
|
|
1894
1911
|
let switchValue = (json["switch_value"] as? Int) ?? (json["switchValue"] as? Int) ?? -1
|
|
1895
1912
|
let timestamp = parseTimestamp(json["timestamp"])
|
|
1896
|
-
|
|
1897
|
-
switchType: switchType, value: switchValue, timestamp: timestamp
|
|
1898
|
-
)
|
|
1913
|
+
handleSwitchStatus(switchType: switchType, value: switchValue, timestamp: timestamp)
|
|
1899
1914
|
|
|
1900
1915
|
case "rgb_led_control_response":
|
|
1901
1916
|
let requestId = json["requestId"] as? String ?? ""
|
|
@@ -2087,6 +2102,9 @@ class MentraLive: NSObject, SGCManager {
|
|
|
2087
2102
|
// Note: Stored separately from BES version for OTA patch matching
|
|
2088
2103
|
DeviceStore.shared.apply("glasses", "mtkFirmwareVersion", mtkFirmwareVersion)
|
|
2089
2104
|
}
|
|
2105
|
+
if let systemTimeMs = fields["system_time_ms"] as? NSNumber {
|
|
2106
|
+
DeviceStore.shared.apply("glasses", "systemTimeMs", systemTimeMs.int64Value)
|
|
2107
|
+
}
|
|
2090
2108
|
if let bluetoothMacAddress = fields["bt_mac_address"] as? String {
|
|
2091
2109
|
DeviceStore.shared.apply("glasses", "bluetoothMacAddress", bluetoothMacAddress)
|
|
2092
2110
|
}
|
|
@@ -2241,6 +2259,25 @@ class MentraLive: NSObject, SGCManager {
|
|
|
2241
2259
|
case "sr_vol":
|
|
2242
2260
|
handleSrVol(json)
|
|
2243
2261
|
|
|
2262
|
+
case "sr_vad":
|
|
2263
|
+
if let bodyObj = k900ParseBody(json["B"]),
|
|
2264
|
+
let on = k900JsonInt(bodyObj, "on"),
|
|
2265
|
+
on == 0 || on == 1
|
|
2266
|
+
{
|
|
2267
|
+
handleSpeakingStatus(speaking: on == 1)
|
|
2268
|
+
}
|
|
2269
|
+
|
|
2270
|
+
case "sr_swit":
|
|
2271
|
+
if let body = k900ParseBody(json["B"]) {
|
|
2272
|
+
let switchType = k900JsonInt(body, "type") ?? -1
|
|
2273
|
+
let switchValue = k900JsonInt(body, "switch") ?? -1
|
|
2274
|
+
handleSwitchStatus(
|
|
2275
|
+
switchType: switchType,
|
|
2276
|
+
value: switchValue,
|
|
2277
|
+
timestamp: Int64(Date().timeIntervalSince1970 * 1000)
|
|
2278
|
+
)
|
|
2279
|
+
}
|
|
2280
|
+
|
|
2244
2281
|
case "sr_shut":
|
|
2245
2282
|
Bridge.log("K900 shutdown command received - glasses shutting down")
|
|
2246
2283
|
// Mark as killed to prevent reconnection attempts
|
|
@@ -2461,7 +2498,7 @@ class MentraLive: NSObject, SGCManager {
|
|
|
2461
2498
|
}
|
|
2462
2499
|
|
|
2463
2500
|
func sendGalleryMode() {
|
|
2464
|
-
let active = DeviceStore.shared.get("bluetooth", "
|
|
2501
|
+
let active = DeviceStore.shared.get("bluetooth", "gallery_mode") as! Bool
|
|
2465
2502
|
Bridge.log("LIVE: 📸 Sending gallery mode active to glasses: \(active)")
|
|
2466
2503
|
|
|
2467
2504
|
let json: [String: Any] = [
|
|
@@ -2656,7 +2693,7 @@ class MentraLive: NSObject, SGCManager {
|
|
|
2656
2693
|
// return
|
|
2657
2694
|
// }
|
|
2658
2695
|
|
|
2659
|
-
// // Forward PCM data to DeviceManager for
|
|
2696
|
+
// // Forward PCM data to DeviceManager for audio events and server transmission (same as Android)
|
|
2660
2697
|
// DeviceManager.shared.handlePcm(pcmData)
|
|
2661
2698
|
|
|
2662
2699
|
// Bridge.log(
|
|
@@ -3237,6 +3274,9 @@ class MentraLive: NSObject, SGCManager {
|
|
|
3237
3274
|
Bridge.log("LIVE: All chunks queued for transmission")
|
|
3238
3275
|
} else {
|
|
3239
3276
|
// Normal single message transmission
|
|
3277
|
+
if (json["type"] as? String) == "take_photo" {
|
|
3278
|
+
Bridge.log("LIVE: PHOTO PIPELINE BLE handoff — sendJson -> queueSend take_photo")
|
|
3279
|
+
}
|
|
3240
3280
|
Bridge.log("LIVE: Sending data to glasses: \(jsonString)")
|
|
3241
3281
|
let packedData = packJson(jsonString, wakeUp: wakeUp) ?? Data()
|
|
3242
3282
|
queueSend(packedData, id: trackingId)
|
|
@@ -3508,6 +3548,27 @@ class MentraLive: NSObject, SGCManager {
|
|
|
3508
3548
|
}
|
|
3509
3549
|
}
|
|
3510
3550
|
|
|
3551
|
+
private func handleVoiceActivityDetectionStatus(enabled: Bool) {
|
|
3552
|
+
Bridge.log("LIVE: Voice Activity Detection \(enabled ? "enabled" : "disabled")")
|
|
3553
|
+
Bridge.sendVoiceActivityDetectionStatus(enabled)
|
|
3554
|
+
}
|
|
3555
|
+
|
|
3556
|
+
private func handleSpeakingStatus(speaking: Bool) {
|
|
3557
|
+
guard voiceActivityDetectionEnabled else {
|
|
3558
|
+
Bridge.log("LIVE: Ignoring speaking status because Voice Activity Detection is disabled")
|
|
3559
|
+
return
|
|
3560
|
+
}
|
|
3561
|
+
Bridge.log("LIVE: Speaking status \(speaking ? "speaking" : "not speaking")")
|
|
3562
|
+
Bridge.sendSpeakingStatus(speaking)
|
|
3563
|
+
}
|
|
3564
|
+
|
|
3565
|
+
private func handleSwitchStatus(switchType: Int, value: Int, timestamp: Int64) {
|
|
3566
|
+
Bridge.sendSwitchStatus(switchType: switchType, value: value, timestamp: timestamp)
|
|
3567
|
+
if switchType == Self.voiceActivityDetectionSwitchType, value == 0 || value == 1 {
|
|
3568
|
+
handleVoiceActivityDetectionStatus(enabled: value == 1)
|
|
3569
|
+
}
|
|
3570
|
+
}
|
|
3571
|
+
|
|
3511
3572
|
private func updateWifiStatus(connected: Bool, ssid: String, ip: String) {
|
|
3512
3573
|
Bridge.log("LIVE: 🌐 Updating WiFi status - connected: \(connected), ssid: \(ssid)")
|
|
3513
3574
|
DeviceStore.shared.apply("glasses", "wifiConnected", connected)
|
|
@@ -4493,6 +4554,42 @@ extension MentraLive {
|
|
|
4493
4554
|
|
|
4494
4555
|
// Send gallery mode state (camera app running status)
|
|
4495
4556
|
sendGalleryMode()
|
|
4557
|
+
|
|
4558
|
+
// Send glasses-side Voice Activity Detection setting.
|
|
4559
|
+
sendVoiceActivityDetectionSetting()
|
|
4560
|
+
}
|
|
4561
|
+
|
|
4562
|
+
func sendVoiceActivityDetectionSetting() {
|
|
4563
|
+
let enabled = DeviceStore.shared.get("bluetooth", "voice_activity_detection_enabled") as? Bool ?? true
|
|
4564
|
+
Bridge.log("LIVE: 🎤 Sending Voice Activity Detection setting to glasses: \(enabled)")
|
|
4565
|
+
|
|
4566
|
+
guard connectedPeripheral != nil, txCharacteristic != nil else {
|
|
4567
|
+
Bridge.log("Cannot send Voice Activity Detection setting - BLE write path not ready")
|
|
4568
|
+
return
|
|
4569
|
+
}
|
|
4570
|
+
|
|
4571
|
+
do {
|
|
4572
|
+
let bodyData = try JSONSerialization.data(withJSONObject: [
|
|
4573
|
+
"type": Self.voiceActivityDetectionSwitchType,
|
|
4574
|
+
"switch": enabled ? 1 : 0,
|
|
4575
|
+
])
|
|
4576
|
+
guard let bodyString = String(data: bodyData, encoding: .utf8) else {
|
|
4577
|
+
Bridge.log("LIVE: Failed to encode Voice Activity Detection payload")
|
|
4578
|
+
return
|
|
4579
|
+
}
|
|
4580
|
+
let command: [String: Any] = [
|
|
4581
|
+
"C": "cs_swit",
|
|
4582
|
+
"V": 1,
|
|
4583
|
+
"B": bodyString,
|
|
4584
|
+
]
|
|
4585
|
+
if sendRawK900Command(command, wakeUp: true) {
|
|
4586
|
+
Bridge.sendVoiceActivityDetectionStatus(enabled)
|
|
4587
|
+
} else {
|
|
4588
|
+
Bridge.log("LIVE: Failed to send Voice Activity Detection setting command")
|
|
4589
|
+
}
|
|
4590
|
+
} catch {
|
|
4591
|
+
Bridge.log("LIVE: Error encoding Voice Activity Detection payload: \(error)")
|
|
4592
|
+
}
|
|
4496
4593
|
}
|
|
4497
4594
|
|
|
4498
4595
|
func sendButtonVideoRecordingSettings() {
|
|
@@ -4500,11 +4597,11 @@ extension MentraLive {
|
|
|
4500
4597
|
DeviceStore.shared.get("bluetooth", "button_video_settings") as? [String: Any] ?? [
|
|
4501
4598
|
"width": 1280,
|
|
4502
4599
|
"height": 720,
|
|
4503
|
-
"
|
|
4600
|
+
"fps": 30,
|
|
4504
4601
|
]
|
|
4505
4602
|
let width = settings["width"] as? Int ?? 1280
|
|
4506
4603
|
let height = settings["height"] as? Int ?? 720
|
|
4507
|
-
let fps = settings["
|
|
4604
|
+
let fps = settings["fps"] as? Int ?? 30
|
|
4508
4605
|
|
|
4509
4606
|
// Use defaults if not set
|
|
4510
4607
|
let finalWidth = width > 0 ? width : 1280
|
|
@@ -4582,9 +4679,9 @@ extension MentraLive {
|
|
|
4582
4679
|
}
|
|
4583
4680
|
|
|
4584
4681
|
func sendCameraFovSetting() {
|
|
4585
|
-
let settings = DeviceStore.shared.get("bluetooth", "camera_fov") as? [String: Any] ?? ["fov": 118, "
|
|
4682
|
+
let settings = DeviceStore.shared.get("bluetooth", "camera_fov") as? [String: Any] ?? ["fov": 118, "roi_position": 0]
|
|
4586
4683
|
let fov = settings["fov"] as? Int ?? 118
|
|
4587
|
-
let roiPosition = settings["
|
|
4684
|
+
let roiPosition = settings["roi_position"] as? Int ?? 0
|
|
4588
4685
|
|
|
4589
4686
|
Bridge.log("Sending camera FOV setting: fov=\(fov), roiPosition=\(roiPosition)")
|
|
4590
4687
|
|
|
@@ -46,7 +46,7 @@ class MentraNexSGC: NSObject, CBCentralManagerDelegate, CBPeripheralDelegate, SG
|
|
|
46
46
|
|
|
47
47
|
func requestPhoto(
|
|
48
48
|
_: String, appId _: String, size _: String?, webhookUrl _: String?, authToken _: String?,
|
|
49
|
-
compress _: String?, flash _: Bool, sound _: Bool
|
|
49
|
+
compress _: String?, flash _: Bool, sound _: Bool, exposureTimeNs _: Double?
|
|
50
50
|
) {}
|
|
51
51
|
|
|
52
52
|
func startStream(_: [String: Any]) {}
|
|
@@ -20,7 +20,7 @@ protocol SGCManager {
|
|
|
20
20
|
|
|
21
21
|
func requestPhoto(
|
|
22
22
|
_ requestId: String, appId: String, size: String?, webhookUrl: String?, authToken: String?,
|
|
23
|
-
compress: String?, flash: Bool, sound: Bool
|
|
23
|
+
compress: String?, flash: Bool, sound: Bool, exposureTimeNs: Double?
|
|
24
24
|
)
|
|
25
25
|
func startStream(_ message: [String: Any])
|
|
26
26
|
func stopStream()
|
|
@@ -103,6 +103,10 @@ protocol SGCManager {
|
|
|
103
103
|
func queryGalleryStatus()
|
|
104
104
|
func sendGalleryMode()
|
|
105
105
|
|
|
106
|
+
// MARK: - Voice Activity Detection
|
|
107
|
+
|
|
108
|
+
func sendVoiceActivityDetectionSetting()
|
|
109
|
+
|
|
106
110
|
// MARK: - Version Info
|
|
107
111
|
|
|
108
112
|
func requestVersionInfo()
|
|
@@ -127,6 +131,10 @@ extension SGCManager {
|
|
|
127
131
|
|
|
128
132
|
func setDashboardMenu(_: [[String: Any]]) {}
|
|
129
133
|
|
|
134
|
+
// MARK: - Voice Activity Detection (default no-op — Mentra Live supports this)
|
|
135
|
+
|
|
136
|
+
func sendVoiceActivityDetectionSetting() {}
|
|
137
|
+
|
|
130
138
|
// MARK: - Default DeviceStore-backed property implementations
|
|
131
139
|
|
|
132
140
|
var fullyBooted: Bool {
|
|
@@ -181,8 +189,8 @@ extension SGCManager {
|
|
|
181
189
|
DeviceStore.shared.get("glasses", "micEnabled") as? Bool ?? false
|
|
182
190
|
}
|
|
183
191
|
|
|
184
|
-
var
|
|
185
|
-
DeviceStore.shared.get("glasses", "
|
|
192
|
+
var voiceActivityDetectionEnabled: Bool {
|
|
193
|
+
DeviceStore.shared.get("glasses", "voiceActivityDetectionEnabled") as? Bool ?? true
|
|
186
194
|
}
|
|
187
195
|
|
|
188
196
|
var batteryLevel: Int {
|
|
@@ -12,7 +12,7 @@ class Simulated: SGCManager {
|
|
|
12
12
|
DeviceStore.shared.apply("glasses", "connected", true)
|
|
13
13
|
DeviceStore.shared.apply("glasses", "connectionState", ConnTypes.CONNECTED)
|
|
14
14
|
DeviceStore.shared.apply("glasses", "micEnabled", false)
|
|
15
|
-
DeviceStore.shared.apply("glasses", "
|
|
15
|
+
DeviceStore.shared.apply("glasses", "voiceActivityDetectionEnabled", true)
|
|
16
16
|
DeviceStore.shared.apply("glasses", "bluetoothClassicConnected", false)
|
|
17
17
|
}
|
|
18
18
|
|
|
@@ -75,7 +75,7 @@ class Simulated: SGCManager {
|
|
|
75
75
|
|
|
76
76
|
// MARK: - Camera & Media
|
|
77
77
|
|
|
78
|
-
func requestPhoto(_: String, appId _: String, size _: String?, webhookUrl _: String?, authToken _: String?, compress _: String?, flash _: Bool, sound _: Bool) {
|
|
78
|
+
func requestPhoto(_: String, appId _: String, size _: String?, webhookUrl _: String?, authToken _: String?, compress _: String?, flash _: Bool, sound _: Bool, exposureTimeNs _: Double?) {
|
|
79
79
|
Bridge.log("requestPhoto")
|
|
80
80
|
}
|
|
81
81
|
|
|
@@ -214,16 +214,13 @@ class SherpaOnnxTranscriber {
|
|
|
214
214
|
* Feed PCM audio data (16-bit little endian) into the transcriber.
|
|
215
215
|
* This method should be called continuously with short chunks (e.g., 100-300ms).
|
|
216
216
|
*
|
|
217
|
-
*
|
|
218
|
-
* so it's directly queued for processing without additional VAD checks.
|
|
217
|
+
* Audio is queued directly; microphone VAD gating is not applied in the SDK.
|
|
219
218
|
*/
|
|
220
219
|
func acceptAudio(pcm16le: Data) {
|
|
221
220
|
guard isRunning else {
|
|
222
221
|
return
|
|
223
222
|
}
|
|
224
223
|
|
|
225
|
-
// Directly queue the audio data for processing
|
|
226
|
-
// No VAD check here as it's assumed to be done upstream
|
|
227
224
|
queueAudioData(pcm16le)
|
|
228
225
|
}
|
|
229
226
|
|