@mentra/bluetooth-sdk 0.1.6 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -22
- package/android/.project +28 -0
- package/android/build.gradle +5 -9
- package/android/lc3Lib/.project +28 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +54 -33
- package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +48 -10
- package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +27 -36
- package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt +8 -4
- package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +170 -14
- package/android/src/main/java/com/mentra/bluetoothsdk/audio/AudioModels.kt +6 -6
- package/android/src/main/java/com/mentra/bluetoothsdk/camera/CameraModels.kt +69 -10
- package/android/src/main/java/com/mentra/bluetoothsdk/controllers/ControllerManager.kt +5 -3
- package/android/src/main/java/com/mentra/bluetoothsdk/controllers/R1.kt +2 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/debug/BleTraceLogger.kt +149 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/events/BluetoothEvents.kt +13 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoUploadServer.kt +297 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/MentraPhotoReceiverModule.kt +120 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/services/Foreground.kt +13 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G1.java +1 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt +3 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Mach1.java +1 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraLive.java +269 -28
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraNex.kt +1 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt +8 -3
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Simulated.kt +4 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/status/DeviceStatus.kt +14 -12
- package/android/src/main/java/com/mentra/bluetoothsdk/status/RuntimeState.kt +4 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +131 -6
- package/android/src/main/java/com/mentra/bluetoothsdk/stt/SherpaOnnxTranscriber.kt +1 -3
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunkReassembler.java +105 -0
- package/build/BluetoothSdk.types.d.ts +107 -48
- package/build/BluetoothSdk.types.d.ts.map +1 -1
- package/build/BluetoothSdk.types.js +4 -0
- package/build/BluetoothSdk.types.js.map +1 -1
- package/build/_private/BluetoothSdkModule.d.ts +8 -6
- package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
- package/build/_private/BluetoothSdkModule.js +58 -9
- package/build/_private/BluetoothSdkModule.js.map +1 -1
- package/build/index.d.ts +2 -2
- package/build/index.d.ts.map +1 -1
- package/build/index.js +6 -5
- package/build/index.js.map +1 -1
- package/build/photo-receiver/MentraPhotoReceiver.types.d.ts +18 -0
- package/build/photo-receiver/MentraPhotoReceiver.types.d.ts.map +1 -0
- package/build/photo-receiver/MentraPhotoReceiver.types.js +2 -0
- package/build/photo-receiver/MentraPhotoReceiver.types.js.map +1 -0
- package/build/photo-receiver/MentraPhotoReceiverModule.d.ts +10 -0
- package/build/photo-receiver/MentraPhotoReceiverModule.d.ts.map +1 -0
- package/build/photo-receiver/MentraPhotoReceiverModule.js +3 -0
- package/build/photo-receiver/MentraPhotoReceiverModule.js.map +1 -0
- package/build/photo-receiver/MentraPhotoReceiverModule.web.d.ts +10 -0
- package/build/photo-receiver/MentraPhotoReceiverModule.web.d.ts.map +1 -0
- package/build/photo-receiver/MentraPhotoReceiverModule.web.js +12 -0
- package/build/photo-receiver/MentraPhotoReceiverModule.web.js.map +1 -0
- package/build/photo-receiver/index.d.ts +3 -0
- package/build/photo-receiver/index.d.ts.map +1 -0
- package/build/photo-receiver/index.js +3 -0
- package/build/photo-receiver/index.js.map +1 -0
- package/build/react/useMentraBluetooth.d.ts +5 -3
- package/build/react/useMentraBluetooth.d.ts.map +1 -1
- package/build/react/useMentraBluetooth.js +15 -4
- package/build/react/useMentraBluetooth.js.map +1 -1
- package/expo-module.config.json +5 -2
- package/ios/BluetoothSdkModule.swift +59 -26
- package/ios/LocalPhotoUploadServer.swift +411 -0
- package/ios/MentraBluetoothSDK.podspec +8 -5
- package/ios/MentraPhotoReceiverModule.swift +147 -0
- package/ios/Source/Audio/AudioModels.swift +10 -10
- package/ios/Source/Bridge.swift +42 -12
- package/ios/Source/Camera/CameraModels.swift +74 -19
- package/ios/Source/DeviceManager.swift +23 -98
- package/ios/Source/DeviceStore.swift +12 -4
- package/ios/Source/Events/BluetoothEvents.swift +37 -0
- package/ios/Source/MentraBluetoothSDK.swift +135 -14
- package/ios/Source/Status/DeviceStatus.swift +12 -4
- package/ios/Source/Status/RuntimeState.swift +22 -13
- package/ios/Source/Streaming/StreamModels.swift +239 -45
- package/ios/Source/controllers/ControllerManager.swift +3 -3
- package/ios/Source/controllers/R1.swift +9 -3
- package/ios/Source/sgcs/G1.swift +5 -4
- package/ios/Source/sgcs/G2.swift +5 -6
- package/ios/Source/sgcs/Mach1.swift +1 -1
- package/ios/Source/sgcs/MentraLive.swift +183 -22
- package/ios/Source/sgcs/MentraNex.swift +1 -1
- package/ios/Source/sgcs/SGCManager.swift +11 -3
- package/ios/Source/sgcs/Simulated.swift +2 -2
- package/ios/Source/stt/SherpaOnnxTranscriber.swift +1 -4
- package/ios/Source/utils/MessageChunkReassembler.swift +89 -0
- package/package.json +6 -1
- package/src/BluetoothSdk.types.ts +133 -62
- package/src/_private/BluetoothSdkModule.ts +79 -26
- package/src/index.ts +15 -8
- package/src/photo-receiver/MentraPhotoReceiver.types.ts +20 -0
- package/src/photo-receiver/MentraPhotoReceiverModule.ts +14 -0
- package/src/photo-receiver/MentraPhotoReceiverModule.web.ts +20 -0
- package/src/photo-receiver/index.ts +2 -0
- package/src/react/useMentraBluetooth.ts +19 -7
- package/android/src/main/java/com/mentra/bluetoothsdk/stt/VadGateSpeechPolicy.kt +0 -221
- package/ios/Packages/VAD/Common/VAD/VADQuality.swift +0 -41
- package/ios/Packages/VAD/Common/VAD/VADState.swift +0 -26
- package/ios/Packages/VAD/Common/VAD/VADStrategy.swift +0 -29
- package/ios/Packages/VAD/Common/VAD/VADType.swift +0 -14
- package/ios/Packages/VAD/Data/Configuration.swift +0 -68
- package/ios/Packages/VAD/Data/FrameSize.swift +0 -39
- package/ios/Packages/VAD/Data/Record.swift +0 -13
- package/ios/Packages/VAD/Data/Result.swift +0 -22
- package/ios/Packages/VAD/Data/SampleRate.swift +0 -48
- package/ios/Packages/VAD/Silero/Model/silero_vad.onnx +0 -0
- package/ios/Packages/VAD/Silero/SileroVAD.swift +0 -284
- package/ios/Packages/VAD/Silero/SileroVADStrategy.swift +0 -64
|
@@ -768,9 +768,12 @@ extension MentraLive: CBPeripheralDelegate {
|
|
|
768
768
|
// Keep state as connecting until glasses are ready
|
|
769
769
|
updateConnectionState(ConnTypes.CONNECTING)
|
|
770
770
|
|
|
771
|
-
|
|
772
|
-
let
|
|
773
|
-
|
|
771
|
+
let withResponseMtu = peripheral.maximumWriteValueLength(for: .withResponse) + 3
|
|
772
|
+
let withoutResponseMtu = peripheral.maximumWriteValueLength(for: .withoutResponse) + 3
|
|
773
|
+
currentMtu = max(23, min(withResponseMtu, withoutResponseMtu))
|
|
774
|
+
Bridge.log(
|
|
775
|
+
"LIVE: Current MTU estimate: withResponse=\(withResponseMtu), withoutResponse=\(withoutResponseMtu), selected=\(currentMtu)"
|
|
776
|
+
)
|
|
774
777
|
|
|
775
778
|
// Enable notifications on RX characteristic
|
|
776
779
|
peripheral.setNotifyValue(true, for: rx)
|
|
@@ -903,6 +906,7 @@ class MentraLive: NSObject, SGCManager {
|
|
|
903
906
|
// BLOCK_AUDIO_DUPLEX: When true, suspends LC3 mic while phone is playing audio via A2DP
|
|
904
907
|
// to avoid overloading the MCU. Set to false to allow simultaneous A2DP + LC3 mic.
|
|
905
908
|
private let BLOCK_AUDIO_DUPLEX = false
|
|
909
|
+
private static let voiceActivityDetectionSwitchType = 8
|
|
906
910
|
|
|
907
911
|
var connectionState: String = ConnTypes.DISCONNECTED
|
|
908
912
|
|
|
@@ -1071,6 +1075,7 @@ class MentraLive: NSObject, SGCManager {
|
|
|
1071
1075
|
private var connectedPeripheral: CBPeripheral?
|
|
1072
1076
|
private var txCharacteristic: CBCharacteristic?
|
|
1073
1077
|
private var rxCharacteristic: CBCharacteristic?
|
|
1078
|
+
private let bes2700MtuLimit = 256
|
|
1074
1079
|
private var currentMtu: Int = 23 // Default BLE MTU
|
|
1075
1080
|
|
|
1076
1081
|
// State Tracking
|
|
@@ -1095,6 +1100,7 @@ class MentraLive: NSObject, SGCManager {
|
|
|
1095
1100
|
// Queue Management
|
|
1096
1101
|
private let commandQueue = CommandQueue()
|
|
1097
1102
|
private let bluetoothQueue = DispatchQueue(label: "MentraLiveBluetooth", qos: .userInitiated)
|
|
1103
|
+
private let incomingChunkReassembler = MessageChunkReassembler()
|
|
1098
1104
|
private var lastSendTimeMs: TimeInterval = 0
|
|
1099
1105
|
|
|
1100
1106
|
// Timers
|
|
@@ -1311,9 +1317,11 @@ class MentraLive: NSObject, SGCManager {
|
|
|
1311
1317
|
|
|
1312
1318
|
func requestPhoto(
|
|
1313
1319
|
_ requestId: String, appId: String, size: String?, webhookUrl: String?, authToken: String?,
|
|
1314
|
-
compress: String?, flash: Bool, sound: Bool
|
|
1320
|
+
compress: String?, flash: Bool, sound: Bool, exposureTimeNs: Double?, iso: Int?
|
|
1315
1321
|
) {
|
|
1316
|
-
Bridge.log(
|
|
1322
|
+
Bridge.log(
|
|
1323
|
+
"LIVE: PHOTO PIPELINE [5/6] requestPhoto() entry requestId=\(requestId) appId=\(appId) flash=\(flash) sound=\(sound) iso=\(iso.map { String($0) } ?? "auto")"
|
|
1324
|
+
)
|
|
1317
1325
|
|
|
1318
1326
|
var json: [String: Any] = [
|
|
1319
1327
|
"type": "take_photo",
|
|
@@ -1360,7 +1368,17 @@ class MentraLive: NSObject, SGCManager {
|
|
|
1360
1368
|
json["flash"] = flash
|
|
1361
1369
|
json["sound"] = sound
|
|
1362
1370
|
|
|
1363
|
-
|
|
1371
|
+
if let e = exposureTimeNs, e.isFinite, e > 0, e <= Double(Int64.max) {
|
|
1372
|
+
Bridge.log("LIVE: Using manual exposure time for photo request \(requestId): \(Int64(e)) ns")
|
|
1373
|
+
json["exposureTimeNs"] = Int64(e)
|
|
1374
|
+
}
|
|
1375
|
+
if let iso, iso > 0 {
|
|
1376
|
+
Bridge.log("LIVE: Using manual ISO for photo request \(requestId): ISO \(iso)")
|
|
1377
|
+
json["iso"] = iso
|
|
1378
|
+
}
|
|
1379
|
+
|
|
1380
|
+
Bridge.log("LIVE: PHOTO PIPELINE [5b/6] take_photo JSON ready bleImgId=\(bleImgId) transferMethod=auto")
|
|
1381
|
+
Bridge.log("LIVE: PHOTO PIPELINE [6/6] Dispatching take_photo to sendJson()")
|
|
1364
1382
|
|
|
1365
1383
|
sendJson(json, wakeUp: true)
|
|
1366
1384
|
}
|
|
@@ -1768,6 +1786,11 @@ class MentraLive: NSObject, SGCManager {
|
|
|
1768
1786
|
// Log ALL incoming JSON objects for debugging
|
|
1769
1787
|
// Bridge.log("LIVE: DEBUG: processJsonObject: \(json)")
|
|
1770
1788
|
|
|
1789
|
+
if MessageChunker.isChunkedMessage(json) {
|
|
1790
|
+
processChunkedJsonObject(json)
|
|
1791
|
+
return
|
|
1792
|
+
}
|
|
1793
|
+
|
|
1771
1794
|
// Check for K900 command format
|
|
1772
1795
|
if let command = json["C"] as? String {
|
|
1773
1796
|
processK900JsonMessage(json)
|
|
@@ -1814,6 +1837,14 @@ class MentraLive: NSObject, SGCManager {
|
|
|
1814
1837
|
let isCharging = json["charging"] as? Bool ?? charging
|
|
1815
1838
|
updateBatteryStatus(level: level, isCharging: isCharging)
|
|
1816
1839
|
|
|
1840
|
+
case "voice_activity_detection_status":
|
|
1841
|
+
let enabled = json["voiceActivityDetectionEnabled"] as? Bool ?? true
|
|
1842
|
+
handleVoiceActivityDetectionStatus(enabled: enabled)
|
|
1843
|
+
|
|
1844
|
+
case "speaking_status":
|
|
1845
|
+
let speaking = json["speaking"] as? Bool ?? false
|
|
1846
|
+
handleSpeakingStatus(speaking: speaking)
|
|
1847
|
+
|
|
1817
1848
|
case "wifi_status":
|
|
1818
1849
|
let connected = json["connected"] as? Bool ?? false
|
|
1819
1850
|
let ssid = json["ssid"] as? String ?? ""
|
|
@@ -1839,6 +1870,9 @@ class MentraLive: NSObject, SGCManager {
|
|
|
1839
1870
|
case "stream_status":
|
|
1840
1871
|
emitRtmpStreamStatus(json)
|
|
1841
1872
|
|
|
1873
|
+
case "photo_status":
|
|
1874
|
+
emitPhotoStatus(json)
|
|
1875
|
+
|
|
1842
1876
|
case "gallery_status":
|
|
1843
1877
|
let photoCount = json["photos"] as? Int ?? 0
|
|
1844
1878
|
let videoCount = json["videos"] as? Int ?? 0
|
|
@@ -1893,9 +1927,7 @@ class MentraLive: NSObject, SGCManager {
|
|
|
1893
1927
|
let switchType = (json["switch_type"] as? Int) ?? (json["switchType"] as? Int) ?? -1
|
|
1894
1928
|
let switchValue = (json["switch_value"] as? Int) ?? (json["switchValue"] as? Int) ?? -1
|
|
1895
1929
|
let timestamp = parseTimestamp(json["timestamp"])
|
|
1896
|
-
|
|
1897
|
-
switchType: switchType, value: switchValue, timestamp: timestamp
|
|
1898
|
-
)
|
|
1930
|
+
handleSwitchStatus(switchType: switchType, value: switchValue, timestamp: timestamp)
|
|
1899
1931
|
|
|
1900
1932
|
case "rgb_led_control_response":
|
|
1901
1933
|
let requestId = json["requestId"] as? String ?? ""
|
|
@@ -2087,6 +2119,9 @@ class MentraLive: NSObject, SGCManager {
|
|
|
2087
2119
|
// Note: Stored separately from BES version for OTA patch matching
|
|
2088
2120
|
DeviceStore.shared.apply("glasses", "mtkFirmwareVersion", mtkFirmwareVersion)
|
|
2089
2121
|
}
|
|
2122
|
+
if let systemTimeMs = fields["system_time_ms"] as? NSNumber {
|
|
2123
|
+
DeviceStore.shared.apply("glasses", "systemTimeMs", systemTimeMs.int64Value)
|
|
2124
|
+
}
|
|
2090
2125
|
if let bluetoothMacAddress = fields["bt_mac_address"] as? String {
|
|
2091
2126
|
DeviceStore.shared.apply("glasses", "bluetoothMacAddress", bluetoothMacAddress)
|
|
2092
2127
|
}
|
|
@@ -2102,6 +2137,26 @@ class MentraLive: NSObject, SGCManager {
|
|
|
2102
2137
|
}
|
|
2103
2138
|
}
|
|
2104
2139
|
|
|
2140
|
+
private func processChunkedJsonObject(_ json: [String: Any]) {
|
|
2141
|
+
guard let info = MessageChunker.getChunkInfo(json) else {
|
|
2142
|
+
Bridge.log("LIVE: Received malformed chunked message from glasses")
|
|
2143
|
+
return
|
|
2144
|
+
}
|
|
2145
|
+
|
|
2146
|
+
guard let reassembled = incomingChunkReassembler.addChunk(info) else {
|
|
2147
|
+
return
|
|
2148
|
+
}
|
|
2149
|
+
|
|
2150
|
+
guard let data = reassembled.data(using: .utf8),
|
|
2151
|
+
let reassembledJson = try? JSONSerialization.jsonObject(with: data) as? [String: Any]
|
|
2152
|
+
else {
|
|
2153
|
+
Bridge.log("LIVE: Failed to parse reassembled chunked message")
|
|
2154
|
+
return
|
|
2155
|
+
}
|
|
2156
|
+
|
|
2157
|
+
processJsonObject(reassembledJson)
|
|
2158
|
+
}
|
|
2159
|
+
|
|
2105
2160
|
/// Maps K900 gesture type codes to gesture names
|
|
2106
2161
|
/// Compute the weighted overall OTA percentage for a BES progress event arriving via sr_adota.
|
|
2107
2162
|
/// Mirrors the weight table in OtaSessionManager.computeStepWeights() on the glasses side.
|
|
@@ -2241,6 +2296,25 @@ class MentraLive: NSObject, SGCManager {
|
|
|
2241
2296
|
case "sr_vol":
|
|
2242
2297
|
handleSrVol(json)
|
|
2243
2298
|
|
|
2299
|
+
case "sr_vad":
|
|
2300
|
+
if let bodyObj = k900ParseBody(json["B"]),
|
|
2301
|
+
let on = k900JsonInt(bodyObj, "on"),
|
|
2302
|
+
on == 0 || on == 1
|
|
2303
|
+
{
|
|
2304
|
+
handleSpeakingStatus(speaking: on == 1)
|
|
2305
|
+
}
|
|
2306
|
+
|
|
2307
|
+
case "sr_swit":
|
|
2308
|
+
if let body = k900ParseBody(json["B"]) {
|
|
2309
|
+
let switchType = k900JsonInt(body, "type") ?? -1
|
|
2310
|
+
let switchValue = k900JsonInt(body, "switch") ?? -1
|
|
2311
|
+
handleSwitchStatus(
|
|
2312
|
+
switchType: switchType,
|
|
2313
|
+
value: switchValue,
|
|
2314
|
+
timestamp: Int64(Date().timeIntervalSince1970 * 1000)
|
|
2315
|
+
)
|
|
2316
|
+
}
|
|
2317
|
+
|
|
2244
2318
|
case "sr_shut":
|
|
2245
2319
|
Bridge.log("K900 shutdown command received - glasses shutting down")
|
|
2246
2320
|
// Mark as killed to prevent reconnection attempts
|
|
@@ -2461,7 +2535,7 @@ class MentraLive: NSObject, SGCManager {
|
|
|
2461
2535
|
}
|
|
2462
2536
|
|
|
2463
2537
|
func sendGalleryMode() {
|
|
2464
|
-
let active = DeviceStore.shared.get("bluetooth", "
|
|
2538
|
+
let active = DeviceStore.shared.get("bluetooth", "gallery_mode") as! Bool
|
|
2465
2539
|
Bridge.log("LIVE: 📸 Sending gallery mode active to glasses: \(active)")
|
|
2466
2540
|
|
|
2467
2541
|
let json: [String: Any] = [
|
|
@@ -2515,6 +2589,7 @@ class MentraLive: NSObject, SGCManager {
|
|
|
2515
2589
|
Bridge.log("LIVE: 🎉 Received glasses_ready message - SOC is booted and ready!")
|
|
2516
2590
|
|
|
2517
2591
|
stopReadinessCheckLoop()
|
|
2592
|
+
sendBleMtuConfig()
|
|
2518
2593
|
|
|
2519
2594
|
// Invalidate any version fields from a prior link session so the next version_info
|
|
2520
2595
|
// cannot leave a stale build number in RN (ASG is source of truth for PackageInfo).
|
|
@@ -2656,7 +2731,7 @@ class MentraLive: NSObject, SGCManager {
|
|
|
2656
2731
|
// return
|
|
2657
2732
|
// }
|
|
2658
2733
|
|
|
2659
|
-
// // Forward PCM data to DeviceManager for
|
|
2734
|
+
// // Forward PCM data to DeviceManager for audio events and server transmission (same as Android)
|
|
2660
2735
|
// DeviceManager.shared.handlePcm(pcmData)
|
|
2661
2736
|
|
|
2662
2737
|
// Bridge.log(
|
|
@@ -2762,8 +2837,12 @@ class MentraLive: NSObject, SGCManager {
|
|
|
2762
2837
|
activeFileTransfers.removeValue(forKey: fileName)
|
|
2763
2838
|
|
|
2764
2839
|
let bleImgId = fileName.split(separator: ".").first.map(String.init) ?? ""
|
|
2765
|
-
if blePhotoTransfers.removeValue(forKey: bleImgId)
|
|
2840
|
+
if let transfer = blePhotoTransfers.removeValue(forKey: bleImgId) {
|
|
2766
2841
|
Bridge.log("LIVE: 🧹 Cleaned up timed out BLE photo transfer for: \(bleImgId)")
|
|
2842
|
+
Bridge.sendPhotoError(
|
|
2843
|
+
requestId: transfer.requestId, errorCode: "TRANSFER_TIMEOUT",
|
|
2844
|
+
errorMessage: "Transfer timed out for: \(fileName)"
|
|
2845
|
+
)
|
|
2767
2846
|
}
|
|
2768
2847
|
if bleIncidentLogRelays.removeValue(forKey: bleImgId) != nil {
|
|
2769
2848
|
Bridge.log("LIVE: 🧹 Cleaned up timed out BLE incident log relay for: \(bleImgId)")
|
|
@@ -2785,8 +2864,11 @@ class MentraLive: NSObject, SGCManager {
|
|
|
2785
2864
|
}
|
|
2786
2865
|
|
|
2787
2866
|
Bridge.log("LIVE: ❌ Transfer failed for: \(fileName) (reason: \(reason))")
|
|
2867
|
+
let bleImgId = fileName.split(separator: ".").first.map(String.init) ?? ""
|
|
2868
|
+
let transfer = blePhotoTransfers[bleImgId]
|
|
2869
|
+
let effectiveRequestId = requestId.isEmpty ? transfer?.requestId ?? "" : requestId
|
|
2788
2870
|
Bridge.sendPhotoError(
|
|
2789
|
-
requestId:
|
|
2871
|
+
requestId: effectiveRequestId, errorCode: "TRANSFER_FAILED",
|
|
2790
2872
|
errorMessage: "Transfer failed for: \(fileName) (reason: \(reason))"
|
|
2791
2873
|
)
|
|
2792
2874
|
|
|
@@ -2796,7 +2878,6 @@ class MentraLive: NSObject, SGCManager {
|
|
|
2796
2878
|
)
|
|
2797
2879
|
}
|
|
2798
2880
|
|
|
2799
|
-
let bleImgId = fileName.split(separator: ".").first.map(String.init) ?? ""
|
|
2800
2881
|
if let transfer = blePhotoTransfers.removeValue(forKey: bleImgId) {
|
|
2801
2882
|
Bridge.log(
|
|
2802
2883
|
"LIVE: 🧹 Cleaned up failed BLE photo transfer for: \(bleImgId) (requestId: \(transfer.requestId))"
|
|
@@ -2933,11 +3014,13 @@ class MentraLive: NSObject, SGCManager {
|
|
|
2933
3014
|
)
|
|
2934
3015
|
Bridge.log("❌ Telling glasses to retry entire transfer")
|
|
2935
3016
|
|
|
2936
|
-
// Tell glasses transfer failed, they will retry
|
|
3017
|
+
// Tell glasses transfer failed, they will retry. Keep the photo
|
|
3018
|
+
// transfer entry so the retry maps back to the original requestId.
|
|
2937
3019
|
sendTransferCompleteConfirmation(
|
|
2938
3020
|
fileName: packetInfo.fileName, success: false
|
|
2939
3021
|
)
|
|
2940
|
-
|
|
3022
|
+
photoTransfer.session = nil
|
|
3023
|
+
blePhotoTransfers[bleImgId] = photoTransfer
|
|
2941
3024
|
}
|
|
2942
3025
|
}
|
|
2943
3026
|
}
|
|
@@ -3170,6 +3253,20 @@ class MentraLive: NSObject, SGCManager {
|
|
|
3170
3253
|
)
|
|
3171
3254
|
}
|
|
3172
3255
|
|
|
3256
|
+
private func sendBleMtuConfig() {
|
|
3257
|
+
let effectiveMtu = min(currentMtu, bes2700MtuLimit)
|
|
3258
|
+
let json: [String: Any] = [
|
|
3259
|
+
"type": "set_ble_mtu",
|
|
3260
|
+
"mtu": effectiveMtu,
|
|
3261
|
+
"timestamp": Int64(Date().timeIntervalSince1970 * 1000),
|
|
3262
|
+
]
|
|
3263
|
+
|
|
3264
|
+
sendJson(json)
|
|
3265
|
+
Bridge.log(
|
|
3266
|
+
"LIVE: Sent BLE MTU config to glasses: negotiated=\(currentMtu), BES2700 limit=\(bes2700MtuLimit), effective=\(effectiveMtu)"
|
|
3267
|
+
)
|
|
3268
|
+
}
|
|
3269
|
+
|
|
3173
3270
|
// MARK: - Sending Data
|
|
3174
3271
|
|
|
3175
3272
|
func queueSend(_ data: Data, id: String) {
|
|
@@ -3237,6 +3334,9 @@ class MentraLive: NSObject, SGCManager {
|
|
|
3237
3334
|
Bridge.log("LIVE: All chunks queued for transmission")
|
|
3238
3335
|
} else {
|
|
3239
3336
|
// Normal single message transmission
|
|
3337
|
+
if (json["type"] as? String) == "take_photo" {
|
|
3338
|
+
Bridge.log("LIVE: PHOTO PIPELINE BLE handoff — sendJson -> queueSend take_photo")
|
|
3339
|
+
}
|
|
3240
3340
|
Bridge.log("LIVE: Sending data to glasses: \(jsonString)")
|
|
3241
3341
|
let packedData = packJson(jsonString, wakeUp: wakeUp) ?? Data()
|
|
3242
3342
|
queueSend(packedData, id: trackingId)
|
|
@@ -3508,6 +3608,27 @@ class MentraLive: NSObject, SGCManager {
|
|
|
3508
3608
|
}
|
|
3509
3609
|
}
|
|
3510
3610
|
|
|
3611
|
+
private func handleVoiceActivityDetectionStatus(enabled: Bool) {
|
|
3612
|
+
Bridge.log("LIVE: Voice Activity Detection \(enabled ? "enabled" : "disabled")")
|
|
3613
|
+
Bridge.sendVoiceActivityDetectionStatus(enabled)
|
|
3614
|
+
}
|
|
3615
|
+
|
|
3616
|
+
private func handleSpeakingStatus(speaking: Bool) {
|
|
3617
|
+
guard voiceActivityDetectionEnabled else {
|
|
3618
|
+
Bridge.log("LIVE: Ignoring speaking status because Voice Activity Detection is disabled")
|
|
3619
|
+
return
|
|
3620
|
+
}
|
|
3621
|
+
Bridge.log("LIVE: Speaking status \(speaking ? "speaking" : "not speaking")")
|
|
3622
|
+
Bridge.sendSpeakingStatus(speaking)
|
|
3623
|
+
}
|
|
3624
|
+
|
|
3625
|
+
private func handleSwitchStatus(switchType: Int, value: Int, timestamp: Int64) {
|
|
3626
|
+
Bridge.sendSwitchStatus(switchType: switchType, value: value, timestamp: timestamp)
|
|
3627
|
+
if switchType == Self.voiceActivityDetectionSwitchType, value == 0 || value == 1 {
|
|
3628
|
+
handleVoiceActivityDetectionStatus(enabled: value == 1)
|
|
3629
|
+
}
|
|
3630
|
+
}
|
|
3631
|
+
|
|
3511
3632
|
private func updateWifiStatus(connected: Bool, ssid: String, ip: String) {
|
|
3512
3633
|
Bridge.log("LIVE: 🌐 Updating WiFi status - connected: \(connected), ssid: \(ssid)")
|
|
3513
3634
|
DeviceStore.shared.apply("glasses", "wifiConnected", connected)
|
|
@@ -3796,6 +3917,10 @@ class MentraLive: NSObject, SGCManager {
|
|
|
3796
3917
|
Bridge.sendTypedMessage("stream_status", body: json)
|
|
3797
3918
|
}
|
|
3798
3919
|
|
|
3920
|
+
private func emitPhotoStatus(_ json: [String: Any]) {
|
|
3921
|
+
Bridge.sendPhotoStatus(json)
|
|
3922
|
+
}
|
|
3923
|
+
|
|
3799
3924
|
private func emitButtonPress(buttonId: String, pressType: String, timestamp: Int64) {
|
|
3800
3925
|
let eventBody: [String: Any] = [
|
|
3801
3926
|
"device_model": "Mentra Live",
|
|
@@ -4493,6 +4618,42 @@ extension MentraLive {
|
|
|
4493
4618
|
|
|
4494
4619
|
// Send gallery mode state (camera app running status)
|
|
4495
4620
|
sendGalleryMode()
|
|
4621
|
+
|
|
4622
|
+
// Send glasses-side Voice Activity Detection setting.
|
|
4623
|
+
sendVoiceActivityDetectionSetting()
|
|
4624
|
+
}
|
|
4625
|
+
|
|
4626
|
+
func sendVoiceActivityDetectionSetting() {
|
|
4627
|
+
let enabled = DeviceStore.shared.get("bluetooth", "voice_activity_detection_enabled") as? Bool ?? true
|
|
4628
|
+
Bridge.log("LIVE: 🎤 Sending Voice Activity Detection setting to glasses: \(enabled)")
|
|
4629
|
+
|
|
4630
|
+
guard connectedPeripheral != nil, txCharacteristic != nil else {
|
|
4631
|
+
Bridge.log("Cannot send Voice Activity Detection setting - BLE write path not ready")
|
|
4632
|
+
return
|
|
4633
|
+
}
|
|
4634
|
+
|
|
4635
|
+
do {
|
|
4636
|
+
let bodyData = try JSONSerialization.data(withJSONObject: [
|
|
4637
|
+
"type": Self.voiceActivityDetectionSwitchType,
|
|
4638
|
+
"switch": enabled ? 1 : 0,
|
|
4639
|
+
])
|
|
4640
|
+
guard let bodyString = String(data: bodyData, encoding: .utf8) else {
|
|
4641
|
+
Bridge.log("LIVE: Failed to encode Voice Activity Detection payload")
|
|
4642
|
+
return
|
|
4643
|
+
}
|
|
4644
|
+
let command: [String: Any] = [
|
|
4645
|
+
"C": "cs_swit",
|
|
4646
|
+
"V": 1,
|
|
4647
|
+
"B": bodyString,
|
|
4648
|
+
]
|
|
4649
|
+
if sendRawK900Command(command, wakeUp: true) {
|
|
4650
|
+
Bridge.sendVoiceActivityDetectionStatus(enabled)
|
|
4651
|
+
} else {
|
|
4652
|
+
Bridge.log("LIVE: Failed to send Voice Activity Detection setting command")
|
|
4653
|
+
}
|
|
4654
|
+
} catch {
|
|
4655
|
+
Bridge.log("LIVE: Error encoding Voice Activity Detection payload: \(error)")
|
|
4656
|
+
}
|
|
4496
4657
|
}
|
|
4497
4658
|
|
|
4498
4659
|
func sendButtonVideoRecordingSettings() {
|
|
@@ -4500,11 +4661,11 @@ extension MentraLive {
|
|
|
4500
4661
|
DeviceStore.shared.get("bluetooth", "button_video_settings") as? [String: Any] ?? [
|
|
4501
4662
|
"width": 1280,
|
|
4502
4663
|
"height": 720,
|
|
4503
|
-
"
|
|
4664
|
+
"fps": 30,
|
|
4504
4665
|
]
|
|
4505
4666
|
let width = settings["width"] as? Int ?? 1280
|
|
4506
4667
|
let height = settings["height"] as? Int ?? 720
|
|
4507
|
-
let fps = settings["
|
|
4668
|
+
let fps = settings["fps"] as? Int ?? 30
|
|
4508
4669
|
|
|
4509
4670
|
// Use defaults if not set
|
|
4510
4671
|
let finalWidth = width > 0 ? width : 1280
|
|
@@ -4582,9 +4743,9 @@ extension MentraLive {
|
|
|
4582
4743
|
}
|
|
4583
4744
|
|
|
4584
4745
|
func sendCameraFovSetting() {
|
|
4585
|
-
let settings = DeviceStore.shared.get("bluetooth", "camera_fov") as? [String: Any] ?? ["fov": 118, "
|
|
4746
|
+
let settings = DeviceStore.shared.get("bluetooth", "camera_fov") as? [String: Any] ?? ["fov": 118, "roi_position": 0]
|
|
4586
4747
|
let fov = settings["fov"] as? Int ?? 118
|
|
4587
|
-
let roiPosition = settings["
|
|
4748
|
+
let roiPosition = settings["roi_position"] as? Int ?? 0
|
|
4588
4749
|
|
|
4589
4750
|
Bridge.log("Sending camera FOV setting: fov=\(fov), roiPosition=\(roiPosition)")
|
|
4590
4751
|
|
|
@@ -4642,7 +4803,7 @@ extension MentraLive {
|
|
|
4642
4803
|
|
|
4643
4804
|
var json: [String: Any] = [
|
|
4644
4805
|
"type": "start_video_recording",
|
|
4645
|
-
"
|
|
4806
|
+
"requestId": requestId,
|
|
4646
4807
|
"save": save,
|
|
4647
4808
|
"flash": flash,
|
|
4648
4809
|
"sound": sound,
|
|
@@ -4669,7 +4830,7 @@ extension MentraLive {
|
|
|
4669
4830
|
|
|
4670
4831
|
let json: [String: Any] = [
|
|
4671
4832
|
"type": "stop_video_recording",
|
|
4672
|
-
"
|
|
4833
|
+
"requestId": requestId,
|
|
4673
4834
|
]
|
|
4674
4835
|
sendJson(json)
|
|
4675
4836
|
}
|
|
@@ -46,7 +46,7 @@ class MentraNexSGC: NSObject, CBCentralManagerDelegate, CBPeripheralDelegate, SG
|
|
|
46
46
|
|
|
47
47
|
func requestPhoto(
|
|
48
48
|
_: String, appId _: String, size _: String?, webhookUrl _: String?, authToken _: String?,
|
|
49
|
-
compress _: String?, flash _: Bool, sound _: Bool
|
|
49
|
+
compress _: String?, flash _: Bool, sound _: Bool, exposureTimeNs _: Double?, iso _: Int?
|
|
50
50
|
) {}
|
|
51
51
|
|
|
52
52
|
func startStream(_: [String: Any]) {}
|
|
@@ -20,7 +20,7 @@ protocol SGCManager {
|
|
|
20
20
|
|
|
21
21
|
func requestPhoto(
|
|
22
22
|
_ requestId: String, appId: String, size: String?, webhookUrl: String?, authToken: String?,
|
|
23
|
-
compress: String?, flash: Bool, sound: Bool
|
|
23
|
+
compress: String?, flash: Bool, sound: Bool, exposureTimeNs: Double?, iso: Int?
|
|
24
24
|
)
|
|
25
25
|
func startStream(_ message: [String: Any])
|
|
26
26
|
func stopStream()
|
|
@@ -103,6 +103,10 @@ protocol SGCManager {
|
|
|
103
103
|
func queryGalleryStatus()
|
|
104
104
|
func sendGalleryMode()
|
|
105
105
|
|
|
106
|
+
// MARK: - Voice Activity Detection
|
|
107
|
+
|
|
108
|
+
func sendVoiceActivityDetectionSetting()
|
|
109
|
+
|
|
106
110
|
// MARK: - Version Info
|
|
107
111
|
|
|
108
112
|
func requestVersionInfo()
|
|
@@ -127,6 +131,10 @@ extension SGCManager {
|
|
|
127
131
|
|
|
128
132
|
func setDashboardMenu(_: [[String: Any]]) {}
|
|
129
133
|
|
|
134
|
+
// MARK: - Voice Activity Detection (default no-op — Mentra Live supports this)
|
|
135
|
+
|
|
136
|
+
func sendVoiceActivityDetectionSetting() {}
|
|
137
|
+
|
|
130
138
|
// MARK: - Default DeviceStore-backed property implementations
|
|
131
139
|
|
|
132
140
|
var fullyBooted: Bool {
|
|
@@ -181,8 +189,8 @@ extension SGCManager {
|
|
|
181
189
|
DeviceStore.shared.get("glasses", "micEnabled") as? Bool ?? false
|
|
182
190
|
}
|
|
183
191
|
|
|
184
|
-
var
|
|
185
|
-
DeviceStore.shared.get("glasses", "
|
|
192
|
+
var voiceActivityDetectionEnabled: Bool {
|
|
193
|
+
DeviceStore.shared.get("glasses", "voiceActivityDetectionEnabled") as? Bool ?? true
|
|
186
194
|
}
|
|
187
195
|
|
|
188
196
|
var batteryLevel: Int {
|
|
@@ -12,7 +12,7 @@ class Simulated: SGCManager {
|
|
|
12
12
|
DeviceStore.shared.apply("glasses", "connected", true)
|
|
13
13
|
DeviceStore.shared.apply("glasses", "connectionState", ConnTypes.CONNECTED)
|
|
14
14
|
DeviceStore.shared.apply("glasses", "micEnabled", false)
|
|
15
|
-
DeviceStore.shared.apply("glasses", "
|
|
15
|
+
DeviceStore.shared.apply("glasses", "voiceActivityDetectionEnabled", true)
|
|
16
16
|
DeviceStore.shared.apply("glasses", "bluetoothClassicConnected", false)
|
|
17
17
|
}
|
|
18
18
|
|
|
@@ -75,7 +75,7 @@ class Simulated: SGCManager {
|
|
|
75
75
|
|
|
76
76
|
// MARK: - Camera & Media
|
|
77
77
|
|
|
78
|
-
func requestPhoto(_: String, appId _: String, size _: String?, webhookUrl _: String?, authToken _: String?, compress _: String?, flash _: Bool, sound _: Bool) {
|
|
78
|
+
func requestPhoto(_: String, appId _: String, size _: String?, webhookUrl _: String?, authToken _: String?, compress _: String?, flash _: Bool, sound _: Bool, exposureTimeNs _: Double?, iso _: Int?) {
|
|
79
79
|
Bridge.log("requestPhoto")
|
|
80
80
|
}
|
|
81
81
|
|
|
@@ -214,16 +214,13 @@ class SherpaOnnxTranscriber {
|
|
|
214
214
|
* Feed PCM audio data (16-bit little endian) into the transcriber.
|
|
215
215
|
* This method should be called continuously with short chunks (e.g., 100-300ms).
|
|
216
216
|
*
|
|
217
|
-
*
|
|
218
|
-
* so it's directly queued for processing without additional VAD checks.
|
|
217
|
+
* Audio is queued directly; microphone VAD gating is not applied in the SDK.
|
|
219
218
|
*/
|
|
220
219
|
func acceptAudio(pcm16le: Data) {
|
|
221
220
|
guard isRunning else {
|
|
222
221
|
return
|
|
223
222
|
}
|
|
224
223
|
|
|
225
|
-
// Directly queue the audio data for processing
|
|
226
|
-
// No VAD check here as it's assumed to be done upstream
|
|
227
224
|
queueAudioData(pcm16le)
|
|
228
225
|
}
|
|
229
226
|
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import Foundation
|
|
2
|
+
|
|
3
|
+
final class MessageChunkReassembler {
|
|
4
|
+
private static let chunkTimeoutMs: TimeInterval = 30_000
|
|
5
|
+
private static let maxConcurrentSessions = 10
|
|
6
|
+
|
|
7
|
+
private var activeSessions: [String: ChunkSession] = [:]
|
|
8
|
+
|
|
9
|
+
func addChunk(_ info: MessageChunker.ChunkInfo) -> String? {
|
|
10
|
+
cleanupTimedOutSessions()
|
|
11
|
+
|
|
12
|
+
if activeSessions.count >= Self.maxConcurrentSessions,
|
|
13
|
+
activeSessions[info.chunkId] == nil
|
|
14
|
+
{
|
|
15
|
+
removeOldestSession()
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
let session = activeSessions[info.chunkId] ?? ChunkSession(
|
|
19
|
+
chunkId: info.chunkId,
|
|
20
|
+
totalChunks: info.totalChunks
|
|
21
|
+
)
|
|
22
|
+
activeSessions[info.chunkId] = session
|
|
23
|
+
|
|
24
|
+
guard session.addChunk(index: info.chunkIndex, data: info.data) else {
|
|
25
|
+
print("MessageChunkReassembler: Failed to add chunk \(info.chunkIndex) for \(info.chunkId)")
|
|
26
|
+
return nil
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
guard session.isComplete else {
|
|
30
|
+
return nil
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
let reassembled = session.reassemble()
|
|
34
|
+
activeSessions.removeValue(forKey: info.chunkId)
|
|
35
|
+
print("MessageChunkReassembler: Reassembled \(reassembled.count) bytes from \(info.totalChunks) chunks")
|
|
36
|
+
return reassembled
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
func clear() {
|
|
40
|
+
activeSessions.removeAll()
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
private func cleanupTimedOutSessions() {
|
|
44
|
+
let now = Date().timeIntervalSince1970 * 1000
|
|
45
|
+
activeSessions = activeSessions.filter { entry in
|
|
46
|
+
now - entry.value.createdAtMs <= Self.chunkTimeoutMs
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
private func removeOldestSession() {
|
|
51
|
+
guard let oldest = activeSessions.min(by: { $0.value.createdAtMs < $1.value.createdAtMs }) else {
|
|
52
|
+
return
|
|
53
|
+
}
|
|
54
|
+
activeSessions.removeValue(forKey: oldest.key)
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
private final class ChunkSession {
|
|
58
|
+
let chunkId: String
|
|
59
|
+
let totalChunks: Int
|
|
60
|
+
let createdAtMs: TimeInterval
|
|
61
|
+
private var chunks: [Int: String] = [:]
|
|
62
|
+
|
|
63
|
+
init(chunkId: String, totalChunks: Int) {
|
|
64
|
+
self.chunkId = chunkId
|
|
65
|
+
self.totalChunks = totalChunks
|
|
66
|
+
self.createdAtMs = Date().timeIntervalSince1970 * 1000
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
func addChunk(index: Int, data: String) -> Bool {
|
|
70
|
+
guard index >= 0, index < totalChunks else {
|
|
71
|
+
return false
|
|
72
|
+
}
|
|
73
|
+
chunks[index] = data
|
|
74
|
+
return true
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
var isComplete: Bool {
|
|
78
|
+
chunks.count == totalChunks
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
func reassemble() -> String {
|
|
82
|
+
var result = ""
|
|
83
|
+
for index in 0 ..< totalChunks {
|
|
84
|
+
result += chunks[index] ?? ""
|
|
85
|
+
}
|
|
86
|
+
return result
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mentra/bluetooth-sdk",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.8",
|
|
4
4
|
"description": "SDK for communicating with smart glasses",
|
|
5
5
|
"main": "build/index.js",
|
|
6
6
|
"react-native": "src/index.ts",
|
|
@@ -17,6 +17,11 @@
|
|
|
17
17
|
"react-native": "./src/react/index.ts",
|
|
18
18
|
"default": "./build/react/index.js"
|
|
19
19
|
},
|
|
20
|
+
"./photo-receiver": {
|
|
21
|
+
"types": "./build/photo-receiver/index.d.ts",
|
|
22
|
+
"react-native": "./src/photo-receiver/index.ts",
|
|
23
|
+
"default": "./build/photo-receiver/index.js"
|
|
24
|
+
},
|
|
20
25
|
"./_private/*": null,
|
|
21
26
|
"./build/*": null,
|
|
22
27
|
"./src/*": null,
|