@mentra/bluetooth-sdk 3.2.1-beta.476 → 3.2.1-beta.481
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/android/src/main/java/com/mentra/bluetoothsdk/GeneratedReleaseMetadata.kt +5 -5
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SerializedGattCallback.kt +5 -0
- package/android/src/test/java/com/mentra/bluetoothsdk/sgcs/MentraLiveGattCallbackTest.kt +33 -0
- package/build/generated/releaseMetadata.js +5 -5
- package/build/generated/releaseMetadata.js.map +1 -1
- package/ios/Source/BluetoothSdkDefaults.swift +1 -1
- package/ios/Source/DeviceManager.swift +7 -23
- package/ios/Source/GeneratedReleaseMetadata.swift +5 -5
- package/ios/Source/sgcs/Nimo.swift +62 -37
- package/ios/Source/sgcs/NimoDiscovery.swift +49 -0
- package/ios/Source/stt/SherpaOnnxTranscriber.swift +49 -189
- package/ios/Source/utils/DemandDrivenAudioWorker.swift +150 -0
- package/ios/Tests/DemandDrivenAudioWorkerTests.swift +157 -0
- package/ios/Tests/NimoDiscoveryTests.swift +79 -0
- package/package.json +2 -2
- package/src/generated/releaseMetadata.ts +5 -5
|
@@ -3,9 +3,9 @@ package com.mentra.bluetoothsdk
|
|
|
3
3
|
/** Generated by release CI. Do not edit in a release checkout. */
|
|
4
4
|
internal object GeneratedReleaseMetadata {
|
|
5
5
|
const val FAMILY_BASE_VERSION: String = "3.2.1"
|
|
6
|
-
const val RELEASE_IDENTITY: String = "3.2.1-beta.
|
|
7
|
-
const val RELEASE_SET_ID: String = "mentra-3.2.1-beta.
|
|
8
|
-
const val SOURCE_COMMIT: String = "
|
|
9
|
-
const val OTA_MANIFEST_URL: String = "https://artifactscdn.mentraglass.com/Mentra-Community/MentraOS/releases/mentra-builds-v3.2.1/mentra-live-ota-3.2.1-beta.
|
|
10
|
-
const val OTA_MANIFEST_SHA256: String = "
|
|
6
|
+
const val RELEASE_IDENTITY: String = "3.2.1-beta.481"
|
|
7
|
+
const val RELEASE_SET_ID: String = "mentra-3.2.1-beta.481"
|
|
8
|
+
const val SOURCE_COMMIT: String = "c3269f1bc026d0b1967e8f18cae271c0ef396c11"
|
|
9
|
+
const val OTA_MANIFEST_URL: String = "https://artifactscdn.mentraglass.com/Mentra-Community/MentraOS/releases/mentra-builds-v3.2.1/mentra-live-ota-3.2.1-beta.481.json"
|
|
10
|
+
const val OTA_MANIFEST_SHA256: String = "be95e7f2ec85c32d385f66ed8d1c319320e1b5f61f59ea3d1d76199f269834ea"
|
|
11
11
|
}
|
|
@@ -4,6 +4,7 @@ import android.bluetooth.BluetoothGatt
|
|
|
4
4
|
import android.bluetooth.BluetoothGattCallback
|
|
5
5
|
import android.bluetooth.BluetoothGattCharacteristic
|
|
6
6
|
import android.bluetooth.BluetoothGattDescriptor
|
|
7
|
+
import android.os.Build
|
|
7
8
|
|
|
8
9
|
/**
|
|
9
10
|
* Binder callbacks only capture their arguments. Session validation and all callback effects run
|
|
@@ -45,6 +46,10 @@ internal abstract class SerializedGattCallback(
|
|
|
45
46
|
final override fun onCharacteristicChanged(
|
|
46
47
|
gatt: BluetoothGatt, characteristic: BluetoothGattCharacteristic,
|
|
47
48
|
) {
|
|
49
|
+
// Android 13 introduced the value callback; some framework versions also call
|
|
50
|
+
// this deprecated overload for the same notification. Decoding both replays
|
|
51
|
+
// each LC3 packet and doubles recorded audio. Older Android uses this path.
|
|
52
|
+
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) return
|
|
48
53
|
val value = characteristic.value?.copyOf() ?: return
|
|
49
54
|
dispatch(gatt) { handleCharacteristicChanged(gatt, characteristic, value) }
|
|
50
55
|
}
|
|
@@ -78,6 +78,39 @@ class MentraLiveGattCallbackTest {
|
|
|
78
78
|
assertArrayEquals(byteArrayOf(3, 2), callback.packets[1])
|
|
79
79
|
}
|
|
80
80
|
|
|
81
|
+
@Test
|
|
82
|
+
@Config(sdk = [33])
|
|
83
|
+
@Suppress("DEPRECATION")
|
|
84
|
+
fun `modern and legacy callbacks for one notification deliver one packet`() {
|
|
85
|
+
val queue = ArrayDeque<() -> Unit>()
|
|
86
|
+
val callback = RecordingCallback({ work -> queue.addLast(work) })
|
|
87
|
+
val gatt = gatt()
|
|
88
|
+
val characteristic = BluetoothGattCharacteristic(UUID.randomUUID(), 0, 0)
|
|
89
|
+
val value = byteArrayOf(0xf1.toByte(), 7, 1, 2)
|
|
90
|
+
characteristic.value = value
|
|
91
|
+
callback.onCharacteristicChanged(gatt, characteristic, value)
|
|
92
|
+
callback.onCharacteristicChanged(gatt, characteristic)
|
|
93
|
+
value[2] = 9
|
|
94
|
+
while (queue.isNotEmpty()) queue.removeFirst().invoke()
|
|
95
|
+
assertEquals(1, callback.packets.size)
|
|
96
|
+
assertArrayEquals(byteArrayOf(0xf1.toByte(), 7, 1, 2), callback.packets.single())
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
@Test
|
|
100
|
+
@Config(sdk = [33])
|
|
101
|
+
fun `separate modern notifications retain identical payloads`() {
|
|
102
|
+
val queue = ArrayDeque<() -> Unit>()
|
|
103
|
+
val callback = RecordingCallback({ work -> queue.addLast(work) })
|
|
104
|
+
val gatt = gatt()
|
|
105
|
+
val characteristic = BluetoothGattCharacteristic(UUID.randomUUID(), 0, 0)
|
|
106
|
+
val value = byteArrayOf(1, 2)
|
|
107
|
+
repeat(2) { callback.onCharacteristicChanged(gatt, characteristic, value) }
|
|
108
|
+
value[0] = 9
|
|
109
|
+
while (queue.isNotEmpty()) queue.removeFirst().invoke()
|
|
110
|
+
assertEquals(2, callback.packets.size)
|
|
111
|
+
callback.packets.forEach { assertArrayEquals(byteArrayOf(1, 2), it) }
|
|
112
|
+
}
|
|
113
|
+
|
|
81
114
|
@Test
|
|
82
115
|
fun `replacement invalidates callbacks already waiting on the lifecycle queue`() {
|
|
83
116
|
val queue = ArrayDeque<() -> Unit>()
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
export const BLUETOOTH_SDK_RELEASE_METADATA = Object.freeze({
|
|
2
2
|
"schemaVersion": 1,
|
|
3
3
|
"familyBaseVersion": "3.2.1",
|
|
4
|
-
"releaseIdentity": "3.2.1-beta.
|
|
5
|
-
"releaseSetId": "mentra-3.2.1-beta.
|
|
6
|
-
"sourceCommit": "
|
|
7
|
-
"otaManifestUrl": "https://artifactscdn.mentraglass.com/Mentra-Community/MentraOS/releases/mentra-builds-v3.2.1/mentra-live-ota-3.2.1-beta.
|
|
8
|
-
"otaManifestSha256": "
|
|
4
|
+
"releaseIdentity": "3.2.1-beta.481",
|
|
5
|
+
"releaseSetId": "mentra-3.2.1-beta.481",
|
|
6
|
+
"sourceCommit": "c3269f1bc026d0b1967e8f18cae271c0ef396c11",
|
|
7
|
+
"otaManifestUrl": "https://artifactscdn.mentraglass.com/Mentra-Community/MentraOS/releases/mentra-builds-v3.2.1/mentra-live-ota-3.2.1-beta.481.json",
|
|
8
|
+
"otaManifestSha256": "be95e7f2ec85c32d385f66ed8d1c319320e1b5f61f59ea3d1d76199f269834ea"
|
|
9
9
|
});
|
|
10
10
|
//# sourceMappingURL=releaseMetadata.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"releaseMetadata.js","sourceRoot":"","sources":["../../src/generated/releaseMetadata.ts"],"names":[],"mappings":"AAWA,MAAM,CAAC,MAAM,8BAA8B,GAA0C,MAAM,CAAC,MAAM,CAAC;IACjG,eAAe,EAAE,CAAC;IAClB,mBAAmB,EAAE,OAAO;IAC5B,iBAAiB,EAAE,gBAAgB;IACnC,cAAc,EAAE,uBAAuB;IACvC,cAAc,EAAE,0CAA0C;IAC1D,gBAAgB,EAAE,kIAAkI;IACpJ,mBAAmB,EAAE,kEAAkE;CACxF,CAAC,CAAA","sourcesContent":["/** Generated by release CI. Do not edit in a release checkout. */\nexport interface BluetoothSdkReleaseMetadata {\n schemaVersion: 1\n familyBaseVersion: string | null\n releaseIdentity: string | null\n releaseSetId: string | null\n sourceCommit: string | null\n otaManifestUrl: string | null\n otaManifestSha256: string | null\n}\n\nexport const BLUETOOTH_SDK_RELEASE_METADATA: Readonly<BluetoothSdkReleaseMetadata> = Object.freeze({\n \"schemaVersion\": 1,\n \"familyBaseVersion\": \"3.2.1\",\n \"releaseIdentity\": \"3.2.1-beta.
|
|
1
|
+
{"version":3,"file":"releaseMetadata.js","sourceRoot":"","sources":["../../src/generated/releaseMetadata.ts"],"names":[],"mappings":"AAWA,MAAM,CAAC,MAAM,8BAA8B,GAA0C,MAAM,CAAC,MAAM,CAAC;IACjG,eAAe,EAAE,CAAC;IAClB,mBAAmB,EAAE,OAAO;IAC5B,iBAAiB,EAAE,gBAAgB;IACnC,cAAc,EAAE,uBAAuB;IACvC,cAAc,EAAE,0CAA0C;IAC1D,gBAAgB,EAAE,kIAAkI;IACpJ,mBAAmB,EAAE,kEAAkE;CACxF,CAAC,CAAA","sourcesContent":["/** Generated by release CI. Do not edit in a release checkout. */\nexport interface BluetoothSdkReleaseMetadata {\n schemaVersion: 1\n familyBaseVersion: string | null\n releaseIdentity: string | null\n releaseSetId: string | null\n sourceCommit: string | null\n otaManifestUrl: string | null\n otaManifestSha256: string | null\n}\n\nexport const BLUETOOTH_SDK_RELEASE_METADATA: Readonly<BluetoothSdkReleaseMetadata> = Object.freeze({\n \"schemaVersion\": 1,\n \"familyBaseVersion\": \"3.2.1\",\n \"releaseIdentity\": \"3.2.1-beta.481\",\n \"releaseSetId\": \"mentra-3.2.1-beta.481\",\n \"sourceCommit\": \"c3269f1bc026d0b1967e8f18cae271c0ef396c11\",\n \"otaManifestUrl\": \"https://artifactscdn.mentraglass.com/Mentra-Community/MentraOS/releases/mentra-builds-v3.2.1/mentra-live-ota-3.2.1-beta.481.json\",\n \"otaManifestSha256\": \"be95e7f2ec85c32d385f66ed8d1c319320e1b5f61f59ea3d1d76199f269834ea\"\n})\n"]}
|
|
@@ -16,7 +16,7 @@ enum BluetoothSdkDefaults {
|
|
|
16
16
|
/// silently disable it on every pair the app touches.
|
|
17
17
|
static let autoPowerOffEnabled = false
|
|
18
18
|
private static let infoSdkVersionKey = "MentraBluetoothSdkVersion"
|
|
19
|
-
private static let swiftPackageSdkVersion = "3.2.1-beta.
|
|
19
|
+
private static let swiftPackageSdkVersion = "3.2.1-beta.481"
|
|
20
20
|
private static let swiftPackageSdkVersionPlaceholder = "__MENTRA" + "_BLUETOOTH_SDK_VERSION__"
|
|
21
21
|
|
|
22
22
|
private static func normalizedSdkVersion(_ value: String?) -> String? {
|
|
@@ -330,7 +330,7 @@ struct ViewState {
|
|
|
330
330
|
|
|
331
331
|
/// STT:
|
|
332
332
|
#if !os(macOS) && (!SWIFT_PACKAGE || MENTRA_FEATURE_LOCAL_STT)
|
|
333
|
-
private
|
|
333
|
+
private let transcriber = SherpaOnnxTranscriber()
|
|
334
334
|
#endif
|
|
335
335
|
|
|
336
336
|
var viewStates: [ViewState] = [
|
|
@@ -383,24 +383,6 @@ struct ViewState {
|
|
|
383
383
|
// Start memory monitoring (logs every 30s to help detect leaks)
|
|
384
384
|
// MemoryMonitor.start()
|
|
385
385
|
|
|
386
|
-
// Initialize SherpaOnnx Transcriber
|
|
387
|
-
#if !os(macOS) && (!SWIFT_PACKAGE || MENTRA_FEATURE_LOCAL_STT)
|
|
388
|
-
if let windowScene = UIApplication.shared.connectedScenes.first as? UIWindowScene,
|
|
389
|
-
let window = windowScene.windows.first,
|
|
390
|
-
let rootViewController = window.rootViewController
|
|
391
|
-
{
|
|
392
|
-
transcriber = SherpaOnnxTranscriber(context: rootViewController)
|
|
393
|
-
} else {
|
|
394
|
-
Bridge.log("Failed to create SherpaOnnxTranscriber - no root view controller found")
|
|
395
|
-
}
|
|
396
|
-
|
|
397
|
-
// Initialize the transcriber
|
|
398
|
-
if let transcriber = transcriber {
|
|
399
|
-
transcriber.initialize()
|
|
400
|
-
Bridge.log("SherpaOnnxTranscriber fully initialized")
|
|
401
|
-
}
|
|
402
|
-
#endif
|
|
403
|
-
|
|
404
386
|
// Initialize persistent LC3 converter for unified audio encoding
|
|
405
387
|
lc3Converter = PcmConverter()
|
|
406
388
|
Bridge.log("LC3 converter initialized for unified audio encoding")
|
|
@@ -542,7 +524,7 @@ struct ViewState {
|
|
|
542
524
|
// Send PCM to local transcriber.
|
|
543
525
|
#if !os(macOS) && (!SWIFT_PACKAGE || MENTRA_FEATURE_LOCAL_STT)
|
|
544
526
|
if shouldSendTranscript || localSttFallbackActive {
|
|
545
|
-
transcriber
|
|
527
|
+
transcriber.acceptAudio(pcm16le: pcmData)
|
|
546
528
|
}
|
|
547
529
|
#endif
|
|
548
530
|
}
|
|
@@ -984,7 +966,7 @@ struct ViewState {
|
|
|
984
966
|
func restartTranscriber() {
|
|
985
967
|
#if !os(macOS) && (!SWIFT_PACKAGE || MENTRA_FEATURE_LOCAL_STT)
|
|
986
968
|
Bridge.log("MAN: Restarting SherpaOnnxTranscriber via command")
|
|
987
|
-
transcriber
|
|
969
|
+
transcriber.restart()
|
|
988
970
|
#else
|
|
989
971
|
Bridge.log("MAN: Local STT is not included in this SwiftPM build")
|
|
990
972
|
#endif
|
|
@@ -1762,6 +1744,9 @@ struct ViewState {
|
|
|
1762
1744
|
func setMicState() {
|
|
1763
1745
|
let willSendPcm = shouldSendPcm || shouldSendLc3
|
|
1764
1746
|
let willSendTranscript = shouldSendTranscript || localSttFallbackActive
|
|
1747
|
+
#if !os(macOS) && (!SWIFT_PACKAGE || MENTRA_FEATURE_LOCAL_STT)
|
|
1748
|
+
transcriber.setActive(willSendTranscript)
|
|
1749
|
+
#endif
|
|
1765
1750
|
micEnabled = willSendPcm || willSendTranscript
|
|
1766
1751
|
updateMicState()
|
|
1767
1752
|
}
|
|
@@ -2080,8 +2065,7 @@ struct ViewState {
|
|
|
2080
2065
|
#endif
|
|
2081
2066
|
// Clean up transcriber resources
|
|
2082
2067
|
#if !os(macOS) && (!SWIFT_PACKAGE || MENTRA_FEATURE_LOCAL_STT)
|
|
2083
|
-
transcriber
|
|
2084
|
-
transcriber = nil
|
|
2068
|
+
transcriber.shutdown()
|
|
2085
2069
|
#endif
|
|
2086
2070
|
|
|
2087
2071
|
// Clean up LC3 converter
|
|
@@ -3,9 +3,9 @@ import Foundation
|
|
|
3
3
|
/// Generated by release CI. Do not edit in a release checkout.
|
|
4
4
|
enum GeneratedReleaseMetadata {
|
|
5
5
|
static let familyBaseVersion = "3.2.1"
|
|
6
|
-
static let releaseIdentity = "3.2.1-beta.
|
|
7
|
-
static let releaseSetId = "mentra-3.2.1-beta.
|
|
8
|
-
static let sourceCommit = "
|
|
9
|
-
static let otaManifestUrl = "https://artifactscdn.mentraglass.com/Mentra-Community/MentraOS/releases/mentra-builds-v3.2.1/mentra-live-ota-3.2.1-beta.
|
|
10
|
-
static let otaManifestSha256 = "
|
|
6
|
+
static let releaseIdentity = "3.2.1-beta.481"
|
|
7
|
+
static let releaseSetId = "mentra-3.2.1-beta.481"
|
|
8
|
+
static let sourceCommit = "c3269f1bc026d0b1967e8f18cae271c0ef396c11"
|
|
9
|
+
static let otaManifestUrl = "https://artifactscdn.mentraglass.com/Mentra-Community/MentraOS/releases/mentra-builds-v3.2.1/mentra-live-ota-3.2.1-beta.481.json"
|
|
10
|
+
static let otaManifestSha256 = "be95e7f2ec85c32d385f66ed8d1c319320e1b5f61f59ea3d1d76199f269834ea"
|
|
11
11
|
}
|
|
@@ -20,10 +20,6 @@ enum NimoBLE {
|
|
|
20
20
|
static let CHAR_RX = CBUUID(string: "00002022-0000-1000-8000-00805F9B34FB")
|
|
21
21
|
static let CHAR_MIC = CBUUID(string: "00002025-0000-1000-8000-00805F9B34FB")
|
|
22
22
|
|
|
23
|
-
static let NAME_PREFIX = "nimo"
|
|
24
|
-
/// iOS ANCS side-channel devices advertise "<name>_ble" — never the data channel.
|
|
25
|
-
static let BLE_NAME_SUFFIX = "_ble"
|
|
26
|
-
|
|
27
23
|
static let CHUNK_SIZE = 501
|
|
28
24
|
static let INTER_FRAME_DELAY_MS = 5
|
|
29
25
|
}
|
|
@@ -591,6 +587,27 @@ class Nimo: NSObject, SGCManager {
|
|
|
591
587
|
private var rxChar: CBCharacteristic?
|
|
592
588
|
private var micChar: CBCharacteristic?
|
|
593
589
|
private var isDisconnecting = false
|
|
590
|
+
private var scanRequested = false
|
|
591
|
+
private lazy var discovery = NimoDiscovery<CBPeripheral>(
|
|
592
|
+
register: { [weak self] in
|
|
593
|
+
#if os(iOS)
|
|
594
|
+
self?.centralManager?.registerForConnectionEvents(options: [.serviceUUIDs: [NimoBLE.SERVICE_UUID]])
|
|
595
|
+
#endif
|
|
596
|
+
}, connected: { [weak self] in
|
|
597
|
+
self?.centralManager?.retrieveConnectedPeripherals(withServices: [NimoBLE.SERVICE_UUID]) ?? []
|
|
598
|
+
}, scan: { [weak self] in
|
|
599
|
+
self?.centralManager?.scanForPeripherals(
|
|
600
|
+
withServices: nil, options: [CBCentralManagerScanOptionAllowDuplicatesKey: false]
|
|
601
|
+
)
|
|
602
|
+
}, stop: { [weak self] in
|
|
603
|
+
self?.centralManager?.stopScan()
|
|
604
|
+
#if os(iOS)
|
|
605
|
+
self?.centralManager?.registerForConnectionEvents(options: [:])
|
|
606
|
+
#endif
|
|
607
|
+
}, name: { $0.name }, found: { [weak self] peripheral, name, rssi in
|
|
608
|
+
self?.onDeviceFound(peripheral, name: name, rssi: rssi)
|
|
609
|
+
}
|
|
610
|
+
)
|
|
594
611
|
|
|
595
612
|
/// Device search
|
|
596
613
|
private var DEVICE_SEARCH_ID = "NOT_SET"
|
|
@@ -680,6 +697,8 @@ class Nimo: NSObject, SGCManager {
|
|
|
680
697
|
func findCompatibleDevices() {
|
|
681
698
|
Bridge.log("NIMO: findCompatibleDevices()")
|
|
682
699
|
DEVICE_SEARCH_ID = "NOT_SET"
|
|
700
|
+
isDisconnecting = false
|
|
701
|
+
scanRequested = true
|
|
683
702
|
DeviceStore.shared.apply("glasses", "connectionState", ConnTypes.SCANNING)
|
|
684
703
|
startScan()
|
|
685
704
|
}
|
|
@@ -687,6 +706,7 @@ class Nimo: NSObject, SGCManager {
|
|
|
687
706
|
func connectById(_ id: String) {
|
|
688
707
|
Bridge.log("NIMO: connectById(\(id))")
|
|
689
708
|
DEVICE_SEARCH_ID = id
|
|
709
|
+
scanRequested = true
|
|
690
710
|
DeviceStore.shared.apply("glasses", "connectionState", ConnTypes.CONNECTING)
|
|
691
711
|
isDisconnecting = false
|
|
692
712
|
startPairingTimeout()
|
|
@@ -694,7 +714,8 @@ class Nimo: NSObject, SGCManager {
|
|
|
694
714
|
}
|
|
695
715
|
|
|
696
716
|
func stopScan() {
|
|
697
|
-
|
|
717
|
+
scanRequested = false
|
|
718
|
+
discovery.cancel()
|
|
698
719
|
}
|
|
699
720
|
|
|
700
721
|
func disconnect() {
|
|
@@ -1061,13 +1082,9 @@ class Nimo: NSObject, SGCManager {
|
|
|
1061
1082
|
|
|
1062
1083
|
// MARK: - BLE Scanning
|
|
1063
1084
|
|
|
1064
|
-
private func isNimoMainDevice(_ name: String) -> Bool {
|
|
1065
|
-
let lower = name.lowercased()
|
|
1066
|
-
return lower.hasPrefix(NimoBLE.NAME_PREFIX) && !lower.hasSuffix(NimoBLE.BLE_NAME_SUFFIX)
|
|
1067
|
-
}
|
|
1068
|
-
|
|
1069
1085
|
@discardableResult
|
|
1070
1086
|
private func startScan() -> Bool {
|
|
1087
|
+
guard scanRequested, !isDisconnecting else { return false }
|
|
1071
1088
|
Bridge.log("NIMO: startScan()")
|
|
1072
1089
|
if centralManager == nil {
|
|
1073
1090
|
centralManager = CBCentralManager(
|
|
@@ -1083,17 +1100,32 @@ class Nimo: NSObject, SGCManager {
|
|
|
1083
1100
|
|
|
1084
1101
|
// Fast path: reconnect to the cached peripheral UUID.
|
|
1085
1102
|
if connectByUUID() {
|
|
1103
|
+
stopScan()
|
|
1086
1104
|
return true
|
|
1087
1105
|
}
|
|
1088
1106
|
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
)
|
|
1107
|
+
// NIMO's Companion service is GATT over BR/EDR. The main device must
|
|
1108
|
+
// first be paired in Settings; it does not appear in a BLE-only scan.
|
|
1109
|
+
discovery.start()
|
|
1093
1110
|
return true
|
|
1094
1111
|
}
|
|
1095
1112
|
|
|
1113
|
+
private func onDeviceFound(_ peripheral: CBPeripheral, name: String, rssi: Int?) {
|
|
1114
|
+
Bridge.log("NIMO: found Companion device: \(name)")
|
|
1115
|
+
Bridge.sendDiscoveredDevice(DeviceTypes.NIMO, name, rssi: rssi)
|
|
1116
|
+
guard DEVICE_SEARCH_ID != "NOT_SET", name == DEVICE_SEARCH_ID,
|
|
1117
|
+
self.peripheral == nil else { return }
|
|
1118
|
+
Bridge.log("NIMO: Connecting to \(name)")
|
|
1119
|
+
stopScan()
|
|
1120
|
+
lastDeviceName = name
|
|
1121
|
+
lastDeviceUUID = peripheral.identifier.uuidString
|
|
1122
|
+
self.peripheral = peripheral
|
|
1123
|
+
peripheral.delegate = self
|
|
1124
|
+
centralManager?.connect(peripheral, options: nil)
|
|
1125
|
+
}
|
|
1126
|
+
|
|
1096
1127
|
private func connectByUUID() -> Bool {
|
|
1128
|
+
guard peripheral == nil else { return false }
|
|
1097
1129
|
guard DEVICE_SEARCH_ID != "NOT_SET", !DEVICE_SEARCH_ID.isEmpty else { return false }
|
|
1098
1130
|
guard lastDeviceName == DEVICE_SEARCH_ID,
|
|
1099
1131
|
let uuidString = lastDeviceUUID,
|
|
@@ -1299,7 +1331,8 @@ class Nimo: NSObject, SGCManager {
|
|
|
1299
1331
|
}
|
|
1300
1332
|
Bridge.log("NIMO: Attempting reconnection...")
|
|
1301
1333
|
await MainActor.run {
|
|
1302
|
-
self.isDisconnecting
|
|
1334
|
+
guard !self.isDisconnecting else { return }
|
|
1335
|
+
self.scanRequested = true
|
|
1303
1336
|
self.startScan()
|
|
1304
1337
|
}
|
|
1305
1338
|
return false
|
|
@@ -1564,37 +1597,29 @@ extension Nimo: CBCentralManagerDelegate {
|
|
|
1564
1597
|
}
|
|
1565
1598
|
|
|
1566
1599
|
nonisolated func centralManager(
|
|
1567
|
-
_
|
|
1600
|
+
_: CBCentralManager,
|
|
1568
1601
|
didDiscover peripheral: CBPeripheral,
|
|
1569
1602
|
advertisementData: [String: Any],
|
|
1570
1603
|
rssi: NSNumber
|
|
1571
1604
|
) {
|
|
1572
|
-
|
|
1573
|
-
let name = peripheral.name
|
|
1574
|
-
?? advertisementData[CBAdvertisementDataLocalNameKey] as? String
|
|
1575
|
-
else { return }
|
|
1605
|
+
let advertisedName = advertisementData[CBAdvertisementDataLocalNameKey] as? String
|
|
1576
1606
|
let rssiValue = rssi.intValue
|
|
1577
|
-
|
|
1578
1607
|
DispatchQueue.main.async { [weak self] in
|
|
1579
|
-
|
|
1580
|
-
guard self.isNimoMainDevice(name) else { return }
|
|
1581
|
-
|
|
1582
|
-
Bridge.sendDiscoveredDevice(DeviceTypes.NIMO, name, rssi: rssiValue)
|
|
1583
|
-
|
|
1584
|
-
guard self.DEVICE_SEARCH_ID != "NOT_SET" else { return }
|
|
1585
|
-
guard name == self.DEVICE_SEARCH_ID else { return }
|
|
1586
|
-
guard self.peripheral == nil else { return }
|
|
1587
|
-
|
|
1588
|
-
Bridge.log("NIMO: Connecting to \(name)")
|
|
1589
|
-
self.stopScan()
|
|
1590
|
-
self.lastDeviceName = name
|
|
1591
|
-
self.lastDeviceUUID = peripheral.identifier.uuidString
|
|
1592
|
-
self.peripheral = peripheral
|
|
1593
|
-
peripheral.delegate = self
|
|
1594
|
-
central.connect(peripheral, options: nil)
|
|
1608
|
+
self?.discovery.receive(peripheral, advertisedName: advertisedName, rssi: rssiValue)
|
|
1595
1609
|
}
|
|
1596
1610
|
}
|
|
1597
1611
|
|
|
1612
|
+
#if os(iOS)
|
|
1613
|
+
nonisolated func centralManager(
|
|
1614
|
+
_: CBCentralManager, connectionEventDidOccur event: CBConnectionEvent, for peripheral: CBPeripheral
|
|
1615
|
+
) {
|
|
1616
|
+
guard event == .peerConnected else { return }
|
|
1617
|
+
DispatchQueue.main.async { [weak self] in
|
|
1618
|
+
self?.discovery.receive(peripheral)
|
|
1619
|
+
}
|
|
1620
|
+
}
|
|
1621
|
+
#endif
|
|
1622
|
+
|
|
1598
1623
|
nonisolated func centralManager(_: CBCentralManager, didConnect peripheral: CBPeripheral) {
|
|
1599
1624
|
DispatchQueue.main.async { [weak self] in
|
|
1600
1625
|
guard let self else { return }
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import Foundation
|
|
2
|
+
|
|
3
|
+
/// Combines system-connected classic devices with incoming CoreBluetooth discoveries.
|
|
4
|
+
/// Kept independent of the radio so discovery can be exercised without hardware.
|
|
5
|
+
@MainActor
|
|
6
|
+
final class NimoDiscovery<Peripheral> {
|
|
7
|
+
private(set) var active = false
|
|
8
|
+
private let register: () -> Void
|
|
9
|
+
private let connected: () -> [Peripheral]
|
|
10
|
+
private let scan: () -> Void
|
|
11
|
+
private let stop: () -> Void
|
|
12
|
+
private let name: (Peripheral) -> String?
|
|
13
|
+
private let found: (Peripheral, String, Int?) -> Void
|
|
14
|
+
|
|
15
|
+
init(register: @escaping () -> Void, connected: @escaping () -> [Peripheral],
|
|
16
|
+
scan: @escaping () -> Void, stop: @escaping () -> Void,
|
|
17
|
+
name: @escaping (Peripheral) -> String?, found: @escaping (Peripheral, String, Int?) -> Void)
|
|
18
|
+
{
|
|
19
|
+
self.register = register
|
|
20
|
+
self.connected = connected
|
|
21
|
+
self.scan = scan
|
|
22
|
+
self.stop = stop
|
|
23
|
+
self.name = name
|
|
24
|
+
self.found = found
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
func start() {
|
|
28
|
+
active = true
|
|
29
|
+
// Register first so pairing in Settings cannot race the connected snapshot.
|
|
30
|
+
register()
|
|
31
|
+
for peripheral in connected() {
|
|
32
|
+
receive(peripheral)
|
|
33
|
+
}
|
|
34
|
+
if active { scan() }
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
func cancel() {
|
|
38
|
+
active = false
|
|
39
|
+
stop()
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
func receive(_ peripheral: Peripheral, advertisedName: String? = nil, rssi: Int? = nil) {
|
|
43
|
+
guard active, let name = advertisedName ?? name(peripheral) else { return }
|
|
44
|
+
let lower = name.lowercased()
|
|
45
|
+
// The separate ANCS peripheral does not host the Companion data service.
|
|
46
|
+
guard lower.hasPrefix("nimo"), !lower.hasSuffix("_ble") else { return }
|
|
47
|
+
found(peripheral, name, rssi)
|
|
48
|
+
}
|
|
49
|
+
}
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import Foundation
|
|
2
|
-
import UIKit
|
|
3
2
|
|
|
4
3
|
/**
|
|
5
4
|
* SherpaOnnxTranscriber handles real-time audio transcription using Sherpa-ONNX.
|
|
@@ -11,24 +10,30 @@ class SherpaOnnxTranscriber {
|
|
|
11
10
|
private static let TAG = "SherpaOnnxTranscriber"
|
|
12
11
|
|
|
13
12
|
private static let SAMPLE_RATE = 16000 // Sherpa-ONNX model's required sample rate
|
|
14
|
-
private
|
|
13
|
+
private final class Session {
|
|
14
|
+
let recognizer: SherpaOnnxRecognizer
|
|
15
|
+
var lastPartialResult = ""
|
|
15
16
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
private var processingTask: DispatchWorkItem?
|
|
21
|
-
|
|
22
|
-
/// The underlying Sherpa-ONNX objects
|
|
23
|
-
private var recognizer: SherpaOnnxRecognizer?
|
|
17
|
+
init(_ recognizer: SherpaOnnxRecognizer) {
|
|
18
|
+
self.recognizer = recognizer
|
|
19
|
+
}
|
|
20
|
+
}
|
|
24
21
|
|
|
25
|
-
private var
|
|
22
|
+
private lazy var worker = DemandDrivenAudioWorker<Session>(
|
|
23
|
+
makeProcessor: { Self.makeRecognizer().map(Session.init) },
|
|
24
|
+
process: { [weak self] session, data, generation in
|
|
25
|
+
self?.process(session, data: data, generation: generation)
|
|
26
|
+
}
|
|
27
|
+
)
|
|
26
28
|
|
|
27
|
-
///
|
|
28
|
-
|
|
29
|
+
/// Construct the worker on the manager's actor before audio can arrive.
|
|
30
|
+
init() {
|
|
31
|
+
_ = worker
|
|
32
|
+
}
|
|
29
33
|
|
|
30
|
-
|
|
31
|
-
|
|
34
|
+
func setActive(_ active: Bool) {
|
|
35
|
+
worker.setActive(active)
|
|
36
|
+
}
|
|
32
37
|
|
|
33
38
|
/// Dynamic model path support
|
|
34
39
|
private static var customModelPath: String? {
|
|
@@ -65,23 +70,9 @@ class SherpaOnnxTranscriber {
|
|
|
65
70
|
return nil
|
|
66
71
|
}
|
|
67
72
|
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
init(context: UIViewController) {
|
|
72
|
-
self.context = context
|
|
73
|
-
transcriptionSessionStart = Date()
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
deinit {
|
|
77
|
-
shutdown()
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
/**
|
|
81
|
-
* Initialize the Sherpa-ONNX recognizer.
|
|
82
|
-
* Loads models and configuration, sets up processing thread.
|
|
83
|
-
*/
|
|
84
|
-
func initialize() {
|
|
73
|
+
/// Runs exclusively on the worker queue, never on the main/audio delivery thread.
|
|
74
|
+
private static func makeRecognizer() -> SherpaOnnxRecognizer? {
|
|
75
|
+
var recognizer: SherpaOnnxRecognizer?
|
|
85
76
|
do {
|
|
86
77
|
var tokensPath: String
|
|
87
78
|
var modelType = "unknown"
|
|
@@ -201,30 +192,26 @@ class SherpaOnnxTranscriber {
|
|
|
201
192
|
Bridge.log("No Sherpa ONNX model available. Transcription will be disabled.")
|
|
202
193
|
Bridge.log("Please download a model using the model downloader in settings.")
|
|
203
194
|
recognizer = nil
|
|
204
|
-
|
|
205
|
-
return
|
|
195
|
+
return nil
|
|
206
196
|
}
|
|
207
197
|
|
|
208
198
|
if recognizer == nil {
|
|
209
199
|
throw NSError(domain: "SherpaOnnxTranscriber", code: 2, userInfo: [NSLocalizedDescriptionKey: "Failed to create recognizer"])
|
|
210
200
|
}
|
|
211
201
|
|
|
212
|
-
startProcessingTask()
|
|
213
|
-
isRunning = true
|
|
214
|
-
|
|
215
202
|
Bridge.log("Sherpa-ONNX ASR initialized successfully with \(modelType) model")
|
|
203
|
+
return recognizer
|
|
216
204
|
|
|
217
205
|
} catch {
|
|
218
206
|
Bridge.log("Failed to initialize Sherpa-ONNX: \(error.localizedDescription)")
|
|
207
|
+
return nil
|
|
219
208
|
}
|
|
220
209
|
}
|
|
221
210
|
|
|
222
|
-
|
|
223
|
-
* Handle transcription results - send only to delegate
|
|
224
|
-
*/
|
|
225
|
-
private func handleTranscriptionResult(text: String, isFinal: Bool) {
|
|
226
|
-
// Forward to delegate if set
|
|
211
|
+
private func handleTranscriptionResult(text: String, isFinal: Bool, generation: UInt64) {
|
|
227
212
|
DispatchQueue.main.async { [weak self] in
|
|
213
|
+
// A decode or main-queue delivery may finish after stop/model replacement.
|
|
214
|
+
guard let self, self.worker.isCurrent(generation) else { return }
|
|
228
215
|
if isFinal {
|
|
229
216
|
STTTools.didReceiveFinalTranscription(text)
|
|
230
217
|
} else {
|
|
@@ -233,123 +220,30 @@ class SherpaOnnxTranscriber {
|
|
|
233
220
|
}
|
|
234
221
|
}
|
|
235
222
|
|
|
236
|
-
/**
|
|
237
|
-
* Feed PCM audio data (16-bit little endian) into the transcriber.
|
|
238
|
-
* This method should be called continuously with short chunks (e.g., 100-300ms).
|
|
239
|
-
*
|
|
240
|
-
* Audio is queued directly; microphone VAD gating is not applied in the SDK.
|
|
241
|
-
*/
|
|
242
223
|
func acceptAudio(pcm16le: Data) {
|
|
243
|
-
|
|
244
|
-
return
|
|
245
|
-
}
|
|
246
|
-
|
|
247
|
-
queueAudioData(pcm16le)
|
|
248
|
-
}
|
|
249
|
-
|
|
250
|
-
private func queueAudioData(_ pcm16le: Data) {
|
|
251
|
-
pcmQueue.async { [weak self] in
|
|
252
|
-
guard let self = self else { return }
|
|
253
|
-
|
|
254
|
-
let queueSizeBefore = self.pcmBuffers.count
|
|
255
|
-
self.pcmBuffers.append(pcm16le)
|
|
256
|
-
|
|
257
|
-
// Keep queue size manageable
|
|
258
|
-
if self.pcmBuffers.count > Self.QUEUE_CAPACITY {
|
|
259
|
-
let removedBuffer = self.pcmBuffers.removeFirst()
|
|
260
|
-
Bridge.log("⚠️ Audio queue overflow - dropped buffer of \(removedBuffer.count) bytes")
|
|
261
|
-
}
|
|
262
|
-
}
|
|
224
|
+
worker.accept(pcm16le)
|
|
263
225
|
}
|
|
264
226
|
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
let workItem = DispatchWorkItem { [weak self] in
|
|
274
|
-
self?.runLoop()
|
|
227
|
+
private func process(_ session: Session, data: Data, generation: UInt64) {
|
|
228
|
+
guard worker.isCurrent(generation) else { return }
|
|
229
|
+
let recognizer = session.recognizer
|
|
230
|
+
recognizer.acceptWaveform(samples: toFloatArray(from: data), sampleRate: Self.SAMPLE_RATE)
|
|
231
|
+
while recognizer.isReady() {
|
|
232
|
+
guard worker.isCurrent(generation) else { return }
|
|
233
|
+
recognizer.decode()
|
|
275
234
|
}
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
/**
|
|
282
|
-
* Main processing loop that handles transcription in real-time.
|
|
283
|
-
* Pulls audio from queue, feeds into Sherpa, emits partial/final results.
|
|
284
|
-
*/
|
|
285
|
-
private func runLoop() {
|
|
286
|
-
Bridge.log("🔄 Sherpa-ONNX processing loop started")
|
|
287
|
-
|
|
288
|
-
while isRunning {
|
|
289
|
-
// Pull data from queue
|
|
290
|
-
var audioData: Data?
|
|
291
|
-
|
|
292
|
-
pcmQueue.sync {
|
|
293
|
-
if !self.pcmBuffers.isEmpty {
|
|
294
|
-
audioData = self.pcmBuffers.removeFirst()
|
|
295
|
-
}
|
|
296
|
-
}
|
|
297
|
-
|
|
298
|
-
if let data = audioData {
|
|
299
|
-
// Synchronize access to recognizer to prevent race conditions
|
|
300
|
-
objc_sync_enter(self)
|
|
301
|
-
defer { objc_sync_exit(self) }
|
|
302
|
-
|
|
303
|
-
guard let recognizer = recognizer else {
|
|
304
|
-
Bridge.log("⚠️ Recognizer not available, skipping audio chunk")
|
|
305
|
-
continue
|
|
306
|
-
}
|
|
307
|
-
|
|
308
|
-
do {
|
|
309
|
-
// Convert PCM to float [-1.0, 1.0]
|
|
310
|
-
let floatBuf = toFloatArray(from: data)
|
|
311
|
-
|
|
312
|
-
// Pass audio data to the Sherpa-ONNX stream
|
|
313
|
-
recognizer.acceptWaveform(samples: floatBuf, sampleRate: Self.SAMPLE_RATE)
|
|
314
|
-
|
|
315
|
-
// Decode continuously while model is ready
|
|
316
|
-
var decodeCount = 0
|
|
317
|
-
while recognizer.isReady() {
|
|
318
|
-
recognizer.decode()
|
|
319
|
-
decodeCount += 1
|
|
320
|
-
}
|
|
321
|
-
|
|
322
|
-
// If utterance endpoint detected
|
|
323
|
-
if recognizer.isEndpoint() {
|
|
324
|
-
let result = recognizer.getResult()
|
|
325
|
-
let finalText = result.text.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
326
|
-
|
|
327
|
-
if !finalText.isEmpty {
|
|
328
|
-
handleTranscriptionResult(text: finalText, isFinal: true)
|
|
329
|
-
}
|
|
330
|
-
|
|
331
|
-
recognizer.reset() // Start new utterance
|
|
332
|
-
lastPartialResult = ""
|
|
333
|
-
} else {
|
|
334
|
-
// Emit partial results if changed
|
|
335
|
-
let result = recognizer.getResult()
|
|
336
|
-
let partial = result.text.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
337
|
-
|
|
338
|
-
if partial != lastPartialResult, !partial.isEmpty {
|
|
339
|
-
handleTranscriptionResult(text: partial, isFinal: false)
|
|
340
|
-
lastPartialResult = partial
|
|
341
|
-
}
|
|
342
|
-
}
|
|
343
|
-
} catch {
|
|
344
|
-
Bridge.log("❌ Error processing audio: \(error.localizedDescription)")
|
|
345
|
-
}
|
|
346
|
-
} else {
|
|
347
|
-
// Sleep briefly to avoid tight CPU loop if no audio is available
|
|
348
|
-
Thread.sleep(forTimeInterval: 0.01)
|
|
235
|
+
guard worker.isCurrent(generation) else { return }
|
|
236
|
+
let text = recognizer.getResult().text.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
237
|
+
if recognizer.isEndpoint() {
|
|
238
|
+
if !text.isEmpty {
|
|
239
|
+
handleTranscriptionResult(text: text, isFinal: true, generation: generation)
|
|
349
240
|
}
|
|
241
|
+
recognizer.reset()
|
|
242
|
+
session.lastPartialResult = ""
|
|
243
|
+
} else if !text.isEmpty, text != session.lastPartialResult {
|
|
244
|
+
handleTranscriptionResult(text: text, isFinal: false, generation: generation)
|
|
245
|
+
session.lastPartialResult = text
|
|
350
246
|
}
|
|
351
|
-
|
|
352
|
-
Bridge.log("ASR processing thread stopped")
|
|
353
247
|
}
|
|
354
248
|
|
|
355
249
|
/**
|
|
@@ -377,45 +271,11 @@ class SherpaOnnxTranscriber {
|
|
|
377
271
|
return samples
|
|
378
272
|
}
|
|
379
273
|
|
|
380
|
-
/**
|
|
381
|
-
* Stop transcription processing.
|
|
382
|
-
* This shuts down the processing thread and releases Sherpa-ONNX resources.
|
|
383
|
-
*/
|
|
384
274
|
func shutdown() {
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
isRunning = false
|
|
388
|
-
processingTask?.cancel()
|
|
389
|
-
|
|
390
|
-
// Synchronize access to recognizer during shutdown
|
|
391
|
-
objc_sync_enter(self)
|
|
392
|
-
defer { objc_sync_exit(self) }
|
|
393
|
-
|
|
394
|
-
// The recognizer will be automatically cleaned up by ARC when set to nil
|
|
395
|
-
if recognizer != nil {
|
|
396
|
-
Bridge.log("🧹 Cleaning up Sherpa-ONNX recognizer")
|
|
397
|
-
recognizer = nil
|
|
398
|
-
}
|
|
399
|
-
|
|
400
|
-
// Clear any remaining audio buffers
|
|
401
|
-
pcmQueue.sync {
|
|
402
|
-
let remainingBuffers = self.pcmBuffers.count
|
|
403
|
-
if remainingBuffers > 0 {
|
|
404
|
-
Bridge.log("🗑️ Clearing \(remainingBuffers) remaining audio buffers")
|
|
405
|
-
}
|
|
406
|
-
self.pcmBuffers.removeAll()
|
|
407
|
-
}
|
|
408
|
-
|
|
409
|
-
Bridge.log("✅ SherpaOnnxTranscriber shutdown complete")
|
|
275
|
+
worker.setActive(false)
|
|
410
276
|
}
|
|
411
277
|
|
|
412
|
-
/**
|
|
413
|
-
* Restarts the transcriber after a model change.
|
|
414
|
-
* Shuts down existing resources, clears buffers, and reinitializes the recognizer.
|
|
415
|
-
*/
|
|
416
278
|
func restart() {
|
|
417
|
-
|
|
418
|
-
shutdown()
|
|
419
|
-
initialize()
|
|
279
|
+
worker.restart()
|
|
420
280
|
}
|
|
421
281
|
}
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
import Foundation
|
|
2
|
+
|
|
3
|
+
/// Owns a processor on one serial queue. Empty queues schedule no recurring work.
|
|
4
|
+
/// Demand changes invalidate queued audio/results immediately; creation, processing,
|
|
5
|
+
/// and destruction finish in order without blocking the caller (including the UI).
|
|
6
|
+
final class DemandDrivenAudioWorker<Processor> {
|
|
7
|
+
struct Snapshot {
|
|
8
|
+
let active: Bool
|
|
9
|
+
let initialized: Bool
|
|
10
|
+
let processing: Bool
|
|
11
|
+
let queuedBuffers: Int
|
|
12
|
+
let processedBuffers: UInt64
|
|
13
|
+
let droppedBuffers: UInt64
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
private let lock = NSLock()
|
|
17
|
+
private let queue: DispatchQueue
|
|
18
|
+
private let capacity: Int
|
|
19
|
+
private let makeProcessor: () -> Processor?
|
|
20
|
+
private let process: (Processor, Data, UInt64) -> Void
|
|
21
|
+
private var active = false
|
|
22
|
+
private var generation: UInt64 = 0
|
|
23
|
+
private var buffers = [Data]()
|
|
24
|
+
private var scheduled = false
|
|
25
|
+
private var initialized = false
|
|
26
|
+
private var processing = false
|
|
27
|
+
private var processedBuffers: UInt64 = 0
|
|
28
|
+
private var droppedBuffers: UInt64 = 0
|
|
29
|
+
|
|
30
|
+
// Accessed only on queue, including release of the previous processor.
|
|
31
|
+
private var processor: Processor?
|
|
32
|
+
private var processorGeneration: UInt64?
|
|
33
|
+
private var attemptedInitialization = false
|
|
34
|
+
|
|
35
|
+
init(capacity: Int = 100,
|
|
36
|
+
queue: DispatchQueue = DispatchQueue(label: "com.mentra.speech.processor", qos: .userInitiated),
|
|
37
|
+
makeProcessor: @escaping () -> Processor?,
|
|
38
|
+
process: @escaping (Processor, Data, UInt64) -> Void)
|
|
39
|
+
{
|
|
40
|
+
precondition(capacity > 0)
|
|
41
|
+
self.capacity = capacity
|
|
42
|
+
self.queue = queue
|
|
43
|
+
self.makeProcessor = makeProcessor
|
|
44
|
+
self.process = process
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
func setActive(_ value: Bool) {
|
|
48
|
+
lock.lock()
|
|
49
|
+
defer { lock.unlock() }
|
|
50
|
+
guard active != value else { return }
|
|
51
|
+
active = value
|
|
52
|
+
invalidateLocked()
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/// Reload a changed model only if there is still local-transcription demand.
|
|
56
|
+
func restart() {
|
|
57
|
+
lock.lock()
|
|
58
|
+
defer { lock.unlock() }
|
|
59
|
+
invalidateLocked()
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
private func invalidateLocked() {
|
|
63
|
+
generation &+= 1
|
|
64
|
+
buffers.removeAll(keepingCapacity: true)
|
|
65
|
+
initialized = false
|
|
66
|
+
scheduleLocked()
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
func accept(_ data: Data) {
|
|
70
|
+
guard !data.isEmpty else { return }
|
|
71
|
+
lock.lock()
|
|
72
|
+
defer { lock.unlock() }
|
|
73
|
+
guard active else { return }
|
|
74
|
+
// Bound even audio arriving while model loading or decoding is blocked.
|
|
75
|
+
if buffers.count == capacity {
|
|
76
|
+
buffers.removeFirst()
|
|
77
|
+
droppedBuffers &+= 1
|
|
78
|
+
}
|
|
79
|
+
buffers.append(data)
|
|
80
|
+
scheduleLocked()
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
func isCurrent(_ token: UInt64) -> Bool {
|
|
84
|
+
lock.lock()
|
|
85
|
+
defer { lock.unlock() }
|
|
86
|
+
return active && generation == token
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
func snapshot() -> Snapshot {
|
|
90
|
+
lock.lock()
|
|
91
|
+
defer { lock.unlock() }
|
|
92
|
+
return Snapshot(active: active, initialized: initialized, processing: processing,
|
|
93
|
+
queuedBuffers: buffers.count, processedBuffers: processedBuffers,
|
|
94
|
+
droppedBuffers: droppedBuffers)
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
private func scheduleLocked() {
|
|
98
|
+
guard !scheduled else { return }
|
|
99
|
+
scheduled = true
|
|
100
|
+
queue.async { self.drain() }
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
private func drain() {
|
|
104
|
+
while true {
|
|
105
|
+
lock.lock()
|
|
106
|
+
let token = generation
|
|
107
|
+
if processorGeneration != token {
|
|
108
|
+
lock.unlock()
|
|
109
|
+
// ARC destruction stays serialized with inference and model loading.
|
|
110
|
+
processor = nil
|
|
111
|
+
processorGeneration = token
|
|
112
|
+
attemptedInitialization = false
|
|
113
|
+
continue
|
|
114
|
+
}
|
|
115
|
+
guard active else {
|
|
116
|
+
scheduled = false
|
|
117
|
+
lock.unlock()
|
|
118
|
+
return
|
|
119
|
+
}
|
|
120
|
+
if !attemptedInitialization {
|
|
121
|
+
attemptedInitialization = true
|
|
122
|
+
lock.unlock()
|
|
123
|
+
processor = makeProcessor()
|
|
124
|
+
lock.lock()
|
|
125
|
+
if generation == token { initialized = processor != nil }
|
|
126
|
+
lock.unlock()
|
|
127
|
+
continue
|
|
128
|
+
}
|
|
129
|
+
guard let processor else {
|
|
130
|
+
buffers.removeAll(keepingCapacity: true)
|
|
131
|
+
scheduled = false
|
|
132
|
+
lock.unlock()
|
|
133
|
+
return // Missing model: retry on a demand transition or model restart.
|
|
134
|
+
}
|
|
135
|
+
guard !buffers.isEmpty else {
|
|
136
|
+
scheduled = false
|
|
137
|
+
lock.unlock()
|
|
138
|
+
return
|
|
139
|
+
}
|
|
140
|
+
let data = buffers.removeFirst()
|
|
141
|
+
processing = true
|
|
142
|
+
lock.unlock()
|
|
143
|
+
process(processor, data, token)
|
|
144
|
+
lock.lock()
|
|
145
|
+
processedBuffers &+= 1
|
|
146
|
+
processing = false
|
|
147
|
+
lock.unlock()
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
import Foundation
|
|
2
|
+
@testable import MentraBluetoothSDK
|
|
3
|
+
import XCTest
|
|
4
|
+
|
|
5
|
+
final class DemandDrivenAudioWorkerTests: XCTestCase {
|
|
6
|
+
private final class Processor {
|
|
7
|
+
let onRelease: () -> Void
|
|
8
|
+
init(onRelease: @escaping () -> Void = {}) {
|
|
9
|
+
self.onRelease = onRelease
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
deinit { onRelease() }
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
func testNoDemandAndEmptyActiveQueueScheduleNoRecurringWork() {
|
|
16
|
+
let queue = DispatchQueue(label: "test.speech")
|
|
17
|
+
var creations = 0
|
|
18
|
+
var processed = [Data]()
|
|
19
|
+
let worker = DemandDrivenAudioWorker(queue: queue, makeProcessor: {
|
|
20
|
+
creations += 1
|
|
21
|
+
return Processor()
|
|
22
|
+
}, process: { _, data, _ in processed.append(data) })
|
|
23
|
+
worker.accept(Data([1]))
|
|
24
|
+
queue.sync {}
|
|
25
|
+
XCTAssertEqual(creations, 0)
|
|
26
|
+
worker.setActive(true)
|
|
27
|
+
worker.setActive(true)
|
|
28
|
+
queue.sync {}
|
|
29
|
+
XCTAssertEqual(creations, 1)
|
|
30
|
+
XCTAssertTrue(worker.snapshot().initialized)
|
|
31
|
+
XCTAssertFalse(worker.snapshot().processing)
|
|
32
|
+
// A serial barrier completes with an empty queue; no sleeping consumer holds it.
|
|
33
|
+
queue.sync {}
|
|
34
|
+
XCTAssertTrue(processed.isEmpty)
|
|
35
|
+
worker.setActive(false)
|
|
36
|
+
queue.sync {}
|
|
37
|
+
XCTAssertFalse(worker.snapshot().initialized)
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
func testFirstAudioIsRetainedDuringAsynchronousInitializationAndBounded() {
|
|
41
|
+
let queue = DispatchQueue(label: "test.speech")
|
|
42
|
+
let loading = DispatchSemaphore(value: 0)
|
|
43
|
+
let release = DispatchSemaphore(value: 0)
|
|
44
|
+
var processed = [Data]()
|
|
45
|
+
let worker = DemandDrivenAudioWorker(capacity: 3, queue: queue, makeProcessor: {
|
|
46
|
+
XCTAssertFalse(Thread.isMainThread)
|
|
47
|
+
loading.signal()
|
|
48
|
+
release.wait()
|
|
49
|
+
return Processor()
|
|
50
|
+
}, process: { _, data, _ in processed.append(data) })
|
|
51
|
+
worker.setActive(true)
|
|
52
|
+
XCTAssertEqual(loading.wait(timeout: .now() + 2), .success)
|
|
53
|
+
worker.accept(Data([1]))
|
|
54
|
+
worker.accept(Data([2]))
|
|
55
|
+
XCTAssertEqual(worker.snapshot().queuedBuffers, 2)
|
|
56
|
+
release.signal()
|
|
57
|
+
queue.sync {}
|
|
58
|
+
XCTAssertEqual(processed, [Data([1]), Data([2])])
|
|
59
|
+
worker.setActive(false)
|
|
60
|
+
queue.sync {}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
func testStopDuringLoadingDropsOldAudioAndCannotResurrectProcessor() {
|
|
64
|
+
let queue = DispatchQueue(label: "test.speech")
|
|
65
|
+
let loading = DispatchSemaphore(value: 0)
|
|
66
|
+
let release = DispatchSemaphore(value: 0)
|
|
67
|
+
var released = 0
|
|
68
|
+
var processed = 0
|
|
69
|
+
let worker = DemandDrivenAudioWorker(capacity: 2, queue: queue, makeProcessor: {
|
|
70
|
+
loading.signal()
|
|
71
|
+
release.wait()
|
|
72
|
+
return Processor { released += 1 }
|
|
73
|
+
}, process: { _, _, _ in processed += 1 })
|
|
74
|
+
worker.setActive(true)
|
|
75
|
+
XCTAssertEqual(loading.wait(timeout: .now() + 2), .success)
|
|
76
|
+
for byte: UInt8 in 1 ... 10 {
|
|
77
|
+
worker.accept(Data([byte]))
|
|
78
|
+
}
|
|
79
|
+
XCTAssertEqual(worker.snapshot().queuedBuffers, 2)
|
|
80
|
+
XCTAssertEqual(worker.snapshot().droppedBuffers, 8)
|
|
81
|
+
worker.setActive(false)
|
|
82
|
+
XCTAssertEqual(worker.snapshot().queuedBuffers, 0)
|
|
83
|
+
release.signal()
|
|
84
|
+
queue.sync {}
|
|
85
|
+
XCTAssertEqual(processed, 0)
|
|
86
|
+
XCTAssertEqual(released, 1)
|
|
87
|
+
XCTAssertFalse(worker.snapshot().initialized)
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
func testRestartAndRapidDemandChangesSerializeOldAndNewProcessors() {
|
|
91
|
+
let queue = DispatchQueue(label: "test.speech")
|
|
92
|
+
let processing = DispatchSemaphore(value: 0)
|
|
93
|
+
let release = DispatchSemaphore(value: 0)
|
|
94
|
+
var creations = 0
|
|
95
|
+
var destructions = 0
|
|
96
|
+
var tokens = [UInt64]()
|
|
97
|
+
var dataSeen = [Data]()
|
|
98
|
+
let worker = DemandDrivenAudioWorker(queue: queue, makeProcessor: {
|
|
99
|
+
XCTAssertEqual(creations, destructions)
|
|
100
|
+
creations += 1
|
|
101
|
+
return Processor { destructions += 1 }
|
|
102
|
+
}, process: { _, data, token in
|
|
103
|
+
dataSeen.append(data)
|
|
104
|
+
tokens.append(token)
|
|
105
|
+
if data == Data([1]) {
|
|
106
|
+
processing.signal()
|
|
107
|
+
release.wait()
|
|
108
|
+
}
|
|
109
|
+
})
|
|
110
|
+
worker.setActive(true)
|
|
111
|
+
worker.accept(Data([1]))
|
|
112
|
+
XCTAssertEqual(processing.wait(timeout: .now() + 2), .success)
|
|
113
|
+
worker.accept(Data([2]))
|
|
114
|
+
worker.restart()
|
|
115
|
+
worker.setActive(false)
|
|
116
|
+
worker.setActive(true)
|
|
117
|
+
worker.accept(Data([3]))
|
|
118
|
+
release.signal()
|
|
119
|
+
queue.sync {}
|
|
120
|
+
XCTAssertEqual(dataSeen, [Data([1]), Data([3])])
|
|
121
|
+
XCTAssertFalse(worker.isCurrent(tokens[0]))
|
|
122
|
+
XCTAssertTrue(worker.isCurrent(tokens[1]))
|
|
123
|
+
XCTAssertEqual(creations, 2)
|
|
124
|
+
worker.setActive(false)
|
|
125
|
+
queue.sync {}
|
|
126
|
+
XCTAssertEqual(destructions, 2)
|
|
127
|
+
XCTAssertFalse(worker.isCurrent(tokens[1]))
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
func testMissingModelDoesNotRetryEveryFrameAndRestartWithoutDemandDoesNotLoad() {
|
|
131
|
+
let queue = DispatchQueue(label: "test.speech")
|
|
132
|
+
var attempts = 0
|
|
133
|
+
let worker = DemandDrivenAudioWorker<Processor>(queue: queue, makeProcessor: {
|
|
134
|
+
attempts += 1
|
|
135
|
+
return nil
|
|
136
|
+
}, process: { _, _, _ in XCTFail("Missing processor") })
|
|
137
|
+
worker.restart()
|
|
138
|
+
queue.sync {}
|
|
139
|
+
XCTAssertEqual(attempts, 0)
|
|
140
|
+
worker.setActive(true)
|
|
141
|
+
for _ in 0 ..< 5 {
|
|
142
|
+
worker.accept(Data([1]))
|
|
143
|
+
queue.sync {}
|
|
144
|
+
}
|
|
145
|
+
XCTAssertEqual(attempts, 1)
|
|
146
|
+
XCTAssertEqual(worker.snapshot().queuedBuffers, 0)
|
|
147
|
+
worker.restart()
|
|
148
|
+
queue.sync {}
|
|
149
|
+
XCTAssertEqual(attempts, 2)
|
|
150
|
+
worker.setActive(false)
|
|
151
|
+
worker.setActive(true)
|
|
152
|
+
queue.sync {}
|
|
153
|
+
XCTAssertEqual(attempts, 3)
|
|
154
|
+
worker.setActive(false)
|
|
155
|
+
queue.sync {}
|
|
156
|
+
}
|
|
157
|
+
}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
@testable import MentraBluetoothSDK
|
|
2
|
+
import XCTest
|
|
3
|
+
|
|
4
|
+
@MainActor
|
|
5
|
+
final class NimoDiscoveryTests: XCTestCase {
|
|
6
|
+
@MainActor
|
|
7
|
+
private final class Radio {
|
|
8
|
+
var connected: [String] = []
|
|
9
|
+
var events: [String] = []
|
|
10
|
+
var found: [String] = []
|
|
11
|
+
var selected: String?
|
|
12
|
+
lazy var discovery: NimoDiscovery<String> = NimoDiscovery(
|
|
13
|
+
register: { [unowned self] in self.events.append("register") },
|
|
14
|
+
connected: { [unowned self] in self.events.append("retrieve"); return self.connected },
|
|
15
|
+
scan: { [unowned self] in self.events.append("scan") },
|
|
16
|
+
stop: { [unowned self] in self.events.append("stop") },
|
|
17
|
+
name: { $0 },
|
|
18
|
+
found: { [unowned self] _, name, _ in
|
|
19
|
+
found.append(name)
|
|
20
|
+
if selected == name { discovery.cancel() }
|
|
21
|
+
}
|
|
22
|
+
)
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
func testAlreadyConnectedClassicDeviceIsFoundWithoutAdvertisement() {
|
|
26
|
+
let radio = Radio()
|
|
27
|
+
radio.connected = ["NIMO_1234", "NIMO_1234_BLE", "Unrelated"]
|
|
28
|
+
radio.discovery.start()
|
|
29
|
+
XCTAssertEqual(radio.events, ["register", "retrieve", "scan"])
|
|
30
|
+
XCTAssertEqual(radio.found, ["NIMO_1234"])
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
func testPairingInSettingsAfterScanStartedDeliversDevice() {
|
|
34
|
+
let radio = Radio()
|
|
35
|
+
radio.discovery.start()
|
|
36
|
+
XCTAssertTrue(radio.found.isEmpty)
|
|
37
|
+
// Same entry point as connectionEventDidOccur(.peerConnected).
|
|
38
|
+
radio.discovery.receive("NIMO_5678")
|
|
39
|
+
XCTAssertEqual(radio.found, ["NIMO_5678"])
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
func testSelectionFromConnectedSnapshotStopsBeforeStartingBleScan() {
|
|
43
|
+
let radio = Radio()
|
|
44
|
+
radio.selected = "NIMO_5678"
|
|
45
|
+
radio.connected = ["NIMO_1234", "NIMO_5678", "NIMO_9999"]
|
|
46
|
+
radio.discovery.start()
|
|
47
|
+
XCTAssertEqual(radio.found, ["NIMO_1234", "NIMO_5678"])
|
|
48
|
+
XCTAssertEqual(radio.events, ["register", "retrieve", "stop"])
|
|
49
|
+
XCTAssertFalse(radio.discovery.active)
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
func testCancelledDiscoveryIgnoresQueuedEventsAndCanRestart() {
|
|
53
|
+
let radio = Radio()
|
|
54
|
+
radio.discovery.start()
|
|
55
|
+
radio.discovery.cancel()
|
|
56
|
+
radio.discovery.receive("NIMO_1234")
|
|
57
|
+
XCTAssertTrue(radio.found.isEmpty)
|
|
58
|
+
radio.connected = ["NIMO_1234"]
|
|
59
|
+
radio.discovery.start()
|
|
60
|
+
XCTAssertEqual(radio.found, ["NIMO_1234"])
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
func testBleFallbackUsesAdvertisementNameAndPreservesSignalStrength() {
|
|
64
|
+
var found: [String] = []
|
|
65
|
+
var signal: Int?
|
|
66
|
+
let discovery = NimoDiscovery<String>(
|
|
67
|
+
register: {}, connected: { [] }, scan: {}, stop: {}, name: { _ in nil },
|
|
68
|
+
found: { _, name, rssi in found.append(name); signal = rssi }
|
|
69
|
+
)
|
|
70
|
+
discovery.start()
|
|
71
|
+
discovery.receive("uuid", advertisedName: "NiMo_1234_bLe", rssi: -60)
|
|
72
|
+
discovery.receive("uuid", advertisedName: "Unrelated", rssi: -60)
|
|
73
|
+
discovery.receive("uuid")
|
|
74
|
+
XCTAssertTrue(found.isEmpty)
|
|
75
|
+
discovery.receive("uuid", advertisedName: "NiMo_1234", rssi: -55)
|
|
76
|
+
XCTAssertEqual(found, ["NiMo_1234"])
|
|
77
|
+
XCTAssertEqual(signal, -55)
|
|
78
|
+
}
|
|
79
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mentra/bluetooth-sdk",
|
|
3
|
-
"version": "3.2.1-beta.
|
|
3
|
+
"version": "3.2.1-beta.481",
|
|
4
4
|
"description": "SDK for communicating with smart glasses",
|
|
5
5
|
"main": "build/index.js",
|
|
6
6
|
"react-native": "src/index.ts",
|
|
@@ -114,7 +114,7 @@
|
|
|
114
114
|
"registry": "https://registry.npmjs.org/"
|
|
115
115
|
},
|
|
116
116
|
"dependencies": {
|
|
117
|
-
"@mentra/cloud-protocol": "3.2.1-beta.
|
|
117
|
+
"@mentra/cloud-protocol": "3.2.1-beta.481"
|
|
118
118
|
},
|
|
119
119
|
"devDependencies": {
|
|
120
120
|
"@types/node": "^25.9.3",
|
|
@@ -12,9 +12,9 @@ export interface BluetoothSdkReleaseMetadata {
|
|
|
12
12
|
export const BLUETOOTH_SDK_RELEASE_METADATA: Readonly<BluetoothSdkReleaseMetadata> = Object.freeze({
|
|
13
13
|
"schemaVersion": 1,
|
|
14
14
|
"familyBaseVersion": "3.2.1",
|
|
15
|
-
"releaseIdentity": "3.2.1-beta.
|
|
16
|
-
"releaseSetId": "mentra-3.2.1-beta.
|
|
17
|
-
"sourceCommit": "
|
|
18
|
-
"otaManifestUrl": "https://artifactscdn.mentraglass.com/Mentra-Community/MentraOS/releases/mentra-builds-v3.2.1/mentra-live-ota-3.2.1-beta.
|
|
19
|
-
"otaManifestSha256": "
|
|
15
|
+
"releaseIdentity": "3.2.1-beta.481",
|
|
16
|
+
"releaseSetId": "mentra-3.2.1-beta.481",
|
|
17
|
+
"sourceCommit": "c3269f1bc026d0b1967e8f18cae271c0ef396c11",
|
|
18
|
+
"otaManifestUrl": "https://artifactscdn.mentraglass.com/Mentra-Community/MentraOS/releases/mentra-builds-v3.2.1/mentra-live-ota-3.2.1-beta.481.json",
|
|
19
|
+
"otaManifestSha256": "be95e7f2ec85c32d385f66ed8d1c319320e1b5f61f59ea3d1d76199f269834ea"
|
|
20
20
|
})
|