@mentra/bluetooth-sdk 3.2.0-dev.221 → 3.2.0-dev.223
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +12 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +4 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +120 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/GeneratedReleaseMetadata.kt +5 -5
- package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +12 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/audio/AudioModels.kt +11 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +38 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/MicSourcePin.kt +31 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/audio/PcmStreamPlayer.kt +39 -5
- package/android/src/test/java/com/mentra/bluetoothsdk/MicPcmEventSourceTest.kt +75 -0
- package/android/src/test/java/com/mentra/bluetoothsdk/StreamRequestCaptureAudioTest.kt +44 -0
- package/android/src/test/java/com/mentra/bluetoothsdk/utils/MicSourcePinTest.kt +68 -0
- package/build/BluetoothSdk.types.d.ts +27 -0
- package/build/BluetoothSdk.types.d.ts.map +1 -1
- package/build/BluetoothSdk.types.js.map +1 -1
- package/build/_private/BluetoothSdkModule.d.ts +18 -2
- package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
- package/build/_private/BluetoothSdkModule.js.map +1 -1
- package/build/generated/releaseMetadata.js +5 -5
- package/build/generated/releaseMetadata.js.map +1 -1
- package/ios/BluetoothSdkModule.swift +15 -1
- package/ios/Source/BluetoothSdkDefaults.swift +1 -1
- package/ios/Source/Bridge.swift +4 -0
- package/ios/Source/GeneratedReleaseMetadata.swift +5 -5
- package/ios/Source/streaming/StreamModels.swift +48 -2
- package/package.json +1 -1
- package/src/BluetoothSdk.types.ts +28 -0
- package/src/_private/BluetoothSdkModule.ts +24 -2
- package/src/generated/releaseMetadata.ts +5 -5
|
@@ -835,6 +835,10 @@ class BluetoothSdkModule : Module() {
|
|
|
835
835
|
)
|
|
836
836
|
}
|
|
837
837
|
|
|
838
|
+
SdkAsyncFunction("setMicSourcePin") { source: String? ->
|
|
839
|
+
sdk?.setMicSourcePin(source)
|
|
840
|
+
}
|
|
841
|
+
|
|
838
842
|
// Runs on Dispatchers.IO, not the shared Expo AsyncFunctionQueue: restart()
|
|
839
843
|
// does a synchronous JNI model reload that would otherwise block every other
|
|
840
844
|
// native call in the app until it completes.
|
|
@@ -857,8 +861,14 @@ class BluetoothSdkModule : Module() {
|
|
|
857
861
|
// close blocks until the backlog drains — either would otherwise stall
|
|
858
862
|
// every other native call queued behind them.
|
|
859
863
|
|
|
860
|
-
AsyncFunction("pcmStreamOpen") {
|
|
861
|
-
|
|
864
|
+
AsyncFunction("pcmStreamOpen") {
|
|
865
|
+
streamId: String,
|
|
866
|
+
sampleRate: Int,
|
|
867
|
+
channels: Int,
|
|
868
|
+
volume: Double,
|
|
869
|
+
jitterMs: Int?,
|
|
870
|
+
->
|
|
871
|
+
PcmStreamManager.open(streamId, sampleRate, channels, volume.toFloat(), jitterMs)
|
|
862
872
|
}
|
|
863
873
|
|
|
864
874
|
AsyncFunction("pcmStreamWrite") Coroutine { streamId: String, base64: String ->
|
|
@@ -194,6 +194,10 @@ public class Bridge private constructor() {
|
|
|
194
194
|
body["channels"] = MIC_CHANNELS
|
|
195
195
|
body["encoding"] = "pcm_s16le"
|
|
196
196
|
body["voiceActivityDetectionEnabled"] = voiceActivityDetectionEnabled
|
|
197
|
+
// Stamped per frame so a consumer that pinned the source can verify it rather than
|
|
198
|
+
// assume it. Read from the store, not from a captured value: the whole point is to
|
|
199
|
+
// report the microphone that is selected right now.
|
|
200
|
+
body["source"] = DeviceStore.store.get("bluetooth", "currentMic") as? String ?: ""
|
|
197
201
|
return body
|
|
198
202
|
}
|
|
199
203
|
|
|
@@ -28,6 +28,7 @@ import com.mentra.bluetoothsdk.sgcs.Simulated
|
|
|
28
28
|
import com.mentra.bluetoothsdk.utils.ControllerTypes
|
|
29
29
|
import com.mentra.bluetoothsdk.utils.DeviceTypes
|
|
30
30
|
import com.mentra.bluetoothsdk.utils.MicMap
|
|
31
|
+
import com.mentra.bluetoothsdk.utils.MicSourcePin
|
|
31
32
|
import com.mentra.bluetoothsdk.utils.MicTypes
|
|
32
33
|
import com.mentra.bluetoothsdk.utils.PhoneAudioMonitor
|
|
33
34
|
import com.mentra.lc3Lib.Lc3Cpp
|
|
@@ -850,6 +851,9 @@ class DeviceManager {
|
|
|
850
851
|
}
|
|
851
852
|
}
|
|
852
853
|
if (pcmData != null && pcmData.isNotEmpty()) {
|
|
854
|
+
// #region agent log — per-second RX window: LC3 bytes in, fingerprint, decoded PCM level (H-E)
|
|
855
|
+
micDbgLc3Window(rawLC3Data, sequenceNumber, pcmData)
|
|
856
|
+
// #endregion
|
|
853
857
|
// Re-encode to canonical LC3 via handlePcm (outside lock to avoid deadlock)
|
|
854
858
|
recordMicPcmProduced()
|
|
855
859
|
handlePcm(pcmData)
|
|
@@ -859,6 +863,72 @@ class DeviceManager {
|
|
|
859
863
|
}
|
|
860
864
|
}
|
|
861
865
|
|
|
866
|
+
// #region agent log — glasses LC3 RX diagnostics (debug session 828181)
|
|
867
|
+
private var micDbgWindowStart = 0L
|
|
868
|
+
private var micDbgPkts = 0
|
|
869
|
+
private var micDbgLc3Bytes = 0L
|
|
870
|
+
private var micDbgPcmBytes = 0L
|
|
871
|
+
private var micDbgSumAbs = 0L
|
|
872
|
+
private var micDbgSamples = 0L
|
|
873
|
+
private var micDbgPeak = 0
|
|
874
|
+
private var micDbgDcSum = 0L
|
|
875
|
+
private var micDbgDistinctFrames = HashSet<Int>()
|
|
876
|
+
private var micDbgFirstSeq = -1
|
|
877
|
+
private var micDbgLastSeq = -1
|
|
878
|
+
private var micDbgSeqGaps = 0
|
|
879
|
+
|
|
880
|
+
private fun micDbgLc3Window(lc3: ByteArray, seq: Int?, pcm: ByteArray) {
|
|
881
|
+
val now = System.currentTimeMillis()
|
|
882
|
+
if (micDbgWindowStart == 0L) micDbgWindowStart = now
|
|
883
|
+
micDbgPkts++
|
|
884
|
+
micDbgLc3Bytes += lc3.size
|
|
885
|
+
micDbgPcmBytes += pcm.size
|
|
886
|
+
var i = 0
|
|
887
|
+
while (i + 1 < pcm.size) {
|
|
888
|
+
val v = ((pcm[i + 1].toInt() shl 8) or (pcm[i].toInt() and 0xff)).toShort().toInt()
|
|
889
|
+
val a = if (v < 0) -v else v
|
|
890
|
+
micDbgSumAbs += a
|
|
891
|
+
micDbgDcSum += v
|
|
892
|
+
if (a > micDbgPeak) micDbgPeak = a
|
|
893
|
+
micDbgSamples++
|
|
894
|
+
i += 2
|
|
895
|
+
}
|
|
896
|
+
var off = 0
|
|
897
|
+
while (off + 40 <= lc3.size) {
|
|
898
|
+
var h = 17
|
|
899
|
+
for (k in off until off + 40) h = h * 31 + lc3[k]
|
|
900
|
+
micDbgDistinctFrames.add(h)
|
|
901
|
+
off += 40
|
|
902
|
+
}
|
|
903
|
+
if (seq != null) {
|
|
904
|
+
if (micDbgFirstSeq < 0) micDbgFirstSeq = seq
|
|
905
|
+
if (micDbgLastSeq >= 0 && ((micDbgLastSeq + 1) and 0xff) != seq) micDbgSeqGaps++
|
|
906
|
+
micDbgLastSeq = seq
|
|
907
|
+
}
|
|
908
|
+
if (now - micDbgWindowStart >= 1000) {
|
|
909
|
+
val meanAbs = if (micDbgSamples > 0) micDbgSumAbs / micDbgSamples else 0
|
|
910
|
+
val dc = if (micDbgSamples > 0) micDbgDcSum / micDbgSamples else 0
|
|
911
|
+
val head = lc3.take(8).joinToString("") { String.format("%02x", it.toInt() and 0xff) }
|
|
912
|
+
Bridge.log(
|
|
913
|
+
"MICDBG-RX pkts=$micDbgPkts lc3B=$micDbgLc3Bytes pcmB=$micDbgPcmBytes samples=$micDbgSamples " +
|
|
914
|
+
"meanAbs=$meanAbs peak=$micDbgPeak dc=$dc distinctLc3Frames=${micDbgDistinctFrames.size} " +
|
|
915
|
+
"seq=$micDbgFirstSeq..$micDbgLastSeq gaps=$micDbgSeqGaps frameSizeArg=40 lastLen=${lc3.size} head=$head"
|
|
916
|
+
)
|
|
917
|
+
micDbgWindowStart = now
|
|
918
|
+
micDbgPkts = 0
|
|
919
|
+
micDbgLc3Bytes = 0
|
|
920
|
+
micDbgPcmBytes = 0
|
|
921
|
+
micDbgSumAbs = 0
|
|
922
|
+
micDbgSamples = 0
|
|
923
|
+
micDbgPeak = 0
|
|
924
|
+
micDbgDcSum = 0
|
|
925
|
+
micDbgDistinctFrames = HashSet()
|
|
926
|
+
micDbgFirstSeq = -1
|
|
927
|
+
micDbgSeqGaps = 0
|
|
928
|
+
}
|
|
929
|
+
}
|
|
930
|
+
// #endregion
|
|
931
|
+
|
|
862
932
|
fun handlePcm(pcmData: ByteArray) {
|
|
863
933
|
// Audio always flows. The previous phone-side Silero VAD gate was a
|
|
864
934
|
// bandwidth-saver that ate transcripts when the mic delivered frames
|
|
@@ -909,11 +979,13 @@ class DeviceManager {
|
|
|
909
979
|
|
|
910
980
|
// allow the sgc to make changes to the micRanking:
|
|
911
981
|
micRanking = sgc?.sortMicRanking(micRanking) ?: micRanking
|
|
912
|
-
|
|
982
|
+
val pin = micSourcePin
|
|
983
|
+
val ranking: List<String> = MicSourcePin.selectionOrder(micRanking, pin)
|
|
984
|
+
Bridge.log("MAN: updateMicState() micRanking: $micRanking pin: $pin")
|
|
913
985
|
|
|
914
986
|
if (micEnabled) {
|
|
915
987
|
|
|
916
|
-
for (micMode in
|
|
988
|
+
for (micMode in ranking) {
|
|
917
989
|
if (micMode == MicTypes.PHONE_INTERNAL ||
|
|
918
990
|
micMode == MicTypes.BLUETOOTH_CLASSIC ||
|
|
919
991
|
micMode == MicTypes.BLUETOOTH
|
|
@@ -961,9 +1033,55 @@ class DeviceManager {
|
|
|
961
1033
|
|
|
962
1034
|
if (micUsed == "" && micEnabled) {
|
|
963
1035
|
Bridge.log("MAN: No available mic found!")
|
|
1036
|
+
if (pin == null) return
|
|
1037
|
+
// A pin taken while another microphone was already recording must still close it:
|
|
1038
|
+
// leaving it open would keep feeding PCM that the pinned consumer will reject, with
|
|
1039
|
+
// the phone's indicator lit for audio nobody uses.
|
|
1040
|
+
stopMicsExcept(micUsed)
|
|
1041
|
+
reportPinnedSourceUnavailable(pin)
|
|
964
1042
|
return
|
|
965
1043
|
}
|
|
966
1044
|
|
|
1045
|
+
stopMicsExcept(micUsed)
|
|
1046
|
+
}
|
|
1047
|
+
|
|
1048
|
+
/**
|
|
1049
|
+
* Call-scoped microphone source lock, or null for the normal ranking.
|
|
1050
|
+
*
|
|
1051
|
+
* Only [MicTypes.GLASSES_CUSTOM] is supported today: it exists so an ACS call can promise that
|
|
1052
|
+
* the wearer's own microphone — and nothing else — is what reaches the far end.
|
|
1053
|
+
*/
|
|
1054
|
+
@Volatile private var micSourcePin: String? = null
|
|
1055
|
+
|
|
1056
|
+
/**
|
|
1057
|
+
* Restrict microphone selection to one source for the duration of a call, or release it.
|
|
1058
|
+
*
|
|
1059
|
+
* Releasing re-runs selection so every other consumer (cloud LC3, miniapp `audio_chunk`,
|
|
1060
|
+
* on-device STT) gets the source its own preference asks for back. A `preferred_mic` change
|
|
1061
|
+
* made while the pin is held is stored but not applied until this releases.
|
|
1062
|
+
*/
|
|
1063
|
+
fun setMicSourcePin(source: String?) {
|
|
1064
|
+
val normalized = MicSourcePin.normalize(source)
|
|
1065
|
+
if (micSourcePin == normalized) return
|
|
1066
|
+
micSourcePin = normalized
|
|
1067
|
+
Bridge.log("MAN: setMicSourcePin($normalized)")
|
|
1068
|
+
updateMicState()
|
|
1069
|
+
}
|
|
1070
|
+
|
|
1071
|
+
/** The microphone the SDK is currently recording from, for consumers that must verify it. */
|
|
1072
|
+
fun activeMicSource(): String = currentMic
|
|
1073
|
+
|
|
1074
|
+
/**
|
|
1075
|
+
* Report a pinned source that cannot be opened. Emitted rather than silently fixed, because the
|
|
1076
|
+
* fix — opening a different microphone — is the thing the pin exists to forbid.
|
|
1077
|
+
*/
|
|
1078
|
+
private fun reportPinnedSourceUnavailable(pin: String) {
|
|
1079
|
+
Bridge.log("MAN: pinned mic source '$pin' is unavailable; no fallback will be started")
|
|
1080
|
+
val health = synchronized(micHealthLock) { micHealthSnapshotLocked() }
|
|
1081
|
+
Bridge.sendMicHealth(health, "pinned-source-unavailable")
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
private fun stopMicsExcept(micUsed: String) {
|
|
967
1085
|
// go through and disable all mics after the first used one:
|
|
968
1086
|
val allMics = micRanking
|
|
969
1087
|
// add any missing mics to the list:
|
|
@@ -3,9 +3,9 @@ package com.mentra.bluetoothsdk
|
|
|
3
3
|
/** Generated by release CI. Do not edit in a release checkout. */
|
|
4
4
|
internal object GeneratedReleaseMetadata {
|
|
5
5
|
const val FAMILY_BASE_VERSION: String = "3.2.0"
|
|
6
|
-
const val RELEASE_IDENTITY: String = "3.2.0-dev.
|
|
7
|
-
const val RELEASE_SET_ID: String = "mentra-3.2.0-dev.
|
|
8
|
-
const val SOURCE_COMMIT: String = "
|
|
9
|
-
const val OTA_MANIFEST_URL: String = "https://github.com/Mentra-Community/MentraOS/releases/download/mentra-builds-v3.2.0/mentra-live-ota-3.2.0-dev.
|
|
10
|
-
const val OTA_MANIFEST_SHA256: String = "
|
|
6
|
+
const val RELEASE_IDENTITY: String = "3.2.0-dev.223"
|
|
7
|
+
const val RELEASE_SET_ID: String = "mentra-3.2.0-dev.223"
|
|
8
|
+
const val SOURCE_COMMIT: String = "03c06ea8306e617803295e8707e9a57f2b64a8ef"
|
|
9
|
+
const val OTA_MANIFEST_URL: String = "https://github.com/Mentra-Community/MentraOS/releases/download/mentra-builds-v3.2.0/mentra-live-ota-3.2.0-dev.223.json"
|
|
10
|
+
const val OTA_MANIFEST_SHA256: String = "41657c616a7d88d0f12fb2dddb0f7ead6fa38b6859500c22067faededcd80d4c"
|
|
11
11
|
}
|
|
@@ -806,6 +806,18 @@ class MentraBluetoothSdk private constructor(
|
|
|
806
806
|
DeviceStore.apply(ObservableStore.BLUETOOTH_CATEGORY, "preferred_mic", preferredMic.value)
|
|
807
807
|
}
|
|
808
808
|
|
|
809
|
+
/**
|
|
810
|
+
* Lock microphone selection to one source until it is released with `null`.
|
|
811
|
+
*
|
|
812
|
+
* Distinct from [setPreferredMic]: a preference is a ranking the SDK may fall through, while a
|
|
813
|
+
* pin forbids the fallback. It exists for consumers that told a remote party which microphone
|
|
814
|
+
* they are hearing — an ACS call — where quietly substituting the phone microphone is worse
|
|
815
|
+
* than no audio at all. Only `"glasses"` is supported.
|
|
816
|
+
*/
|
|
817
|
+
fun setMicSourcePin(source: String?) {
|
|
818
|
+
deviceManager.setMicSourcePin(source)
|
|
819
|
+
}
|
|
820
|
+
|
|
809
821
|
fun setOwnAppAudioPlaying(playing: Boolean) {
|
|
810
822
|
PhoneAudioMonitor.getInstance(appContext).setOwnAppAudioPlaying(playing)
|
|
811
823
|
}
|
|
@@ -14,6 +14,15 @@ data class MicPcmEvent(
|
|
|
14
14
|
val channels: Int,
|
|
15
15
|
val encoding: String,
|
|
16
16
|
val voiceActivityDetectionEnabled: Boolean,
|
|
17
|
+
/**
|
|
18
|
+
* Which microphone produced this buffer, as [MicTypes]: `"glasses"`, `"phone"`, `"bluetooth"`.
|
|
19
|
+
*
|
|
20
|
+
* Carried per frame rather than inferred, because the SDK can move the source mid-stream when
|
|
21
|
+
* one becomes unavailable. A consumer that promised the far end a specific microphone — an ACS
|
|
22
|
+
* call, for instance — has no other way to tell that the audio it is forwarding is still the
|
|
23
|
+
* one it advertised. Empty when the SDK has not selected a microphone.
|
|
24
|
+
*/
|
|
25
|
+
val source: String,
|
|
17
26
|
) {
|
|
18
27
|
constructor(values: Map<String, Any>) : this(
|
|
19
28
|
pcm = values["pcm"] as? ByteArray ?: ByteArray(0),
|
|
@@ -24,6 +33,7 @@ data class MicPcmEvent(
|
|
|
24
33
|
voiceActivityDetectionEnabled =
|
|
25
34
|
boolValue(values, "voiceActivityDetectionEnabled")
|
|
26
35
|
?: BluetoothSdkDefaults.VOICE_ACTIVITY_DETECTION_ENABLED,
|
|
36
|
+
source = stringValue(values, "source") ?: "",
|
|
27
37
|
)
|
|
28
38
|
|
|
29
39
|
fun toMap(): Map<String, Any> =
|
|
@@ -35,6 +45,7 @@ data class MicPcmEvent(
|
|
|
35
45
|
"channels" to channels,
|
|
36
46
|
"encoding" to encoding,
|
|
37
47
|
"voiceActivityDetectionEnabled" to voiceActivityDetectionEnabled,
|
|
48
|
+
"source" to source,
|
|
38
49
|
)
|
|
39
50
|
|
|
40
51
|
companion object {
|
|
@@ -63,6 +63,34 @@ data class StreamAudioConfig @JvmOverloads constructor(
|
|
|
63
63
|
}
|
|
64
64
|
}
|
|
65
65
|
|
|
66
|
+
/** ICE overrides for a WHIP stream. Ignored by the RTMP and SRT paths. */
|
|
67
|
+
data class StreamIceConfig @JvmOverloads constructor(
|
|
68
|
+
/**
|
|
69
|
+
* STUN server the glasses use while gathering candidates.
|
|
70
|
+
*
|
|
71
|
+
* Null and empty are different answers, and both have to survive the BLE round trip. Null
|
|
72
|
+
* leaves the glasses on their default Cloudflare STUN server; an empty string is SoftAP
|
|
73
|
+
* calling's explicit request for host-only gathering, because the phone's WHIP server sits on
|
|
74
|
+
* the glasses' own hotspot where a reflexive candidate is meaningless and unreachable.
|
|
75
|
+
*/
|
|
76
|
+
val stun: String? = null,
|
|
77
|
+
) {
|
|
78
|
+
fun toMap(): Map<String, Any> =
|
|
79
|
+
buildMap {
|
|
80
|
+
// Not filtered on blankness, unlike the other optional string fields here: "" is the
|
|
81
|
+
// host-only signal, so dropping it would silently restore the default STUN server.
|
|
82
|
+
stun?.let { put("stun", it) }
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
companion object {
|
|
86
|
+
@JvmStatic
|
|
87
|
+
fun fromMap(values: Map<String, Any>?): StreamIceConfig? {
|
|
88
|
+
values ?: return null
|
|
89
|
+
return StreamIceConfig(stun = stringValue(values, "stun", "s"))
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
66
94
|
/** Effective video settings reported by the glasses after defaults and clamps. */
|
|
67
95
|
data class StreamResolvedVideoConfig @JvmOverloads constructor(
|
|
68
96
|
/** Encoded output width sent to the stream endpoint. */
|
|
@@ -214,6 +242,12 @@ data class StreamRequest @JvmOverloads constructor(
|
|
|
214
242
|
val audio: StreamAudioConfig? = null,
|
|
215
243
|
val authToken: String? = null,
|
|
216
244
|
val captureAudio: Boolean = true,
|
|
245
|
+
val ice: StreamIceConfig? = null,
|
|
246
|
+
/**
|
|
247
|
+
* Correlation id the glasses echo in every SOFTAP_TRACE line, so phone and glasses logs can be
|
|
248
|
+
* joined despite unsynchronised clocks.
|
|
249
|
+
*/
|
|
250
|
+
val traceId: String? = null,
|
|
217
251
|
) {
|
|
218
252
|
fun toMap(): Map<String, Any> =
|
|
219
253
|
buildMap {
|
|
@@ -225,6 +259,8 @@ data class StreamRequest @JvmOverloads constructor(
|
|
|
225
259
|
audio?.toMap()?.takeIf { it.isNotEmpty() }?.let { put("audio", it) }
|
|
226
260
|
authToken?.takeIf { it.isNotEmpty() }?.let { put("authToken", it) }
|
|
227
261
|
if (!captureAudio) put("captureAudio", false)
|
|
262
|
+
ice?.toMap()?.takeIf { it.isNotEmpty() }?.let { put("ice", it) }
|
|
263
|
+
traceId?.takeIf { it.isNotEmpty() }?.let { put("traceId", it) }
|
|
228
264
|
}
|
|
229
265
|
|
|
230
266
|
companion object {
|
|
@@ -240,6 +276,8 @@ data class StreamRequest @JvmOverloads constructor(
|
|
|
240
276
|
audio = StreamAudioConfig.fromMap(stringMapValue(values["audio"])),
|
|
241
277
|
authToken = values["authToken"] as? String ?: values["auth_token"] as? String,
|
|
242
278
|
captureAudio = boolValue(values, "captureAudio") ?: boolValue(values, "ca") ?: true,
|
|
279
|
+
ice = StreamIceConfig.fromMap(stringMapValue(values["ice"] ?: values["i"])),
|
|
280
|
+
traceId = stringValue(values, "traceId"),
|
|
243
281
|
)
|
|
244
282
|
}
|
|
245
283
|
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
package com.mentra.bluetoothsdk.utils
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Call-scoped microphone lock.
|
|
5
|
+
*
|
|
6
|
+
* The ranking in `DeviceManager` answers "which microphone can we get?", and for captions or the
|
|
7
|
+
* cloud uplink that is the right question — any working microphone beats none. A call is the case
|
|
8
|
+
* where it is the wrong question: the wearer agreed to be heard from the glasses, so falling
|
|
9
|
+
* through to the phone puts the room they are standing in onto a Teams call that still reports
|
|
10
|
+
* "glasses". A pin turns the fallback off for exactly as long as the call holds it.
|
|
11
|
+
*/
|
|
12
|
+
object MicSourcePin {
|
|
13
|
+
/**
|
|
14
|
+
* Validate a pin request. Only the glasses microphone can be pinned: every other source is one
|
|
15
|
+
* the ranking would have picked anyway, so pinning it buys nothing and only adds a way to strand
|
|
16
|
+
* the selection on a microphone no consumer asked for.
|
|
17
|
+
*/
|
|
18
|
+
fun normalize(source: String?): String? {
|
|
19
|
+
val trimmed = source?.trim()?.takeIf { it.isNotEmpty() } ?: return null
|
|
20
|
+
require(trimmed == MicTypes.GLASSES_CUSTOM) { "unsupported mic source pin: $trimmed" }
|
|
21
|
+
return trimmed
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* The sources selection may consider. A pin replaces the ranking outright rather than reordering
|
|
26
|
+
* it, so "the pinned source is unavailable" resolves to no microphone instead of the next best
|
|
27
|
+
* one.
|
|
28
|
+
*/
|
|
29
|
+
fun selectionOrder(ranking: List<String>, pin: String?): List<String> =
|
|
30
|
+
if (pin == null) ranking else listOf(pin)
|
|
31
|
+
}
|
|
@@ -29,6 +29,7 @@ class PcmStreamPlayer(
|
|
|
29
29
|
private val sampleRate: Int,
|
|
30
30
|
channels: Int,
|
|
31
31
|
volume: Float,
|
|
32
|
+
jitterMs: Int = TRACK_BUFFER_MS,
|
|
32
33
|
) {
|
|
33
34
|
companion object {
|
|
34
35
|
private const val TAG = "PcmStreamPlayer"
|
|
@@ -41,8 +42,28 @@ class PcmStreamPlayer(
|
|
|
41
42
|
|
|
42
43
|
/** AudioTrack buffer: max(4x minimum, 500ms) of jitter headroom. */
|
|
43
44
|
private const val TRACK_BUFFER_MS = 500
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Floor on the requested headroom.
|
|
48
|
+
*
|
|
49
|
+
* A MODE_STREAM track fills to its buffer size and stays there, so this value is also the
|
|
50
|
+
* steady-state output latency, not just a jitter cushion. Callers that want a conversation
|
|
51
|
+
* to feel live ask for a small number; below this the feeder cannot outrun playback and the
|
|
52
|
+
* track underruns into audible chop, which is worse than the delay it buys back.
|
|
53
|
+
*/
|
|
54
|
+
private const val MIN_TRACK_BUFFER_MS = 80
|
|
44
55
|
}
|
|
45
56
|
|
|
57
|
+
/**
|
|
58
|
+
* Playout headroom, and therefore this stream's floor latency.
|
|
59
|
+
*
|
|
60
|
+
* [TRACK_BUFFER_MS] suits one-way media (a miniapp `speaker.createStream()` clip), where half a
|
|
61
|
+
* second of cushion costs nothing audible. A realtime call is the opposite trade: that same
|
|
62
|
+
* cushion is half a second of delay before the far end's voice reaches the wearer, so the ACS
|
|
63
|
+
* meeting path asks for a much smaller one.
|
|
64
|
+
*/
|
|
65
|
+
private val jitterBufferMs = jitterMs.coerceIn(MIN_TRACK_BUFFER_MS, TRACK_BUFFER_MS)
|
|
66
|
+
|
|
46
67
|
private val bytesPerFrame = 2 * channels
|
|
47
68
|
private val channelMask =
|
|
48
69
|
if (channels >= 2) AudioFormat.CHANNEL_OUT_STEREO else AudioFormat.CHANNEL_OUT_MONO
|
|
@@ -70,7 +91,7 @@ class PcmStreamPlayer(
|
|
|
70
91
|
init {
|
|
71
92
|
val minBuf =
|
|
72
93
|
AudioTrack.getMinBufferSize(sampleRate, channelMask, AudioFormat.ENCODING_PCM_16BIT)
|
|
73
|
-
val jitterBuf = sampleRate * bytesPerFrame *
|
|
94
|
+
val jitterBuf = sampleRate * bytesPerFrame * jitterBufferMs / 1000
|
|
74
95
|
audioTrack =
|
|
75
96
|
AudioTrack.Builder()
|
|
76
97
|
.setAudioAttributes(
|
|
@@ -86,7 +107,11 @@ class PcmStreamPlayer(
|
|
|
86
107
|
.setChannelMask(channelMask)
|
|
87
108
|
.build()
|
|
88
109
|
)
|
|
89
|
-
|
|
110
|
+
// minBuf is AudioTrack's own floor, so it wins over a smaller request; the
|
|
111
|
+
// requested headroom is otherwise authoritative. Any fixed multiple of
|
|
112
|
+
// minBuf here would silently override a caller asking for low latency,
|
|
113
|
+
// since minBuf scales with the rate while the request does not.
|
|
114
|
+
.setBufferSizeInBytes(max(minBuf, jitterBuf))
|
|
90
115
|
.setTransferMode(AudioTrack.MODE_STREAM)
|
|
91
116
|
.build()
|
|
92
117
|
if (audioTrack.state != AudioTrack.STATE_INITIALIZED) {
|
|
@@ -101,7 +126,11 @@ class PcmStreamPlayer(
|
|
|
101
126
|
isDaemon = true
|
|
102
127
|
start()
|
|
103
128
|
}
|
|
104
|
-
Log.d(
|
|
129
|
+
Log.d(
|
|
130
|
+
TAG,
|
|
131
|
+
"[$streamId] opened: rate=$sampleRate ch=$channels vol=$volume " +
|
|
132
|
+
"jitterMs=$jitterBufferMs trackBytes=${max(minBuf, jitterBuf)}",
|
|
133
|
+
)
|
|
105
134
|
}
|
|
106
135
|
|
|
107
136
|
/** Milliseconds of audio accepted but not yet played out. */
|
|
@@ -285,7 +314,7 @@ class PcmStreamPlayer(
|
|
|
285
314
|
object PcmStreamManager {
|
|
286
315
|
private val players = java.util.concurrent.ConcurrentHashMap<String, PcmStreamPlayer>()
|
|
287
316
|
|
|
288
|
-
fun open(streamId: String, sampleRate: Int, channels: Int, volume: Float) {
|
|
317
|
+
fun open(streamId: String, sampleRate: Int, channels: Int, volume: Float, jitterMs: Int? = null) {
|
|
289
318
|
require(streamId.isNotBlank()) { "streamId is required" }
|
|
290
319
|
require(sampleRate == 16000 || sampleRate == 24000 || sampleRate == 48000) {
|
|
291
320
|
"unsupported PCM sample rate $sampleRate"
|
|
@@ -293,7 +322,12 @@ object PcmStreamManager {
|
|
|
293
322
|
require(channels == 1) { "only mono PCM is supported" }
|
|
294
323
|
// Replacing an id is a caller bug, but never leak the old track.
|
|
295
324
|
players.remove(streamId)?.abort()
|
|
296
|
-
players[streamId] =
|
|
325
|
+
players[streamId] =
|
|
326
|
+
if (jitterMs == null) {
|
|
327
|
+
PcmStreamPlayer(streamId, sampleRate, channels, volume)
|
|
328
|
+
} else {
|
|
329
|
+
PcmStreamPlayer(streamId, sampleRate, channels, volume, jitterMs)
|
|
330
|
+
}
|
|
297
331
|
}
|
|
298
332
|
|
|
299
333
|
fun write(streamId: String, base64: String): Long {
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
package com.mentra.bluetoothsdk
|
|
2
|
+
|
|
3
|
+
import com.mentra.bluetoothsdk.utils.MicTypes
|
|
4
|
+
import org.assertj.core.api.Assertions.assertThat
|
|
5
|
+
import org.junit.Test
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* The per-frame source tag, and the JS keys that read it.
|
|
9
|
+
*
|
|
10
|
+
* An ACS call pins the glasses microphone and then forwards every buffer to the far end. The pin is
|
|
11
|
+
* what makes that safe, but a pin the SDK could not honour is exactly the case worth catching, so
|
|
12
|
+
* each frame says which microphone produced it and the call verifies rather than assumes. That is
|
|
13
|
+
* only true if the tag survives the bridge, so the key names here are a contract with
|
|
14
|
+
* `AcsMeetingService`, not an implementation detail.
|
|
15
|
+
*/
|
|
16
|
+
class MicPcmEventSourceTest {
|
|
17
|
+
private fun frame(source: String) = MicPcmEvent(
|
|
18
|
+
pcm = ByteArray(320),
|
|
19
|
+
sampleRate = MicPcmEvent.SAMPLE_RATE,
|
|
20
|
+
bitsPerSample = MicPcmEvent.BITS_PER_SAMPLE,
|
|
21
|
+
channels = MicPcmEvent.CHANNELS,
|
|
22
|
+
encoding = MicPcmEvent.ENCODING,
|
|
23
|
+
voiceActivityDetectionEnabled = false,
|
|
24
|
+
source = source,
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
@Test
|
|
28
|
+
fun theKeysTheCallReadsAreAllPresent() {
|
|
29
|
+
val map = frame(MicTypes.GLASSES_CUSTOM).toMap()
|
|
30
|
+
|
|
31
|
+
assertThat(map).containsKeys("pcm", "sampleRate", "source")
|
|
32
|
+
assertThat(map["source"]).isEqualTo(MicTypes.GLASSES_CUSTOM)
|
|
33
|
+
assertThat(map["sampleRate"]).isEqualTo(16_000)
|
|
34
|
+
assertThat(map["encoding"]).isEqualTo("pcm_s16le")
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
@Test
|
|
38
|
+
fun everySourceTheRankingCanPickRoundTrips() {
|
|
39
|
+
for (source in MicTypes.ALL) {
|
|
40
|
+
val restored = MicPcmEvent(frame(source).toMap())
|
|
41
|
+
assertThat(restored.source).isEqualTo(source)
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Absent must not decode as "glasses". A frame from a build that predates the tag is a frame
|
|
47
|
+
* whose source is unknown, and the call has to drop it rather than forward the phone's room.
|
|
48
|
+
*/
|
|
49
|
+
@Test
|
|
50
|
+
fun anUntaggedFrameDecodesAsUnknownRatherThanGlasses() {
|
|
51
|
+
val legacy = mapOf<String, Any>(
|
|
52
|
+
"pcm" to ByteArray(320),
|
|
53
|
+
"sampleRate" to 16_000,
|
|
54
|
+
"bitsPerSample" to 16,
|
|
55
|
+
"channels" to 1,
|
|
56
|
+
"encoding" to "pcm_s16le",
|
|
57
|
+
"voiceActivityDetectionEnabled" to false,
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
val decoded = MicPcmEvent(legacy)
|
|
61
|
+
|
|
62
|
+
assertThat(decoded.source).isEmpty()
|
|
63
|
+
assertThat(decoded.source).isNotEqualTo(MicTypes.GLASSES_CUSTOM)
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** 16 kHz mono PCM16, 10 ms per LC3 frame: the size the call's base64 round trip has to preserve. */
|
|
67
|
+
@Test
|
|
68
|
+
fun aDecodedLc3FrameIsOneHundredAndSixtySamples() {
|
|
69
|
+
val map = frame(MicTypes.GLASSES_CUSTOM).toMap()
|
|
70
|
+
val pcm = map["pcm"] as ByteArray
|
|
71
|
+
|
|
72
|
+
assertThat(pcm).hasSize(320)
|
|
73
|
+
assertThat(pcm.size / 2).isEqualTo(160)
|
|
74
|
+
}
|
|
75
|
+
}
|
|
@@ -39,4 +39,48 @@ class StreamRequestCaptureAudioTest {
|
|
|
39
39
|
val full = StreamRequest.fromMap(mapOf("streamUrl" to "https://example.com/whip", "captureAudio" to false))
|
|
40
40
|
assertThat(full.captureAudio).isFalse()
|
|
41
41
|
}
|
|
42
|
+
|
|
43
|
+
@Test
|
|
44
|
+
fun hostOnlyIceSurvivesBridgeAndWireSerialization() {
|
|
45
|
+
val request = StreamRequest.fromMap(mapOf(
|
|
46
|
+
"streamUrl" to "http://192.168.43.79:8080/whip",
|
|
47
|
+
"ice" to mapOf("stun" to ""),
|
|
48
|
+
))
|
|
49
|
+
assertThat(request.ice?.stun).isEqualTo("")
|
|
50
|
+
|
|
51
|
+
// The empty string is the host-only request. Dropping it as "blank" is what silently left the
|
|
52
|
+
// glasses gathering against the default Cloudflare STUN server on an internet-less hotspot.
|
|
53
|
+
@Suppress("UNCHECKED_CAST")
|
|
54
|
+
val ice = request.toMap()["ice"] as Map<String, Any>
|
|
55
|
+
assertThat(ice).containsEntry("stun", "")
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
@Test
|
|
59
|
+
fun absentIceStaysAbsentSoGlassesKeepTheirDefault() {
|
|
60
|
+
val request = StreamRequest.fromMap(mapOf("streamUrl" to "https://example.com/whip"))
|
|
61
|
+
assertThat(request.ice).isNull()
|
|
62
|
+
assertThat(request.toMap()).doesNotContainKey("ice")
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
@Test
|
|
66
|
+
fun compactIceKeysParse() {
|
|
67
|
+
val request = StreamRequest.fromMap(mapOf(
|
|
68
|
+
"streamUrl" to "http://192.168.43.79:8080/whip",
|
|
69
|
+
"i" to mapOf("s" to "stun:stun.example.com:3478"),
|
|
70
|
+
))
|
|
71
|
+
assertThat(request.ice?.stun).isEqualTo("stun:stun.example.com:3478")
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
@Test
|
|
75
|
+
fun traceIdSurvivesBridgeAndWireSerialization() {
|
|
76
|
+
val request = StreamRequest.fromMap(mapOf(
|
|
77
|
+
"streamUrl" to "http://192.168.43.79:8080/whip",
|
|
78
|
+
"traceId" to "d34eeb11",
|
|
79
|
+
))
|
|
80
|
+
assertThat(request.traceId).isEqualTo("d34eeb11")
|
|
81
|
+
assertThat(request.toMap()).containsEntry("traceId", "d34eeb11")
|
|
82
|
+
|
|
83
|
+
assertThat(StreamRequest(streamUrl = "https://example.com/whip").toMap())
|
|
84
|
+
.doesNotContainKey("traceId")
|
|
85
|
+
}
|
|
42
86
|
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
package com.mentra.bluetoothsdk.utils
|
|
2
|
+
|
|
3
|
+
import org.assertj.core.api.Assertions.assertThat
|
|
4
|
+
import org.assertj.core.api.Assertions.assertThatThrownBy
|
|
5
|
+
import org.junit.Test
|
|
6
|
+
|
|
7
|
+
class MicSourcePinTest {
|
|
8
|
+
private val auto = MicMap.map.getValue("auto")
|
|
9
|
+
|
|
10
|
+
@Test
|
|
11
|
+
fun noPinLeavesTheRankingAlone() {
|
|
12
|
+
assertThat(MicSourcePin.selectionOrder(auto, null)).isEqualTo(auto)
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* The whole point of the pin: with the glasses microphone unavailable, an unpinned selection
|
|
17
|
+
* walks on to the phone. That fallback is right for captions and wrong for a call, so a pin has
|
|
18
|
+
* to remove the other sources rather than merely prefer the glasses.
|
|
19
|
+
*/
|
|
20
|
+
@Test
|
|
21
|
+
fun aPinRemovesTheFallbacksRatherThanReorderingThem() {
|
|
22
|
+
val order = MicSourcePin.selectionOrder(auto, MicTypes.GLASSES_CUSTOM)
|
|
23
|
+
|
|
24
|
+
assertThat(order).containsExactly(MicTypes.GLASSES_CUSTOM)
|
|
25
|
+
assertThat(order).doesNotContain(MicTypes.PHONE_INTERNAL, MicTypes.BLUETOOTH, MicTypes.BLUETOOTH_CLASSIC)
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
@Test
|
|
29
|
+
fun aPinOutranksAPhoneFirstPreference() {
|
|
30
|
+
// `preferred_mic = phone` is an STT preference. It must not decide who a Teams call hears.
|
|
31
|
+
assertThat(MicSourcePin.selectionOrder(MicMap.map.getValue("phone"), MicTypes.GLASSES_CUSTOM))
|
|
32
|
+
.containsExactly(MicTypes.GLASSES_CUSTOM)
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
@Test
|
|
36
|
+
fun releasingRestoresEveryOtherConsumersPreference() {
|
|
37
|
+
val phoneFirst = MicMap.map.getValue("phone")
|
|
38
|
+
assertThat(MicSourcePin.selectionOrder(phoneFirst, MicSourcePin.normalize(null)))
|
|
39
|
+
.isEqualTo(phoneFirst)
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
@Test
|
|
43
|
+
fun blankAndNullBothMeanUnpinned() {
|
|
44
|
+
assertThat(MicSourcePin.normalize(null)).isNull()
|
|
45
|
+
assertThat(MicSourcePin.normalize("")).isNull()
|
|
46
|
+
assertThat(MicSourcePin.normalize(" ")).isNull()
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
@Test
|
|
50
|
+
fun theGlassesSourceIsAcceptedAndTrimmed() {
|
|
51
|
+
assertThat(MicSourcePin.normalize(MicTypes.GLASSES_CUSTOM)).isEqualTo(MicTypes.GLASSES_CUSTOM)
|
|
52
|
+
assertThat(MicSourcePin.normalize(" glasses ")).isEqualTo(MicTypes.GLASSES_CUSTOM)
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Pinning any other source would strand selection on a microphone no consumer asked for, and
|
|
57
|
+
* failing loudly beats a call that silently pins the phone — the exact outcome the pin exists to
|
|
58
|
+
* prevent.
|
|
59
|
+
*/
|
|
60
|
+
@Test
|
|
61
|
+
fun everyOtherSourceIsRejected() {
|
|
62
|
+
for (source in listOf(MicTypes.PHONE_INTERNAL, MicTypes.BLUETOOTH, MicTypes.BLUETOOTH_CLASSIC, "auto")) {
|
|
63
|
+
assertThatThrownBy { MicSourcePin.normalize(source) }
|
|
64
|
+
.isInstanceOf(IllegalArgumentException::class.java)
|
|
65
|
+
.hasMessageContaining(source)
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|