@mentra/bluetooth-sdk 3.2.0-dev.217 → 3.2.0-dev.222
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +23 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +16 -38
- package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +120 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/GeneratedReleaseMetadata.kt +5 -5
- package/android/src/main/java/com/mentra/bluetoothsdk/LogForwardingBudget.kt +50 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +12 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/audio/AudioModels.kt +11 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +38 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/MicSourcePin.kt +31 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/audio/PcmStreamPlayer.kt +39 -5
- package/android/src/test/java/com/mentra/bluetoothsdk/LogForwardingBudgetTest.kt +49 -0
- package/android/src/test/java/com/mentra/bluetoothsdk/MicPcmEventSourceTest.kt +75 -0
- package/android/src/test/java/com/mentra/bluetoothsdk/StreamRequestCaptureAudioTest.kt +44 -0
- package/android/src/test/java/com/mentra/bluetoothsdk/utils/MicSourcePinTest.kt +68 -0
- package/android/src/test/java/com/mentra/bluetoothsdk/utils/NativeLogTest.kt +15 -0
- package/build/BluetoothSdk.types.d.ts +27 -0
- package/build/BluetoothSdk.types.d.ts.map +1 -1
- package/build/BluetoothSdk.types.js.map +1 -1
- package/build/_private/BluetoothSdkModule.d.ts +18 -2
- package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
- package/build/_private/BluetoothSdkModule.js.map +1 -1
- package/build/generated/releaseMetadata.js +5 -5
- package/build/generated/releaseMetadata.js.map +1 -1
- package/ios/BluetoothSdkModule.swift +15 -1
- package/ios/Source/BluetoothSdkDefaults.swift +1 -1
- package/ios/Source/Bridge.swift +13 -45
- package/ios/Source/GeneratedReleaseMetadata.swift +5 -5
- package/ios/Source/streaming/StreamModels.swift +48 -2
- package/package.json +1 -1
- package/src/BluetoothSdk.types.ts +28 -0
- package/src/_private/BluetoothSdkModule.ts +24 -2
- package/src/generated/releaseMetadata.ts +5 -5
package/README.md
CHANGED
|
@@ -29,6 +29,15 @@ to its existing retention limits. Native diagnostics are also kept in the native
|
|
|
29
29
|
console. Forwarding covers SDK-owned diagnostics while the JS runtime is active,
|
|
30
30
|
not arbitrary OS or third-party native logs or logs from before subscription.
|
|
31
31
|
|
|
32
|
+
On Android, forwarding to JavaScript is capped at 100 messages per second. Each
|
|
33
|
+
forwarded message holds a JNI global reference until the JavaScript thread drains
|
|
34
|
+
it, so an unbounded log source can exhaust the process-wide reference table and
|
|
35
|
+
abort the app rather than merely slow it down. Messages above the cap are withheld
|
|
36
|
+
from the JavaScript stream only — they are still written to the native console —
|
|
37
|
+
and the next forwarded message is preceded by a notice saying how many were
|
|
38
|
+
withheld. Per-frame microphone payload events are not traced at all; audio faults
|
|
39
|
+
are still reported through the `mic_health` event.
|
|
40
|
+
|
|
32
41
|
Rebuild the native app after updating the SDK to pick up both platforms' logging
|
|
33
42
|
changes. Adding this JS package alone cannot reroute logs in an older binary.
|
|
34
43
|
|
|
@@ -133,6 +133,7 @@ private inline fun <
|
|
|
133
133
|
class BluetoothSdkModule : Module() {
|
|
134
134
|
private var sdk: MentraBluetoothSdk? = null
|
|
135
135
|
private var deviceManager: DeviceManager? = null
|
|
136
|
+
private val logForwarding = LogForwardingBudget()
|
|
136
137
|
private val sdkListener =
|
|
137
138
|
object : MentraBluetoothSdkListener {
|
|
138
139
|
override fun onGlassesChanged(glasses: GlassesRuntimeState) {
|
|
@@ -295,6 +296,16 @@ class BluetoothSdkModule : Module() {
|
|
|
295
296
|
}
|
|
296
297
|
|
|
297
298
|
override fun onLog(message: String) {
|
|
299
|
+
// Each event pins a JNI global reference until JavaScript drains it. The
|
|
300
|
+
// log stream is the one source whose rate is unbounded, so it is budgeted.
|
|
301
|
+
val withheld = logForwarding.admit() ?: return
|
|
302
|
+
if (withheld > 0) {
|
|
303
|
+
val notice =
|
|
304
|
+
"[W/BluetoothSdkModule] withheld $withheld native log line(s) from " +
|
|
305
|
+
"JavaScript to stay under ${logForwarding.maxPerWindow}/s; " +
|
|
306
|
+
"logcat has them all"
|
|
307
|
+
sendEvent("log", mapOf("message" to notice))
|
|
308
|
+
}
|
|
298
309
|
sendEvent("log", mapOf("message" to message))
|
|
299
310
|
}
|
|
300
311
|
|
|
@@ -824,6 +835,10 @@ class BluetoothSdkModule : Module() {
|
|
|
824
835
|
)
|
|
825
836
|
}
|
|
826
837
|
|
|
838
|
+
SdkAsyncFunction("setMicSourcePin") { source: String? ->
|
|
839
|
+
sdk?.setMicSourcePin(source)
|
|
840
|
+
}
|
|
841
|
+
|
|
827
842
|
// Runs on Dispatchers.IO, not the shared Expo AsyncFunctionQueue: restart()
|
|
828
843
|
// does a synchronous JNI model reload that would otherwise block every other
|
|
829
844
|
// native call in the app until it completes.
|
|
@@ -846,8 +861,14 @@ class BluetoothSdkModule : Module() {
|
|
|
846
861
|
// close blocks until the backlog drains — either would otherwise stall
|
|
847
862
|
// every other native call queued behind them.
|
|
848
863
|
|
|
849
|
-
AsyncFunction("pcmStreamOpen") {
|
|
850
|
-
|
|
864
|
+
AsyncFunction("pcmStreamOpen") {
|
|
865
|
+
streamId: String,
|
|
866
|
+
sampleRate: Int,
|
|
867
|
+
channels: Int,
|
|
868
|
+
volume: Double,
|
|
869
|
+
jitterMs: Int?,
|
|
870
|
+
->
|
|
871
|
+
PcmStreamManager.open(streamId, sampleRate, channels, volume.toFloat(), jitterMs)
|
|
851
872
|
}
|
|
852
873
|
|
|
853
874
|
AsyncFunction("pcmStreamWrite") Coroutine { streamId: String, base64: String ->
|
|
@@ -30,18 +30,6 @@ public class Bridge private constructor() {
|
|
|
30
30
|
private const val MIC_CHANNELS = 1
|
|
31
31
|
private const val LC3_FRAME_DURATION_MS = 10
|
|
32
32
|
private const val DEFAULT_LC3_FRAME_SIZE_BYTES = 60
|
|
33
|
-
private val AUDIO_TRACE_METADATA_KEYS =
|
|
34
|
-
listOf(
|
|
35
|
-
"sampleRate",
|
|
36
|
-
"bitsPerSample",
|
|
37
|
-
"channels",
|
|
38
|
-
"encoding",
|
|
39
|
-
"frameDurationMs",
|
|
40
|
-
"frameSizeBytes",
|
|
41
|
-
"bitrate",
|
|
42
|
-
"packetizedFromGlasses",
|
|
43
|
-
"voiceActivityDetectionEnabled",
|
|
44
|
-
)
|
|
45
33
|
|
|
46
34
|
@Volatile private var instance: Bridge? = null
|
|
47
35
|
|
|
@@ -206,6 +194,10 @@ public class Bridge private constructor() {
|
|
|
206
194
|
body["channels"] = MIC_CHANNELS
|
|
207
195
|
body["encoding"] = "pcm_s16le"
|
|
208
196
|
body["voiceActivityDetectionEnabled"] = voiceActivityDetectionEnabled
|
|
197
|
+
// Stamped per frame so a consumer that pinned the source can verify it rather than
|
|
198
|
+
// assume it. Read from the store, not from a captured value: the whole point is to
|
|
199
|
+
// report the microphone that is selected right now.
|
|
200
|
+
body["source"] = DeviceStore.store.get("bluetooth", "currentMic") as? String ?: ""
|
|
209
201
|
return body
|
|
210
202
|
}
|
|
211
203
|
|
|
@@ -943,43 +935,29 @@ public class Bridge private constructor() {
|
|
|
943
935
|
}
|
|
944
936
|
}
|
|
945
937
|
|
|
938
|
+
/**
|
|
939
|
+
* Returns null for events that must not be traced.
|
|
940
|
+
*
|
|
941
|
+
* "log" is excluded so tracing never recurses back through NativeLog. Audio payload
|
|
942
|
+
* events are excluded because they arrive at frame rate: tracing them turned every
|
|
943
|
+
* microphone frame into a second bridge event, and each of those pins a JNI global
|
|
944
|
+
* reference until JavaScript drains it. A JavaScript thread busy with call audio
|
|
945
|
+
* cannot keep up, so the references accumulated until the process-wide table
|
|
946
|
+
* overflowed and the runtime aborted. Audio faults (sequence gaps, decode failures)
|
|
947
|
+
* are still reported through "mic_health", and healthy frames need no trace.
|
|
948
|
+
*/
|
|
946
949
|
private fun tracePayloadForTypedMessage(
|
|
947
950
|
type: String,
|
|
948
951
|
body: Map<String, Any>
|
|
949
952
|
): Map<String, Any>? =
|
|
950
953
|
when {
|
|
951
954
|
type == "log" -> null
|
|
952
|
-
isAudioPayloadEvent(type) ->
|
|
955
|
+
isAudioPayloadEvent(type) -> null
|
|
953
956
|
else -> body
|
|
954
957
|
}
|
|
955
958
|
|
|
956
959
|
private fun isAudioPayloadEvent(type: String): Boolean =
|
|
957
960
|
type == "mic_pcm" || type == "mic_lc3"
|
|
958
|
-
|
|
959
|
-
private fun audioTracePayload(type: String, body: Map<String, Any>): Map<String, Any> {
|
|
960
|
-
val payload = HashMap<String, Any>()
|
|
961
|
-
payload["type"] = type
|
|
962
|
-
payload["timestamp"] = System.currentTimeMillis()
|
|
963
|
-
payload["payloadOmitted"] = true
|
|
964
|
-
payload["payloadOmittedReason"] = "audio"
|
|
965
|
-
|
|
966
|
-
val audioBytes =
|
|
967
|
-
when (type) {
|
|
968
|
-
"mic_pcm" -> (body["pcm"] as? ByteArray)?.size
|
|
969
|
-
"mic_lc3" -> (body["lc3"] as? ByteArray)?.size
|
|
970
|
-
else -> null
|
|
971
|
-
}
|
|
972
|
-
audioBytes?.let { payload["audioBytes"] = it }
|
|
973
|
-
|
|
974
|
-
AUDIO_TRACE_METADATA_KEYS.forEach { key ->
|
|
975
|
-
val value = body[key]
|
|
976
|
-
if (value != null) {
|
|
977
|
-
payload[key] = value
|
|
978
|
-
}
|
|
979
|
-
}
|
|
980
|
-
|
|
981
|
-
return payload
|
|
982
|
-
}
|
|
983
961
|
}
|
|
984
962
|
|
|
985
963
|
init {
|
|
@@ -28,6 +28,7 @@ import com.mentra.bluetoothsdk.sgcs.Simulated
|
|
|
28
28
|
import com.mentra.bluetoothsdk.utils.ControllerTypes
|
|
29
29
|
import com.mentra.bluetoothsdk.utils.DeviceTypes
|
|
30
30
|
import com.mentra.bluetoothsdk.utils.MicMap
|
|
31
|
+
import com.mentra.bluetoothsdk.utils.MicSourcePin
|
|
31
32
|
import com.mentra.bluetoothsdk.utils.MicTypes
|
|
32
33
|
import com.mentra.bluetoothsdk.utils.PhoneAudioMonitor
|
|
33
34
|
import com.mentra.lc3Lib.Lc3Cpp
|
|
@@ -850,6 +851,9 @@ class DeviceManager {
|
|
|
850
851
|
}
|
|
851
852
|
}
|
|
852
853
|
if (pcmData != null && pcmData.isNotEmpty()) {
|
|
854
|
+
// #region agent log — per-second RX window: LC3 bytes in, fingerprint, decoded PCM level (H-E)
|
|
855
|
+
micDbgLc3Window(rawLC3Data, sequenceNumber, pcmData)
|
|
856
|
+
// #endregion
|
|
853
857
|
// Re-encode to canonical LC3 via handlePcm (outside lock to avoid deadlock)
|
|
854
858
|
recordMicPcmProduced()
|
|
855
859
|
handlePcm(pcmData)
|
|
@@ -859,6 +863,72 @@ class DeviceManager {
|
|
|
859
863
|
}
|
|
860
864
|
}
|
|
861
865
|
|
|
866
|
+
// #region agent log — glasses LC3 RX diagnostics (debug session 828181)
|
|
867
|
+
private var micDbgWindowStart = 0L
|
|
868
|
+
private var micDbgPkts = 0
|
|
869
|
+
private var micDbgLc3Bytes = 0L
|
|
870
|
+
private var micDbgPcmBytes = 0L
|
|
871
|
+
private var micDbgSumAbs = 0L
|
|
872
|
+
private var micDbgSamples = 0L
|
|
873
|
+
private var micDbgPeak = 0
|
|
874
|
+
private var micDbgDcSum = 0L
|
|
875
|
+
private var micDbgDistinctFrames = HashSet<Int>()
|
|
876
|
+
private var micDbgFirstSeq = -1
|
|
877
|
+
private var micDbgLastSeq = -1
|
|
878
|
+
private var micDbgSeqGaps = 0
|
|
879
|
+
|
|
880
|
+
private fun micDbgLc3Window(lc3: ByteArray, seq: Int?, pcm: ByteArray) {
|
|
881
|
+
val now = System.currentTimeMillis()
|
|
882
|
+
if (micDbgWindowStart == 0L) micDbgWindowStart = now
|
|
883
|
+
micDbgPkts++
|
|
884
|
+
micDbgLc3Bytes += lc3.size
|
|
885
|
+
micDbgPcmBytes += pcm.size
|
|
886
|
+
var i = 0
|
|
887
|
+
while (i + 1 < pcm.size) {
|
|
888
|
+
val v = ((pcm[i + 1].toInt() shl 8) or (pcm[i].toInt() and 0xff)).toShort().toInt()
|
|
889
|
+
val a = if (v < 0) -v else v
|
|
890
|
+
micDbgSumAbs += a
|
|
891
|
+
micDbgDcSum += v
|
|
892
|
+
if (a > micDbgPeak) micDbgPeak = a
|
|
893
|
+
micDbgSamples++
|
|
894
|
+
i += 2
|
|
895
|
+
}
|
|
896
|
+
var off = 0
|
|
897
|
+
while (off + 40 <= lc3.size) {
|
|
898
|
+
var h = 17
|
|
899
|
+
for (k in off until off + 40) h = h * 31 + lc3[k]
|
|
900
|
+
micDbgDistinctFrames.add(h)
|
|
901
|
+
off += 40
|
|
902
|
+
}
|
|
903
|
+
if (seq != null) {
|
|
904
|
+
if (micDbgFirstSeq < 0) micDbgFirstSeq = seq
|
|
905
|
+
if (micDbgLastSeq >= 0 && ((micDbgLastSeq + 1) and 0xff) != seq) micDbgSeqGaps++
|
|
906
|
+
micDbgLastSeq = seq
|
|
907
|
+
}
|
|
908
|
+
if (now - micDbgWindowStart >= 1000) {
|
|
909
|
+
val meanAbs = if (micDbgSamples > 0) micDbgSumAbs / micDbgSamples else 0
|
|
910
|
+
val dc = if (micDbgSamples > 0) micDbgDcSum / micDbgSamples else 0
|
|
911
|
+
val head = lc3.take(8).joinToString("") { String.format("%02x", it.toInt() and 0xff) }
|
|
912
|
+
Bridge.log(
|
|
913
|
+
"MICDBG-RX pkts=$micDbgPkts lc3B=$micDbgLc3Bytes pcmB=$micDbgPcmBytes samples=$micDbgSamples " +
|
|
914
|
+
"meanAbs=$meanAbs peak=$micDbgPeak dc=$dc distinctLc3Frames=${micDbgDistinctFrames.size} " +
|
|
915
|
+
"seq=$micDbgFirstSeq..$micDbgLastSeq gaps=$micDbgSeqGaps frameSizeArg=40 lastLen=${lc3.size} head=$head"
|
|
916
|
+
)
|
|
917
|
+
micDbgWindowStart = now
|
|
918
|
+
micDbgPkts = 0
|
|
919
|
+
micDbgLc3Bytes = 0
|
|
920
|
+
micDbgPcmBytes = 0
|
|
921
|
+
micDbgSumAbs = 0
|
|
922
|
+
micDbgSamples = 0
|
|
923
|
+
micDbgPeak = 0
|
|
924
|
+
micDbgDcSum = 0
|
|
925
|
+
micDbgDistinctFrames = HashSet()
|
|
926
|
+
micDbgFirstSeq = -1
|
|
927
|
+
micDbgSeqGaps = 0
|
|
928
|
+
}
|
|
929
|
+
}
|
|
930
|
+
// #endregion
|
|
931
|
+
|
|
862
932
|
fun handlePcm(pcmData: ByteArray) {
|
|
863
933
|
// Audio always flows. The previous phone-side Silero VAD gate was a
|
|
864
934
|
// bandwidth-saver that ate transcripts when the mic delivered frames
|
|
@@ -909,11 +979,13 @@ class DeviceManager {
|
|
|
909
979
|
|
|
910
980
|
// allow the sgc to make changes to the micRanking:
|
|
911
981
|
micRanking = sgc?.sortMicRanking(micRanking) ?: micRanking
|
|
912
|
-
|
|
982
|
+
val pin = micSourcePin
|
|
983
|
+
val ranking: List<String> = MicSourcePin.selectionOrder(micRanking, pin)
|
|
984
|
+
Bridge.log("MAN: updateMicState() micRanking: $micRanking pin: $pin")
|
|
913
985
|
|
|
914
986
|
if (micEnabled) {
|
|
915
987
|
|
|
916
|
-
for (micMode in
|
|
988
|
+
for (micMode in ranking) {
|
|
917
989
|
if (micMode == MicTypes.PHONE_INTERNAL ||
|
|
918
990
|
micMode == MicTypes.BLUETOOTH_CLASSIC ||
|
|
919
991
|
micMode == MicTypes.BLUETOOTH
|
|
@@ -961,9 +1033,55 @@ class DeviceManager {
|
|
|
961
1033
|
|
|
962
1034
|
if (micUsed == "" && micEnabled) {
|
|
963
1035
|
Bridge.log("MAN: No available mic found!")
|
|
1036
|
+
if (pin == null) return
|
|
1037
|
+
// A pin taken while another microphone was already recording must still close it:
|
|
1038
|
+
// leaving it open would keep feeding PCM that the pinned consumer will reject, with
|
|
1039
|
+
// the phone's indicator lit for audio nobody uses.
|
|
1040
|
+
stopMicsExcept(micUsed)
|
|
1041
|
+
reportPinnedSourceUnavailable(pin)
|
|
964
1042
|
return
|
|
965
1043
|
}
|
|
966
1044
|
|
|
1045
|
+
stopMicsExcept(micUsed)
|
|
1046
|
+
}
|
|
1047
|
+
|
|
1048
|
+
/**
|
|
1049
|
+
* Call-scoped microphone source lock, or null for the normal ranking.
|
|
1050
|
+
*
|
|
1051
|
+
* Only [MicTypes.GLASSES_CUSTOM] is supported today: it exists so an ACS call can promise that
|
|
1052
|
+
* the wearer's own microphone — and nothing else — is what reaches the far end.
|
|
1053
|
+
*/
|
|
1054
|
+
@Volatile private var micSourcePin: String? = null
|
|
1055
|
+
|
|
1056
|
+
/**
|
|
1057
|
+
* Restrict microphone selection to one source for the duration of a call, or release it.
|
|
1058
|
+
*
|
|
1059
|
+
* Releasing re-runs selection so every other consumer (cloud LC3, miniapp `audio_chunk`,
|
|
1060
|
+
* on-device STT) gets the source its own preference asks for back. A `preferred_mic` change
|
|
1061
|
+
* made while the pin is held is stored but not applied until this releases.
|
|
1062
|
+
*/
|
|
1063
|
+
fun setMicSourcePin(source: String?) {
|
|
1064
|
+
val normalized = MicSourcePin.normalize(source)
|
|
1065
|
+
if (micSourcePin == normalized) return
|
|
1066
|
+
micSourcePin = normalized
|
|
1067
|
+
Bridge.log("MAN: setMicSourcePin($normalized)")
|
|
1068
|
+
updateMicState()
|
|
1069
|
+
}
|
|
1070
|
+
|
|
1071
|
+
/** The microphone the SDK is currently recording from, for consumers that must verify it. */
|
|
1072
|
+
fun activeMicSource(): String = currentMic
|
|
1073
|
+
|
|
1074
|
+
/**
|
|
1075
|
+
* Report a pinned source that cannot be opened. Emitted rather than silently fixed, because the
|
|
1076
|
+
* fix — opening a different microphone — is the thing the pin exists to forbid.
|
|
1077
|
+
*/
|
|
1078
|
+
private fun reportPinnedSourceUnavailable(pin: String) {
|
|
1079
|
+
Bridge.log("MAN: pinned mic source '$pin' is unavailable; no fallback will be started")
|
|
1080
|
+
val health = synchronized(micHealthLock) { micHealthSnapshotLocked() }
|
|
1081
|
+
Bridge.sendMicHealth(health, "pinned-source-unavailable")
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
private fun stopMicsExcept(micUsed: String) {
|
|
967
1085
|
// go through and disable all mics after the first used one:
|
|
968
1086
|
val allMics = micRanking
|
|
969
1087
|
// add any missing mics to the list:
|
|
@@ -3,9 +3,9 @@ package com.mentra.bluetoothsdk
|
|
|
3
3
|
/** Generated by release CI. Do not edit in a release checkout. */
|
|
4
4
|
internal object GeneratedReleaseMetadata {
|
|
5
5
|
const val FAMILY_BASE_VERSION: String = "3.2.0"
|
|
6
|
-
const val RELEASE_IDENTITY: String = "3.2.0-dev.
|
|
7
|
-
const val RELEASE_SET_ID: String = "mentra-3.2.0-dev.
|
|
8
|
-
const val SOURCE_COMMIT: String = "
|
|
9
|
-
const val OTA_MANIFEST_URL: String = "https://github.com/Mentra-Community/MentraOS/releases/download/mentra-builds-v3.2.0/mentra-live-ota-3.2.0-dev.
|
|
10
|
-
const val OTA_MANIFEST_SHA256: String = "
|
|
6
|
+
const val RELEASE_IDENTITY: String = "3.2.0-dev.222"
|
|
7
|
+
const val RELEASE_SET_ID: String = "mentra-3.2.0-dev.222"
|
|
8
|
+
const val SOURCE_COMMIT: String = "7865d7f192a2d9a732269857bb877a3a8a6bf4b8"
|
|
9
|
+
const val OTA_MANIFEST_URL: String = "https://github.com/Mentra-Community/MentraOS/releases/download/mentra-builds-v3.2.0/mentra-live-ota-3.2.0-dev.222.json"
|
|
10
|
+
const val OTA_MANIFEST_SHA256: String = "8c33e8beb5ba61eb0821d89b58d81fe2bbf05641d0a99b2e1972926863cc6ab7"
|
|
11
11
|
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
package com.mentra.bluetoothsdk
|
|
2
|
+
|
|
3
|
+
import android.os.SystemClock
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Bounds how many native log lines per second the module hands to the JavaScript runtime.
|
|
7
|
+
*
|
|
8
|
+
* Every event the module emits pins a JNI global reference until the JavaScript thread
|
|
9
|
+
* drains it, and the process-wide table holds about 51,200 references. Native code can log
|
|
10
|
+
* faster than a busy JavaScript thread drains, so an unbounded log stream aborts the process
|
|
11
|
+
* instead of merely lagging. Lines over the budget are withheld from JavaScript only; the
|
|
12
|
+
* native console still has every one of them. The first line forwarded after a shortfall
|
|
13
|
+
* reports how many were withheld, so the gap is visible in the JavaScript stream.
|
|
14
|
+
*/
|
|
15
|
+
internal class LogForwardingBudget(
|
|
16
|
+
val maxPerWindow: Int = DEFAULT_MAX_PER_SECOND,
|
|
17
|
+
private val windowMs: Long = 1_000L,
|
|
18
|
+
private val clock: () -> Long = SystemClock::elapsedRealtime,
|
|
19
|
+
) {
|
|
20
|
+
private var windowEndsAtMs = Long.MIN_VALUE
|
|
21
|
+
private var forwardedInWindow = 0
|
|
22
|
+
private var withheld = 0
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Returns null when the current line must be withheld, otherwise the number of lines
|
|
26
|
+
* withheld since the previous forwarded one (zero in steady state).
|
|
27
|
+
*/
|
|
28
|
+
@Synchronized
|
|
29
|
+
fun admit(): Int? {
|
|
30
|
+
val now = clock()
|
|
31
|
+
if (now >= windowEndsAtMs) {
|
|
32
|
+
windowEndsAtMs = now + windowMs
|
|
33
|
+
forwardedInWindow = 0
|
|
34
|
+
}
|
|
35
|
+
if (forwardedInWindow >= maxPerWindow) {
|
|
36
|
+
withheld++
|
|
37
|
+
return null
|
|
38
|
+
}
|
|
39
|
+
forwardedInWindow++
|
|
40
|
+
return withheld.also { withheld = 0 }
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
companion object {
|
|
44
|
+
/**
|
|
45
|
+
* Far above the SDK's steady-state diagnostic rate, and low enough that a
|
|
46
|
+
* JavaScript thread has to stay stalled for minutes before the table could fill.
|
|
47
|
+
*/
|
|
48
|
+
const val DEFAULT_MAX_PER_SECOND = 100
|
|
49
|
+
}
|
|
50
|
+
}
|
|
@@ -806,6 +806,18 @@ class MentraBluetoothSdk private constructor(
|
|
|
806
806
|
DeviceStore.apply(ObservableStore.BLUETOOTH_CATEGORY, "preferred_mic", preferredMic.value)
|
|
807
807
|
}
|
|
808
808
|
|
|
809
|
+
/**
|
|
810
|
+
* Lock microphone selection to one source until it is released with `null`.
|
|
811
|
+
*
|
|
812
|
+
* Distinct from [setPreferredMic]: a preference is a ranking the SDK may fall through, while a
|
|
813
|
+
* pin forbids the fallback. It exists for consumers that told a remote party which microphone
|
|
814
|
+
* they are hearing — an ACS call — where quietly substituting the phone microphone is worse
|
|
815
|
+
* than no audio at all. Only `"glasses"` is supported.
|
|
816
|
+
*/
|
|
817
|
+
fun setMicSourcePin(source: String?) {
|
|
818
|
+
deviceManager.setMicSourcePin(source)
|
|
819
|
+
}
|
|
820
|
+
|
|
809
821
|
fun setOwnAppAudioPlaying(playing: Boolean) {
|
|
810
822
|
PhoneAudioMonitor.getInstance(appContext).setOwnAppAudioPlaying(playing)
|
|
811
823
|
}
|
|
@@ -14,6 +14,15 @@ data class MicPcmEvent(
|
|
|
14
14
|
val channels: Int,
|
|
15
15
|
val encoding: String,
|
|
16
16
|
val voiceActivityDetectionEnabled: Boolean,
|
|
17
|
+
/**
|
|
18
|
+
* Which microphone produced this buffer, as [MicTypes]: `"glasses"`, `"phone"`, `"bluetooth"`.
|
|
19
|
+
*
|
|
20
|
+
* Carried per frame rather than inferred, because the SDK can move the source mid-stream when
|
|
21
|
+
* one becomes unavailable. A consumer that promised the far end a specific microphone — an ACS
|
|
22
|
+
* call, for instance — has no other way to tell that the audio it is forwarding is still the
|
|
23
|
+
* one it advertised. Empty when the SDK has not selected a microphone.
|
|
24
|
+
*/
|
|
25
|
+
val source: String,
|
|
17
26
|
) {
|
|
18
27
|
constructor(values: Map<String, Any>) : this(
|
|
19
28
|
pcm = values["pcm"] as? ByteArray ?: ByteArray(0),
|
|
@@ -24,6 +33,7 @@ data class MicPcmEvent(
|
|
|
24
33
|
voiceActivityDetectionEnabled =
|
|
25
34
|
boolValue(values, "voiceActivityDetectionEnabled")
|
|
26
35
|
?: BluetoothSdkDefaults.VOICE_ACTIVITY_DETECTION_ENABLED,
|
|
36
|
+
source = stringValue(values, "source") ?: "",
|
|
27
37
|
)
|
|
28
38
|
|
|
29
39
|
fun toMap(): Map<String, Any> =
|
|
@@ -35,6 +45,7 @@ data class MicPcmEvent(
|
|
|
35
45
|
"channels" to channels,
|
|
36
46
|
"encoding" to encoding,
|
|
37
47
|
"voiceActivityDetectionEnabled" to voiceActivityDetectionEnabled,
|
|
48
|
+
"source" to source,
|
|
38
49
|
)
|
|
39
50
|
|
|
40
51
|
companion object {
|
|
@@ -63,6 +63,34 @@ data class StreamAudioConfig @JvmOverloads constructor(
|
|
|
63
63
|
}
|
|
64
64
|
}
|
|
65
65
|
|
|
66
|
+
/** ICE overrides for a WHIP stream. Ignored by the RTMP and SRT paths. */
|
|
67
|
+
data class StreamIceConfig @JvmOverloads constructor(
|
|
68
|
+
/**
|
|
69
|
+
* STUN server the glasses use while gathering candidates.
|
|
70
|
+
*
|
|
71
|
+
* Null and empty are different answers, and both have to survive the BLE round trip. Null
|
|
72
|
+
* leaves the glasses on their default Cloudflare STUN server; an empty string is SoftAP
|
|
73
|
+
* calling's explicit request for host-only gathering, because the phone's WHIP server sits on
|
|
74
|
+
* the glasses' own hotspot where a reflexive candidate is meaningless and unreachable.
|
|
75
|
+
*/
|
|
76
|
+
val stun: String? = null,
|
|
77
|
+
) {
|
|
78
|
+
fun toMap(): Map<String, Any> =
|
|
79
|
+
buildMap {
|
|
80
|
+
// Not filtered on blankness, unlike the other optional string fields here: "" is the
|
|
81
|
+
// host-only signal, so dropping it would silently restore the default STUN server.
|
|
82
|
+
stun?.let { put("stun", it) }
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
companion object {
|
|
86
|
+
@JvmStatic
|
|
87
|
+
fun fromMap(values: Map<String, Any>?): StreamIceConfig? {
|
|
88
|
+
values ?: return null
|
|
89
|
+
return StreamIceConfig(stun = stringValue(values, "stun", "s"))
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
66
94
|
/** Effective video settings reported by the glasses after defaults and clamps. */
|
|
67
95
|
data class StreamResolvedVideoConfig @JvmOverloads constructor(
|
|
68
96
|
/** Encoded output width sent to the stream endpoint. */
|
|
@@ -214,6 +242,12 @@ data class StreamRequest @JvmOverloads constructor(
|
|
|
214
242
|
val audio: StreamAudioConfig? = null,
|
|
215
243
|
val authToken: String? = null,
|
|
216
244
|
val captureAudio: Boolean = true,
|
|
245
|
+
val ice: StreamIceConfig? = null,
|
|
246
|
+
/**
|
|
247
|
+
* Correlation id the glasses echo in every SOFTAP_TRACE line, so phone and glasses logs can be
|
|
248
|
+
* joined despite unsynchronised clocks.
|
|
249
|
+
*/
|
|
250
|
+
val traceId: String? = null,
|
|
217
251
|
) {
|
|
218
252
|
fun toMap(): Map<String, Any> =
|
|
219
253
|
buildMap {
|
|
@@ -225,6 +259,8 @@ data class StreamRequest @JvmOverloads constructor(
|
|
|
225
259
|
audio?.toMap()?.takeIf { it.isNotEmpty() }?.let { put("audio", it) }
|
|
226
260
|
authToken?.takeIf { it.isNotEmpty() }?.let { put("authToken", it) }
|
|
227
261
|
if (!captureAudio) put("captureAudio", false)
|
|
262
|
+
ice?.toMap()?.takeIf { it.isNotEmpty() }?.let { put("ice", it) }
|
|
263
|
+
traceId?.takeIf { it.isNotEmpty() }?.let { put("traceId", it) }
|
|
228
264
|
}
|
|
229
265
|
|
|
230
266
|
companion object {
|
|
@@ -240,6 +276,8 @@ data class StreamRequest @JvmOverloads constructor(
|
|
|
240
276
|
audio = StreamAudioConfig.fromMap(stringMapValue(values["audio"])),
|
|
241
277
|
authToken = values["authToken"] as? String ?: values["auth_token"] as? String,
|
|
242
278
|
captureAudio = boolValue(values, "captureAudio") ?: boolValue(values, "ca") ?: true,
|
|
279
|
+
ice = StreamIceConfig.fromMap(stringMapValue(values["ice"] ?: values["i"])),
|
|
280
|
+
traceId = stringValue(values, "traceId"),
|
|
243
281
|
)
|
|
244
282
|
}
|
|
245
283
|
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
package com.mentra.bluetoothsdk.utils
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Call-scoped microphone lock.
|
|
5
|
+
*
|
|
6
|
+
* The ranking in `DeviceManager` answers "which microphone can we get?", and for captions or the
|
|
7
|
+
* cloud uplink that is the right question — any working microphone beats none. A call is the case
|
|
8
|
+
* where it is the wrong question: the wearer agreed to be heard from the glasses, so falling
|
|
9
|
+
* through to the phone puts the room they are standing in onto a Teams call that still reports
|
|
10
|
+
* "glasses". A pin turns the fallback off for exactly as long as the call holds it.
|
|
11
|
+
*/
|
|
12
|
+
object MicSourcePin {
|
|
13
|
+
/**
|
|
14
|
+
* Validate a pin request. Only the glasses microphone can be pinned: every other source is one
|
|
15
|
+
* the ranking would have picked anyway, so pinning it buys nothing and only adds a way to strand
|
|
16
|
+
* the selection on a microphone no consumer asked for.
|
|
17
|
+
*/
|
|
18
|
+
fun normalize(source: String?): String? {
|
|
19
|
+
val trimmed = source?.trim()?.takeIf { it.isNotEmpty() } ?: return null
|
|
20
|
+
require(trimmed == MicTypes.GLASSES_CUSTOM) { "unsupported mic source pin: $trimmed" }
|
|
21
|
+
return trimmed
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* The sources selection may consider. A pin replaces the ranking outright rather than reordering
|
|
26
|
+
* it, so "the pinned source is unavailable" resolves to no microphone instead of the next best
|
|
27
|
+
* one.
|
|
28
|
+
*/
|
|
29
|
+
fun selectionOrder(ranking: List<String>, pin: String?): List<String> =
|
|
30
|
+
if (pin == null) ranking else listOf(pin)
|
|
31
|
+
}
|
|
@@ -29,6 +29,7 @@ class PcmStreamPlayer(
|
|
|
29
29
|
private val sampleRate: Int,
|
|
30
30
|
channels: Int,
|
|
31
31
|
volume: Float,
|
|
32
|
+
jitterMs: Int = TRACK_BUFFER_MS,
|
|
32
33
|
) {
|
|
33
34
|
companion object {
|
|
34
35
|
private const val TAG = "PcmStreamPlayer"
|
|
@@ -41,8 +42,28 @@ class PcmStreamPlayer(
|
|
|
41
42
|
|
|
42
43
|
/** AudioTrack buffer: max(4x minimum, 500ms) of jitter headroom. */
|
|
43
44
|
private const val TRACK_BUFFER_MS = 500
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Floor on the requested headroom.
|
|
48
|
+
*
|
|
49
|
+
* A MODE_STREAM track fills to its buffer size and stays there, so this value is also the
|
|
50
|
+
* steady-state output latency, not just a jitter cushion. Callers that want a conversation
|
|
51
|
+
* to feel live ask for a small number; below this the feeder cannot outrun playback and the
|
|
52
|
+
* track underruns into audible chop, which is worse than the delay it buys back.
|
|
53
|
+
*/
|
|
54
|
+
private const val MIN_TRACK_BUFFER_MS = 80
|
|
44
55
|
}
|
|
45
56
|
|
|
57
|
+
/**
|
|
58
|
+
* Playout headroom, and therefore this stream's floor latency.
|
|
59
|
+
*
|
|
60
|
+
* [TRACK_BUFFER_MS] suits one-way media (a miniapp `speaker.createStream()` clip), where half a
|
|
61
|
+
* second of cushion costs nothing audible. A realtime call is the opposite trade: that same
|
|
62
|
+
* cushion is half a second of delay before the far end's voice reaches the wearer, so the ACS
|
|
63
|
+
* meeting path asks for a much smaller one.
|
|
64
|
+
*/
|
|
65
|
+
private val jitterBufferMs = jitterMs.coerceIn(MIN_TRACK_BUFFER_MS, TRACK_BUFFER_MS)
|
|
66
|
+
|
|
46
67
|
private val bytesPerFrame = 2 * channels
|
|
47
68
|
private val channelMask =
|
|
48
69
|
if (channels >= 2) AudioFormat.CHANNEL_OUT_STEREO else AudioFormat.CHANNEL_OUT_MONO
|
|
@@ -70,7 +91,7 @@ class PcmStreamPlayer(
|
|
|
70
91
|
init {
|
|
71
92
|
val minBuf =
|
|
72
93
|
AudioTrack.getMinBufferSize(sampleRate, channelMask, AudioFormat.ENCODING_PCM_16BIT)
|
|
73
|
-
val jitterBuf = sampleRate * bytesPerFrame *
|
|
94
|
+
val jitterBuf = sampleRate * bytesPerFrame * jitterBufferMs / 1000
|
|
74
95
|
audioTrack =
|
|
75
96
|
AudioTrack.Builder()
|
|
76
97
|
.setAudioAttributes(
|
|
@@ -86,7 +107,11 @@ class PcmStreamPlayer(
|
|
|
86
107
|
.setChannelMask(channelMask)
|
|
87
108
|
.build()
|
|
88
109
|
)
|
|
89
|
-
|
|
110
|
+
// minBuf is AudioTrack's own floor, so it wins over a smaller request; the
|
|
111
|
+
// requested headroom is otherwise authoritative. Any fixed multiple of
|
|
112
|
+
// minBuf here would silently override a caller asking for low latency,
|
|
113
|
+
// since minBuf scales with the rate while the request does not.
|
|
114
|
+
.setBufferSizeInBytes(max(minBuf, jitterBuf))
|
|
90
115
|
.setTransferMode(AudioTrack.MODE_STREAM)
|
|
91
116
|
.build()
|
|
92
117
|
if (audioTrack.state != AudioTrack.STATE_INITIALIZED) {
|
|
@@ -101,7 +126,11 @@ class PcmStreamPlayer(
|
|
|
101
126
|
isDaemon = true
|
|
102
127
|
start()
|
|
103
128
|
}
|
|
104
|
-
Log.d(
|
|
129
|
+
Log.d(
|
|
130
|
+
TAG,
|
|
131
|
+
"[$streamId] opened: rate=$sampleRate ch=$channels vol=$volume " +
|
|
132
|
+
"jitterMs=$jitterBufferMs trackBytes=${max(minBuf, jitterBuf)}",
|
|
133
|
+
)
|
|
105
134
|
}
|
|
106
135
|
|
|
107
136
|
/** Milliseconds of audio accepted but not yet played out. */
|
|
@@ -285,7 +314,7 @@ class PcmStreamPlayer(
|
|
|
285
314
|
object PcmStreamManager {
|
|
286
315
|
private val players = java.util.concurrent.ConcurrentHashMap<String, PcmStreamPlayer>()
|
|
287
316
|
|
|
288
|
-
fun open(streamId: String, sampleRate: Int, channels: Int, volume: Float) {
|
|
317
|
+
fun open(streamId: String, sampleRate: Int, channels: Int, volume: Float, jitterMs: Int? = null) {
|
|
289
318
|
require(streamId.isNotBlank()) { "streamId is required" }
|
|
290
319
|
require(sampleRate == 16000 || sampleRate == 24000 || sampleRate == 48000) {
|
|
291
320
|
"unsupported PCM sample rate $sampleRate"
|
|
@@ -293,7 +322,12 @@ object PcmStreamManager {
|
|
|
293
322
|
require(channels == 1) { "only mono PCM is supported" }
|
|
294
323
|
// Replacing an id is a caller bug, but never leak the old track.
|
|
295
324
|
players.remove(streamId)?.abort()
|
|
296
|
-
players[streamId] =
|
|
325
|
+
players[streamId] =
|
|
326
|
+
if (jitterMs == null) {
|
|
327
|
+
PcmStreamPlayer(streamId, sampleRate, channels, volume)
|
|
328
|
+
} else {
|
|
329
|
+
PcmStreamPlayer(streamId, sampleRate, channels, volume, jitterMs)
|
|
330
|
+
}
|
|
297
331
|
}
|
|
298
332
|
|
|
299
333
|
fun write(streamId: String, base64: String): Long {
|