@mentra/acs-meeting 3.2.0-dev.217 → 3.2.0-dev.222
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/android/build.gradle +23 -0
- package/android/src/androidTest/java/com/mentra/acsmeeting/network/SoftApFeasibilityGateTest.kt +291 -0
- package/android/src/main/AndroidManifest.xml +2 -0
- package/android/src/main/java/com/mentra/acsmeeting/AbandonedCallAgent.kt +36 -0
- package/android/src/main/java/com/mentra/acsmeeting/AcsMeetingModule.kt +392 -22
- package/android/src/main/java/com/mentra/acsmeeting/AcsMeetingSession.kt +909 -56
- package/android/src/main/java/com/mentra/acsmeeting/CallCapabilities.kt +33 -0
- package/android/src/main/java/com/mentra/acsmeeting/audio/AcsAudioPolicy.kt +21 -0
- package/android/src/main/java/com/mentra/acsmeeting/audio/AudioUplinkChain.kt +126 -0
- package/android/src/main/java/com/mentra/acsmeeting/audio/PcmBridge.kt +27 -4
- package/android/src/main/java/com/mentra/acsmeeting/audio/UplinkPacer.kt +45 -10
- package/android/src/main/java/com/mentra/acsmeeting/audio/UplinkSender.kt +110 -5
- package/android/src/main/java/com/mentra/acsmeeting/network/InternetHold.kt +501 -0
- package/android/src/main/java/com/mentra/acsmeeting/network/Ipv4Prefix.kt +51 -0
- package/android/src/main/java/com/mentra/acsmeeting/network/NetworkFacts.kt +81 -0
- package/android/src/main/java/com/mentra/acsmeeting/network/ScopedNetworkChangeDetector.kt +355 -0
- package/android/src/main/java/com/mentra/acsmeeting/network/ScopedNetworkError.kt +109 -0
- package/android/src/main/java/com/mentra/acsmeeting/network/ScopedNetworkObserver.kt +160 -0
- package/android/src/main/java/com/mentra/acsmeeting/network/ScopedNetworkReadiness.kt +103 -0
- package/android/src/main/java/com/mentra/acsmeeting/network/ScopedNetworkRequestSpec.kt +53 -0
- package/android/src/main/java/com/mentra/acsmeeting/network/ScopedNetworkState.kt +127 -0
- package/android/src/main/java/com/mentra/acsmeeting/network/ScopedSoftApNetwork.kt +573 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/CloudflareWhepSource.kt +10 -101
- package/android/src/main/java/com/mentra/acsmeeting/source/DecodedTrackRelay.kt +147 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/GlassesMediaSource.kt +47 -5
- package/android/src/main/java/com/mentra/acsmeeting/source/LocalWhipIngestSource.kt +772 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/MeetingVideoSourceSpec.kt +73 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/SdpAdapter.kt +17 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/SelectedIcePair.kt +139 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/SoftApIceDiagnosis.kt +144 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/SoftApIcePolicy.kt +46 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/SoftApSdpGuard.kt +160 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/TrackRegistry.kt +3 -1
- package/android/src/main/java/com/mentra/acsmeeting/source/VideoSourceArm.kt +69 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/WhipIngestProtocol.kt +207 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/WhipIngestServer.kt +356 -0
- package/android/src/main/java/com/mentra/acsmeeting/telemetry/AvSyncProbe.kt +155 -0
- package/android/src/main/java/com/mentra/acsmeeting/telemetry/PipelineStats.kt +94 -3
- package/android/src/main/java/com/mentra/acsmeeting/telemetry/PipelineTicker.kt +33 -5
- package/android/src/main/java/com/mentra/acsmeeting/telemetry/RingPercentile.kt +15 -0
- package/android/src/main/java/com/mentra/acsmeeting/telemetry/VideoQuality.kt +178 -0
- package/android/src/main/java/com/mentra/acsmeeting/telemetry/VideoRateVerdict.kt +145 -0
- package/android/src/main/java/com/mentra/acsmeeting/trace/SoftApTrace.kt +143 -0
- package/android/src/main/java/com/mentra/acsmeeting/video/AcsFrameSender.kt +23 -4
- package/android/src/main/java/com/mentra/acsmeeting/video/FramePacer.kt +70 -0
- package/android/src/main/java/com/mentra/acsmeeting/video/I420FormatSpec.kt +1 -1
- package/android/src/main/java/com/mentra/acsmeeting/video/VideoProfile.kt +36 -3
- package/android/src/test/java/com/mentra/acsmeeting/AbandonedCallAgentTest.kt +74 -0
- package/android/src/test/java/com/mentra/acsmeeting/CallCapabilitiesTest.kt +47 -0
- package/android/src/test/java/com/mentra/acsmeeting/audio/AudioUplinkChainTest.kt +250 -0
- package/android/src/test/java/com/mentra/acsmeeting/audio/CapturePolicyTest.kt +44 -0
- package/android/src/test/java/com/mentra/acsmeeting/audio/UplinkPacerTest.kt +34 -4
- package/android/src/test/java/com/mentra/acsmeeting/audio/UplinkSenderTest.kt +96 -0
- package/android/src/test/java/com/mentra/acsmeeting/network/InternetHoldLifecycleTest.kt +102 -0
- package/android/src/test/java/com/mentra/acsmeeting/network/InternetHoldTest.kt +148 -0
- package/android/src/test/java/com/mentra/acsmeeting/network/Ipv4PrefixTest.kt +59 -0
- package/android/src/test/java/com/mentra/acsmeeting/network/NetworkFactsTest.kt +59 -0
- package/android/src/test/java/com/mentra/acsmeeting/network/ScopedLocalNetworkPermissionTest.kt +35 -0
- package/android/src/test/java/com/mentra/acsmeeting/network/ScopedNetworkChangeDetectorTest.kt +165 -0
- package/android/src/test/java/com/mentra/acsmeeting/network/ScopedNetworkErrorTest.kt +96 -0
- package/android/src/test/java/com/mentra/acsmeeting/network/ScopedNetworkLifecycleTest.kt +187 -0
- package/android/src/test/java/com/mentra/acsmeeting/network/ScopedNetworkObserverTest.kt +172 -0
- package/android/src/test/java/com/mentra/acsmeeting/network/ScopedNetworkReadinessTest.kt +141 -0
- package/android/src/test/java/com/mentra/acsmeeting/network/ScopedNetworkRequestSpecTest.kt +69 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/AcsInvestigationTest.kt +40 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/DecodedTrackRelayTest.kt +41 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/GlassesMediaSourceTest.kt +26 -8
- package/android/src/test/java/com/mentra/acsmeeting/source/MeetingVideoSourceSpecTest.kt +109 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/SelectedIcePairTest.kt +165 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/SoftApIceDiagnosisTest.kt +148 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/SoftApIcePolicyTest.kt +76 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/SoftApSdpGuardTest.kt +275 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/TrackRegistryTest.kt +9 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/WhipIngestProtocolTest.kt +356 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/WhipIngestServerLoopbackTest.kt +411 -0
- package/android/src/test/java/com/mentra/acsmeeting/telemetry/AvSyncProbeTest.kt +83 -0
- package/android/src/test/java/com/mentra/acsmeeting/telemetry/PipelineStatsTest.kt +69 -3
- package/android/src/test/java/com/mentra/acsmeeting/telemetry/VideoQualityTest.kt +182 -0
- package/android/src/test/java/com/mentra/acsmeeting/telemetry/VideoRateVerdictTest.kt +149 -0
- package/android/src/test/java/com/mentra/acsmeeting/trace/SoftApTraceTest.kt +97 -0
- package/android/src/test/java/com/mentra/acsmeeting/video/AcsTimestampTest.kt +34 -0
- package/android/src/test/java/com/mentra/acsmeeting/video/FramePacerTest.kt +105 -0
- package/android/src/test/java/com/mentra/acsmeeting/video/I420FormatSpecTest.kt +5 -0
- package/android/src/test/java/com/mentra/acsmeeting/video/VideoProfileTest.kt +38 -3
- package/build/AcsMeeting.types.d.ts +5 -0
- package/build/AcsMeeting.types.d.ts.map +1 -1
- package/build/AcsMeeting.types.js.map +1 -1
- package/build/AcsMeetingModule.d.ts +23 -0
- package/build/AcsMeetingModule.d.ts.map +1 -1
- package/build/AcsMeetingModule.js.map +1 -1
- package/build/AcsMeetingModule.web.d.ts +15 -0
- package/build/AcsMeetingModule.web.d.ts.map +1 -1
- package/build/AcsMeetingModule.web.js +13 -0
- package/build/AcsMeetingModule.web.js.map +1 -1
- package/ios/AcsMeetingModule.swift +213 -2
- package/package.json +1 -1
- package/src/AcsMeeting.types.ts +5 -0
- package/src/AcsMeetingModule.ts +13 -0
- package/src/AcsMeetingModule.web.ts +13 -0
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
package com.mentra.acsmeeting
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* One ACS participant capability, as reported to the host.
|
|
5
|
+
*
|
|
6
|
+
* [allowed] is nullable on purpose. "Denied" and "not known yet" are different facts and the UI has
|
|
7
|
+
* to treat them differently: a denied End must be hidden, while an unknown one is what every call
|
|
8
|
+
* looks like for the moment between joining and the first capabilities event.
|
|
9
|
+
*/
|
|
10
|
+
data class CapabilityStatus(val allowed: Boolean? = null, val reason: String? = null) {
|
|
11
|
+
fun toMap(): Map<String, Any?> = mapOf("allowed" to allowed, "reason" to reason)
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Whether an End-for-everyone attempt should be made at all.
|
|
16
|
+
*
|
|
17
|
+
* Teams only lets a presenter end a meeting for everyone, and ACS surfaces that as the runtime
|
|
18
|
+
* capability `HANG_UP_FOR_EVERYONE`. Refusing a known-denied End locally is better than sending it:
|
|
19
|
+
* ACS rejects it with an opaque error, and the wearer would have watched a confirm sheet for
|
|
20
|
+
* nothing. An *unknown* capability is not a refusal — it is the ordinary state before the first
|
|
21
|
+
* capabilities event lands, and letting the service decide is more honest than guessing.
|
|
22
|
+
*/
|
|
23
|
+
object EndForEveryonePolicy {
|
|
24
|
+
|
|
25
|
+
const val REFUSED_PREFIX = "hang_up_for_everyone_not_allowed"
|
|
26
|
+
|
|
27
|
+
/** The refusal to report, or null when the End should be attempted. */
|
|
28
|
+
fun refusalFor(status: CapabilityStatus): String? {
|
|
29
|
+
if (status.allowed != false) return null
|
|
30
|
+
val reason = status.reason?.takeIf { it.isNotBlank() } ?: "unknown"
|
|
31
|
+
return "$REFUSED_PREFIX:$reason"
|
|
32
|
+
}
|
|
33
|
+
}
|
|
@@ -82,6 +82,27 @@ object CapturePolicy {
|
|
|
82
82
|
fun captureGlassesMic(source: AudioSourceKind): Boolean = source == AudioSourceKind.GLASSES
|
|
83
83
|
}
|
|
84
84
|
|
|
85
|
+
/**
|
|
86
|
+
* Which of the two possible sources of the wearer's voice this call listens to.
|
|
87
|
+
*
|
|
88
|
+
* SoftAP takes BLE LC3 from the host ([AcsMeetingSession.pushOutgoingPcm]) and ignores the WHIP
|
|
89
|
+
* track, which the glasses publish video-only (`captureAudio=false`). WHEP has no such flag: the
|
|
90
|
+
* wearer's voice is already mixed into the subscribed track, so that call takes the relay.
|
|
91
|
+
* Exactly one source is ever on.
|
|
92
|
+
*
|
|
93
|
+
* A previous revision routed SoftAP to the relay too, on a soak that read LC3 `meanAbs` ~15 as
|
|
94
|
+
* "analog-silent". Bench measurement of the same path puts the quiet-room floor at ~30–60, and
|
|
95
|
+
* that soak had already published with `captureAudio=false`, so neither the BES capture nor the
|
|
96
|
+
* SoC microphone was at fault. The JS gate `SOFTAP_BLE_LC3_UPLINK` and this routing move together.
|
|
97
|
+
*/
|
|
98
|
+
data class GlassesPcmRouting(val relayPcm: Boolean, val externalPcm: Boolean) {
|
|
99
|
+
companion object {
|
|
100
|
+
fun decide(softap: Boolean, enabled: Boolean): GlassesPcmRouting =
|
|
101
|
+
if (softap) GlassesPcmRouting(relayPcm = false, externalPcm = enabled)
|
|
102
|
+
else GlassesPcmRouting(relayPcm = enabled, externalPcm = false)
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
85
106
|
/** Null ACS call handles must fail, never report success via `call?.mute(); Unit`. */
|
|
86
107
|
object CallGuard {
|
|
87
108
|
fun <T : Any> require(value: T?): Result<T> =
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
package com.mentra.acsmeeting.audio
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Everything between "a microphone buffer arrived" and "the pacer has it", behind one lock.
|
|
5
|
+
*
|
|
6
|
+
* The lock is the point. Muting an audio path is not one flag: the wearer's voice is spread across
|
|
7
|
+
* a delay ring, a resampler's filter history, a partial 20 ms frame, and the pacer's ring, and
|
|
8
|
+
* clearing them one at a time leaves whichever stage the producer thread happened to be writing to.
|
|
9
|
+
* Serializing ingest against mute is what makes "mute" mean the same thing at every stage.
|
|
10
|
+
*
|
|
11
|
+
* It cannot make mute instantaneous at the far end. Buffers already handed to ACS cannot be
|
|
12
|
+
* recalled, so the residual tail is bounded by [UplinkSender.MAX_IN_FLIGHT] frames and measured
|
|
13
|
+
* rather than assumed — see the sender's `muteTailMs`.
|
|
14
|
+
*
|
|
15
|
+
* @param bridge downmix/resample to the 48 kHz mono 20 ms frames ACS wants
|
|
16
|
+
* @param pacer the clock-domain adapter the sender drains
|
|
17
|
+
* @param delayMs how long to hold input before the bridge, for A/V alignment (0 = passthrough)
|
|
18
|
+
* @param onIngest raw input plus arrival time, before the delay line, for A/V clap measurement
|
|
19
|
+
*/
|
|
20
|
+
class AudioUplinkChain(
|
|
21
|
+
private val bridge: PcmBridge,
|
|
22
|
+
private val pacer: UplinkPacer,
|
|
23
|
+
private val delayMs: Int = 0,
|
|
24
|
+
private val clock: () -> Long = System::nanoTime,
|
|
25
|
+
private val onIngest: ((pcm: ByteArray, nowNs: Long) -> Unit)? = null,
|
|
26
|
+
) {
|
|
27
|
+
private val lock = Any()
|
|
28
|
+
private val delay = DelayLine(delayMs.coerceIn(0, MAX_DELAY_MS))
|
|
29
|
+
@Volatile private var muted = false
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Accept one microphone buffer. A no-op while muted, so muted audio never enters any stage and
|
|
33
|
+
* an unmute cannot resurrect words the wearer said while muted.
|
|
34
|
+
*/
|
|
35
|
+
fun ingest(pcm: ByteArray, sampleRate: Int, channels: Int) {
|
|
36
|
+
synchronized(lock) {
|
|
37
|
+
if (muted) return
|
|
38
|
+
val now = clock()
|
|
39
|
+
onIngest?.invoke(pcm, now)
|
|
40
|
+
for (chunk in delay.push(pcm, now)) {
|
|
41
|
+
for (frame in bridge.ingest(chunk, sampleRate, channels)) pacer.push(frame)
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** Stop accepting audio and drop everything already buffered, atomically against [ingest]. */
|
|
47
|
+
fun mute() {
|
|
48
|
+
synchronized(lock) {
|
|
49
|
+
muted = true
|
|
50
|
+
resetLocked()
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** Accept audio again. The pacer re-prerolls, so Teams hears voice one preroll later. */
|
|
55
|
+
fun unmute() {
|
|
56
|
+
synchronized(lock) { muted = false }
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
fun isMuted(): Boolean = muted
|
|
60
|
+
|
|
61
|
+
/** Drop buffered audio without changing the mute state. For leave and source rebuilds. */
|
|
62
|
+
fun reset() {
|
|
63
|
+
synchronized(lock) { resetLocked() }
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** Buffered input held for A/V alignment, in bytes. Diagnostics and tests. */
|
|
67
|
+
fun delayedBytes(): Int = synchronized(lock) { delay.heldBytes() }
|
|
68
|
+
|
|
69
|
+
private fun resetLocked() {
|
|
70
|
+
delay.reset()
|
|
71
|
+
bridge.reset()
|
|
72
|
+
pacer.reset()
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Holds input until it is [delayMs] old.
|
|
77
|
+
*
|
|
78
|
+
* In front of the resampler rather than behind it so the delay is expressed in the input's own
|
|
79
|
+
* arrival times, which is what a receiver-side A/V measurement calibrates against.
|
|
80
|
+
*/
|
|
81
|
+
private class DelayLine(private val delayMs: Int) {
|
|
82
|
+
private class Chunk(val bytes: ByteArray, val atNanos: Long)
|
|
83
|
+
|
|
84
|
+
private val queue = ArrayDeque<Chunk>()
|
|
85
|
+
private var bytesHeld = 0
|
|
86
|
+
|
|
87
|
+
fun push(pcm: ByteArray, nowNanos: Long): List<ByteArray> {
|
|
88
|
+
if (delayMs <= 0) return listOf(pcm)
|
|
89
|
+
queue.addLast(Chunk(pcm, nowNanos))
|
|
90
|
+
bytesHeld += pcm.size
|
|
91
|
+
val threshold = delayMs * 1_000_000L
|
|
92
|
+
val released = mutableListOf<ByteArray>()
|
|
93
|
+
while (queue.isNotEmpty()) {
|
|
94
|
+
val head = queue.first()
|
|
95
|
+
if (nowNanos - head.atNanos < threshold) break
|
|
96
|
+
queue.removeFirst()
|
|
97
|
+
bytesHeld -= head.bytes.size
|
|
98
|
+
released.add(head.bytes)
|
|
99
|
+
}
|
|
100
|
+
// Age alone bounds this in normal operation. The cap is for the case it cannot see: a clock
|
|
101
|
+
// that does not advance would otherwise turn a held buffer into an unbounded one.
|
|
102
|
+
val cap = delayMs * MAX_BYTES_PER_MS
|
|
103
|
+
while (bytesHeld > cap && queue.isNotEmpty()) {
|
|
104
|
+
val dropped = queue.removeFirst()
|
|
105
|
+
bytesHeld -= dropped.bytes.size
|
|
106
|
+
released.add(dropped.bytes)
|
|
107
|
+
}
|
|
108
|
+
return released
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
fun reset() {
|
|
112
|
+
queue.clear()
|
|
113
|
+
bytesHeld = 0
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
fun heldBytes(): Int = bytesHeld
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
companion object {
|
|
120
|
+
/** Half a second is already unusable as conversation; past that it is a bug, not a setting. */
|
|
121
|
+
const val MAX_DELAY_MS = 500
|
|
122
|
+
|
|
123
|
+
/** 48 kHz stereo PCM16, the widest input the bridge accepts. Sizes the safety cap only. */
|
|
124
|
+
internal const val MAX_BYTES_PER_MS = 48_000 * 2 * 2 / 1000
|
|
125
|
+
}
|
|
126
|
+
}
|
|
@@ -20,6 +20,8 @@ class PcmBridge(
|
|
|
20
20
|
private val outgoing = ByteArrayOutputStream()
|
|
21
21
|
private val resampler = PcmResampler(TARGET_RATE)
|
|
22
22
|
private var lastRmsLogMs = 0L
|
|
23
|
+
@Volatile var lastMeanAbs: Int = -1
|
|
24
|
+
private set
|
|
23
25
|
|
|
24
26
|
fun ingest(pcm16Le: ByteArray, sampleRate: Int, channels: Int): List<ByteArray> {
|
|
25
27
|
if (dump != null && !dumped) {
|
|
@@ -47,6 +49,19 @@ class PcmBridge(
|
|
|
47
49
|
return frames
|
|
48
50
|
}
|
|
49
51
|
|
|
52
|
+
/**
|
|
53
|
+
* Drop the state that spans calls to [ingest]: the resampler's filter history and fractional
|
|
54
|
+
* phase, and the partial 20 ms frame not yet emitted.
|
|
55
|
+
*
|
|
56
|
+
* What makes this necessary is that both are, by design, audio the wearer already spoke. A mute
|
|
57
|
+
* that left them behind would emit them on the next unmute — a fraction of a word, from before
|
|
58
|
+
* the wearer asked not to be heard. Serialization is the caller's job; see [AudioUplinkChain].
|
|
59
|
+
*/
|
|
60
|
+
fun reset() {
|
|
61
|
+
resampler.reset()
|
|
62
|
+
outgoing.reset()
|
|
63
|
+
}
|
|
64
|
+
|
|
50
65
|
@Synchronized
|
|
51
66
|
fun finishDump() {
|
|
52
67
|
if (dumped) return
|
|
@@ -60,9 +75,6 @@ class PcmBridge(
|
|
|
60
75
|
}
|
|
61
76
|
|
|
62
77
|
private fun logLevel(pcm: ByteArray, sampleRate: Int, channels: Int) {
|
|
63
|
-
val now = System.currentTimeMillis()
|
|
64
|
-
if (now - lastRmsLogMs < 1000) return
|
|
65
|
-
lastRmsLogMs = now
|
|
66
78
|
var acc = 0L
|
|
67
79
|
var n = 0
|
|
68
80
|
var i = 0
|
|
@@ -74,7 +86,12 @@ class PcmBridge(
|
|
|
74
86
|
i += 2
|
|
75
87
|
}
|
|
76
88
|
val mean = if (n == 0) 0 else acc / n
|
|
77
|
-
|
|
89
|
+
lastMeanAbs = mean.toInt()
|
|
90
|
+
val now = System.currentTimeMillis()
|
|
91
|
+
if (now - lastRmsLogMs < 1000) return
|
|
92
|
+
lastRmsLogMs = now
|
|
93
|
+
val verdict = if (mean < SILENCE_MEAN_ABS) "SILENCE" else "voice"
|
|
94
|
+
Log.i(TAG, "P4 pcm rate=$sampleRate ch=$channels meanAbs=$mean bytes=${pcm.size} $verdict")
|
|
78
95
|
}
|
|
79
96
|
|
|
80
97
|
companion object {
|
|
@@ -104,5 +121,11 @@ class PcmBridge(
|
|
|
104
121
|
}
|
|
105
122
|
|
|
106
123
|
fun encodeBase64(pcm: ByteArray): String = Base64.encodeToString(pcm, Base64.NO_WRAP)
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* 16-bit speech sits in the hundreds to thousands. Below this is the LC3/ADC noise floor
|
|
127
|
+
* Teams hears as mute even when ACS is unmuted and the pacer is RUNNING.
|
|
128
|
+
*/
|
|
129
|
+
const val SILENCE_MEAN_ABS = 50
|
|
107
130
|
}
|
|
108
131
|
}
|
|
@@ -13,7 +13,8 @@ import kotlin.math.min
|
|
|
13
13
|
*
|
|
14
14
|
* - sustained depth above [HIGH_MS] (producer fast) discards a [CORRECTION_MS] slice
|
|
15
15
|
* - sustained depth below [LOW_MS] (producer slow) inserts a [CORRECTION_MS] slice of silence
|
|
16
|
-
* - depth above [EMERGENCY_CAP_MS]
|
|
16
|
+
* - depth above [EMERGENCY_CAP_MS] shears only the peak above the cap — not back to target.
|
|
17
|
+
* Shearing to target was a 150 ms hole every time the SoftAP delay line released a burst.
|
|
17
18
|
*
|
|
18
19
|
* Corrections read an EMA over roughly two seconds and are rate limited to one
|
|
19
20
|
* per [CORRECTION_INTERVAL_MS], so ordinary jitter never triggers one; only
|
|
@@ -28,7 +29,10 @@ import kotlin.math.min
|
|
|
28
29
|
* [push] runs on the WebRTC audio thread and [tick] on the pacing thread, so
|
|
29
30
|
* every entry point is synchronized on this instance.
|
|
30
31
|
*/
|
|
31
|
-
class UplinkPacer(
|
|
32
|
+
class UplinkPacer(
|
|
33
|
+
private var targetMs: Int = TARGET_MS,
|
|
34
|
+
private val log: ((String) -> Unit)? = null,
|
|
35
|
+
) {
|
|
32
36
|
enum class State { PREROLLING, RUNNING, STARVED }
|
|
33
37
|
|
|
34
38
|
/** One [FRAME_MS] frame. [silence] marks a frame the pacer manufactured. */
|
|
@@ -72,6 +76,15 @@ class UplinkPacer(private val targetMs: Int = TARGET_MS) {
|
|
|
72
76
|
private var driftCorrections = 0L
|
|
73
77
|
private var driftDroppedBytes = 0L
|
|
74
78
|
private var driftInsertedBytes = 0L
|
|
79
|
+
private var lastUnderrunLogNanos = 0L
|
|
80
|
+
|
|
81
|
+
/** Select headroom for the producer before starting a call; discard the previous call's PCM. */
|
|
82
|
+
@Synchronized
|
|
83
|
+
fun configureTarget(ms: Int) {
|
|
84
|
+
require(ms in FRAME_MS..EMERGENCY_CAP_MS)
|
|
85
|
+
targetMs = ms
|
|
86
|
+
reset()
|
|
87
|
+
}
|
|
75
88
|
|
|
76
89
|
/** Appends 48 kHz mono PCM16 from the producer thread. */
|
|
77
90
|
@Synchronized
|
|
@@ -83,12 +96,12 @@ class UplinkPacer(private val targetMs: Int = TARGET_MS) {
|
|
|
83
96
|
// A single burst larger than the whole ring: keep only the newest audio.
|
|
84
97
|
offset = len - CAPACITY_BYTES
|
|
85
98
|
len = CAPACITY_BYTES
|
|
86
|
-
|
|
99
|
+
noteDrop("overflow-push", offset)
|
|
87
100
|
}
|
|
88
101
|
val overflow = used + len - CAPACITY_BYTES
|
|
89
102
|
if (overflow > 0) {
|
|
90
103
|
dropOldest(overflow)
|
|
91
|
-
|
|
104
|
+
noteDrop("overflow-push", overflow)
|
|
92
105
|
}
|
|
93
106
|
write(pcm16Le, offset, len)
|
|
94
107
|
}
|
|
@@ -164,6 +177,7 @@ class UplinkPacer(private val targetMs: Int = TARGET_MS) {
|
|
|
164
177
|
driftCorrections = 0
|
|
165
178
|
driftDroppedBytes = 0
|
|
166
179
|
driftInsertedBytes = 0
|
|
180
|
+
lastUnderrunLogNanos = 0
|
|
167
181
|
}
|
|
168
182
|
|
|
169
183
|
@Synchronized
|
|
@@ -201,9 +215,14 @@ class UplinkPacer(private val targetMs: Int = TARGET_MS) {
|
|
|
201
215
|
val since = starvingSinceNanos
|
|
202
216
|
if (since == null) {
|
|
203
217
|
starvingSinceNanos = nowNanos
|
|
204
|
-
|
|
218
|
+
if (lastUnderrunLogNanos == 0L || nowNanos - lastUnderrunLogNanos >= 1_000_000_000L) {
|
|
219
|
+
lastUnderrunLogNanos = nowNanos
|
|
220
|
+
log?.invoke("P8 audio-up drop reason=underrun depthMs=0 state=$state")
|
|
221
|
+
}
|
|
222
|
+
} else if (nowNanos - since >= STARVE_MS * 1_000_000L && state != State.STARVED) {
|
|
205
223
|
state = State.STARVED
|
|
206
224
|
depthEmaBytes = -1.0
|
|
225
|
+
log?.invoke("P8 audio-up drop reason=starved depthMs=0 silentMs=$STARVE_MS")
|
|
207
226
|
}
|
|
208
227
|
return silence()
|
|
209
228
|
}
|
|
@@ -211,9 +230,9 @@ class UplinkPacer(private val targetMs: Int = TARGET_MS) {
|
|
|
211
230
|
private fun enforceCap() {
|
|
212
231
|
val cap = EMERGENCY_CAP_MS * BYTES_PER_MS
|
|
213
232
|
if (used <= cap) return
|
|
214
|
-
val excess = used -
|
|
233
|
+
val excess = used - cap
|
|
215
234
|
dropOldest(excess)
|
|
216
|
-
|
|
235
|
+
noteDrop("overflow-cap", excess)
|
|
217
236
|
depthEmaBytes = used.toDouble()
|
|
218
237
|
}
|
|
219
238
|
|
|
@@ -224,11 +243,11 @@ class UplinkPacer(private val targetMs: Int = TARGET_MS) {
|
|
|
224
243
|
val emaMs = depthEmaBytes / BYTES_PER_MS
|
|
225
244
|
val slice = CORRECTION_MS * BYTES_PER_MS
|
|
226
245
|
when {
|
|
227
|
-
emaMs > HIGH_MS -> {
|
|
246
|
+
emaMs > targetMs + (HIGH_MS - TARGET_MS) -> {
|
|
228
247
|
dropOldest(slice)
|
|
229
248
|
driftDroppedBytes += slice
|
|
230
249
|
}
|
|
231
|
-
emaMs < LOW_MS -> {
|
|
250
|
+
emaMs < targetMs - (TARGET_MS - LOW_MS) -> {
|
|
232
251
|
if (used + slice > CAPACITY_BYTES) return
|
|
233
252
|
write(SILENCE_SLICE, 0, slice)
|
|
234
253
|
driftInsertedBytes += slice
|
|
@@ -240,6 +259,15 @@ class UplinkPacer(private val targetMs: Int = TARGET_MS) {
|
|
|
240
259
|
depthEmaBytes = used.toDouble()
|
|
241
260
|
}
|
|
242
261
|
|
|
262
|
+
private fun noteDrop(reason: String, bytes: Int) {
|
|
263
|
+
if (bytes <= 0) return
|
|
264
|
+
overflowDroppedBytes += bytes
|
|
265
|
+
log?.invoke(
|
|
266
|
+
"P8 audio-up drop reason=$reason droppedMs=${bytes / BYTES_PER_MS} " +
|
|
267
|
+
"depthMs=${used / BYTES_PER_MS} state=$state",
|
|
268
|
+
)
|
|
269
|
+
}
|
|
270
|
+
|
|
243
271
|
private fun dropOldest(bytes: Int) {
|
|
244
272
|
val n = min(bytes, used)
|
|
245
273
|
head = (head + n) % CAPACITY_BYTES
|
|
@@ -285,9 +313,16 @@ class UplinkPacer(private val targetMs: Int = TARGET_MS) {
|
|
|
285
313
|
const val FRAME_BYTES = FRAME_MS * BYTES_PER_MS
|
|
286
314
|
|
|
287
315
|
const val TARGET_MS = 60
|
|
316
|
+
// Device traces contain 90–130 ms BLE delivery gaps. Preserve enough real PCM for those gaps.
|
|
317
|
+
const val LC3_TARGET_MS = 160
|
|
288
318
|
const val LOW_MS = 40
|
|
289
319
|
const val HIGH_MS = 100
|
|
290
|
-
|
|
320
|
+
/**
|
|
321
|
+
* Peak the ring may hold. SoftAP+LC3 parks ~170 ms in the delay line and then
|
|
322
|
+
* releases it as a burst; 200 ms used to shear that dump back to [TARGET_MS]
|
|
323
|
+
* and punch a hole the wearer hears as a dropout.
|
|
324
|
+
*/
|
|
325
|
+
const val EMERGENCY_CAP_MS = 400
|
|
291
326
|
|
|
292
327
|
const val CORRECTION_MS = 10
|
|
293
328
|
const val CORRECTION_INTERVAL_MS = 500L
|
|
@@ -3,7 +3,9 @@ package com.mentra.acsmeeting.audio
|
|
|
3
3
|
import android.util.Log
|
|
4
4
|
import com.azure.android.communication.calling.RawAudioBuffer
|
|
5
5
|
import com.azure.android.communication.calling.RawOutgoingAudioStream
|
|
6
|
+
import com.mentra.acsmeeting.source.AcsInvestigation
|
|
6
7
|
import com.mentra.acsmeeting.telemetry.RingPercentile
|
|
8
|
+
import com.mentra.acsmeeting.video.AcsTimestamp
|
|
7
9
|
import java.nio.ByteBuffer
|
|
8
10
|
import java.util.concurrent.atomic.AtomicBoolean
|
|
9
11
|
import java.util.concurrent.atomic.AtomicInteger
|
|
@@ -22,8 +24,28 @@ fun interface UplinkTransport {
|
|
|
22
24
|
fun send(frame: ByteArray, onComplete: (Throwable?) -> Unit)
|
|
23
25
|
}
|
|
24
26
|
|
|
25
|
-
/**
|
|
26
|
-
|
|
27
|
+
/**
|
|
28
|
+
* [UplinkTransport] over ACS's raw outgoing audio stream.
|
|
29
|
+
*
|
|
30
|
+
* Every buffer is stamped on the same 100-nanosecond tick base the outgoing video uses
|
|
31
|
+
* ([AcsTimestamp]), because `System.nanoTime()` is the only reference the two streams share — the
|
|
32
|
+
* audio arrives over BLE and the video over the SoftAP peer, so submission order says nothing about
|
|
33
|
+
* capture order. Unlike `VirtualOutgoingVideoStream`, `RawOutgoingAudioStream` in 2.16.0 offers no
|
|
34
|
+
* stream clock to prefer, so the capture time is always the source.
|
|
35
|
+
*
|
|
36
|
+
* Whether ACS actually *uses* these ticks to lip-sync raw audio at the receiver is an open question
|
|
37
|
+
* that only a receiver-side recording answers. Stamping them is the precondition for asking it, not
|
|
38
|
+
* the answer, and it is separately worth doing: a run of zero timestamps is what makes the Teams
|
|
39
|
+
* jitter buffer hold. [AcsInvestigation.acsAudioTimestamps] turns it off so the question can be
|
|
40
|
+
* asked both ways on the same device.
|
|
41
|
+
*/
|
|
42
|
+
class AcsUplinkTransport(
|
|
43
|
+
private val stream: RawOutgoingAudioStream,
|
|
44
|
+
private val clock: () -> Long = System::nanoTime,
|
|
45
|
+
private val stamp: Boolean = AcsInvestigation.acsAudioTimestamps,
|
|
46
|
+
) : UplinkTransport {
|
|
47
|
+
private var lastTicks = 0L
|
|
48
|
+
|
|
27
49
|
override fun send(frame: ByteArray, onComplete: (Throwable?) -> Unit) {
|
|
28
50
|
// Fresh direct storage per submission. Pooling is a valid follow-up, but
|
|
29
51
|
// only if buffers return to the pool on completion rather than on return.
|
|
@@ -32,6 +54,11 @@ class AcsUplinkTransport(private val stream: RawOutgoingAudioStream) : UplinkTra
|
|
|
32
54
|
direct.flip()
|
|
33
55
|
val buffer = RawAudioBuffer()
|
|
34
56
|
buffer.buffer = direct
|
|
57
|
+
if (stamp) {
|
|
58
|
+
val ticks = AcsTimestamp.resolve(streamTicks = 0L, captureNs = clock(), lastTicks = lastTicks)
|
|
59
|
+
lastTicks = ticks
|
|
60
|
+
buffer.timestampInTicks = ticks
|
|
61
|
+
}
|
|
35
62
|
stream.sendRawAudioBuffer(buffer).whenComplete { _, error ->
|
|
36
63
|
try {
|
|
37
64
|
buffer.close()
|
|
@@ -60,6 +87,19 @@ class UplinkSender(
|
|
|
60
87
|
private val pacer: UplinkPacer,
|
|
61
88
|
private val transport: UplinkTransport,
|
|
62
89
|
private val clock: () -> Long = System::nanoTime,
|
|
90
|
+
/**
|
|
91
|
+
* Read immediately before every submission, not once per period.
|
|
92
|
+
*
|
|
93
|
+
* The last gate on the path. The pacer pops a frame and the transport accepts it on the same
|
|
94
|
+
* thread but not at the same instant, and a mute that lands in between would otherwise put the
|
|
95
|
+
* wearer's last word on the call after they asked not to be heard.
|
|
96
|
+
*/
|
|
97
|
+
private val muted: () -> Boolean = { false },
|
|
98
|
+
/**
|
|
99
|
+
* Latest 16-bit mean-abs from [PcmBridge]. Mute flags and a RUNNING pacer both look healthy on
|
|
100
|
+
* analog-silent LC3; this is what tells a soak that Teams is hearing the noise floor.
|
|
101
|
+
*/
|
|
102
|
+
private val pcmMeanAbs: () -> Int = { -1 },
|
|
63
103
|
private val log: (String) -> Unit = { Log.i(TAG, it) },
|
|
64
104
|
private val logError: (String, Throwable?) -> Unit = { message, error -> Log.e(TAG, message, error) },
|
|
65
105
|
) {
|
|
@@ -70,6 +110,14 @@ class UplinkSender(
|
|
|
70
110
|
val inFlight: Int,
|
|
71
111
|
val sendFailures: Long,
|
|
72
112
|
val backpressureDrops: Long,
|
|
113
|
+
/** Frames replaced by silence by the mute gate, i.e. voice that never reached Teams. */
|
|
114
|
+
val mutedFrames: Long,
|
|
115
|
+
/**
|
|
116
|
+
* How long after a mute the buffers ACS had already accepted took to drain, or null while no
|
|
117
|
+
* mute has completed. This is the honest measure of "when did Teams stop hearing the wearer";
|
|
118
|
+
* the gate itself is instantaneous, `sendRawAudioBuffer` is not.
|
|
119
|
+
*/
|
|
120
|
+
val muteTailMs: Long?,
|
|
73
121
|
)
|
|
74
122
|
|
|
75
123
|
private val running = AtomicBoolean(false)
|
|
@@ -86,6 +134,11 @@ class UplinkSender(
|
|
|
86
134
|
@Volatile private var backpressureDrops = 0L
|
|
87
135
|
@Volatile private var lastLogNanos: Long? = null
|
|
88
136
|
@Volatile private var lastFailureLogNanos: Long? = null
|
|
137
|
+
@Volatile private var mutedFrames = 0L
|
|
138
|
+
@Volatile private var mutedSinceNanos: Long? = null
|
|
139
|
+
@Volatile private var muteTailNanos: Long? = null
|
|
140
|
+
private var lastLoggedOverflowMs = 0L
|
|
141
|
+
private var lastLoggedSilence = 0L
|
|
89
142
|
|
|
90
143
|
/** Idempotent: a second call is a no-op so a session can only ever pace once. */
|
|
91
144
|
@Synchronized
|
|
@@ -96,6 +149,8 @@ class UplinkSender(
|
|
|
96
149
|
}
|
|
97
150
|
nextDeadlineNanos = null
|
|
98
151
|
lastLogNanos = null
|
|
152
|
+
lastLoggedOverflowMs = 0
|
|
153
|
+
lastLoggedSilence = 0
|
|
99
154
|
running.set(true)
|
|
100
155
|
// Audio cadence: a late frame is an artifact, so outrank the RN and
|
|
101
156
|
// decoder threads this competes with.
|
|
@@ -131,6 +186,8 @@ class UplinkSender(
|
|
|
131
186
|
inFlight = inFlight.get(),
|
|
132
187
|
sendFailures = sendFailures.get(),
|
|
133
188
|
backpressureDrops = backpressureDrops,
|
|
189
|
+
mutedFrames = mutedFrames,
|
|
190
|
+
muteTailMs = muteTailNanos?.let { it / 1_000_000L },
|
|
134
191
|
)
|
|
135
192
|
|
|
136
193
|
private fun loop() {
|
|
@@ -166,6 +223,9 @@ class UplinkSender(
|
|
|
166
223
|
deadline + PERIOD_NANOS
|
|
167
224
|
}
|
|
168
225
|
nextDeadlineNanos = next
|
|
226
|
+
// Before the submission, because this period's own frame is silence once muted and counting it
|
|
227
|
+
// as in flight would make the tail look like it never finished draining.
|
|
228
|
+
trackMuteTail(nowNanos)
|
|
169
229
|
submit(pacer.tick(nowNanos))
|
|
170
230
|
maybeLog(nowNanos)
|
|
171
231
|
return next
|
|
@@ -177,11 +237,19 @@ class UplinkSender(
|
|
|
177
237
|
backpressureDrops += 1
|
|
178
238
|
return
|
|
179
239
|
}
|
|
240
|
+
// Silence rather than nothing: the cadence ACS reads as a healthy stream has to continue, and
|
|
241
|
+
// a gap is a glitch on the far end rather than a mute.
|
|
242
|
+
val bytes = if (!frame.silence && muted()) {
|
|
243
|
+
mutedFrames += 1
|
|
244
|
+
SILENCE_FRAME
|
|
245
|
+
} else {
|
|
246
|
+
frame.bytes
|
|
247
|
+
}
|
|
180
248
|
inFlight.incrementAndGet()
|
|
181
249
|
framesSubmitted.incrementAndGet()
|
|
182
250
|
val startedNanos = clock()
|
|
183
251
|
try {
|
|
184
|
-
transport.send(
|
|
252
|
+
transport.send(bytes) { error ->
|
|
185
253
|
inFlight.decrementAndGet()
|
|
186
254
|
completion.record(clock() - startedNanos)
|
|
187
255
|
if (error != null) recordFailure(error)
|
|
@@ -192,6 +260,30 @@ class UplinkSender(
|
|
|
192
260
|
}
|
|
193
261
|
}
|
|
194
262
|
|
|
263
|
+
/**
|
|
264
|
+
* Time the drain of the buffers ACS accepted before the mute.
|
|
265
|
+
*
|
|
266
|
+
* Measured from the first period that observed the mute to the first period with nothing left in
|
|
267
|
+
* flight, which is the last moment any pre-mute voice can still be played out at the receiver.
|
|
268
|
+
*/
|
|
269
|
+
private fun trackMuteTail(nowNanos: Long) {
|
|
270
|
+
if (!muted()) {
|
|
271
|
+
mutedSinceNanos = null
|
|
272
|
+
return
|
|
273
|
+
}
|
|
274
|
+
val since = mutedSinceNanos
|
|
275
|
+
if (since == null) {
|
|
276
|
+
mutedSinceNanos = nowNanos
|
|
277
|
+
muteTailNanos = null
|
|
278
|
+
return
|
|
279
|
+
}
|
|
280
|
+
if (muteTailNanos == null && inFlight.get() == 0) {
|
|
281
|
+
val tail = nowNanos - since
|
|
282
|
+
muteTailNanos = tail
|
|
283
|
+
log("P8 audio-up muteTailMs=${tail / 1_000_000L} mutedFrames=$mutedFrames")
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
|
|
195
287
|
private fun recordFailure(error: Throwable) {
|
|
196
288
|
sendFailures.incrementAndGet()
|
|
197
289
|
val now = clock()
|
|
@@ -207,15 +299,22 @@ class UplinkSender(
|
|
|
207
299
|
lastLogNanos = nowNanos
|
|
208
300
|
if (last == null) return
|
|
209
301
|
val pace = pacer.snapshot(nowNanos)
|
|
302
|
+
val overflowDelta = pace.overflowDroppedMs - lastLoggedOverflowMs
|
|
303
|
+
val silenceDelta = pace.silenceFrames - lastLoggedSilence
|
|
304
|
+
lastLoggedOverflowMs = pace.overflowDroppedMs
|
|
305
|
+
lastLoggedSilence = pace.silenceFrames
|
|
210
306
|
log(
|
|
211
307
|
"P8 audio-up state=${pace.state} depthMs=${pace.depthMs} targetMs=${pace.targetMs} " +
|
|
212
308
|
"sentFps=${"%.1f".format(pace.sentFps)} silenceFrames=${pace.silenceFrames} " +
|
|
213
|
-
"overflowDroppedMs=${pace.overflowDroppedMs}
|
|
309
|
+
"overflowDroppedMs=${pace.overflowDroppedMs} dropMs=$overflowDelta silenceDelta=$silenceDelta " +
|
|
310
|
+
"driftCorrections=${pace.driftCorrections} " +
|
|
214
311
|
"driftDroppedMs=${pace.driftDroppedMs} driftInsertedMs=${pace.driftInsertedMs} " +
|
|
215
312
|
"tickLateP95Ms=${lateness.p95()} tickLateMaxMs=${maxLateNanos / 1_000_000L} " +
|
|
216
313
|
"skippedTicks=$skippedTicks inFlight=${inFlight.get()} " +
|
|
217
314
|
"sendFailures=${sendFailures.get()} sendCompletionP95Ms=${completion.p95()} " +
|
|
218
|
-
"backpressureDrops=$backpressureDrops"
|
|
315
|
+
"backpressureDrops=$backpressureDrops mutedFrames=$mutedFrames " +
|
|
316
|
+
"muteTailMs=${muteTailNanos?.let { it / 1_000_000L } ?: -1} " +
|
|
317
|
+
"pcmMeanAbs=${pcmMeanAbs()}",
|
|
219
318
|
)
|
|
220
319
|
}
|
|
221
320
|
|
|
@@ -226,5 +325,11 @@ class UplinkSender(
|
|
|
226
325
|
const val MAX_IN_FLIGHT = 10
|
|
227
326
|
private const val JOIN_TIMEOUT_MS = 500L
|
|
228
327
|
private const val LOG_INTERVAL_NANOS = 1_000_000_000L
|
|
328
|
+
|
|
329
|
+
/**
|
|
330
|
+
* Shared, and safe to share: [UplinkTransport] contracts to copy into its own storage before
|
|
331
|
+
* returning, so nothing downstream retains this array.
|
|
332
|
+
*/
|
|
333
|
+
private val SILENCE_FRAME = ByteArray(UplinkPacer.FRAME_BYTES)
|
|
229
334
|
}
|
|
230
335
|
}
|