@mentra/acs-meeting 3.2.0-dev.217 → 3.2.0-dev.222

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/README.md +4 -2
  2. package/android/build.gradle +23 -0
  3. package/android/src/androidTest/java/com/mentra/acsmeeting/network/SoftApFeasibilityGateTest.kt +291 -0
  4. package/android/src/main/AndroidManifest.xml +2 -0
  5. package/android/src/main/java/com/mentra/acsmeeting/AbandonedCallAgent.kt +36 -0
  6. package/android/src/main/java/com/mentra/acsmeeting/AcsMeetingModule.kt +392 -22
  7. package/android/src/main/java/com/mentra/acsmeeting/AcsMeetingSession.kt +909 -56
  8. package/android/src/main/java/com/mentra/acsmeeting/CallCapabilities.kt +33 -0
  9. package/android/src/main/java/com/mentra/acsmeeting/audio/AcsAudioPolicy.kt +21 -0
  10. package/android/src/main/java/com/mentra/acsmeeting/audio/AudioUplinkChain.kt +126 -0
  11. package/android/src/main/java/com/mentra/acsmeeting/audio/PcmBridge.kt +27 -4
  12. package/android/src/main/java/com/mentra/acsmeeting/audio/UplinkPacer.kt +45 -10
  13. package/android/src/main/java/com/mentra/acsmeeting/audio/UplinkSender.kt +110 -5
  14. package/android/src/main/java/com/mentra/acsmeeting/network/InternetHold.kt +501 -0
  15. package/android/src/main/java/com/mentra/acsmeeting/network/Ipv4Prefix.kt +51 -0
  16. package/android/src/main/java/com/mentra/acsmeeting/network/NetworkFacts.kt +81 -0
  17. package/android/src/main/java/com/mentra/acsmeeting/network/ScopedNetworkChangeDetector.kt +355 -0
  18. package/android/src/main/java/com/mentra/acsmeeting/network/ScopedNetworkError.kt +109 -0
  19. package/android/src/main/java/com/mentra/acsmeeting/network/ScopedNetworkObserver.kt +160 -0
  20. package/android/src/main/java/com/mentra/acsmeeting/network/ScopedNetworkReadiness.kt +103 -0
  21. package/android/src/main/java/com/mentra/acsmeeting/network/ScopedNetworkRequestSpec.kt +53 -0
  22. package/android/src/main/java/com/mentra/acsmeeting/network/ScopedNetworkState.kt +127 -0
  23. package/android/src/main/java/com/mentra/acsmeeting/network/ScopedSoftApNetwork.kt +573 -0
  24. package/android/src/main/java/com/mentra/acsmeeting/source/CloudflareWhepSource.kt +10 -101
  25. package/android/src/main/java/com/mentra/acsmeeting/source/DecodedTrackRelay.kt +147 -0
  26. package/android/src/main/java/com/mentra/acsmeeting/source/GlassesMediaSource.kt +47 -5
  27. package/android/src/main/java/com/mentra/acsmeeting/source/LocalWhipIngestSource.kt +772 -0
  28. package/android/src/main/java/com/mentra/acsmeeting/source/MeetingVideoSourceSpec.kt +73 -0
  29. package/android/src/main/java/com/mentra/acsmeeting/source/SdpAdapter.kt +17 -0
  30. package/android/src/main/java/com/mentra/acsmeeting/source/SelectedIcePair.kt +139 -0
  31. package/android/src/main/java/com/mentra/acsmeeting/source/SoftApIceDiagnosis.kt +144 -0
  32. package/android/src/main/java/com/mentra/acsmeeting/source/SoftApIcePolicy.kt +46 -0
  33. package/android/src/main/java/com/mentra/acsmeeting/source/SoftApSdpGuard.kt +160 -0
  34. package/android/src/main/java/com/mentra/acsmeeting/source/TrackRegistry.kt +3 -1
  35. package/android/src/main/java/com/mentra/acsmeeting/source/VideoSourceArm.kt +69 -0
  36. package/android/src/main/java/com/mentra/acsmeeting/source/WhipIngestProtocol.kt +207 -0
  37. package/android/src/main/java/com/mentra/acsmeeting/source/WhipIngestServer.kt +356 -0
  38. package/android/src/main/java/com/mentra/acsmeeting/telemetry/AvSyncProbe.kt +155 -0
  39. package/android/src/main/java/com/mentra/acsmeeting/telemetry/PipelineStats.kt +94 -3
  40. package/android/src/main/java/com/mentra/acsmeeting/telemetry/PipelineTicker.kt +33 -5
  41. package/android/src/main/java/com/mentra/acsmeeting/telemetry/RingPercentile.kt +15 -0
  42. package/android/src/main/java/com/mentra/acsmeeting/telemetry/VideoQuality.kt +178 -0
  43. package/android/src/main/java/com/mentra/acsmeeting/telemetry/VideoRateVerdict.kt +145 -0
  44. package/android/src/main/java/com/mentra/acsmeeting/trace/SoftApTrace.kt +143 -0
  45. package/android/src/main/java/com/mentra/acsmeeting/video/AcsFrameSender.kt +23 -4
  46. package/android/src/main/java/com/mentra/acsmeeting/video/FramePacer.kt +70 -0
  47. package/android/src/main/java/com/mentra/acsmeeting/video/I420FormatSpec.kt +1 -1
  48. package/android/src/main/java/com/mentra/acsmeeting/video/VideoProfile.kt +36 -3
  49. package/android/src/test/java/com/mentra/acsmeeting/AbandonedCallAgentTest.kt +74 -0
  50. package/android/src/test/java/com/mentra/acsmeeting/CallCapabilitiesTest.kt +47 -0
  51. package/android/src/test/java/com/mentra/acsmeeting/audio/AudioUplinkChainTest.kt +250 -0
  52. package/android/src/test/java/com/mentra/acsmeeting/audio/CapturePolicyTest.kt +44 -0
  53. package/android/src/test/java/com/mentra/acsmeeting/audio/UplinkPacerTest.kt +34 -4
  54. package/android/src/test/java/com/mentra/acsmeeting/audio/UplinkSenderTest.kt +96 -0
  55. package/android/src/test/java/com/mentra/acsmeeting/network/InternetHoldLifecycleTest.kt +102 -0
  56. package/android/src/test/java/com/mentra/acsmeeting/network/InternetHoldTest.kt +148 -0
  57. package/android/src/test/java/com/mentra/acsmeeting/network/Ipv4PrefixTest.kt +59 -0
  58. package/android/src/test/java/com/mentra/acsmeeting/network/NetworkFactsTest.kt +59 -0
  59. package/android/src/test/java/com/mentra/acsmeeting/network/ScopedLocalNetworkPermissionTest.kt +35 -0
  60. package/android/src/test/java/com/mentra/acsmeeting/network/ScopedNetworkChangeDetectorTest.kt +165 -0
  61. package/android/src/test/java/com/mentra/acsmeeting/network/ScopedNetworkErrorTest.kt +96 -0
  62. package/android/src/test/java/com/mentra/acsmeeting/network/ScopedNetworkLifecycleTest.kt +187 -0
  63. package/android/src/test/java/com/mentra/acsmeeting/network/ScopedNetworkObserverTest.kt +172 -0
  64. package/android/src/test/java/com/mentra/acsmeeting/network/ScopedNetworkReadinessTest.kt +141 -0
  65. package/android/src/test/java/com/mentra/acsmeeting/network/ScopedNetworkRequestSpecTest.kt +69 -0
  66. package/android/src/test/java/com/mentra/acsmeeting/source/AcsInvestigationTest.kt +40 -0
  67. package/android/src/test/java/com/mentra/acsmeeting/source/DecodedTrackRelayTest.kt +41 -0
  68. package/android/src/test/java/com/mentra/acsmeeting/source/GlassesMediaSourceTest.kt +26 -8
  69. package/android/src/test/java/com/mentra/acsmeeting/source/MeetingVideoSourceSpecTest.kt +109 -0
  70. package/android/src/test/java/com/mentra/acsmeeting/source/SelectedIcePairTest.kt +165 -0
  71. package/android/src/test/java/com/mentra/acsmeeting/source/SoftApIceDiagnosisTest.kt +148 -0
  72. package/android/src/test/java/com/mentra/acsmeeting/source/SoftApIcePolicyTest.kt +76 -0
  73. package/android/src/test/java/com/mentra/acsmeeting/source/SoftApSdpGuardTest.kt +275 -0
  74. package/android/src/test/java/com/mentra/acsmeeting/source/TrackRegistryTest.kt +9 -0
  75. package/android/src/test/java/com/mentra/acsmeeting/source/WhipIngestProtocolTest.kt +356 -0
  76. package/android/src/test/java/com/mentra/acsmeeting/source/WhipIngestServerLoopbackTest.kt +411 -0
  77. package/android/src/test/java/com/mentra/acsmeeting/telemetry/AvSyncProbeTest.kt +83 -0
  78. package/android/src/test/java/com/mentra/acsmeeting/telemetry/PipelineStatsTest.kt +69 -3
  79. package/android/src/test/java/com/mentra/acsmeeting/telemetry/VideoQualityTest.kt +182 -0
  80. package/android/src/test/java/com/mentra/acsmeeting/telemetry/VideoRateVerdictTest.kt +149 -0
  81. package/android/src/test/java/com/mentra/acsmeeting/trace/SoftApTraceTest.kt +97 -0
  82. package/android/src/test/java/com/mentra/acsmeeting/video/AcsTimestampTest.kt +34 -0
  83. package/android/src/test/java/com/mentra/acsmeeting/video/FramePacerTest.kt +105 -0
  84. package/android/src/test/java/com/mentra/acsmeeting/video/I420FormatSpecTest.kt +5 -0
  85. package/android/src/test/java/com/mentra/acsmeeting/video/VideoProfileTest.kt +38 -3
  86. package/build/AcsMeeting.types.d.ts +5 -0
  87. package/build/AcsMeeting.types.d.ts.map +1 -1
  88. package/build/AcsMeeting.types.js.map +1 -1
  89. package/build/AcsMeetingModule.d.ts +23 -0
  90. package/build/AcsMeetingModule.d.ts.map +1 -1
  91. package/build/AcsMeetingModule.js.map +1 -1
  92. package/build/AcsMeetingModule.web.d.ts +15 -0
  93. package/build/AcsMeetingModule.web.d.ts.map +1 -1
  94. package/build/AcsMeetingModule.web.js +13 -0
  95. package/build/AcsMeetingModule.web.js.map +1 -1
  96. package/ios/AcsMeetingModule.swift +213 -2
  97. package/package.json +1 -1
  98. package/src/AcsMeeting.types.ts +5 -0
  99. package/src/AcsMeetingModule.ts +13 -0
  100. package/src/AcsMeetingModule.web.ts +13 -0
@@ -0,0 +1,33 @@
1
+ package com.mentra.acsmeeting
2
+
3
+ /**
4
+ * One ACS participant capability, as reported to the host.
5
+ *
6
+ * [allowed] is nullable on purpose. "Denied" and "not known yet" are different facts and the UI has
7
+ * to treat them differently: a denied End must be hidden, while an unknown one is what every call
8
+ * looks like for the moment between joining and the first capabilities event.
9
+ */
10
+ data class CapabilityStatus(val allowed: Boolean? = null, val reason: String? = null) {
11
+ fun toMap(): Map<String, Any?> = mapOf("allowed" to allowed, "reason" to reason)
12
+ }
13
+
14
+ /**
15
+ * Whether an End-for-everyone attempt should be made at all.
16
+ *
17
+ * Teams only lets a presenter end a meeting for everyone, and ACS surfaces that as the runtime
18
+ * capability `HANG_UP_FOR_EVERYONE`. Refusing a known-denied End locally is better than sending it:
19
+ * ACS rejects it with an opaque error, and the wearer would have watched a confirm sheet for
20
+ * nothing. An *unknown* capability is not a refusal — it is the ordinary state before the first
21
+ * capabilities event lands, and letting the service decide is more honest than guessing.
22
+ */
23
+ object EndForEveryonePolicy {
24
+
25
+ const val REFUSED_PREFIX = "hang_up_for_everyone_not_allowed"
26
+
27
+ /** The refusal to report, or null when the End should be attempted. */
28
+ fun refusalFor(status: CapabilityStatus): String? {
29
+ if (status.allowed != false) return null
30
+ val reason = status.reason?.takeIf { it.isNotBlank() } ?: "unknown"
31
+ return "$REFUSED_PREFIX:$reason"
32
+ }
33
+ }
@@ -82,6 +82,27 @@ object CapturePolicy {
82
82
  fun captureGlassesMic(source: AudioSourceKind): Boolean = source == AudioSourceKind.GLASSES
83
83
  }
84
84
 
85
+ /**
86
+ * Which of the two possible sources of the wearer's voice this call listens to.
87
+ *
88
+ * SoftAP takes BLE LC3 from the host ([AcsMeetingSession.pushOutgoingPcm]) and ignores the WHIP
89
+ * track, which the glasses publish video-only (`captureAudio=false`). WHEP has no such flag: the
90
+ * wearer's voice is already mixed into the subscribed track, so that call takes the relay.
91
+ * Exactly one source is ever on.
92
+ *
93
+ * A previous revision routed SoftAP to the relay too, on a soak that read LC3 `meanAbs` ~15 as
94
+ * "analog-silent". Bench measurement of the same path puts the quiet-room floor at ~30–60, and
95
+ * that soak had already published with `captureAudio=false`, so neither the BES capture nor the
96
+ * SoC microphone was at fault. The JS gate `SOFTAP_BLE_LC3_UPLINK` and this routing move together.
97
+ */
98
+ data class GlassesPcmRouting(val relayPcm: Boolean, val externalPcm: Boolean) {
99
+ companion object {
100
+ fun decide(softap: Boolean, enabled: Boolean): GlassesPcmRouting =
101
+ if (softap) GlassesPcmRouting(relayPcm = false, externalPcm = enabled)
102
+ else GlassesPcmRouting(relayPcm = enabled, externalPcm = false)
103
+ }
104
+ }
105
+
85
106
  /** Null ACS call handles must fail, never report success via `call?.mute(); Unit`. */
86
107
  object CallGuard {
87
108
  fun <T : Any> require(value: T?): Result<T> =
@@ -0,0 +1,126 @@
1
+ package com.mentra.acsmeeting.audio
2
+
3
+ /**
4
+ * Everything between "a microphone buffer arrived" and "the pacer has it", behind one lock.
5
+ *
6
+ * The lock is the point. Muting an audio path is not one flag: the wearer's voice is spread across
7
+ * a delay ring, a resampler's filter history, a partial 20 ms frame, and the pacer's ring, and
8
+ * clearing them one at a time leaves whichever stage the producer thread happened to be writing to.
9
+ * Serializing ingest against mute is what makes "mute" mean the same thing at every stage.
10
+ *
11
+ * It cannot make mute instantaneous at the far end. Buffers already handed to ACS cannot be
12
+ * recalled, so the residual tail is bounded by [UplinkSender.MAX_IN_FLIGHT] frames and measured
13
+ * rather than assumed — see the sender's `muteTailMs`.
14
+ *
15
+ * @param bridge downmix/resample to the 48 kHz mono 20 ms frames ACS wants
16
+ * @param pacer the clock-domain adapter the sender drains
17
+ * @param delayMs how long to hold input before the bridge, for A/V alignment (0 = passthrough)
18
+ * @param onIngest raw input plus arrival time, before the delay line, for A/V clap measurement
19
+ */
20
+ class AudioUplinkChain(
21
+ private val bridge: PcmBridge,
22
+ private val pacer: UplinkPacer,
23
+ private val delayMs: Int = 0,
24
+ private val clock: () -> Long = System::nanoTime,
25
+ private val onIngest: ((pcm: ByteArray, nowNs: Long) -> Unit)? = null,
26
+ ) {
27
+ private val lock = Any()
28
+ private val delay = DelayLine(delayMs.coerceIn(0, MAX_DELAY_MS))
29
+ @Volatile private var muted = false
30
+
31
+ /**
32
+ * Accept one microphone buffer. A no-op while muted, so muted audio never enters any stage and
33
+ * an unmute cannot resurrect words the wearer said while muted.
34
+ */
35
+ fun ingest(pcm: ByteArray, sampleRate: Int, channels: Int) {
36
+ synchronized(lock) {
37
+ if (muted) return
38
+ val now = clock()
39
+ onIngest?.invoke(pcm, now)
40
+ for (chunk in delay.push(pcm, now)) {
41
+ for (frame in bridge.ingest(chunk, sampleRate, channels)) pacer.push(frame)
42
+ }
43
+ }
44
+ }
45
+
46
+ /** Stop accepting audio and drop everything already buffered, atomically against [ingest]. */
47
+ fun mute() {
48
+ synchronized(lock) {
49
+ muted = true
50
+ resetLocked()
51
+ }
52
+ }
53
+
54
+ /** Accept audio again. The pacer re-prerolls, so Teams hears voice one preroll later. */
55
+ fun unmute() {
56
+ synchronized(lock) { muted = false }
57
+ }
58
+
59
+ fun isMuted(): Boolean = muted
60
+
61
+ /** Drop buffered audio without changing the mute state. For leave and source rebuilds. */
62
+ fun reset() {
63
+ synchronized(lock) { resetLocked() }
64
+ }
65
+
66
+ /** Buffered input held for A/V alignment, in bytes. Diagnostics and tests. */
67
+ fun delayedBytes(): Int = synchronized(lock) { delay.heldBytes() }
68
+
69
+ private fun resetLocked() {
70
+ delay.reset()
71
+ bridge.reset()
72
+ pacer.reset()
73
+ }
74
+
75
+ /**
76
+ * Holds input until it is [delayMs] old.
77
+ *
78
+ * In front of the resampler rather than behind it so the delay is expressed in the input's own
79
+ * arrival times, which is what a receiver-side A/V measurement calibrates against.
80
+ */
81
+ private class DelayLine(private val delayMs: Int) {
82
+ private class Chunk(val bytes: ByteArray, val atNanos: Long)
83
+
84
+ private val queue = ArrayDeque<Chunk>()
85
+ private var bytesHeld = 0
86
+
87
+ fun push(pcm: ByteArray, nowNanos: Long): List<ByteArray> {
88
+ if (delayMs <= 0) return listOf(pcm)
89
+ queue.addLast(Chunk(pcm, nowNanos))
90
+ bytesHeld += pcm.size
91
+ val threshold = delayMs * 1_000_000L
92
+ val released = mutableListOf<ByteArray>()
93
+ while (queue.isNotEmpty()) {
94
+ val head = queue.first()
95
+ if (nowNanos - head.atNanos < threshold) break
96
+ queue.removeFirst()
97
+ bytesHeld -= head.bytes.size
98
+ released.add(head.bytes)
99
+ }
100
+ // Age alone bounds this in normal operation. The cap is for the case it cannot see: a clock
101
+ // that does not advance would otherwise turn a held buffer into an unbounded one.
102
+ val cap = delayMs * MAX_BYTES_PER_MS
103
+ while (bytesHeld > cap && queue.isNotEmpty()) {
104
+ val dropped = queue.removeFirst()
105
+ bytesHeld -= dropped.bytes.size
106
+ released.add(dropped.bytes)
107
+ }
108
+ return released
109
+ }
110
+
111
+ fun reset() {
112
+ queue.clear()
113
+ bytesHeld = 0
114
+ }
115
+
116
+ fun heldBytes(): Int = bytesHeld
117
+ }
118
+
119
+ companion object {
120
+ /** Half a second is already unusable as conversation; past that it is a bug, not a setting. */
121
+ const val MAX_DELAY_MS = 500
122
+
123
+ /** 48 kHz stereo PCM16, the widest input the bridge accepts. Sizes the safety cap only. */
124
+ internal const val MAX_BYTES_PER_MS = 48_000 * 2 * 2 / 1000
125
+ }
126
+ }
@@ -20,6 +20,8 @@ class PcmBridge(
20
20
  private val outgoing = ByteArrayOutputStream()
21
21
  private val resampler = PcmResampler(TARGET_RATE)
22
22
  private var lastRmsLogMs = 0L
23
+ @Volatile var lastMeanAbs: Int = -1
24
+ private set
23
25
 
24
26
  fun ingest(pcm16Le: ByteArray, sampleRate: Int, channels: Int): List<ByteArray> {
25
27
  if (dump != null && !dumped) {
@@ -47,6 +49,19 @@ class PcmBridge(
47
49
  return frames
48
50
  }
49
51
 
52
+ /**
53
+ * Drop the state that spans calls to [ingest]: the resampler's filter history and fractional
54
+ * phase, and the partial 20 ms frame not yet emitted.
55
+ *
56
+ * What makes this necessary is that both are, by design, audio the wearer already spoke. A mute
57
+ * that left them behind would emit them on the next unmute — a fraction of a word, from before
58
+ * the wearer asked not to be heard. Serialization is the caller's job; see [AudioUplinkChain].
59
+ */
60
+ fun reset() {
61
+ resampler.reset()
62
+ outgoing.reset()
63
+ }
64
+
50
65
  @Synchronized
51
66
  fun finishDump() {
52
67
  if (dumped) return
@@ -60,9 +75,6 @@ class PcmBridge(
60
75
  }
61
76
 
62
77
  private fun logLevel(pcm: ByteArray, sampleRate: Int, channels: Int) {
63
- val now = System.currentTimeMillis()
64
- if (now - lastRmsLogMs < 1000) return
65
- lastRmsLogMs = now
66
78
  var acc = 0L
67
79
  var n = 0
68
80
  var i = 0
@@ -74,7 +86,12 @@ class PcmBridge(
74
86
  i += 2
75
87
  }
76
88
  val mean = if (n == 0) 0 else acc / n
77
- Log.i(TAG, "P4 pcm rate=$sampleRate ch=$channels meanAbs=$mean bytes=${pcm.size}")
89
+ lastMeanAbs = mean.toInt()
90
+ val now = System.currentTimeMillis()
91
+ if (now - lastRmsLogMs < 1000) return
92
+ lastRmsLogMs = now
93
+ val verdict = if (mean < SILENCE_MEAN_ABS) "SILENCE" else "voice"
94
+ Log.i(TAG, "P4 pcm rate=$sampleRate ch=$channels meanAbs=$mean bytes=${pcm.size} $verdict")
78
95
  }
79
96
 
80
97
  companion object {
@@ -104,5 +121,11 @@ class PcmBridge(
104
121
  }
105
122
 
106
123
  fun encodeBase64(pcm: ByteArray): String = Base64.encodeToString(pcm, Base64.NO_WRAP)
124
+
125
+ /**
126
+ * 16-bit speech sits in the hundreds to thousands. Below this is the LC3/ADC noise floor
127
+ * Teams hears as mute even when ACS is unmuted and the pacer is RUNNING.
128
+ */
129
+ const val SILENCE_MEAN_ABS = 50
107
130
  }
108
131
  }
@@ -13,7 +13,8 @@ import kotlin.math.min
13
13
  *
14
14
  * - sustained depth above [HIGH_MS] (producer fast) discards a [CORRECTION_MS] slice
15
15
  * - sustained depth below [LOW_MS] (producer slow) inserts a [CORRECTION_MS] slice of silence
16
- * - depth above [EMERGENCY_CAP_MS] drops oldest down to target immediately
16
+ * - depth above [EMERGENCY_CAP_MS] shears only the peak above the cap — not back to target.
17
+ * Shearing to target was a 150 ms hole every time the SoftAP delay line released a burst.
17
18
  *
18
19
  * Corrections read an EMA over roughly two seconds and are rate limited to one
19
20
  * per [CORRECTION_INTERVAL_MS], so ordinary jitter never triggers one; only
@@ -28,7 +29,10 @@ import kotlin.math.min
28
29
  * [push] runs on the WebRTC audio thread and [tick] on the pacing thread, so
29
30
  * every entry point is synchronized on this instance.
30
31
  */
31
- class UplinkPacer(private val targetMs: Int = TARGET_MS) {
32
+ class UplinkPacer(
33
+ private var targetMs: Int = TARGET_MS,
34
+ private val log: ((String) -> Unit)? = null,
35
+ ) {
32
36
  enum class State { PREROLLING, RUNNING, STARVED }
33
37
 
34
38
  /** One [FRAME_MS] frame. [silence] marks a frame the pacer manufactured. */
@@ -72,6 +76,15 @@ class UplinkPacer(private val targetMs: Int = TARGET_MS) {
72
76
  private var driftCorrections = 0L
73
77
  private var driftDroppedBytes = 0L
74
78
  private var driftInsertedBytes = 0L
79
+ private var lastUnderrunLogNanos = 0L
80
+
81
+ /** Select headroom for the producer before starting a call; discard the previous call's PCM. */
82
+ @Synchronized
83
+ fun configureTarget(ms: Int) {
84
+ require(ms in FRAME_MS..EMERGENCY_CAP_MS)
85
+ targetMs = ms
86
+ reset()
87
+ }
75
88
 
76
89
  /** Appends 48 kHz mono PCM16 from the producer thread. */
77
90
  @Synchronized
@@ -83,12 +96,12 @@ class UplinkPacer(private val targetMs: Int = TARGET_MS) {
83
96
  // A single burst larger than the whole ring: keep only the newest audio.
84
97
  offset = len - CAPACITY_BYTES
85
98
  len = CAPACITY_BYTES
86
- overflowDroppedBytes += offset
99
+ noteDrop("overflow-push", offset)
87
100
  }
88
101
  val overflow = used + len - CAPACITY_BYTES
89
102
  if (overflow > 0) {
90
103
  dropOldest(overflow)
91
- overflowDroppedBytes += overflow
104
+ noteDrop("overflow-push", overflow)
92
105
  }
93
106
  write(pcm16Le, offset, len)
94
107
  }
@@ -164,6 +177,7 @@ class UplinkPacer(private val targetMs: Int = TARGET_MS) {
164
177
  driftCorrections = 0
165
178
  driftDroppedBytes = 0
166
179
  driftInsertedBytes = 0
180
+ lastUnderrunLogNanos = 0
167
181
  }
168
182
 
169
183
  @Synchronized
@@ -201,9 +215,14 @@ class UplinkPacer(private val targetMs: Int = TARGET_MS) {
201
215
  val since = starvingSinceNanos
202
216
  if (since == null) {
203
217
  starvingSinceNanos = nowNanos
204
- } else if (nowNanos - since >= STARVE_MS * 1_000_000L) {
218
+ if (lastUnderrunLogNanos == 0L || nowNanos - lastUnderrunLogNanos >= 1_000_000_000L) {
219
+ lastUnderrunLogNanos = nowNanos
220
+ log?.invoke("P8 audio-up drop reason=underrun depthMs=0 state=$state")
221
+ }
222
+ } else if (nowNanos - since >= STARVE_MS * 1_000_000L && state != State.STARVED) {
205
223
  state = State.STARVED
206
224
  depthEmaBytes = -1.0
225
+ log?.invoke("P8 audio-up drop reason=starved depthMs=0 silentMs=$STARVE_MS")
207
226
  }
208
227
  return silence()
209
228
  }
@@ -211,9 +230,9 @@ class UplinkPacer(private val targetMs: Int = TARGET_MS) {
211
230
  private fun enforceCap() {
212
231
  val cap = EMERGENCY_CAP_MS * BYTES_PER_MS
213
232
  if (used <= cap) return
214
- val excess = used - targetMs * BYTES_PER_MS
233
+ val excess = used - cap
215
234
  dropOldest(excess)
216
- overflowDroppedBytes += excess
235
+ noteDrop("overflow-cap", excess)
217
236
  depthEmaBytes = used.toDouble()
218
237
  }
219
238
 
@@ -224,11 +243,11 @@ class UplinkPacer(private val targetMs: Int = TARGET_MS) {
224
243
  val emaMs = depthEmaBytes / BYTES_PER_MS
225
244
  val slice = CORRECTION_MS * BYTES_PER_MS
226
245
  when {
227
- emaMs > HIGH_MS -> {
246
+ emaMs > targetMs + (HIGH_MS - TARGET_MS) -> {
228
247
  dropOldest(slice)
229
248
  driftDroppedBytes += slice
230
249
  }
231
- emaMs < LOW_MS -> {
250
+ emaMs < targetMs - (TARGET_MS - LOW_MS) -> {
232
251
  if (used + slice > CAPACITY_BYTES) return
233
252
  write(SILENCE_SLICE, 0, slice)
234
253
  driftInsertedBytes += slice
@@ -240,6 +259,15 @@ class UplinkPacer(private val targetMs: Int = TARGET_MS) {
240
259
  depthEmaBytes = used.toDouble()
241
260
  }
242
261
 
262
+ private fun noteDrop(reason: String, bytes: Int) {
263
+ if (bytes <= 0) return
264
+ overflowDroppedBytes += bytes
265
+ log?.invoke(
266
+ "P8 audio-up drop reason=$reason droppedMs=${bytes / BYTES_PER_MS} " +
267
+ "depthMs=${used / BYTES_PER_MS} state=$state",
268
+ )
269
+ }
270
+
243
271
  private fun dropOldest(bytes: Int) {
244
272
  val n = min(bytes, used)
245
273
  head = (head + n) % CAPACITY_BYTES
@@ -285,9 +313,16 @@ class UplinkPacer(private val targetMs: Int = TARGET_MS) {
285
313
  const val FRAME_BYTES = FRAME_MS * BYTES_PER_MS
286
314
 
287
315
  const val TARGET_MS = 60
316
+ // Device traces contain 90–130 ms BLE delivery gaps. Preserve enough real PCM for those gaps.
317
+ const val LC3_TARGET_MS = 160
288
318
  const val LOW_MS = 40
289
319
  const val HIGH_MS = 100
290
- const val EMERGENCY_CAP_MS = 200
320
+ /**
321
+ * Peak the ring may hold. SoftAP+LC3 parks ~170 ms in the delay line and then
322
+ * releases it as a burst; 200 ms used to shear that dump back to [TARGET_MS]
323
+ * and punch a hole the wearer hears as a dropout.
324
+ */
325
+ const val EMERGENCY_CAP_MS = 400
291
326
 
292
327
  const val CORRECTION_MS = 10
293
328
  const val CORRECTION_INTERVAL_MS = 500L
@@ -3,7 +3,9 @@ package com.mentra.acsmeeting.audio
3
3
  import android.util.Log
4
4
  import com.azure.android.communication.calling.RawAudioBuffer
5
5
  import com.azure.android.communication.calling.RawOutgoingAudioStream
6
+ import com.mentra.acsmeeting.source.AcsInvestigation
6
7
  import com.mentra.acsmeeting.telemetry.RingPercentile
8
+ import com.mentra.acsmeeting.video.AcsTimestamp
7
9
  import java.nio.ByteBuffer
8
10
  import java.util.concurrent.atomic.AtomicBoolean
9
11
  import java.util.concurrent.atomic.AtomicInteger
@@ -22,8 +24,28 @@ fun interface UplinkTransport {
22
24
  fun send(frame: ByteArray, onComplete: (Throwable?) -> Unit)
23
25
  }
24
26
 
25
- /** [UplinkTransport] over ACS's raw outgoing audio stream. */
26
- class AcsUplinkTransport(private val stream: RawOutgoingAudioStream) : UplinkTransport {
27
+ /**
28
+ * [UplinkTransport] over ACS's raw outgoing audio stream.
29
+ *
30
+ * Every buffer is stamped on the same 100-nanosecond tick base the outgoing video uses
31
+ * ([AcsTimestamp]), because `System.nanoTime()` is the only reference the two streams share — the
32
+ * audio arrives over BLE and the video over the SoftAP peer, so submission order says nothing about
33
+ * capture order. Unlike `VirtualOutgoingVideoStream`, `RawOutgoingAudioStream` in 2.16.0 offers no
34
+ * stream clock to prefer, so the capture time is always the source.
35
+ *
36
+ * Whether ACS actually *uses* these ticks to lip-sync raw audio at the receiver is an open question
37
+ * that only a receiver-side recording answers. Stamping them is the precondition for asking it, not
38
+ * the answer, and it is separately worth doing: a run of zero timestamps is what makes the Teams
39
+ * jitter buffer hold. [AcsInvestigation.acsAudioTimestamps] turns it off so the question can be
40
+ * asked both ways on the same device.
41
+ */
42
+ class AcsUplinkTransport(
43
+ private val stream: RawOutgoingAudioStream,
44
+ private val clock: () -> Long = System::nanoTime,
45
+ private val stamp: Boolean = AcsInvestigation.acsAudioTimestamps,
46
+ ) : UplinkTransport {
47
+ private var lastTicks = 0L
48
+
27
49
  override fun send(frame: ByteArray, onComplete: (Throwable?) -> Unit) {
28
50
  // Fresh direct storage per submission. Pooling is a valid follow-up, but
29
51
  // only if buffers return to the pool on completion rather than on return.
@@ -32,6 +54,11 @@ class AcsUplinkTransport(private val stream: RawOutgoingAudioStream) : UplinkTra
32
54
  direct.flip()
33
55
  val buffer = RawAudioBuffer()
34
56
  buffer.buffer = direct
57
+ if (stamp) {
58
+ val ticks = AcsTimestamp.resolve(streamTicks = 0L, captureNs = clock(), lastTicks = lastTicks)
59
+ lastTicks = ticks
60
+ buffer.timestampInTicks = ticks
61
+ }
35
62
  stream.sendRawAudioBuffer(buffer).whenComplete { _, error ->
36
63
  try {
37
64
  buffer.close()
@@ -60,6 +87,19 @@ class UplinkSender(
60
87
  private val pacer: UplinkPacer,
61
88
  private val transport: UplinkTransport,
62
89
  private val clock: () -> Long = System::nanoTime,
90
+ /**
91
+ * Read immediately before every submission, not once per period.
92
+ *
93
+ * The last gate on the path. The pacer pops a frame and the transport accepts it on the same
94
+ * thread but not at the same instant, and a mute that lands in between would otherwise put the
95
+ * wearer's last word on the call after they asked not to be heard.
96
+ */
97
+ private val muted: () -> Boolean = { false },
98
+ /**
99
+ * Latest 16-bit mean-abs from [PcmBridge]. Mute flags and a RUNNING pacer both look healthy on
100
+ * analog-silent LC3; this is what tells a soak that Teams is hearing the noise floor.
101
+ */
102
+ private val pcmMeanAbs: () -> Int = { -1 },
63
103
  private val log: (String) -> Unit = { Log.i(TAG, it) },
64
104
  private val logError: (String, Throwable?) -> Unit = { message, error -> Log.e(TAG, message, error) },
65
105
  ) {
@@ -70,6 +110,14 @@ class UplinkSender(
70
110
  val inFlight: Int,
71
111
  val sendFailures: Long,
72
112
  val backpressureDrops: Long,
113
+ /** Frames replaced by silence by the mute gate, i.e. voice that never reached Teams. */
114
+ val mutedFrames: Long,
115
+ /**
116
+ * How long after a mute the buffers ACS had already accepted took to drain, or null while no
117
+ * mute has completed. This is the honest measure of "when did Teams stop hearing the wearer";
118
+ * the gate itself is instantaneous, `sendRawAudioBuffer` is not.
119
+ */
120
+ val muteTailMs: Long?,
73
121
  )
74
122
 
75
123
  private val running = AtomicBoolean(false)
@@ -86,6 +134,11 @@ class UplinkSender(
86
134
  @Volatile private var backpressureDrops = 0L
87
135
  @Volatile private var lastLogNanos: Long? = null
88
136
  @Volatile private var lastFailureLogNanos: Long? = null
137
+ @Volatile private var mutedFrames = 0L
138
+ @Volatile private var mutedSinceNanos: Long? = null
139
+ @Volatile private var muteTailNanos: Long? = null
140
+ private var lastLoggedOverflowMs = 0L
141
+ private var lastLoggedSilence = 0L
89
142
 
90
143
  /** Idempotent: a second call is a no-op so a session can only ever pace once. */
91
144
  @Synchronized
@@ -96,6 +149,8 @@ class UplinkSender(
96
149
  }
97
150
  nextDeadlineNanos = null
98
151
  lastLogNanos = null
152
+ lastLoggedOverflowMs = 0
153
+ lastLoggedSilence = 0
99
154
  running.set(true)
100
155
  // Audio cadence: a late frame is an artifact, so outrank the RN and
101
156
  // decoder threads this competes with.
@@ -131,6 +186,8 @@ class UplinkSender(
131
186
  inFlight = inFlight.get(),
132
187
  sendFailures = sendFailures.get(),
133
188
  backpressureDrops = backpressureDrops,
189
+ mutedFrames = mutedFrames,
190
+ muteTailMs = muteTailNanos?.let { it / 1_000_000L },
134
191
  )
135
192
 
136
193
  private fun loop() {
@@ -166,6 +223,9 @@ class UplinkSender(
166
223
  deadline + PERIOD_NANOS
167
224
  }
168
225
  nextDeadlineNanos = next
226
+ // Before the submission, because this period's own frame is silence once muted and counting it
227
+ // as in flight would make the tail look like it never finished draining.
228
+ trackMuteTail(nowNanos)
169
229
  submit(pacer.tick(nowNanos))
170
230
  maybeLog(nowNanos)
171
231
  return next
@@ -177,11 +237,19 @@ class UplinkSender(
177
237
  backpressureDrops += 1
178
238
  return
179
239
  }
240
+ // Silence rather than nothing: the cadence ACS reads as a healthy stream has to continue, and
241
+ // a gap is a glitch on the far end rather than a mute.
242
+ val bytes = if (!frame.silence && muted()) {
243
+ mutedFrames += 1
244
+ SILENCE_FRAME
245
+ } else {
246
+ frame.bytes
247
+ }
180
248
  inFlight.incrementAndGet()
181
249
  framesSubmitted.incrementAndGet()
182
250
  val startedNanos = clock()
183
251
  try {
184
- transport.send(frame.bytes) { error ->
252
+ transport.send(bytes) { error ->
185
253
  inFlight.decrementAndGet()
186
254
  completion.record(clock() - startedNanos)
187
255
  if (error != null) recordFailure(error)
@@ -192,6 +260,30 @@ class UplinkSender(
192
260
  }
193
261
  }
194
262
 
263
+ /**
264
+ * Time the drain of the buffers ACS accepted before the mute.
265
+ *
266
+ * Measured from the first period that observed the mute to the first period with nothing left in
267
+ * flight, which is the last moment any pre-mute voice can still be played out at the receiver.
268
+ */
269
+ private fun trackMuteTail(nowNanos: Long) {
270
+ if (!muted()) {
271
+ mutedSinceNanos = null
272
+ return
273
+ }
274
+ val since = mutedSinceNanos
275
+ if (since == null) {
276
+ mutedSinceNanos = nowNanos
277
+ muteTailNanos = null
278
+ return
279
+ }
280
+ if (muteTailNanos == null && inFlight.get() == 0) {
281
+ val tail = nowNanos - since
282
+ muteTailNanos = tail
283
+ log("P8 audio-up muteTailMs=${tail / 1_000_000L} mutedFrames=$mutedFrames")
284
+ }
285
+ }
286
+
195
287
  private fun recordFailure(error: Throwable) {
196
288
  sendFailures.incrementAndGet()
197
289
  val now = clock()
@@ -207,15 +299,22 @@ class UplinkSender(
207
299
  lastLogNanos = nowNanos
208
300
  if (last == null) return
209
301
  val pace = pacer.snapshot(nowNanos)
302
+ val overflowDelta = pace.overflowDroppedMs - lastLoggedOverflowMs
303
+ val silenceDelta = pace.silenceFrames - lastLoggedSilence
304
+ lastLoggedOverflowMs = pace.overflowDroppedMs
305
+ lastLoggedSilence = pace.silenceFrames
210
306
  log(
211
307
  "P8 audio-up state=${pace.state} depthMs=${pace.depthMs} targetMs=${pace.targetMs} " +
212
308
  "sentFps=${"%.1f".format(pace.sentFps)} silenceFrames=${pace.silenceFrames} " +
213
- "overflowDroppedMs=${pace.overflowDroppedMs} driftCorrections=${pace.driftCorrections} " +
309
+ "overflowDroppedMs=${pace.overflowDroppedMs} dropMs=$overflowDelta silenceDelta=$silenceDelta " +
310
+ "driftCorrections=${pace.driftCorrections} " +
214
311
  "driftDroppedMs=${pace.driftDroppedMs} driftInsertedMs=${pace.driftInsertedMs} " +
215
312
  "tickLateP95Ms=${lateness.p95()} tickLateMaxMs=${maxLateNanos / 1_000_000L} " +
216
313
  "skippedTicks=$skippedTicks inFlight=${inFlight.get()} " +
217
314
  "sendFailures=${sendFailures.get()} sendCompletionP95Ms=${completion.p95()} " +
218
- "backpressureDrops=$backpressureDrops",
315
+ "backpressureDrops=$backpressureDrops mutedFrames=$mutedFrames " +
316
+ "muteTailMs=${muteTailNanos?.let { it / 1_000_000L } ?: -1} " +
317
+ "pcmMeanAbs=${pcmMeanAbs()}",
219
318
  )
220
319
  }
221
320
 
@@ -226,5 +325,11 @@ class UplinkSender(
226
325
  const val MAX_IN_FLIGHT = 10
227
326
  private const val JOIN_TIMEOUT_MS = 500L
228
327
  private const val LOG_INTERVAL_NANOS = 1_000_000_000L
328
+
329
+ /**
330
+ * Shared, and safe to share: [UplinkTransport] contracts to copy into its own storage before
331
+ * returning, so nothing downstream retains this array.
332
+ */
333
+ private val SILENCE_FRAME = ByteArray(UplinkPacer.FRAME_BYTES)
229
334
  }
230
335
  }