@mentra/acs-meeting 3.2.0-dev.221 → 3.2.0-dev.223

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/README.md +4 -2
  2. package/android/build.gradle +23 -0
  3. package/android/src/androidTest/java/com/mentra/acsmeeting/network/SoftApFeasibilityGateTest.kt +291 -0
  4. package/android/src/main/AndroidManifest.xml +2 -0
  5. package/android/src/main/java/com/mentra/acsmeeting/AbandonedCallAgent.kt +36 -0
  6. package/android/src/main/java/com/mentra/acsmeeting/AcsMeetingModule.kt +392 -22
  7. package/android/src/main/java/com/mentra/acsmeeting/AcsMeetingSession.kt +909 -56
  8. package/android/src/main/java/com/mentra/acsmeeting/CallCapabilities.kt +33 -0
  9. package/android/src/main/java/com/mentra/acsmeeting/audio/AcsAudioPolicy.kt +21 -0
  10. package/android/src/main/java/com/mentra/acsmeeting/audio/AudioUplinkChain.kt +126 -0
  11. package/android/src/main/java/com/mentra/acsmeeting/audio/PcmBridge.kt +27 -4
  12. package/android/src/main/java/com/mentra/acsmeeting/audio/UplinkPacer.kt +45 -10
  13. package/android/src/main/java/com/mentra/acsmeeting/audio/UplinkSender.kt +110 -5
  14. package/android/src/main/java/com/mentra/acsmeeting/network/InternetHold.kt +501 -0
  15. package/android/src/main/java/com/mentra/acsmeeting/network/Ipv4Prefix.kt +51 -0
  16. package/android/src/main/java/com/mentra/acsmeeting/network/NetworkFacts.kt +81 -0
  17. package/android/src/main/java/com/mentra/acsmeeting/network/ScopedNetworkChangeDetector.kt +355 -0
  18. package/android/src/main/java/com/mentra/acsmeeting/network/ScopedNetworkError.kt +109 -0
  19. package/android/src/main/java/com/mentra/acsmeeting/network/ScopedNetworkObserver.kt +160 -0
  20. package/android/src/main/java/com/mentra/acsmeeting/network/ScopedNetworkReadiness.kt +103 -0
  21. package/android/src/main/java/com/mentra/acsmeeting/network/ScopedNetworkRequestSpec.kt +53 -0
  22. package/android/src/main/java/com/mentra/acsmeeting/network/ScopedNetworkState.kt +127 -0
  23. package/android/src/main/java/com/mentra/acsmeeting/network/ScopedSoftApNetwork.kt +573 -0
  24. package/android/src/main/java/com/mentra/acsmeeting/source/CloudflareWhepSource.kt +10 -101
  25. package/android/src/main/java/com/mentra/acsmeeting/source/DecodedTrackRelay.kt +147 -0
  26. package/android/src/main/java/com/mentra/acsmeeting/source/GlassesMediaSource.kt +47 -5
  27. package/android/src/main/java/com/mentra/acsmeeting/source/LocalWhipIngestSource.kt +772 -0
  28. package/android/src/main/java/com/mentra/acsmeeting/source/MeetingVideoSourceSpec.kt +73 -0
  29. package/android/src/main/java/com/mentra/acsmeeting/source/SdpAdapter.kt +17 -0
  30. package/android/src/main/java/com/mentra/acsmeeting/source/SelectedIcePair.kt +139 -0
  31. package/android/src/main/java/com/mentra/acsmeeting/source/SoftApIceDiagnosis.kt +144 -0
  32. package/android/src/main/java/com/mentra/acsmeeting/source/SoftApIcePolicy.kt +46 -0
  33. package/android/src/main/java/com/mentra/acsmeeting/source/SoftApSdpGuard.kt +160 -0
  34. package/android/src/main/java/com/mentra/acsmeeting/source/TrackRegistry.kt +3 -1
  35. package/android/src/main/java/com/mentra/acsmeeting/source/VideoSourceArm.kt +69 -0
  36. package/android/src/main/java/com/mentra/acsmeeting/source/WhipIngestProtocol.kt +207 -0
  37. package/android/src/main/java/com/mentra/acsmeeting/source/WhipIngestServer.kt +356 -0
  38. package/android/src/main/java/com/mentra/acsmeeting/telemetry/AvSyncProbe.kt +155 -0
  39. package/android/src/main/java/com/mentra/acsmeeting/telemetry/PipelineStats.kt +94 -3
  40. package/android/src/main/java/com/mentra/acsmeeting/telemetry/PipelineTicker.kt +33 -5
  41. package/android/src/main/java/com/mentra/acsmeeting/telemetry/RingPercentile.kt +15 -0
  42. package/android/src/main/java/com/mentra/acsmeeting/telemetry/VideoQuality.kt +178 -0
  43. package/android/src/main/java/com/mentra/acsmeeting/telemetry/VideoRateVerdict.kt +145 -0
  44. package/android/src/main/java/com/mentra/acsmeeting/trace/SoftApTrace.kt +143 -0
  45. package/android/src/main/java/com/mentra/acsmeeting/video/AcsFrameSender.kt +23 -4
  46. package/android/src/main/java/com/mentra/acsmeeting/video/FramePacer.kt +70 -0
  47. package/android/src/main/java/com/mentra/acsmeeting/video/I420FormatSpec.kt +1 -1
  48. package/android/src/main/java/com/mentra/acsmeeting/video/VideoProfile.kt +36 -3
  49. package/android/src/test/java/com/mentra/acsmeeting/AbandonedCallAgentTest.kt +74 -0
  50. package/android/src/test/java/com/mentra/acsmeeting/CallCapabilitiesTest.kt +47 -0
  51. package/android/src/test/java/com/mentra/acsmeeting/audio/AudioUplinkChainTest.kt +250 -0
  52. package/android/src/test/java/com/mentra/acsmeeting/audio/CapturePolicyTest.kt +44 -0
  53. package/android/src/test/java/com/mentra/acsmeeting/audio/UplinkPacerTest.kt +34 -4
  54. package/android/src/test/java/com/mentra/acsmeeting/audio/UplinkSenderTest.kt +96 -0
  55. package/android/src/test/java/com/mentra/acsmeeting/network/InternetHoldLifecycleTest.kt +102 -0
  56. package/android/src/test/java/com/mentra/acsmeeting/network/InternetHoldTest.kt +148 -0
  57. package/android/src/test/java/com/mentra/acsmeeting/network/Ipv4PrefixTest.kt +59 -0
  58. package/android/src/test/java/com/mentra/acsmeeting/network/NetworkFactsTest.kt +59 -0
  59. package/android/src/test/java/com/mentra/acsmeeting/network/ScopedLocalNetworkPermissionTest.kt +35 -0
  60. package/android/src/test/java/com/mentra/acsmeeting/network/ScopedNetworkChangeDetectorTest.kt +165 -0
  61. package/android/src/test/java/com/mentra/acsmeeting/network/ScopedNetworkErrorTest.kt +96 -0
  62. package/android/src/test/java/com/mentra/acsmeeting/network/ScopedNetworkLifecycleTest.kt +187 -0
  63. package/android/src/test/java/com/mentra/acsmeeting/network/ScopedNetworkObserverTest.kt +172 -0
  64. package/android/src/test/java/com/mentra/acsmeeting/network/ScopedNetworkReadinessTest.kt +141 -0
  65. package/android/src/test/java/com/mentra/acsmeeting/network/ScopedNetworkRequestSpecTest.kt +69 -0
  66. package/android/src/test/java/com/mentra/acsmeeting/source/AcsInvestigationTest.kt +40 -0
  67. package/android/src/test/java/com/mentra/acsmeeting/source/DecodedTrackRelayTest.kt +41 -0
  68. package/android/src/test/java/com/mentra/acsmeeting/source/GlassesMediaSourceTest.kt +26 -8
  69. package/android/src/test/java/com/mentra/acsmeeting/source/MeetingVideoSourceSpecTest.kt +109 -0
  70. package/android/src/test/java/com/mentra/acsmeeting/source/SelectedIcePairTest.kt +165 -0
  71. package/android/src/test/java/com/mentra/acsmeeting/source/SoftApIceDiagnosisTest.kt +148 -0
  72. package/android/src/test/java/com/mentra/acsmeeting/source/SoftApIcePolicyTest.kt +76 -0
  73. package/android/src/test/java/com/mentra/acsmeeting/source/SoftApSdpGuardTest.kt +275 -0
  74. package/android/src/test/java/com/mentra/acsmeeting/source/TrackRegistryTest.kt +9 -0
  75. package/android/src/test/java/com/mentra/acsmeeting/source/WhipIngestProtocolTest.kt +356 -0
  76. package/android/src/test/java/com/mentra/acsmeeting/source/WhipIngestServerLoopbackTest.kt +411 -0
  77. package/android/src/test/java/com/mentra/acsmeeting/telemetry/AvSyncProbeTest.kt +83 -0
  78. package/android/src/test/java/com/mentra/acsmeeting/telemetry/PipelineStatsTest.kt +69 -3
  79. package/android/src/test/java/com/mentra/acsmeeting/telemetry/VideoQualityTest.kt +182 -0
  80. package/android/src/test/java/com/mentra/acsmeeting/telemetry/VideoRateVerdictTest.kt +149 -0
  81. package/android/src/test/java/com/mentra/acsmeeting/trace/SoftApTraceTest.kt +97 -0
  82. package/android/src/test/java/com/mentra/acsmeeting/video/AcsTimestampTest.kt +34 -0
  83. package/android/src/test/java/com/mentra/acsmeeting/video/FramePacerTest.kt +105 -0
  84. package/android/src/test/java/com/mentra/acsmeeting/video/I420FormatSpecTest.kt +5 -0
  85. package/android/src/test/java/com/mentra/acsmeeting/video/VideoProfileTest.kt +38 -3
  86. package/build/AcsMeeting.types.d.ts +5 -0
  87. package/build/AcsMeeting.types.d.ts.map +1 -1
  88. package/build/AcsMeeting.types.js.map +1 -1
  89. package/build/AcsMeetingModule.d.ts +23 -0
  90. package/build/AcsMeetingModule.d.ts.map +1 -1
  91. package/build/AcsMeetingModule.js.map +1 -1
  92. package/build/AcsMeetingModule.web.d.ts +15 -0
  93. package/build/AcsMeetingModule.web.d.ts.map +1 -1
  94. package/build/AcsMeetingModule.web.js +13 -0
  95. package/build/AcsMeetingModule.web.js.map +1 -1
  96. package/ios/AcsMeetingModule.swift +213 -2
  97. package/package.json +1 -1
  98. package/src/AcsMeeting.types.ts +5 -0
  99. package/src/AcsMeetingModule.ts +13 -0
  100. package/src/AcsMeetingModule.web.ts +13 -0
@@ -0,0 +1,155 @@
1
+ package com.mentra.acsmeeting.telemetry
2
+
3
+ import android.util.Log
4
+ import kotlin.math.abs
5
+
6
+ /**
7
+ * Measures how far glasses audio leads glasses video at ACS ingest.
8
+ *
9
+ * SoftAP video and BLE LC3 do not share a capture clock, so first-frame arrival is startup
10
+ * skew, not lip-sync. This probe watches both paths on [System.nanoTime]: a clap in frame is a
11
+ * loud PCM transient and a luma jump. Their difference is [lastAudioLeadMs] — the number the
12
+ * delay line can cancel. ACS/Teams jitter after ingest is still a receiver recording.
13
+ *
14
+ * Audio is sampled at uplink ingest, in front of any configured delay, so the lead is the raw
15
+ * path offset even when the host already holds PCM for alignment.
16
+ */
17
+ class AvSyncProbe(
18
+ private val clock: () -> Long = System::nanoTime,
19
+ private val log: (String) -> Unit = { Log.i(TAG, it) },
20
+ ) {
21
+ private val lock = Any()
22
+ private var lastMeanAbs = -1
23
+ private var lastLumaY = -1
24
+ private var pendingAudioNs = 0L
25
+ private var pendingVideoNs = 0L
26
+ private var pendingMeanAbs = -1
27
+ private var pendingLumaY = -1
28
+ private var cooldownUntilNs = 0L
29
+ @Volatile private var clapCount = 0
30
+ @Volatile private var lastAudioLeadMs: Int? = null
31
+ @Volatile private var publishedMeanAbs = -1
32
+ @Volatile private var publishedLumaY = -1
33
+
34
+ fun onAudio(pcm: ByteArray, nowNs: Long = clock()) {
35
+ onAudioLevel(meanAbs16(pcm), nowNs)
36
+ }
37
+
38
+ fun onAudioLevel(meanAbs: Int, nowNs: Long = clock()) {
39
+ publishedMeanAbs = meanAbs
40
+ synchronized(lock) {
41
+ val prev = lastMeanAbs
42
+ lastMeanAbs = meanAbs
43
+ expire(nowNs)
44
+ if (nowNs < cooldownUntilNs) return
45
+ if (!isAudioTransient(prev, meanAbs)) return
46
+ if (pendingVideoNs > 0L) {
47
+ emit(pendingVideoNs - nowNs, meanAbs, pendingLumaY, nowNs)
48
+ return
49
+ }
50
+ pendingAudioNs = nowNs
51
+ pendingMeanAbs = meanAbs
52
+ }
53
+ }
54
+
55
+ fun onVideoLuma(y: Int, nowNs: Long = clock()) {
56
+ publishedLumaY = y
57
+ synchronized(lock) {
58
+ val prev = lastLumaY
59
+ lastLumaY = y
60
+ expire(nowNs)
61
+ if (nowNs < cooldownUntilNs) return
62
+ if (!isVideoTransient(prev, y)) return
63
+ if (pendingAudioNs > 0L) {
64
+ emit(nowNs - pendingAudioNs, pendingMeanAbs, y, nowNs)
65
+ return
66
+ }
67
+ pendingVideoNs = nowNs
68
+ pendingLumaY = y
69
+ }
70
+ }
71
+
72
+ fun lastAudioLeadMs(): Int? = lastAudioLeadMs
73
+
74
+ fun clapCount(): Int = clapCount
75
+
76
+ fun tick(videoE2eP50: String, videoAgeP50: String): String {
77
+ val lead = lastAudioLeadMs
78
+ return "AVSYNC live videoE2eP50=$videoE2eP50 " +
79
+ "videoAgeP50=$videoAgeP50 " +
80
+ "lastClapLeadMs=${lead?.toString() ?: "na"} clapN=$clapCount " +
81
+ "meanAbs=$publishedMeanAbs lumaY=$publishedLumaY"
82
+ }
83
+
84
+ fun reset() {
85
+ synchronized(lock) {
86
+ lastMeanAbs = -1
87
+ lastLumaY = -1
88
+ pendingAudioNs = 0L
89
+ pendingVideoNs = 0L
90
+ pendingMeanAbs = -1
91
+ pendingLumaY = -1
92
+ cooldownUntilNs = 0L
93
+ clapCount = 0
94
+ lastAudioLeadMs = null
95
+ publishedMeanAbs = -1
96
+ publishedLumaY = -1
97
+ }
98
+ }
99
+
100
+ private fun emit(leadNs: Long, meanAbs: Int, lumaY: Int, nowNs: Long) {
101
+ val leadMs = (leadNs / 1_000_000L).toInt()
102
+ lastAudioLeadMs = leadMs
103
+ clapCount += 1
104
+ pendingAudioNs = 0L
105
+ pendingVideoNs = 0L
106
+ pendingMeanAbs = -1
107
+ pendingLumaY = -1
108
+ cooldownUntilNs = nowNs + COOLDOWN_NS
109
+ log("AVSYNC clap audioLeadMs=$leadMs meanAbs=$meanAbs lumaY=$lumaY clapN=$clapCount")
110
+ }
111
+
112
+ private fun expire(nowNs: Long) {
113
+ if (pendingAudioNs > 0L && nowNs - pendingAudioNs > MATCH_WINDOW_NS) {
114
+ pendingAudioNs = 0L
115
+ pendingMeanAbs = -1
116
+ }
117
+ if (pendingVideoNs > 0L && nowNs - pendingVideoNs > MATCH_WINDOW_NS) {
118
+ pendingVideoNs = 0L
119
+ pendingLumaY = -1
120
+ }
121
+ }
122
+
123
+ companion object {
124
+ private const val TAG = "ACS-SPIKE"
125
+ internal const val AUDIO_CLAP = 2_000
126
+ internal const val AUDIO_RISE = 1_500
127
+ internal const val VIDEO_JUMP = 10
128
+ internal const val MATCH_WINDOW_NS = 500L * 1_000_000L
129
+ internal const val COOLDOWN_NS = 750L * 1_000_000L
130
+
131
+ fun meanAbs16(pcm: ByteArray): Int {
132
+ var acc = 0L
133
+ var n = 0
134
+ var i = 0
135
+ while (i + 1 < pcm.size) {
136
+ val s = (pcm[i].toInt() and 0xff) or (pcm[i + 1].toInt() shl 8)
137
+ val signed = if (s >= 0x8000) s - 0x10000 else s
138
+ acc += abs(signed)
139
+ n++
140
+ i += 2
141
+ }
142
+ return if (n == 0) 0 else (acc / n).toInt()
143
+ }
144
+
145
+ internal fun isAudioTransient(previous: Int, current: Int): Boolean {
146
+ if (previous < 0 || current < AUDIO_CLAP) return false
147
+ return current - previous >= AUDIO_RISE
148
+ }
149
+
150
+ internal fun isVideoTransient(previous: Int, current: Int): Boolean {
151
+ if (previous < 0) return false
152
+ return abs(current - previous) >= VIDEO_JUMP
153
+ }
154
+ }
155
+ }
@@ -24,6 +24,7 @@ class PipelineStats(
24
24
  private val dropNotStarted = AtomicInteger(0)
25
25
  private val dropFail = AtomicInteger(0)
26
26
  private val dropNullI420 = AtomicInteger(0)
27
+ private val dropPaced = AtomicInteger(0)
27
28
  private val abandoned = AtomicInteger(0)
28
29
  private val dup = AtomicInteger(0)
29
30
  private val rot = AtomicInteger(0)
@@ -33,6 +34,8 @@ class PipelineStats(
33
34
 
34
35
  private val lastTickSink = AtomicInteger(0)
35
36
  private val lastTickSub = AtomicInteger(0)
37
+ private val lastTickPackets = AtomicInteger(0)
38
+ private val lastPacketsAtMs = AtomicLong(0)
36
39
  private val lastTickMs = AtomicLong(0)
37
40
 
38
41
  private val lastArrivalNs = AtomicLong(0)
@@ -46,6 +49,28 @@ class PipelineStats(
46
49
  val copy = RingPercentile()
47
50
  val send = RingPercentile()
48
51
 
52
+ /**
53
+ * How stale a frame already is when the decoder hands it to us.
54
+ *
55
+ * Everything else on this ladder measures work we do, so all of it can read healthy while the
56
+ * wearer's picture lags seconds behind: capture, encode, and the network happen before our first
57
+ * timestamp. WebRTC's extrapolated capture time is the only upstream clock we get, and it is an
58
+ * estimate — read the trend, not the absolute value.
59
+ */
60
+ val age = RingPercentile()
61
+
62
+ /**
63
+ * Wait between handing a frame to the ACS sender and its single send thread picking it up.
64
+ *
65
+ * The one stage with no other symptom. `sendP95` times the ACS call itself, so a backed-up
66
+ * `acs-i420-send` queue shows only as drops at the gate, with every timing on this ladder still
67
+ * looking fast.
68
+ */
69
+ val queue = RingPercentile()
70
+
71
+ /** Decoder-estimated capture time to ACS accepting the frame: the number the far end feels. */
72
+ val e2e = RingPercentile()
73
+
49
74
  /**
50
75
  * WebRTC inbound-rtp disposition. Cumulative; the ladder prints deltas so a
51
76
  * climbing counter is visible without reading absolute values.
@@ -74,12 +99,30 @@ class PipelineStats(
74
99
  @Volatile var pix: String = "i420"
75
100
  /** MEDIA_STATISTICS codecName, underscored. Empty until the first report. */
76
101
  @Volatile var codecName: String = ""
102
+ /** The fps we declared to ACS, so [VideoRateVerdict] can score against it. */
103
+ @Volatile var advertisedFps: Double = 0.0
104
+ /** The bitrate ceiling we granted ACS, for the starved-versus-CPU-bound split. */
105
+ @Volatile var budgetBps: Int = 0
106
+ /** Last rates computed by [tick], kept so the verdict scores the same numbers the ladder printed. */
107
+ @Volatile var lastSinkFps: Double = 0.0
108
+ @Volatile var lastSubFps: Double = 0.0
77
109
  @Volatile var decodedFps: Double? = null
78
110
  @Volatile var recvFps: Double? = null
79
111
  @Volatile var wireFps: Double? = null
80
112
  @Volatile var wireWidth: Int? = null
81
113
  @Volatile var wireHeight: Int? = null
82
114
  @Volatile var wireBitrateBps: Long? = null
115
+ /** ACS cumulative outgoing packet count; differenced per tick into [lastPacketsPerSecond]. */
116
+ @Volatile var wirePacketCount: Int? = null
117
+ @Volatile var lastPacketsPerSecond: Double? = null
118
+
119
+ /**
120
+ * Latest ACS `networkSendQuality`, the only send-side network signal the SDK
121
+ * exposes. It arrives on change rather than on a schedule, so it is latched here
122
+ * and reprinted every tick — a GOOD/POOR that fired once before the bitrate
123
+ * started sliding is exactly the line nobody scrolls back far enough to find.
124
+ */
125
+ @Volatile var sendQuality: String = ""
83
126
  @Volatile private var recv: RecvHealth? = null
84
127
  @Volatile private var recvPrev: RecvHealth? = null
85
128
  @Volatile var width: Int = 0
@@ -125,6 +168,15 @@ class PipelineStats(
125
168
  fun onDropFail() = dropFail.incrementAndGet()
126
169
  fun onDropMalformed() = dropFail.incrementAndGet()
127
170
 
171
+ /**
172
+ * Dropped because it arrived sooner than the rate we advertised to ACS.
173
+ *
174
+ * Distinct from [onDropBusy]: the send thread is idle, we just refuse to
175
+ * over-feed the software encoder. Counted in [dropCount] so conservation
176
+ * still holds after the sink has already seen the frame.
177
+ */
178
+ fun onDropPaced() = dropPaced.incrementAndGet()
179
+
128
180
  /**
129
181
  * A send that timed out. The future is not cancelled, so ACS may still read those
130
182
  * direct buffers; they are never recycled. Non-zero here means the pool is leaking.
@@ -221,7 +273,8 @@ class PipelineStats(
221
273
 
222
274
  fun sinkCount(): Int = sink.get()
223
275
  fun subCount(): Int = sub.get()
224
- fun dropCount(): Int = dropSize.get() + dropBusy.get() + dropNotStarted.get() + dropFail.get() + dropNullI420.get()
276
+ fun dropCount(): Int =
277
+ dropSize.get() + dropBusy.get() + dropNotStarted.get() + dropFail.get() + dropNullI420.get() + dropPaced.get()
225
278
 
226
279
  /** Read sink last: every frame hits sink before sub or drop, so this cannot go negative. */
227
280
  fun inFlightCount(): Int {
@@ -234,6 +287,26 @@ class PipelineStats(
234
287
 
235
288
  fun conserved(): Boolean = inFlightCount() >= 0
236
289
 
290
+ /**
291
+ * The phone-side stage with the highest p95, so a soak log names its own bottleneck.
292
+ *
293
+ * Only stages we actually perform are ranked. [age] is excluded on purpose: it is dominated by
294
+ * capture, encode and the network, so including it would make every healthy call blame a stage
295
+ * this device does not run. It is printed beside the verdict instead, which is what separates
296
+ * "the phone is slow" from "the phone is fine and the frames arrive late".
297
+ */
298
+ fun slowestStage(): String {
299
+ val stages = listOf(
300
+ "scale" to scale.p95Ms(),
301
+ "toI420" to toI420.p95Ms(),
302
+ "copy" to copy.p95Ms(),
303
+ "queue" to queue.p95Ms(),
304
+ "send" to send.p95Ms(),
305
+ )
306
+ val worst = stages.filter { it.second >= 0 }.maxByOrNull { it.second } ?: return "na"
307
+ return "${worst.first}=${format(worst.second)}"
308
+ }
309
+
237
310
  fun tick(): String {
238
311
  val now = nowMs()
239
312
  val previous = lastTickMs.get()
@@ -245,6 +318,23 @@ class PipelineStats(
245
318
  val subDelta = subNow - lastTickSub.getAndSet(subNow)
246
319
  val sinkRate = rate(sinkDelta, dt)
247
320
  val subRate = rate(subDelta, dt)
321
+ lastSinkFps = sinkDelta * 1000.0 / dt
322
+ lastSubFps = subDelta * 1000.0 / dt
323
+ // ACS reports packets cumulatively on the MEDIA_STATISTICS interval, which is
324
+ // several times longer than this 1 Hz tick. Differencing every tick reported
325
+ // pps=0.0 on every tick between reports, so rate is measured across the gap
326
+ // between actual changes and held in between rather than recomputed against a
327
+ // count that did not move.
328
+ wirePacketCount?.let { total ->
329
+ val previous = lastTickPackets.get()
330
+ if (total != previous) {
331
+ val previousAt = lastPacketsAtMs.getAndSet(now)
332
+ if (previous in 1..total && previousAt > 0) {
333
+ lastPacketsPerSecond = (total - previous) * 1000.0 / (now - previousAt).coerceAtLeast(1L)
334
+ }
335
+ lastTickPackets.set(total)
336
+ }
337
+ }
248
338
  val dec = decodedFps?.let { formatRate(it) } ?: "na"
249
339
  val rcv = recvFps?.let { formatRate(it) } ?: "na"
250
340
  val wireFpsLabel = wireFps?.let { formatRate(it) } ?: "na"
@@ -257,13 +347,14 @@ class PipelineStats(
257
347
  val inFlightNow = inFlightCount()
258
348
  val conserve = if (inFlightNow >= 0) "" else " CONSERVE_FAIL"
259
349
  return "P6 ladder arm=$arm $sizeLabel recv=$rcv dec=$dec sink=$sinkRate dup=${dup.get()} sub=$subRate wire=$wire rot=${rot.get()} " +
260
- "drop{size=${dropSize.get()} busy=${dropBusy.get()} notStarted=${dropNotStarted.get()} fail=${dropFail.get()} nullI420=${dropNullI420.get()} abandoned=${abandoned.get()}} " +
350
+ "drop{size=${dropSize.get()} busy=${dropBusy.get()} notStarted=${dropNotStarted.get()} fail=${dropFail.get()} nullI420=${dropNullI420.get()} pace=${dropPaced.get()} abandoned=${abandoned.get()}} " +
261
351
  "${recvLabel()} " +
262
352
  "path{mode=$pathMode copy=$pathCopy pix=$pix} " +
263
353
  "buf{tex=${bufTex.get()} i420=${bufI420.get()} other=${bufOther.get()}} " +
264
354
  "stride{y=$strideY u=$strideU v=$strideV tight=${strideTight.get()} padded=${stridePadded.get()}} " +
265
355
  "zc{on=$zcOn used=${zcUsed.get()} fell=${zcFell.get()} padded=${zcPadded.get()} heldMax=${zcHeldMax.get()} timeout=${zcTimeout.get()}} " +
266
- "ms{gapP50=${gap.p50()} gapP95=${gap.p95()} i420P95=${toI420.p95()} packP95=${pack.p95()} scaleP95=${scale.p95()} sinkCbP95=${sinkCb.p95()} splitP95=${split.p95()} copyP95=${copy.p95()} sendP95=${send.p95()}} " +
356
+ "ms{gapP50=${gap.p50()} gapP95=${gap.p95()} i420P95=${toI420.p95()} packP95=${pack.p95()} scaleP95=${scale.p95()} sinkCbP95=${sinkCb.p95()} splitP95=${split.p95()} copyP95=${copy.p95()} queueP95=${queue.p95()} sendP95=${send.p95()}} " +
357
+ "lat{ageP50=${age.p50()} ageP95=${age.p95()} e2eP50=${e2e.p50()} e2eP95=${e2e.p95()} slowest=${slowestStage()}} " +
267
358
  "alloc{dest=${destAlloc.get()} plane=${planeAlloc.get()}} " +
268
359
  "chroma{y=$chromaY u=$chromaU v=$chromaV} " +
269
360
  "cum{sink=$sinkNow sub=$subNow drop=${dropCount()} inFlight=$inFlightNow}" +
@@ -7,6 +7,7 @@ import android.os.Process
7
7
  /** Timer-driven 1 Hz emit so a stall prints sink=0 instead of going silent. */
8
8
  class PipelineTicker(
9
9
  private val stats: PipelineStats,
10
+ private val avSync: AvSyncProbe? = null,
10
11
  looper: Looper = Looper.getMainLooper(),
11
12
  private val elapsedCpuMs: () -> Long = { Process.getElapsedCpuTime() },
12
13
  private val cores: Int = Runtime.getRuntime().availableProcessors(),
@@ -17,7 +18,35 @@ class PipelineTicker(
17
18
  private var lastWallMs = 0L
18
19
  private val tick = object : Runnable {
19
20
  override fun run() {
20
- emit("${stats.tick()} ${cpuLabel()}")
21
+ // CPU is sampled once and shared: the ladder prints it and the verdict uses
22
+ // it to tell an encoder that cannot keep up from one being throttled.
23
+ val percent = samplePercent()
24
+ emit("${stats.tick()} ${ProcessCpu.label(percent, cores)}")
25
+ emit(
26
+ VideoRateVerdict.line(
27
+ advertisedFps = stats.advertisedFps,
28
+ sinkFps = stats.lastSinkFps,
29
+ admittedFps = stats.lastSubFps,
30
+ wireFps = stats.wireFps,
31
+ wireBitrateBps = stats.wireBitrateBps,
32
+ budgetBps = stats.budgetBps,
33
+ cpuPercent = percent,
34
+ codecName = stats.codecName,
35
+ ),
36
+ )
37
+ emit(
38
+ VideoQuality.line(
39
+ wireWidth = stats.wireWidth,
40
+ wireHeight = stats.wireHeight,
41
+ wireFps = stats.wireFps,
42
+ wireBitrateBps = stats.wireBitrateBps,
43
+ budgetBps = stats.budgetBps,
44
+ packetsPerSecond = stats.lastPacketsPerSecond,
45
+ cpuPercent = percent,
46
+ sendQuality = stats.sendQuality,
47
+ ),
48
+ )
49
+ avSync?.let { emit(it.tick(stats.e2e.p50(), stats.age.p50())) }
21
50
  handler.postDelayed(this, INTERVAL_MS)
22
51
  }
23
52
  }
@@ -33,16 +62,15 @@ class PipelineTicker(
33
62
  handler.removeCallbacks(tick)
34
63
  }
35
64
 
36
- private fun cpuLabel(): String {
65
+ private fun samplePercent(): Double? {
37
66
  val now = System.currentTimeMillis()
38
67
  val cpu = elapsedCpuMs()
39
68
  val prevCpu = lastCpuMs
40
69
  val prevWall = lastWallMs
41
70
  lastCpuMs = cpu
42
71
  lastWallMs = now
43
- val percent = if (prevCpu < 0 || prevWall <= 0) null
44
- else ProcessCpu.percent(cpu - prevCpu, now - prevWall)
45
- return ProcessCpu.label(percent, cores)
72
+ if (prevCpu < 0 || prevWall <= 0) return null
73
+ return ProcessCpu.percent(cpu - prevCpu, now - prevWall)
46
74
  }
47
75
 
48
76
  companion object {
@@ -28,6 +28,21 @@ class RingPercentile(private val capacity: Int = 32) {
28
28
  return ((copy[idx] / 1_000_000.0) * 10).roundToInt().div(10.0).toString()
29
29
  }
30
30
 
31
+ /**
32
+ * p95 in milliseconds, or -1 when nothing has been recorded.
33
+ *
34
+ * Numeric rather than the formatted [p95] because the ladder ranks stages against each other to
35
+ * name the slowest one, and comparing the rendered strings would order "9.0" above "10.0".
36
+ */
37
+ @Synchronized
38
+ fun p95Ms(): Double {
39
+ if (count == 0) return -1.0
40
+ val copy = samples.copyOf(count)
41
+ copy.sort()
42
+ val idx = ((copy.size - 1) * 0.95).roundToInt().coerceIn(0, copy.lastIndex)
43
+ return copy[idx] / 1_000_000.0
44
+ }
45
+
31
46
  @Synchronized
32
47
  fun size(): Int = count
33
48
  }
@@ -0,0 +1,178 @@
1
+ package com.mentra.acsmeeting.telemetry
2
+
3
+ import kotlin.math.roundToInt
4
+
5
+ /**
6
+ * How good the picture is, as opposed to how many frames of it there are.
7
+ *
8
+ * Every rate number we log can be healthy while Teams shows mush, because
9
+ * "pixelated" is bits per pixel and nothing in the ladder computed that.
10
+ * `wire=960x540@14.4 kbps=1276` reads fine and is 0.17 bits per pixel, which is
11
+ * under what a head-mounted camera needs. ACS 2.16.0's
12
+ * `OutgoingVideoStatistics` exposes no QP and no quality-limitation reason —
13
+ * only codec, bitrate, packet count, rate, and size — so bits per pixel is the
14
+ * only quality signal available and it has to be derived here.
15
+ *
16
+ * Bands come from the Kush gauge, the standard H.264 estimate
17
+ * `bitrate ≈ width · height · fps · 0.07 · motion`, which rearranges to
18
+ * `bpp ≈ 0.07 · motion`. Motion is 1 for a near-static scene, 2 for moderate,
19
+ * and 4 for high. Glasses are strapped to someone's head, so the honest target
20
+ * is the high-motion one — which is why a number sitting mid-band can still
21
+ * look bad and the log says so rather than printing a bare "OK".
22
+ */
23
+ object VideoQuality {
24
+ enum class Band {
25
+ /** Below what even a static scene needs. Visible blocking. */
26
+ BLOCKY,
27
+
28
+ /** Enough for a near-static scene only. Soft, smeared on movement. */
29
+ SOFT,
30
+
31
+ /** Enough for moderate motion. Still short for a head-mounted camera. */
32
+ OK,
33
+
34
+ /** Enough for high motion, which is what glasses actually produce. */
35
+ GOOD,
36
+
37
+ UNKNOWN,
38
+ }
39
+
40
+ /** bpp ≈ 0.07 · motion, for motion = 1, 2, and 4. */
41
+ const val STATIC_BPP = 0.07
42
+ const val MODERATE_BPP = 0.14
43
+ const val HIGH_MOTION_BPP = 0.28
44
+
45
+ fun bitsPerPixel(bitrateBps: Long?, width: Int?, height: Int?, fps: Double?): Double? {
46
+ if (bitrateBps == null || width == null || height == null || fps == null) return null
47
+ if (bitrateBps <= 0 || width <= 0 || height <= 0 || fps <= 0.0) return null
48
+ return bitrateBps / (width.toDouble() * height * fps)
49
+ }
50
+
51
+ fun band(bpp: Double?): Band = when {
52
+ bpp == null -> Band.UNKNOWN
53
+ bpp < STATIC_BPP -> Band.BLOCKY
54
+ bpp < MODERATE_BPP -> Band.SOFT
55
+ bpp < HIGH_MOTION_BPP -> Band.OK
56
+ else -> Band.GOOD
57
+ }
58
+
59
+ /** Bitrate this geometry needs to reach [bpp], so the log can name the shortfall. */
60
+ fun neededBps(width: Int, height: Int, fps: Double, bpp: Double): Long {
61
+ if (width <= 0 || height <= 0 || fps <= 0.0) return 0L
62
+ return (width.toDouble() * height * fps * bpp).toLong()
63
+ }
64
+
65
+ /**
66
+ * Whether the granted ceiling can even reach high motion at this geometry.
67
+ *
68
+ * This is the check that turns "the picture is bad" into a decision. 540p15
69
+ * against a 1.5 Mbps budget tops out at 0.19 bpp even if ACS spends every bit,
70
+ * so no encoder change and no rate fix can make that combination look sharp on
71
+ * a moving camera — the budget is the ceiling, and it has to move.
72
+ */
73
+ fun budgetCapsQuality(width: Int, height: Int, fps: Double, budgetBps: Int): Boolean {
74
+ if (budgetBps <= 0) return false
75
+ return budgetBps < neededBps(width, height, fps, HIGH_MOTION_BPP)
76
+ }
77
+
78
+ /**
79
+ * Whether ACS is actually spending what it was granted.
80
+ *
81
+ * The grant is a target, not a wall — a device run recorded `used=108%`, the wire
82
+ * at 1623 kbps against a 1.5 Mbps grant — so falling short of it is a choice the
83
+ * rate controller made rather than a limit it hit. That makes the shortfall
84
+ * diagnostic: something upstream of the ceiling is holding the bitrate down, and
85
+ * [hint] uses CPU to say which thing.
86
+ */
87
+ const val SPENDING_FRACTION = 0.85
88
+
89
+ /**
90
+ * Process CPU at which a software encoder is a plausible cap.
91
+ *
92
+ * The figure is a percentage of a *single* core, not of the device, so on an
93
+ * 8-core phone 88% means one core nearly full while seven sit idle. That still
94
+ * binds a single-threaded encoder, which is why the threshold sits near one core
95
+ * rather than near the machine.
96
+ */
97
+ const val ENCODER_BUSY_PERCENT = 90.0
98
+
99
+ fun spendingBudget(wireBitrateBps: Long?, budgetBps: Int): Boolean {
100
+ if (wireBitrateBps == null || budgetBps <= 0) return false
101
+ return wireBitrateBps >= budgetBps * SPENDING_FRACTION
102
+ }
103
+
104
+ fun hint(
105
+ band: Band,
106
+ budgetCapped: Boolean,
107
+ spendingBudget: Boolean,
108
+ encoderBusy: Boolean,
109
+ ): String = when {
110
+ band == Band.UNKNOWN -> "no wire report yet"
111
+ band == Band.GOOD -> "enough for high motion"
112
+ !spendingBudget && encoderBusy ->
113
+ "bits unspent with a core nearly full; encoder is the cap -- cut pixels, not the ceiling"
114
+ !spendingBudget ->
115
+ "bits unspent while CPU is idle; congestion control is holding the rate down, " +
116
+ "so neither a bigger ceiling nor fewer pixels addresses it -- check the network"
117
+ budgetCapped ->
118
+ "budget cannot reach high motion at this geometry; raise maxBitrateBps or drop resolution"
119
+ band == Band.BLOCKY -> "far below any usable bitrate; check for a rate-controller collapse"
120
+ band == Band.SOFT -> "only enough for a static scene; a head camera will smear"
121
+ else -> "fine for moderate motion, still short for a head-mounted camera"
122
+ }
123
+
124
+ /**
125
+ * One line per tick. Prints the derived figures and the bitrate that would be
126
+ * needed, because "0.17 bpp" alone does not tell anyone what to change.
127
+ */
128
+ fun line(
129
+ wireWidth: Int?,
130
+ wireHeight: Int?,
131
+ wireFps: Double?,
132
+ wireBitrateBps: Long?,
133
+ budgetBps: Int,
134
+ packetsPerSecond: Double?,
135
+ cpuPercent: Double?,
136
+ sendQuality: String,
137
+ ): String {
138
+ val bpp = bitsPerPixel(wireBitrateBps, wireWidth, wireHeight, wireFps)
139
+ val band = band(bpp)
140
+ val capped = if (wireWidth != null && wireHeight != null && wireFps != null) {
141
+ budgetCapsQuality(wireWidth, wireHeight, wireFps, budgetBps)
142
+ } else {
143
+ false
144
+ }
145
+ val bitsPerFrame = if (wireBitrateBps != null && wireFps != null && wireFps > 0) {
146
+ (wireBitrateBps / wireFps).roundToInt().toString()
147
+ } else {
148
+ "na"
149
+ }
150
+ val goodKbps = if (wireWidth != null && wireHeight != null && wireFps != null) {
151
+ (neededBps(wireWidth, wireHeight, wireFps, HIGH_MOTION_BPP) / 1000).toString()
152
+ } else {
153
+ "na"
154
+ }
155
+ val usedKbps = wireBitrateBps?.let { (it / 1000).toString() } ?: "na"
156
+ val usedPercent = if (wireBitrateBps != null && budgetBps > 0) {
157
+ ((wireBitrateBps * 100.0 / budgetBps).roundToInt()).toString()
158
+ } else {
159
+ "na"
160
+ }
161
+ return "P9 quality band=$band bpp=${bpp?.let { decimal(it) } ?: "na"} " +
162
+ "size=${wireWidth ?: 0}x${wireHeight ?: 0}@${wireFps?.let { PipelineStats.formatRate(it) } ?: "na"} " +
163
+ "bitsPerFrame=$bitsPerFrame kbps=$usedKbps budgetKbps=${budgetBps / 1000} used=$usedPercent% " +
164
+ "goodNeedsKbps=$goodKbps budgetCapped=$capped " +
165
+ "pps=${packetsPerSecond?.let { PipelineStats.formatRate(it) } ?: "na"} " +
166
+ "cpu=${cpuPercent?.let { PipelineStats.formatRate(it) } ?: "na"} " +
167
+ "sendQuality=${sendQuality.ifBlank { "na" }} -- " +
168
+ hint(
169
+ band = band,
170
+ budgetCapped = capped,
171
+ spendingBudget = spendingBudget(wireBitrateBps, budgetBps),
172
+ encoderBusy = cpuPercent != null && cpuPercent >= ENCODER_BUSY_PERCENT,
173
+ )
174
+ }
175
+
176
+ /** Three places, because bpp differences that matter show up in the third. */
177
+ private fun decimal(value: Double): String = ((value * 1000).roundToInt() / 1000.0).toString()
178
+ }