@mentra/acs-meeting 3.2.0-dev.221 → 3.2.0-dev.223
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/android/build.gradle +23 -0
- package/android/src/androidTest/java/com/mentra/acsmeeting/network/SoftApFeasibilityGateTest.kt +291 -0
- package/android/src/main/AndroidManifest.xml +2 -0
- package/android/src/main/java/com/mentra/acsmeeting/AbandonedCallAgent.kt +36 -0
- package/android/src/main/java/com/mentra/acsmeeting/AcsMeetingModule.kt +392 -22
- package/android/src/main/java/com/mentra/acsmeeting/AcsMeetingSession.kt +909 -56
- package/android/src/main/java/com/mentra/acsmeeting/CallCapabilities.kt +33 -0
- package/android/src/main/java/com/mentra/acsmeeting/audio/AcsAudioPolicy.kt +21 -0
- package/android/src/main/java/com/mentra/acsmeeting/audio/AudioUplinkChain.kt +126 -0
- package/android/src/main/java/com/mentra/acsmeeting/audio/PcmBridge.kt +27 -4
- package/android/src/main/java/com/mentra/acsmeeting/audio/UplinkPacer.kt +45 -10
- package/android/src/main/java/com/mentra/acsmeeting/audio/UplinkSender.kt +110 -5
- package/android/src/main/java/com/mentra/acsmeeting/network/InternetHold.kt +501 -0
- package/android/src/main/java/com/mentra/acsmeeting/network/Ipv4Prefix.kt +51 -0
- package/android/src/main/java/com/mentra/acsmeeting/network/NetworkFacts.kt +81 -0
- package/android/src/main/java/com/mentra/acsmeeting/network/ScopedNetworkChangeDetector.kt +355 -0
- package/android/src/main/java/com/mentra/acsmeeting/network/ScopedNetworkError.kt +109 -0
- package/android/src/main/java/com/mentra/acsmeeting/network/ScopedNetworkObserver.kt +160 -0
- package/android/src/main/java/com/mentra/acsmeeting/network/ScopedNetworkReadiness.kt +103 -0
- package/android/src/main/java/com/mentra/acsmeeting/network/ScopedNetworkRequestSpec.kt +53 -0
- package/android/src/main/java/com/mentra/acsmeeting/network/ScopedNetworkState.kt +127 -0
- package/android/src/main/java/com/mentra/acsmeeting/network/ScopedSoftApNetwork.kt +573 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/CloudflareWhepSource.kt +10 -101
- package/android/src/main/java/com/mentra/acsmeeting/source/DecodedTrackRelay.kt +147 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/GlassesMediaSource.kt +47 -5
- package/android/src/main/java/com/mentra/acsmeeting/source/LocalWhipIngestSource.kt +772 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/MeetingVideoSourceSpec.kt +73 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/SdpAdapter.kt +17 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/SelectedIcePair.kt +139 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/SoftApIceDiagnosis.kt +144 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/SoftApIcePolicy.kt +46 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/SoftApSdpGuard.kt +160 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/TrackRegistry.kt +3 -1
- package/android/src/main/java/com/mentra/acsmeeting/source/VideoSourceArm.kt +69 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/WhipIngestProtocol.kt +207 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/WhipIngestServer.kt +356 -0
- package/android/src/main/java/com/mentra/acsmeeting/telemetry/AvSyncProbe.kt +155 -0
- package/android/src/main/java/com/mentra/acsmeeting/telemetry/PipelineStats.kt +94 -3
- package/android/src/main/java/com/mentra/acsmeeting/telemetry/PipelineTicker.kt +33 -5
- package/android/src/main/java/com/mentra/acsmeeting/telemetry/RingPercentile.kt +15 -0
- package/android/src/main/java/com/mentra/acsmeeting/telemetry/VideoQuality.kt +178 -0
- package/android/src/main/java/com/mentra/acsmeeting/telemetry/VideoRateVerdict.kt +145 -0
- package/android/src/main/java/com/mentra/acsmeeting/trace/SoftApTrace.kt +143 -0
- package/android/src/main/java/com/mentra/acsmeeting/video/AcsFrameSender.kt +23 -4
- package/android/src/main/java/com/mentra/acsmeeting/video/FramePacer.kt +70 -0
- package/android/src/main/java/com/mentra/acsmeeting/video/I420FormatSpec.kt +1 -1
- package/android/src/main/java/com/mentra/acsmeeting/video/VideoProfile.kt +36 -3
- package/android/src/test/java/com/mentra/acsmeeting/AbandonedCallAgentTest.kt +74 -0
- package/android/src/test/java/com/mentra/acsmeeting/CallCapabilitiesTest.kt +47 -0
- package/android/src/test/java/com/mentra/acsmeeting/audio/AudioUplinkChainTest.kt +250 -0
- package/android/src/test/java/com/mentra/acsmeeting/audio/CapturePolicyTest.kt +44 -0
- package/android/src/test/java/com/mentra/acsmeeting/audio/UplinkPacerTest.kt +34 -4
- package/android/src/test/java/com/mentra/acsmeeting/audio/UplinkSenderTest.kt +96 -0
- package/android/src/test/java/com/mentra/acsmeeting/network/InternetHoldLifecycleTest.kt +102 -0
- package/android/src/test/java/com/mentra/acsmeeting/network/InternetHoldTest.kt +148 -0
- package/android/src/test/java/com/mentra/acsmeeting/network/Ipv4PrefixTest.kt +59 -0
- package/android/src/test/java/com/mentra/acsmeeting/network/NetworkFactsTest.kt +59 -0
- package/android/src/test/java/com/mentra/acsmeeting/network/ScopedLocalNetworkPermissionTest.kt +35 -0
- package/android/src/test/java/com/mentra/acsmeeting/network/ScopedNetworkChangeDetectorTest.kt +165 -0
- package/android/src/test/java/com/mentra/acsmeeting/network/ScopedNetworkErrorTest.kt +96 -0
- package/android/src/test/java/com/mentra/acsmeeting/network/ScopedNetworkLifecycleTest.kt +187 -0
- package/android/src/test/java/com/mentra/acsmeeting/network/ScopedNetworkObserverTest.kt +172 -0
- package/android/src/test/java/com/mentra/acsmeeting/network/ScopedNetworkReadinessTest.kt +141 -0
- package/android/src/test/java/com/mentra/acsmeeting/network/ScopedNetworkRequestSpecTest.kt +69 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/AcsInvestigationTest.kt +40 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/DecodedTrackRelayTest.kt +41 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/GlassesMediaSourceTest.kt +26 -8
- package/android/src/test/java/com/mentra/acsmeeting/source/MeetingVideoSourceSpecTest.kt +109 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/SelectedIcePairTest.kt +165 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/SoftApIceDiagnosisTest.kt +148 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/SoftApIcePolicyTest.kt +76 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/SoftApSdpGuardTest.kt +275 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/TrackRegistryTest.kt +9 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/WhipIngestProtocolTest.kt +356 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/WhipIngestServerLoopbackTest.kt +411 -0
- package/android/src/test/java/com/mentra/acsmeeting/telemetry/AvSyncProbeTest.kt +83 -0
- package/android/src/test/java/com/mentra/acsmeeting/telemetry/PipelineStatsTest.kt +69 -3
- package/android/src/test/java/com/mentra/acsmeeting/telemetry/VideoQualityTest.kt +182 -0
- package/android/src/test/java/com/mentra/acsmeeting/telemetry/VideoRateVerdictTest.kt +149 -0
- package/android/src/test/java/com/mentra/acsmeeting/trace/SoftApTraceTest.kt +97 -0
- package/android/src/test/java/com/mentra/acsmeeting/video/AcsTimestampTest.kt +34 -0
- package/android/src/test/java/com/mentra/acsmeeting/video/FramePacerTest.kt +105 -0
- package/android/src/test/java/com/mentra/acsmeeting/video/I420FormatSpecTest.kt +5 -0
- package/android/src/test/java/com/mentra/acsmeeting/video/VideoProfileTest.kt +38 -3
- package/build/AcsMeeting.types.d.ts +5 -0
- package/build/AcsMeeting.types.d.ts.map +1 -1
- package/build/AcsMeeting.types.js.map +1 -1
- package/build/AcsMeetingModule.d.ts +23 -0
- package/build/AcsMeetingModule.d.ts.map +1 -1
- package/build/AcsMeetingModule.js.map +1 -1
- package/build/AcsMeetingModule.web.d.ts +15 -0
- package/build/AcsMeetingModule.web.d.ts.map +1 -1
- package/build/AcsMeetingModule.web.js +13 -0
- package/build/AcsMeetingModule.web.js.map +1 -1
- package/ios/AcsMeetingModule.swift +213 -2
- package/package.json +1 -1
- package/src/AcsMeeting.types.ts +5 -0
- package/src/AcsMeetingModule.ts +13 -0
- package/src/AcsMeetingModule.web.ts +13 -0
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
package com.mentra.acsmeeting.telemetry
|
|
2
|
+
|
|
3
|
+
import android.util.Log
|
|
4
|
+
import kotlin.math.abs
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Measures how far glasses audio leads glasses video at ACS ingest.
|
|
8
|
+
*
|
|
9
|
+
* SoftAP video and BLE LC3 do not share a capture clock, so first-frame arrival is startup
|
|
10
|
+
* skew, not lip-sync. This probe watches both paths on [System.nanoTime]: a clap in frame is a
|
|
11
|
+
* loud PCM transient and a luma jump. Their difference is [lastAudioLeadMs] — the number the
|
|
12
|
+
* delay line can cancel. ACS/Teams jitter after ingest is still a receiver recording.
|
|
13
|
+
*
|
|
14
|
+
* Audio is sampled at uplink ingest, in front of any configured delay, so the lead is the raw
|
|
15
|
+
* path offset even when the host already holds PCM for alignment.
|
|
16
|
+
*/
|
|
17
|
+
class AvSyncProbe(
|
|
18
|
+
private val clock: () -> Long = System::nanoTime,
|
|
19
|
+
private val log: (String) -> Unit = { Log.i(TAG, it) },
|
|
20
|
+
) {
|
|
21
|
+
private val lock = Any()
|
|
22
|
+
private var lastMeanAbs = -1
|
|
23
|
+
private var lastLumaY = -1
|
|
24
|
+
private var pendingAudioNs = 0L
|
|
25
|
+
private var pendingVideoNs = 0L
|
|
26
|
+
private var pendingMeanAbs = -1
|
|
27
|
+
private var pendingLumaY = -1
|
|
28
|
+
private var cooldownUntilNs = 0L
|
|
29
|
+
@Volatile private var clapCount = 0
|
|
30
|
+
@Volatile private var lastAudioLeadMs: Int? = null
|
|
31
|
+
@Volatile private var publishedMeanAbs = -1
|
|
32
|
+
@Volatile private var publishedLumaY = -1
|
|
33
|
+
|
|
34
|
+
fun onAudio(pcm: ByteArray, nowNs: Long = clock()) {
|
|
35
|
+
onAudioLevel(meanAbs16(pcm), nowNs)
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
fun onAudioLevel(meanAbs: Int, nowNs: Long = clock()) {
|
|
39
|
+
publishedMeanAbs = meanAbs
|
|
40
|
+
synchronized(lock) {
|
|
41
|
+
val prev = lastMeanAbs
|
|
42
|
+
lastMeanAbs = meanAbs
|
|
43
|
+
expire(nowNs)
|
|
44
|
+
if (nowNs < cooldownUntilNs) return
|
|
45
|
+
if (!isAudioTransient(prev, meanAbs)) return
|
|
46
|
+
if (pendingVideoNs > 0L) {
|
|
47
|
+
emit(pendingVideoNs - nowNs, meanAbs, pendingLumaY, nowNs)
|
|
48
|
+
return
|
|
49
|
+
}
|
|
50
|
+
pendingAudioNs = nowNs
|
|
51
|
+
pendingMeanAbs = meanAbs
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
fun onVideoLuma(y: Int, nowNs: Long = clock()) {
|
|
56
|
+
publishedLumaY = y
|
|
57
|
+
synchronized(lock) {
|
|
58
|
+
val prev = lastLumaY
|
|
59
|
+
lastLumaY = y
|
|
60
|
+
expire(nowNs)
|
|
61
|
+
if (nowNs < cooldownUntilNs) return
|
|
62
|
+
if (!isVideoTransient(prev, y)) return
|
|
63
|
+
if (pendingAudioNs > 0L) {
|
|
64
|
+
emit(nowNs - pendingAudioNs, pendingMeanAbs, y, nowNs)
|
|
65
|
+
return
|
|
66
|
+
}
|
|
67
|
+
pendingVideoNs = nowNs
|
|
68
|
+
pendingLumaY = y
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
fun lastAudioLeadMs(): Int? = lastAudioLeadMs
|
|
73
|
+
|
|
74
|
+
fun clapCount(): Int = clapCount
|
|
75
|
+
|
|
76
|
+
fun tick(videoE2eP50: String, videoAgeP50: String): String {
|
|
77
|
+
val lead = lastAudioLeadMs
|
|
78
|
+
return "AVSYNC live videoE2eP50=$videoE2eP50 " +
|
|
79
|
+
"videoAgeP50=$videoAgeP50 " +
|
|
80
|
+
"lastClapLeadMs=${lead?.toString() ?: "na"} clapN=$clapCount " +
|
|
81
|
+
"meanAbs=$publishedMeanAbs lumaY=$publishedLumaY"
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
fun reset() {
|
|
85
|
+
synchronized(lock) {
|
|
86
|
+
lastMeanAbs = -1
|
|
87
|
+
lastLumaY = -1
|
|
88
|
+
pendingAudioNs = 0L
|
|
89
|
+
pendingVideoNs = 0L
|
|
90
|
+
pendingMeanAbs = -1
|
|
91
|
+
pendingLumaY = -1
|
|
92
|
+
cooldownUntilNs = 0L
|
|
93
|
+
clapCount = 0
|
|
94
|
+
lastAudioLeadMs = null
|
|
95
|
+
publishedMeanAbs = -1
|
|
96
|
+
publishedLumaY = -1
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
private fun emit(leadNs: Long, meanAbs: Int, lumaY: Int, nowNs: Long) {
|
|
101
|
+
val leadMs = (leadNs / 1_000_000L).toInt()
|
|
102
|
+
lastAudioLeadMs = leadMs
|
|
103
|
+
clapCount += 1
|
|
104
|
+
pendingAudioNs = 0L
|
|
105
|
+
pendingVideoNs = 0L
|
|
106
|
+
pendingMeanAbs = -1
|
|
107
|
+
pendingLumaY = -1
|
|
108
|
+
cooldownUntilNs = nowNs + COOLDOWN_NS
|
|
109
|
+
log("AVSYNC clap audioLeadMs=$leadMs meanAbs=$meanAbs lumaY=$lumaY clapN=$clapCount")
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
private fun expire(nowNs: Long) {
|
|
113
|
+
if (pendingAudioNs > 0L && nowNs - pendingAudioNs > MATCH_WINDOW_NS) {
|
|
114
|
+
pendingAudioNs = 0L
|
|
115
|
+
pendingMeanAbs = -1
|
|
116
|
+
}
|
|
117
|
+
if (pendingVideoNs > 0L && nowNs - pendingVideoNs > MATCH_WINDOW_NS) {
|
|
118
|
+
pendingVideoNs = 0L
|
|
119
|
+
pendingLumaY = -1
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
companion object {
|
|
124
|
+
private const val TAG = "ACS-SPIKE"
|
|
125
|
+
internal const val AUDIO_CLAP = 2_000
|
|
126
|
+
internal const val AUDIO_RISE = 1_500
|
|
127
|
+
internal const val VIDEO_JUMP = 10
|
|
128
|
+
internal const val MATCH_WINDOW_NS = 500L * 1_000_000L
|
|
129
|
+
internal const val COOLDOWN_NS = 750L * 1_000_000L
|
|
130
|
+
|
|
131
|
+
fun meanAbs16(pcm: ByteArray): Int {
|
|
132
|
+
var acc = 0L
|
|
133
|
+
var n = 0
|
|
134
|
+
var i = 0
|
|
135
|
+
while (i + 1 < pcm.size) {
|
|
136
|
+
val s = (pcm[i].toInt() and 0xff) or (pcm[i + 1].toInt() shl 8)
|
|
137
|
+
val signed = if (s >= 0x8000) s - 0x10000 else s
|
|
138
|
+
acc += abs(signed)
|
|
139
|
+
n++
|
|
140
|
+
i += 2
|
|
141
|
+
}
|
|
142
|
+
return if (n == 0) 0 else (acc / n).toInt()
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
internal fun isAudioTransient(previous: Int, current: Int): Boolean {
|
|
146
|
+
if (previous < 0 || current < AUDIO_CLAP) return false
|
|
147
|
+
return current - previous >= AUDIO_RISE
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
internal fun isVideoTransient(previous: Int, current: Int): Boolean {
|
|
151
|
+
if (previous < 0) return false
|
|
152
|
+
return abs(current - previous) >= VIDEO_JUMP
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
@@ -24,6 +24,7 @@ class PipelineStats(
|
|
|
24
24
|
private val dropNotStarted = AtomicInteger(0)
|
|
25
25
|
private val dropFail = AtomicInteger(0)
|
|
26
26
|
private val dropNullI420 = AtomicInteger(0)
|
|
27
|
+
private val dropPaced = AtomicInteger(0)
|
|
27
28
|
private val abandoned = AtomicInteger(0)
|
|
28
29
|
private val dup = AtomicInteger(0)
|
|
29
30
|
private val rot = AtomicInteger(0)
|
|
@@ -33,6 +34,8 @@ class PipelineStats(
|
|
|
33
34
|
|
|
34
35
|
private val lastTickSink = AtomicInteger(0)
|
|
35
36
|
private val lastTickSub = AtomicInteger(0)
|
|
37
|
+
private val lastTickPackets = AtomicInteger(0)
|
|
38
|
+
private val lastPacketsAtMs = AtomicLong(0)
|
|
36
39
|
private val lastTickMs = AtomicLong(0)
|
|
37
40
|
|
|
38
41
|
private val lastArrivalNs = AtomicLong(0)
|
|
@@ -46,6 +49,28 @@ class PipelineStats(
|
|
|
46
49
|
val copy = RingPercentile()
|
|
47
50
|
val send = RingPercentile()
|
|
48
51
|
|
|
52
|
+
/**
|
|
53
|
+
* How stale a frame already is when the decoder hands it to us.
|
|
54
|
+
*
|
|
55
|
+
* Everything else on this ladder measures work we do, so all of it can read healthy while the
|
|
56
|
+
* wearer's picture lags seconds behind: capture, encode, and the network happen before our first
|
|
57
|
+
* timestamp. WebRTC's extrapolated capture time is the only upstream clock we get, and it is an
|
|
58
|
+
* estimate — read the trend, not the absolute value.
|
|
59
|
+
*/
|
|
60
|
+
val age = RingPercentile()
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Wait between handing a frame to the ACS sender and its single send thread picking it up.
|
|
64
|
+
*
|
|
65
|
+
* The one stage with no other symptom. `sendP95` times the ACS call itself, so a backed-up
|
|
66
|
+
* `acs-i420-send` queue shows only as drops at the gate, with every timing on this ladder still
|
|
67
|
+
* looking fast.
|
|
68
|
+
*/
|
|
69
|
+
val queue = RingPercentile()
|
|
70
|
+
|
|
71
|
+
/** Decoder-estimated capture time to ACS accepting the frame: the number the far end feels. */
|
|
72
|
+
val e2e = RingPercentile()
|
|
73
|
+
|
|
49
74
|
/**
|
|
50
75
|
* WebRTC inbound-rtp disposition. Cumulative; the ladder prints deltas so a
|
|
51
76
|
* climbing counter is visible without reading absolute values.
|
|
@@ -74,12 +99,30 @@ class PipelineStats(
|
|
|
74
99
|
@Volatile var pix: String = "i420"
|
|
75
100
|
/** MEDIA_STATISTICS codecName, underscored. Empty until the first report. */
|
|
76
101
|
@Volatile var codecName: String = ""
|
|
102
|
+
/** The fps we declared to ACS, so [VideoRateVerdict] can score against it. */
|
|
103
|
+
@Volatile var advertisedFps: Double = 0.0
|
|
104
|
+
/** The bitrate ceiling we granted ACS, for the starved-versus-CPU-bound split. */
|
|
105
|
+
@Volatile var budgetBps: Int = 0
|
|
106
|
+
/** Last rates computed by [tick], kept so the verdict scores the same numbers the ladder printed. */
|
|
107
|
+
@Volatile var lastSinkFps: Double = 0.0
|
|
108
|
+
@Volatile var lastSubFps: Double = 0.0
|
|
77
109
|
@Volatile var decodedFps: Double? = null
|
|
78
110
|
@Volatile var recvFps: Double? = null
|
|
79
111
|
@Volatile var wireFps: Double? = null
|
|
80
112
|
@Volatile var wireWidth: Int? = null
|
|
81
113
|
@Volatile var wireHeight: Int? = null
|
|
82
114
|
@Volatile var wireBitrateBps: Long? = null
|
|
115
|
+
/** ACS cumulative outgoing packet count; differenced per tick into [lastPacketsPerSecond]. */
|
|
116
|
+
@Volatile var wirePacketCount: Int? = null
|
|
117
|
+
@Volatile var lastPacketsPerSecond: Double? = null
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* Latest ACS `networkSendQuality`, the only send-side network signal the SDK
|
|
121
|
+
* exposes. It arrives on change rather than on a schedule, so it is latched here
|
|
122
|
+
* and reprinted every tick — a GOOD/POOR that fired once before the bitrate
|
|
123
|
+
* started sliding is exactly the line nobody scrolls back far enough to find.
|
|
124
|
+
*/
|
|
125
|
+
@Volatile var sendQuality: String = ""
|
|
83
126
|
@Volatile private var recv: RecvHealth? = null
|
|
84
127
|
@Volatile private var recvPrev: RecvHealth? = null
|
|
85
128
|
@Volatile var width: Int = 0
|
|
@@ -125,6 +168,15 @@ class PipelineStats(
|
|
|
125
168
|
fun onDropFail() = dropFail.incrementAndGet()
|
|
126
169
|
fun onDropMalformed() = dropFail.incrementAndGet()
|
|
127
170
|
|
|
171
|
+
/**
|
|
172
|
+
* Dropped because it arrived sooner than the rate we advertised to ACS.
|
|
173
|
+
*
|
|
174
|
+
* Distinct from [onDropBusy]: the send thread is idle, we just refuse to
|
|
175
|
+
* over-feed the software encoder. Counted in [dropCount] so conservation
|
|
176
|
+
* still holds after the sink has already seen the frame.
|
|
177
|
+
*/
|
|
178
|
+
fun onDropPaced() = dropPaced.incrementAndGet()
|
|
179
|
+
|
|
128
180
|
/**
|
|
129
181
|
* A send that timed out. The future is not cancelled, so ACS may still read those
|
|
130
182
|
* direct buffers; they are never recycled. Non-zero here means the pool is leaking.
|
|
@@ -221,7 +273,8 @@ class PipelineStats(
|
|
|
221
273
|
|
|
222
274
|
fun sinkCount(): Int = sink.get()
|
|
223
275
|
fun subCount(): Int = sub.get()
|
|
224
|
-
fun dropCount(): Int =
|
|
276
|
+
fun dropCount(): Int =
|
|
277
|
+
dropSize.get() + dropBusy.get() + dropNotStarted.get() + dropFail.get() + dropNullI420.get() + dropPaced.get()
|
|
225
278
|
|
|
226
279
|
/** Read sink last: every frame hits sink before sub or drop, so this cannot go negative. */
|
|
227
280
|
fun inFlightCount(): Int {
|
|
@@ -234,6 +287,26 @@ class PipelineStats(
|
|
|
234
287
|
|
|
235
288
|
fun conserved(): Boolean = inFlightCount() >= 0
|
|
236
289
|
|
|
290
|
+
/**
|
|
291
|
+
* The phone-side stage with the highest p95, so a soak log names its own bottleneck.
|
|
292
|
+
*
|
|
293
|
+
* Only stages we actually perform are ranked. [age] is excluded on purpose: it is dominated by
|
|
294
|
+
* capture, encode and the network, so including it would make every healthy call blame a stage
|
|
295
|
+
* this device does not run. It is printed beside the verdict instead, which is what separates
|
|
296
|
+
* "the phone is slow" from "the phone is fine and the frames arrive late".
|
|
297
|
+
*/
|
|
298
|
+
fun slowestStage(): String {
|
|
299
|
+
val stages = listOf(
|
|
300
|
+
"scale" to scale.p95Ms(),
|
|
301
|
+
"toI420" to toI420.p95Ms(),
|
|
302
|
+
"copy" to copy.p95Ms(),
|
|
303
|
+
"queue" to queue.p95Ms(),
|
|
304
|
+
"send" to send.p95Ms(),
|
|
305
|
+
)
|
|
306
|
+
val worst = stages.filter { it.second >= 0 }.maxByOrNull { it.second } ?: return "na"
|
|
307
|
+
return "${worst.first}=${format(worst.second)}"
|
|
308
|
+
}
|
|
309
|
+
|
|
237
310
|
fun tick(): String {
|
|
238
311
|
val now = nowMs()
|
|
239
312
|
val previous = lastTickMs.get()
|
|
@@ -245,6 +318,23 @@ class PipelineStats(
|
|
|
245
318
|
val subDelta = subNow - lastTickSub.getAndSet(subNow)
|
|
246
319
|
val sinkRate = rate(sinkDelta, dt)
|
|
247
320
|
val subRate = rate(subDelta, dt)
|
|
321
|
+
lastSinkFps = sinkDelta * 1000.0 / dt
|
|
322
|
+
lastSubFps = subDelta * 1000.0 / dt
|
|
323
|
+
// ACS reports packets cumulatively on the MEDIA_STATISTICS interval, which is
|
|
324
|
+
// several times longer than this 1 Hz tick. Differencing every tick reported
|
|
325
|
+
// pps=0.0 on every tick between reports, so rate is measured across the gap
|
|
326
|
+
// between actual changes and held in between rather than recomputed against a
|
|
327
|
+
// count that did not move.
|
|
328
|
+
wirePacketCount?.let { total ->
|
|
329
|
+
val previous = lastTickPackets.get()
|
|
330
|
+
if (total != previous) {
|
|
331
|
+
val previousAt = lastPacketsAtMs.getAndSet(now)
|
|
332
|
+
if (previous in 1..total && previousAt > 0) {
|
|
333
|
+
lastPacketsPerSecond = (total - previous) * 1000.0 / (now - previousAt).coerceAtLeast(1L)
|
|
334
|
+
}
|
|
335
|
+
lastTickPackets.set(total)
|
|
336
|
+
}
|
|
337
|
+
}
|
|
248
338
|
val dec = decodedFps?.let { formatRate(it) } ?: "na"
|
|
249
339
|
val rcv = recvFps?.let { formatRate(it) } ?: "na"
|
|
250
340
|
val wireFpsLabel = wireFps?.let { formatRate(it) } ?: "na"
|
|
@@ -257,13 +347,14 @@ class PipelineStats(
|
|
|
257
347
|
val inFlightNow = inFlightCount()
|
|
258
348
|
val conserve = if (inFlightNow >= 0) "" else " CONSERVE_FAIL"
|
|
259
349
|
return "P6 ladder arm=$arm $sizeLabel recv=$rcv dec=$dec sink=$sinkRate dup=${dup.get()} sub=$subRate wire=$wire rot=${rot.get()} " +
|
|
260
|
-
"drop{size=${dropSize.get()} busy=${dropBusy.get()} notStarted=${dropNotStarted.get()} fail=${dropFail.get()} nullI420=${dropNullI420.get()} abandoned=${abandoned.get()}} " +
|
|
350
|
+
"drop{size=${dropSize.get()} busy=${dropBusy.get()} notStarted=${dropNotStarted.get()} fail=${dropFail.get()} nullI420=${dropNullI420.get()} pace=${dropPaced.get()} abandoned=${abandoned.get()}} " +
|
|
261
351
|
"${recvLabel()} " +
|
|
262
352
|
"path{mode=$pathMode copy=$pathCopy pix=$pix} " +
|
|
263
353
|
"buf{tex=${bufTex.get()} i420=${bufI420.get()} other=${bufOther.get()}} " +
|
|
264
354
|
"stride{y=$strideY u=$strideU v=$strideV tight=${strideTight.get()} padded=${stridePadded.get()}} " +
|
|
265
355
|
"zc{on=$zcOn used=${zcUsed.get()} fell=${zcFell.get()} padded=${zcPadded.get()} heldMax=${zcHeldMax.get()} timeout=${zcTimeout.get()}} " +
|
|
266
|
-
"ms{gapP50=${gap.p50()} gapP95=${gap.p95()} i420P95=${toI420.p95()} packP95=${pack.p95()} scaleP95=${scale.p95()} sinkCbP95=${sinkCb.p95()} splitP95=${split.p95()} copyP95=${copy.p95()} sendP95=${send.p95()}} " +
|
|
356
|
+
"ms{gapP50=${gap.p50()} gapP95=${gap.p95()} i420P95=${toI420.p95()} packP95=${pack.p95()} scaleP95=${scale.p95()} sinkCbP95=${sinkCb.p95()} splitP95=${split.p95()} copyP95=${copy.p95()} queueP95=${queue.p95()} sendP95=${send.p95()}} " +
|
|
357
|
+
"lat{ageP50=${age.p50()} ageP95=${age.p95()} e2eP50=${e2e.p50()} e2eP95=${e2e.p95()} slowest=${slowestStage()}} " +
|
|
267
358
|
"alloc{dest=${destAlloc.get()} plane=${planeAlloc.get()}} " +
|
|
268
359
|
"chroma{y=$chromaY u=$chromaU v=$chromaV} " +
|
|
269
360
|
"cum{sink=$sinkNow sub=$subNow drop=${dropCount()} inFlight=$inFlightNow}" +
|
|
@@ -7,6 +7,7 @@ import android.os.Process
|
|
|
7
7
|
/** Timer-driven 1 Hz emit so a stall prints sink=0 instead of going silent. */
|
|
8
8
|
class PipelineTicker(
|
|
9
9
|
private val stats: PipelineStats,
|
|
10
|
+
private val avSync: AvSyncProbe? = null,
|
|
10
11
|
looper: Looper = Looper.getMainLooper(),
|
|
11
12
|
private val elapsedCpuMs: () -> Long = { Process.getElapsedCpuTime() },
|
|
12
13
|
private val cores: Int = Runtime.getRuntime().availableProcessors(),
|
|
@@ -17,7 +18,35 @@ class PipelineTicker(
|
|
|
17
18
|
private var lastWallMs = 0L
|
|
18
19
|
private val tick = object : Runnable {
|
|
19
20
|
override fun run() {
|
|
20
|
-
|
|
21
|
+
// CPU is sampled once and shared: the ladder prints it and the verdict uses
|
|
22
|
+
// it to tell an encoder that cannot keep up from one being throttled.
|
|
23
|
+
val percent = samplePercent()
|
|
24
|
+
emit("${stats.tick()} ${ProcessCpu.label(percent, cores)}")
|
|
25
|
+
emit(
|
|
26
|
+
VideoRateVerdict.line(
|
|
27
|
+
advertisedFps = stats.advertisedFps,
|
|
28
|
+
sinkFps = stats.lastSinkFps,
|
|
29
|
+
admittedFps = stats.lastSubFps,
|
|
30
|
+
wireFps = stats.wireFps,
|
|
31
|
+
wireBitrateBps = stats.wireBitrateBps,
|
|
32
|
+
budgetBps = stats.budgetBps,
|
|
33
|
+
cpuPercent = percent,
|
|
34
|
+
codecName = stats.codecName,
|
|
35
|
+
),
|
|
36
|
+
)
|
|
37
|
+
emit(
|
|
38
|
+
VideoQuality.line(
|
|
39
|
+
wireWidth = stats.wireWidth,
|
|
40
|
+
wireHeight = stats.wireHeight,
|
|
41
|
+
wireFps = stats.wireFps,
|
|
42
|
+
wireBitrateBps = stats.wireBitrateBps,
|
|
43
|
+
budgetBps = stats.budgetBps,
|
|
44
|
+
packetsPerSecond = stats.lastPacketsPerSecond,
|
|
45
|
+
cpuPercent = percent,
|
|
46
|
+
sendQuality = stats.sendQuality,
|
|
47
|
+
),
|
|
48
|
+
)
|
|
49
|
+
avSync?.let { emit(it.tick(stats.e2e.p50(), stats.age.p50())) }
|
|
21
50
|
handler.postDelayed(this, INTERVAL_MS)
|
|
22
51
|
}
|
|
23
52
|
}
|
|
@@ -33,16 +62,15 @@ class PipelineTicker(
|
|
|
33
62
|
handler.removeCallbacks(tick)
|
|
34
63
|
}
|
|
35
64
|
|
|
36
|
-
private fun
|
|
65
|
+
private fun samplePercent(): Double? {
|
|
37
66
|
val now = System.currentTimeMillis()
|
|
38
67
|
val cpu = elapsedCpuMs()
|
|
39
68
|
val prevCpu = lastCpuMs
|
|
40
69
|
val prevWall = lastWallMs
|
|
41
70
|
lastCpuMs = cpu
|
|
42
71
|
lastWallMs = now
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
return ProcessCpu.label(percent, cores)
|
|
72
|
+
if (prevCpu < 0 || prevWall <= 0) return null
|
|
73
|
+
return ProcessCpu.percent(cpu - prevCpu, now - prevWall)
|
|
46
74
|
}
|
|
47
75
|
|
|
48
76
|
companion object {
|
|
@@ -28,6 +28,21 @@ class RingPercentile(private val capacity: Int = 32) {
|
|
|
28
28
|
return ((copy[idx] / 1_000_000.0) * 10).roundToInt().div(10.0).toString()
|
|
29
29
|
}
|
|
30
30
|
|
|
31
|
+
/**
|
|
32
|
+
* p95 in milliseconds, or -1 when nothing has been recorded.
|
|
33
|
+
*
|
|
34
|
+
* Numeric rather than the formatted [p95] because the ladder ranks stages against each other to
|
|
35
|
+
* name the slowest one, and comparing the rendered strings would order "9.0" above "10.0".
|
|
36
|
+
*/
|
|
37
|
+
@Synchronized
|
|
38
|
+
fun p95Ms(): Double {
|
|
39
|
+
if (count == 0) return -1.0
|
|
40
|
+
val copy = samples.copyOf(count)
|
|
41
|
+
copy.sort()
|
|
42
|
+
val idx = ((copy.size - 1) * 0.95).roundToInt().coerceIn(0, copy.lastIndex)
|
|
43
|
+
return copy[idx] / 1_000_000.0
|
|
44
|
+
}
|
|
45
|
+
|
|
31
46
|
@Synchronized
|
|
32
47
|
fun size(): Int = count
|
|
33
48
|
}
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
package com.mentra.acsmeeting.telemetry
|
|
2
|
+
|
|
3
|
+
import kotlin.math.roundToInt
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* How good the picture is, as opposed to how many frames of it there are.
|
|
7
|
+
*
|
|
8
|
+
* Every rate number we log can be healthy while Teams shows mush, because
|
|
9
|
+
* "pixelated" is bits per pixel and nothing in the ladder computed that.
|
|
10
|
+
* `wire=960x540@14.4 kbps=1276` reads fine and is 0.17 bits per pixel, which is
|
|
11
|
+
* under what a head-mounted camera needs. ACS 2.16.0's
|
|
12
|
+
* `OutgoingVideoStatistics` exposes no QP and no quality-limitation reason —
|
|
13
|
+
* only codec, bitrate, packet count, rate, and size — so bits per pixel is the
|
|
14
|
+
* only quality signal available and it has to be derived here.
|
|
15
|
+
*
|
|
16
|
+
* Bands come from the Kush gauge, the standard H.264 estimate
|
|
17
|
+
* `bitrate ≈ width · height · fps · 0.07 · motion`, which rearranges to
|
|
18
|
+
* `bpp ≈ 0.07 · motion`. Motion is 1 for a near-static scene, 2 for moderate,
|
|
19
|
+
* and 4 for high. Glasses are strapped to someone's head, so the honest target
|
|
20
|
+
* is the high-motion one — which is why a number sitting mid-band can still
|
|
21
|
+
* look bad and the log says so rather than printing a bare "OK".
|
|
22
|
+
*/
|
|
23
|
+
object VideoQuality {
|
|
24
|
+
enum class Band {
|
|
25
|
+
/** Below what even a static scene needs. Visible blocking. */
|
|
26
|
+
BLOCKY,
|
|
27
|
+
|
|
28
|
+
/** Enough for a near-static scene only. Soft, smeared on movement. */
|
|
29
|
+
SOFT,
|
|
30
|
+
|
|
31
|
+
/** Enough for moderate motion. Still short for a head-mounted camera. */
|
|
32
|
+
OK,
|
|
33
|
+
|
|
34
|
+
/** Enough for high motion, which is what glasses actually produce. */
|
|
35
|
+
GOOD,
|
|
36
|
+
|
|
37
|
+
UNKNOWN,
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** bpp ≈ 0.07 · motion, for motion = 1, 2, and 4. */
|
|
41
|
+
const val STATIC_BPP = 0.07
|
|
42
|
+
const val MODERATE_BPP = 0.14
|
|
43
|
+
const val HIGH_MOTION_BPP = 0.28
|
|
44
|
+
|
|
45
|
+
fun bitsPerPixel(bitrateBps: Long?, width: Int?, height: Int?, fps: Double?): Double? {
|
|
46
|
+
if (bitrateBps == null || width == null || height == null || fps == null) return null
|
|
47
|
+
if (bitrateBps <= 0 || width <= 0 || height <= 0 || fps <= 0.0) return null
|
|
48
|
+
return bitrateBps / (width.toDouble() * height * fps)
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
fun band(bpp: Double?): Band = when {
|
|
52
|
+
bpp == null -> Band.UNKNOWN
|
|
53
|
+
bpp < STATIC_BPP -> Band.BLOCKY
|
|
54
|
+
bpp < MODERATE_BPP -> Band.SOFT
|
|
55
|
+
bpp < HIGH_MOTION_BPP -> Band.OK
|
|
56
|
+
else -> Band.GOOD
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** Bitrate this geometry needs to reach [bpp], so the log can name the shortfall. */
|
|
60
|
+
fun neededBps(width: Int, height: Int, fps: Double, bpp: Double): Long {
|
|
61
|
+
if (width <= 0 || height <= 0 || fps <= 0.0) return 0L
|
|
62
|
+
return (width.toDouble() * height * fps * bpp).toLong()
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Whether the granted ceiling can even reach high motion at this geometry.
|
|
67
|
+
*
|
|
68
|
+
* This is the check that turns "the picture is bad" into a decision. 540p15
|
|
69
|
+
* against a 1.5 Mbps budget tops out at 0.19 bpp even if ACS spends every bit,
|
|
70
|
+
* so no encoder change and no rate fix can make that combination look sharp on
|
|
71
|
+
* a moving camera — the budget is the ceiling, and it has to move.
|
|
72
|
+
*/
|
|
73
|
+
fun budgetCapsQuality(width: Int, height: Int, fps: Double, budgetBps: Int): Boolean {
|
|
74
|
+
if (budgetBps <= 0) return false
|
|
75
|
+
return budgetBps < neededBps(width, height, fps, HIGH_MOTION_BPP)
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Whether ACS is actually spending what it was granted.
|
|
80
|
+
*
|
|
81
|
+
* The grant is a target, not a wall — a device run recorded `used=108%`, the wire
|
|
82
|
+
* at 1623 kbps against a 1.5 Mbps grant — so falling short of it is a choice the
|
|
83
|
+
* rate controller made rather than a limit it hit. That makes the shortfall
|
|
84
|
+
* diagnostic: something upstream of the ceiling is holding the bitrate down, and
|
|
85
|
+
* [hint] uses CPU to say which thing.
|
|
86
|
+
*/
|
|
87
|
+
const val SPENDING_FRACTION = 0.85
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Process CPU at which a software encoder is a plausible cap.
|
|
91
|
+
*
|
|
92
|
+
* The figure is a percentage of a *single* core, not of the device, so on an
|
|
93
|
+
* 8-core phone 88% means one core nearly full while seven sit idle. That still
|
|
94
|
+
* binds a single-threaded encoder, which is why the threshold sits near one core
|
|
95
|
+
* rather than near the machine.
|
|
96
|
+
*/
|
|
97
|
+
const val ENCODER_BUSY_PERCENT = 90.0
|
|
98
|
+
|
|
99
|
+
fun spendingBudget(wireBitrateBps: Long?, budgetBps: Int): Boolean {
|
|
100
|
+
if (wireBitrateBps == null || budgetBps <= 0) return false
|
|
101
|
+
return wireBitrateBps >= budgetBps * SPENDING_FRACTION
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
fun hint(
|
|
105
|
+
band: Band,
|
|
106
|
+
budgetCapped: Boolean,
|
|
107
|
+
spendingBudget: Boolean,
|
|
108
|
+
encoderBusy: Boolean,
|
|
109
|
+
): String = when {
|
|
110
|
+
band == Band.UNKNOWN -> "no wire report yet"
|
|
111
|
+
band == Band.GOOD -> "enough for high motion"
|
|
112
|
+
!spendingBudget && encoderBusy ->
|
|
113
|
+
"bits unspent with a core nearly full; encoder is the cap -- cut pixels, not the ceiling"
|
|
114
|
+
!spendingBudget ->
|
|
115
|
+
"bits unspent while CPU is idle; congestion control is holding the rate down, " +
|
|
116
|
+
"so neither a bigger ceiling nor fewer pixels addresses it -- check the network"
|
|
117
|
+
budgetCapped ->
|
|
118
|
+
"budget cannot reach high motion at this geometry; raise maxBitrateBps or drop resolution"
|
|
119
|
+
band == Band.BLOCKY -> "far below any usable bitrate; check for a rate-controller collapse"
|
|
120
|
+
band == Band.SOFT -> "only enough for a static scene; a head camera will smear"
|
|
121
|
+
else -> "fine for moderate motion, still short for a head-mounted camera"
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* One line per tick. Prints the derived figures and the bitrate that would be
|
|
126
|
+
* needed, because "0.17 bpp" alone does not tell anyone what to change.
|
|
127
|
+
*/
|
|
128
|
+
fun line(
|
|
129
|
+
wireWidth: Int?,
|
|
130
|
+
wireHeight: Int?,
|
|
131
|
+
wireFps: Double?,
|
|
132
|
+
wireBitrateBps: Long?,
|
|
133
|
+
budgetBps: Int,
|
|
134
|
+
packetsPerSecond: Double?,
|
|
135
|
+
cpuPercent: Double?,
|
|
136
|
+
sendQuality: String,
|
|
137
|
+
): String {
|
|
138
|
+
val bpp = bitsPerPixel(wireBitrateBps, wireWidth, wireHeight, wireFps)
|
|
139
|
+
val band = band(bpp)
|
|
140
|
+
val capped = if (wireWidth != null && wireHeight != null && wireFps != null) {
|
|
141
|
+
budgetCapsQuality(wireWidth, wireHeight, wireFps, budgetBps)
|
|
142
|
+
} else {
|
|
143
|
+
false
|
|
144
|
+
}
|
|
145
|
+
val bitsPerFrame = if (wireBitrateBps != null && wireFps != null && wireFps > 0) {
|
|
146
|
+
(wireBitrateBps / wireFps).roundToInt().toString()
|
|
147
|
+
} else {
|
|
148
|
+
"na"
|
|
149
|
+
}
|
|
150
|
+
val goodKbps = if (wireWidth != null && wireHeight != null && wireFps != null) {
|
|
151
|
+
(neededBps(wireWidth, wireHeight, wireFps, HIGH_MOTION_BPP) / 1000).toString()
|
|
152
|
+
} else {
|
|
153
|
+
"na"
|
|
154
|
+
}
|
|
155
|
+
val usedKbps = wireBitrateBps?.let { (it / 1000).toString() } ?: "na"
|
|
156
|
+
val usedPercent = if (wireBitrateBps != null && budgetBps > 0) {
|
|
157
|
+
((wireBitrateBps * 100.0 / budgetBps).roundToInt()).toString()
|
|
158
|
+
} else {
|
|
159
|
+
"na"
|
|
160
|
+
}
|
|
161
|
+
return "P9 quality band=$band bpp=${bpp?.let { decimal(it) } ?: "na"} " +
|
|
162
|
+
"size=${wireWidth ?: 0}x${wireHeight ?: 0}@${wireFps?.let { PipelineStats.formatRate(it) } ?: "na"} " +
|
|
163
|
+
"bitsPerFrame=$bitsPerFrame kbps=$usedKbps budgetKbps=${budgetBps / 1000} used=$usedPercent% " +
|
|
164
|
+
"goodNeedsKbps=$goodKbps budgetCapped=$capped " +
|
|
165
|
+
"pps=${packetsPerSecond?.let { PipelineStats.formatRate(it) } ?: "na"} " +
|
|
166
|
+
"cpu=${cpuPercent?.let { PipelineStats.formatRate(it) } ?: "na"} " +
|
|
167
|
+
"sendQuality=${sendQuality.ifBlank { "na" }} -- " +
|
|
168
|
+
hint(
|
|
169
|
+
band = band,
|
|
170
|
+
budgetCapped = capped,
|
|
171
|
+
spendingBudget = spendingBudget(wireBitrateBps, budgetBps),
|
|
172
|
+
encoderBusy = cpuPercent != null && cpuPercent >= ENCODER_BUSY_PERCENT,
|
|
173
|
+
)
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/** Three places, because bpp differences that matter show up in the third. */
|
|
177
|
+
private fun decimal(value: Double): String = ((value * 1000).roundToInt() / 1000.0).toString()
|
|
178
|
+
}
|