@mentra/acs-meeting 3.2.0-dev.221 → 3.2.0-dev.223

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/README.md +4 -2
  2. package/android/build.gradle +23 -0
  3. package/android/src/androidTest/java/com/mentra/acsmeeting/network/SoftApFeasibilityGateTest.kt +291 -0
  4. package/android/src/main/AndroidManifest.xml +2 -0
  5. package/android/src/main/java/com/mentra/acsmeeting/AbandonedCallAgent.kt +36 -0
  6. package/android/src/main/java/com/mentra/acsmeeting/AcsMeetingModule.kt +392 -22
  7. package/android/src/main/java/com/mentra/acsmeeting/AcsMeetingSession.kt +909 -56
  8. package/android/src/main/java/com/mentra/acsmeeting/CallCapabilities.kt +33 -0
  9. package/android/src/main/java/com/mentra/acsmeeting/audio/AcsAudioPolicy.kt +21 -0
  10. package/android/src/main/java/com/mentra/acsmeeting/audio/AudioUplinkChain.kt +126 -0
  11. package/android/src/main/java/com/mentra/acsmeeting/audio/PcmBridge.kt +27 -4
  12. package/android/src/main/java/com/mentra/acsmeeting/audio/UplinkPacer.kt +45 -10
  13. package/android/src/main/java/com/mentra/acsmeeting/audio/UplinkSender.kt +110 -5
  14. package/android/src/main/java/com/mentra/acsmeeting/network/InternetHold.kt +501 -0
  15. package/android/src/main/java/com/mentra/acsmeeting/network/Ipv4Prefix.kt +51 -0
  16. package/android/src/main/java/com/mentra/acsmeeting/network/NetworkFacts.kt +81 -0
  17. package/android/src/main/java/com/mentra/acsmeeting/network/ScopedNetworkChangeDetector.kt +355 -0
  18. package/android/src/main/java/com/mentra/acsmeeting/network/ScopedNetworkError.kt +109 -0
  19. package/android/src/main/java/com/mentra/acsmeeting/network/ScopedNetworkObserver.kt +160 -0
  20. package/android/src/main/java/com/mentra/acsmeeting/network/ScopedNetworkReadiness.kt +103 -0
  21. package/android/src/main/java/com/mentra/acsmeeting/network/ScopedNetworkRequestSpec.kt +53 -0
  22. package/android/src/main/java/com/mentra/acsmeeting/network/ScopedNetworkState.kt +127 -0
  23. package/android/src/main/java/com/mentra/acsmeeting/network/ScopedSoftApNetwork.kt +573 -0
  24. package/android/src/main/java/com/mentra/acsmeeting/source/CloudflareWhepSource.kt +10 -101
  25. package/android/src/main/java/com/mentra/acsmeeting/source/DecodedTrackRelay.kt +147 -0
  26. package/android/src/main/java/com/mentra/acsmeeting/source/GlassesMediaSource.kt +47 -5
  27. package/android/src/main/java/com/mentra/acsmeeting/source/LocalWhipIngestSource.kt +772 -0
  28. package/android/src/main/java/com/mentra/acsmeeting/source/MeetingVideoSourceSpec.kt +73 -0
  29. package/android/src/main/java/com/mentra/acsmeeting/source/SdpAdapter.kt +17 -0
  30. package/android/src/main/java/com/mentra/acsmeeting/source/SelectedIcePair.kt +139 -0
  31. package/android/src/main/java/com/mentra/acsmeeting/source/SoftApIceDiagnosis.kt +144 -0
  32. package/android/src/main/java/com/mentra/acsmeeting/source/SoftApIcePolicy.kt +46 -0
  33. package/android/src/main/java/com/mentra/acsmeeting/source/SoftApSdpGuard.kt +160 -0
  34. package/android/src/main/java/com/mentra/acsmeeting/source/TrackRegistry.kt +3 -1
  35. package/android/src/main/java/com/mentra/acsmeeting/source/VideoSourceArm.kt +69 -0
  36. package/android/src/main/java/com/mentra/acsmeeting/source/WhipIngestProtocol.kt +207 -0
  37. package/android/src/main/java/com/mentra/acsmeeting/source/WhipIngestServer.kt +356 -0
  38. package/android/src/main/java/com/mentra/acsmeeting/telemetry/AvSyncProbe.kt +155 -0
  39. package/android/src/main/java/com/mentra/acsmeeting/telemetry/PipelineStats.kt +94 -3
  40. package/android/src/main/java/com/mentra/acsmeeting/telemetry/PipelineTicker.kt +33 -5
  41. package/android/src/main/java/com/mentra/acsmeeting/telemetry/RingPercentile.kt +15 -0
  42. package/android/src/main/java/com/mentra/acsmeeting/telemetry/VideoQuality.kt +178 -0
  43. package/android/src/main/java/com/mentra/acsmeeting/telemetry/VideoRateVerdict.kt +145 -0
  44. package/android/src/main/java/com/mentra/acsmeeting/trace/SoftApTrace.kt +143 -0
  45. package/android/src/main/java/com/mentra/acsmeeting/video/AcsFrameSender.kt +23 -4
  46. package/android/src/main/java/com/mentra/acsmeeting/video/FramePacer.kt +70 -0
  47. package/android/src/main/java/com/mentra/acsmeeting/video/I420FormatSpec.kt +1 -1
  48. package/android/src/main/java/com/mentra/acsmeeting/video/VideoProfile.kt +36 -3
  49. package/android/src/test/java/com/mentra/acsmeeting/AbandonedCallAgentTest.kt +74 -0
  50. package/android/src/test/java/com/mentra/acsmeeting/CallCapabilitiesTest.kt +47 -0
  51. package/android/src/test/java/com/mentra/acsmeeting/audio/AudioUplinkChainTest.kt +250 -0
  52. package/android/src/test/java/com/mentra/acsmeeting/audio/CapturePolicyTest.kt +44 -0
  53. package/android/src/test/java/com/mentra/acsmeeting/audio/UplinkPacerTest.kt +34 -4
  54. package/android/src/test/java/com/mentra/acsmeeting/audio/UplinkSenderTest.kt +96 -0
  55. package/android/src/test/java/com/mentra/acsmeeting/network/InternetHoldLifecycleTest.kt +102 -0
  56. package/android/src/test/java/com/mentra/acsmeeting/network/InternetHoldTest.kt +148 -0
  57. package/android/src/test/java/com/mentra/acsmeeting/network/Ipv4PrefixTest.kt +59 -0
  58. package/android/src/test/java/com/mentra/acsmeeting/network/NetworkFactsTest.kt +59 -0
  59. package/android/src/test/java/com/mentra/acsmeeting/network/ScopedLocalNetworkPermissionTest.kt +35 -0
  60. package/android/src/test/java/com/mentra/acsmeeting/network/ScopedNetworkChangeDetectorTest.kt +165 -0
  61. package/android/src/test/java/com/mentra/acsmeeting/network/ScopedNetworkErrorTest.kt +96 -0
  62. package/android/src/test/java/com/mentra/acsmeeting/network/ScopedNetworkLifecycleTest.kt +187 -0
  63. package/android/src/test/java/com/mentra/acsmeeting/network/ScopedNetworkObserverTest.kt +172 -0
  64. package/android/src/test/java/com/mentra/acsmeeting/network/ScopedNetworkReadinessTest.kt +141 -0
  65. package/android/src/test/java/com/mentra/acsmeeting/network/ScopedNetworkRequestSpecTest.kt +69 -0
  66. package/android/src/test/java/com/mentra/acsmeeting/source/AcsInvestigationTest.kt +40 -0
  67. package/android/src/test/java/com/mentra/acsmeeting/source/DecodedTrackRelayTest.kt +41 -0
  68. package/android/src/test/java/com/mentra/acsmeeting/source/GlassesMediaSourceTest.kt +26 -8
  69. package/android/src/test/java/com/mentra/acsmeeting/source/MeetingVideoSourceSpecTest.kt +109 -0
  70. package/android/src/test/java/com/mentra/acsmeeting/source/SelectedIcePairTest.kt +165 -0
  71. package/android/src/test/java/com/mentra/acsmeeting/source/SoftApIceDiagnosisTest.kt +148 -0
  72. package/android/src/test/java/com/mentra/acsmeeting/source/SoftApIcePolicyTest.kt +76 -0
  73. package/android/src/test/java/com/mentra/acsmeeting/source/SoftApSdpGuardTest.kt +275 -0
  74. package/android/src/test/java/com/mentra/acsmeeting/source/TrackRegistryTest.kt +9 -0
  75. package/android/src/test/java/com/mentra/acsmeeting/source/WhipIngestProtocolTest.kt +356 -0
  76. package/android/src/test/java/com/mentra/acsmeeting/source/WhipIngestServerLoopbackTest.kt +411 -0
  77. package/android/src/test/java/com/mentra/acsmeeting/telemetry/AvSyncProbeTest.kt +83 -0
  78. package/android/src/test/java/com/mentra/acsmeeting/telemetry/PipelineStatsTest.kt +69 -3
  79. package/android/src/test/java/com/mentra/acsmeeting/telemetry/VideoQualityTest.kt +182 -0
  80. package/android/src/test/java/com/mentra/acsmeeting/telemetry/VideoRateVerdictTest.kt +149 -0
  81. package/android/src/test/java/com/mentra/acsmeeting/trace/SoftApTraceTest.kt +97 -0
  82. package/android/src/test/java/com/mentra/acsmeeting/video/AcsTimestampTest.kt +34 -0
  83. package/android/src/test/java/com/mentra/acsmeeting/video/FramePacerTest.kt +105 -0
  84. package/android/src/test/java/com/mentra/acsmeeting/video/I420FormatSpecTest.kt +5 -0
  85. package/android/src/test/java/com/mentra/acsmeeting/video/VideoProfileTest.kt +38 -3
  86. package/build/AcsMeeting.types.d.ts +5 -0
  87. package/build/AcsMeeting.types.d.ts.map +1 -1
  88. package/build/AcsMeeting.types.js.map +1 -1
  89. package/build/AcsMeetingModule.d.ts +23 -0
  90. package/build/AcsMeetingModule.d.ts.map +1 -1
  91. package/build/AcsMeetingModule.js.map +1 -1
  92. package/build/AcsMeetingModule.web.d.ts +15 -0
  93. package/build/AcsMeetingModule.web.d.ts.map +1 -1
  94. package/build/AcsMeetingModule.web.js +13 -0
  95. package/build/AcsMeetingModule.web.js.map +1 -1
  96. package/ios/AcsMeetingModule.swift +213 -2
  97. package/package.json +1 -1
  98. package/src/AcsMeeting.types.ts +5 -0
  99. package/src/AcsMeetingModule.ts +13 -0
  100. package/src/AcsMeetingModule.web.ts +13 -0
@@ -0,0 +1,145 @@
1
+ package com.mentra.acsmeeting.telemetry
2
+
3
+ /**
4
+ * Names what is holding the outgoing frame rate down.
5
+ *
6
+ * The fps question kept being answered by inference: a soak put ~8.8 fps on the
7
+ * wire, so the ACS software encoder "holds 8.8 fps". That does not follow. At
8
+ * least four different things produce a low wire rate, they are trivial to tell
9
+ * apart from numbers we already collect, and two of them want opposite fixes —
10
+ * cutting resolution rescues an encoder that is genuinely CPU-bound and makes a
11
+ * bitrate-starved stream strictly worse.
12
+ *
13
+ * So this prints the answer instead of leaving it to be guessed:
14
+ *
15
+ * - [Bound.SOURCE_SHORT] the glasses are not sending enough frames. Nothing
16
+ * downstream of the decoder can raise the rate.
17
+ * - [Bound.PACER_SHORT] our own gate is the limit — it admitted fewer frames
18
+ * than we advertised while the source had plenty. This is what a greedy
19
+ * `FramePacer` did: 14.6 fps in, 7.3 out, against an advertised 8.
20
+ * - [Bound.ENCODER_SHORT] ACS took the frames and put fewer on the wire while
21
+ * the CPU was busy. The real encoder ceiling. Cut pixels.
22
+ * - [Bound.BITRATE_STARVED] ACS took the frames and put fewer on the wire with
23
+ * CPU to spare and the bitrate far under the budget we granted. The rate
24
+ * controller is throttling an encoder that could keep up. Cutting fps or
25
+ * pixels here makes it worse; fix delivery consistency.
26
+ * - [Bound.CAP_BOUND] every frame is on the wire and the bitrate is pressed
27
+ * against the ceiling we granted. Nothing is broken; the grant is simply the
28
+ * limit, so the picture cannot get sharper until the number does.
29
+ */
30
+ object VideoRateVerdict {
31
+ enum class Bound {
32
+ OK,
33
+ SOURCE_SHORT,
34
+ PACER_SHORT,
35
+ ENCODER_SHORT,
36
+ BITRATE_STARVED,
37
+ CAP_BOUND,
38
+ UNKNOWN,
39
+ }
40
+
41
+ /** Below this fraction of the target a rate counts as short rather than noise. */
42
+ const val SHORT_FRACTION = 0.85
43
+
44
+ /**
45
+ * The pacer is held to a tighter tolerance than anything else, because it is
46
+ * deterministic code we own rather than a jittery encoder or radio. The bug
47
+ * this class was written for delivered 7.3 against an advertised 8 — a 9%
48
+ * chronic shortfall, comfortably inside [SHORT_FRACTION] and still enough to
49
+ * bias the ACS rate controller down on every hiccup.
50
+ */
51
+ const val PACER_SHORT_FRACTION = 0.95
52
+
53
+ /** Process CPU at or above this is treated as the encoder being the cost. */
54
+ const val CPU_BUSY_PERCENT = 80.0
55
+
56
+ /** Wire bitrate below this fraction of the granted budget is a throttled stream. */
57
+ const val STARVED_FRACTION = 0.6
58
+
59
+ /**
60
+ * Wire bitrate at or above this fraction of the grant counts as riding the ceiling.
61
+ *
62
+ * The grant is a target rather than a wall — a device run recorded the wire at 1623 kbps
63
+ * against a 1.5 Mbps grant, 108% — so "pressed against it" has to mean near, not exactly at.
64
+ * 0.95 is below the noise on a controller that routinely overshoots and well above the 0.85
65
+ * that [VideoQuality.SPENDING_FRACTION] treats as merely spending the budget.
66
+ */
67
+ const val CAP_BOUND_FRACTION = 0.95
68
+
69
+ fun of(
70
+ advertisedFps: Double,
71
+ sinkFps: Double,
72
+ admittedFps: Double,
73
+ wireFps: Double?,
74
+ wireBitrateBps: Long?,
75
+ budgetBps: Int,
76
+ cpuPercent: Double?,
77
+ ): Bound {
78
+ if (advertisedFps <= 0.0) return Bound.UNKNOWN
79
+ // Source first: when the glasses are short, every rate below them is short
80
+ // too, and blaming the encoder for that sends the next fix to the wrong leg.
81
+ if (sinkFps < advertisedFps * SHORT_FRACTION) return Bound.SOURCE_SHORT
82
+ // Against what the source actually supplied, not against the advertise. The
83
+ // pacer cannot admit frames that never arrived, and scoring it on the
84
+ // advertise alone would read a merely slow source as our own bug.
85
+ val deliverable = minOf(advertisedFps, sinkFps)
86
+ if (admittedFps < deliverable * PACER_SHORT_FRACTION) return Bound.PACER_SHORT
87
+ if (wireFps == null) return Bound.UNKNOWN
88
+ if (wireFps >= admittedFps * SHORT_FRACTION) {
89
+ // A full frame rate is not the same as a good picture, and this is the one case where the
90
+ // fix is the ceiling itself. Reporting it as a plain OK is how "the quality is low" turns
91
+ // into an argument about the encoder: every rate number looks healthy, so the grant — the
92
+ // one thing actually holding the bits down — never comes up.
93
+ val capBound =
94
+ wireBitrateBps != null && budgetBps > 0 && wireBitrateBps >= budgetBps * CAP_BOUND_FRACTION
95
+ return if (capBound) Bound.CAP_BOUND else Bound.OK
96
+ }
97
+ if (cpuPercent != null && cpuPercent >= CPU_BUSY_PERCENT) return Bound.ENCODER_SHORT
98
+ val starved =
99
+ wireBitrateBps != null && budgetBps > 0 && wireBitrateBps < budgetBps * STARVED_FRACTION
100
+ return if (starved) Bound.BITRATE_STARVED else Bound.ENCODER_SHORT
101
+ }
102
+
103
+ /** What to do about it, so the log line does not need a reader who has this file open. */
104
+ fun hint(bound: Bound): String = when (bound) {
105
+ Bound.OK -> "wire is holding the advertised rate"
106
+ Bound.SOURCE_SHORT -> "glasses/WHIP leg is short; raising ACS fps cannot help"
107
+ Bound.PACER_SHORT -> "our gate drops frames the source supplied; check FramePacer interval"
108
+ Bound.ENCODER_SHORT -> "ACS encoder is the ceiling; cut pixels (VideoProfile.SD) not fps"
109
+ Bound.BITRATE_STARVED -> "rate controller is throttling, not CPU; do not cut fps or pixels"
110
+ Bound.CAP_BOUND -> "wire is riding the granted ceiling; raising maxBitrateBps is the only lever"
111
+ Bound.UNKNOWN -> "not enough evidence yet"
112
+ }
113
+
114
+ /**
115
+ * One line per tick. Prints every input beside the verdict: a verdict whose
116
+ * inputs are not visible is the same guess this class exists to replace.
117
+ */
118
+ fun line(
119
+ advertisedFps: Double,
120
+ sinkFps: Double,
121
+ admittedFps: Double,
122
+ wireFps: Double?,
123
+ wireBitrateBps: Long?,
124
+ budgetBps: Int,
125
+ cpuPercent: Double?,
126
+ codecName: String,
127
+ ): String {
128
+ val bound = of(
129
+ advertisedFps = advertisedFps,
130
+ sinkFps = sinkFps,
131
+ admittedFps = admittedFps,
132
+ wireFps = wireFps,
133
+ wireBitrateBps = wireBitrateBps,
134
+ budgetBps = budgetBps,
135
+ cpuPercent = cpuPercent,
136
+ )
137
+ val wireLabel = wireFps?.let { PipelineStats.formatRate(it) } ?: "na"
138
+ val wireKbps = wireBitrateBps?.let { (it / 1000).toString() } ?: "na"
139
+ return "P7 rate bound=$bound advertised=${PipelineStats.formatRate(advertisedFps)} " +
140
+ "sink=${PipelineStats.formatRate(sinkFps)} admitted=${PipelineStats.formatRate(admittedFps)} " +
141
+ "wire=$wireLabel kbps=$wireKbps budgetKbps=${budgetBps / 1000} " +
142
+ "cpu=${cpuPercent?.let { PipelineStats.formatRate(it) } ?: "na"} " +
143
+ "codec=${codecName.ifBlank { "na" }.replace(' ', '_')} -- ${hint(bound)}"
144
+ }
145
+ }
@@ -0,0 +1,143 @@
1
+ package com.mentra.acsmeeting.trace
2
+
3
+ import android.os.SystemClock
4
+ import android.util.Log
5
+
6
+ /**
7
+ * Correlated stage logging for the SoftAP calling pipeline (phone side).
8
+ *
9
+ * TEMPORARY DIAGNOSTIC. Every line carries the literal `SOFTAP_TRACE` marker so a single
10
+ * `rg -n 'SOFTAP_TRACE'` finds all of them at cleanup time. Flip [ENABLED] to false to mute the
11
+ * layer for a release without deleting call sites.
12
+ *
13
+ * The logcat tag is deliberately distinct from the module's `ACS-SPIKE` tag so that
14
+ * `adb logcat -s SOFTAP-TRACE` isolates the whole SoftAP pipeline in one stream.
15
+ *
16
+ * The phone mints [traceId] when it starts the hotspot and passes it to the glasses in
17
+ * `start_stream`. Both devices stamp the same id on every line, which is the only way to correlate
18
+ * two logs whose clocks were never synchronised.
19
+ */
20
+ object SoftApTrace {
21
+
22
+ /** Master switch. Flip to false to mute the trace without removing call sites. */
23
+ const val ENABLED = true
24
+
25
+ /** Grep marker. Never build this by concatenation — a single grep must be exhaustive. */
26
+ const val MARKER = "SOFTAP_TRACE"
27
+
28
+ const val TAG = "SOFTAP-TRACE"
29
+
30
+ private const val REDACTED = "<redacted>"
31
+
32
+ /** Keys whose values must never reach logcat. Matched case-insensitively as substrings. */
33
+ private val SENSITIVE_KEYS = listOf(
34
+ "password", "passwd", "passphrase", "psk", "token", "secret",
35
+ "credential", "authorization", "bearer", "meetingurl",
36
+ )
37
+
38
+ @Volatile
39
+ private var traceId: String = ""
40
+
41
+ @Volatile
42
+ private var originMs: Long = 0L
43
+
44
+ @Volatile
45
+ private var lastStage: String = ""
46
+
47
+ /** Mint a new trace id and reset the elapsed-time origin. Called when the hotspot request starts. */
48
+ fun begin(id: String = newTraceId()): String {
49
+ traceId = id
50
+ originMs = SystemClock.elapsedRealtime()
51
+ lastStage = ""
52
+ return id
53
+ }
54
+
55
+ fun reset() {
56
+ traceId = ""
57
+ originMs = 0L
58
+ lastStage = ""
59
+ }
60
+
61
+ fun traceId(): String = traceId
62
+
63
+ /** Last stage successfully logged; use as failure context when reporting an error. */
64
+ fun lastStage(): String = lastStage
65
+
66
+ /** Short, collision-resistant enough for correlating one call's logs. */
67
+ fun newTraceId(): String = java.lang.Long.toHexString(System.nanoTime() and 0xFFFFFFFFL)
68
+
69
+ /**
70
+ * Log one pipeline stage transition.
71
+ *
72
+ * @param stage stable snake_case stage name, e.g. `scoped_network_available`
73
+ */
74
+ fun stage(stage: String, vararg fields: Pair<String, Any?>) {
75
+ if (!ENABLED) return
76
+ lastStage = stage
77
+ val elapsedMs = if (originMs == 0L) 0L else SystemClock.elapsedRealtime() - originMs
78
+ Log.i(TAG, format(traceId, stage, elapsedMs, *fields))
79
+ }
80
+
81
+ /** Error-level variant so genuine failures survive a logcat level filter. */
82
+ fun failure(stage: String, vararg fields: Pair<String, Any?>) {
83
+ if (!ENABLED) return
84
+ val elapsedMs = if (originMs == 0L) 0L else SystemClock.elapsedRealtime() - originMs
85
+ Log.e(TAG, format(traceId, stage, elapsedMs, *fields, "afterStage" to lastStage))
86
+ }
87
+
88
+ /**
89
+ * Pure formatter. Split out from [stage] so it can be unit tested without the Android logging
90
+ * framework, which is not available to plain JVM tests in this module.
91
+ */
92
+ fun format(
93
+ traceId: String,
94
+ stage: String,
95
+ elapsedMs: Long,
96
+ vararg fields: Pair<String, Any?>,
97
+ ): String = buildString {
98
+ append('[').append(MARKER).append(']')
99
+ if (traceId.isNotEmpty()) append(" traceId=").append(traceId)
100
+ append(" stage=").append(stage)
101
+ append(" elapsedMs=").append(elapsedMs)
102
+ for ((key, value) in fields) {
103
+ append(' ').append(key).append('=').append(sanitize(key, value))
104
+ }
105
+ }
106
+
107
+ /**
108
+ * Redact secrets outright and strip query strings from URLs. The local WHIP URL is genuinely
109
+ * useful in a trace, but a URL carrying a watch token is not, so query and userinfo go.
110
+ */
111
+ fun sanitize(key: String, value: Any?): String {
112
+ if (isSensitive(key)) return REDACTED
113
+ if (value == null) return "null"
114
+
115
+ val text = value.toString()
116
+ if (text.isEmpty()) return "\"\""
117
+ val cleaned = stripUrlSecrets(text)
118
+ return if (cleaned.contains(' ')) "\"$cleaned\"" else cleaned
119
+ }
120
+
121
+ private fun isSensitive(key: String): Boolean {
122
+ val lower = key.lowercase()
123
+ return SENSITIVE_KEYS.any { lower.contains(it) }
124
+ }
125
+
126
+ /** Drop `?query`, `#fragment`, and `user:pass@` from anything URL-shaped. */
127
+ private fun stripUrlSecrets(text: String): String {
128
+ val schemeIndex = text.indexOf("://")
129
+ if (schemeIndex < 0) return text
130
+
131
+ var result = text
132
+ val query = result.indexOf('?')
133
+ if (query >= 0) result = result.substring(0, query) + "?" + REDACTED
134
+ val fragment = result.indexOf('#')
135
+ if (fragment >= 0) result = result.substring(0, fragment)
136
+
137
+ val at = result.indexOf('@', schemeIndex + 3)
138
+ if (at >= 0) {
139
+ result = result.substring(0, schemeIndex + 3) + REDACTED + "@" + result.substring(at + 1)
140
+ }
141
+ return result
142
+ }
143
+ }
@@ -14,6 +14,7 @@ import com.mentra.acsmeeting.source.AcsInvestigation
14
14
  import com.mentra.acsmeeting.source.I420Planes
15
15
  import com.mentra.acsmeeting.source.PixelFormatArm
16
16
  import com.mentra.acsmeeting.source.TargetSize
17
+ import com.mentra.acsmeeting.telemetry.AvSyncProbe
17
18
  import com.mentra.acsmeeting.telemetry.ChromaProbe
18
19
  import com.mentra.acsmeeting.telemetry.PipelineStats
19
20
  import java.nio.ByteBuffer
@@ -28,12 +29,16 @@ import java.util.concurrent.atomic.AtomicInteger
28
29
  import java.util.concurrent.atomic.AtomicReference
29
30
 
30
31
  /**
31
- * I420 to ACS: three independent direct planes (Y, U, V). No inter-arrival
32
- * pacer — [SendGate] is the only backpressure. Size mismatch drops; the sink
33
- * owns scaling to the negotiated size.
32
+ * I420 to ACS: three independent direct planes (Y, U, V).
33
+ *
34
+ * [SendGate] serializes in-flight sends. [FramePacer] is the time gate: ACS's
35
+ * software encoder holds ~8 fps, so extras have to be dropped here or the rate
36
+ * controller reserves budget for frames that encoder never produces. Size
37
+ * mismatch drops; the sink owns scaling to the negotiated size.
34
38
  */
35
39
  class AcsFrameSender(
36
40
  private val stats: PipelineStats = PipelineStats(),
41
+ private val avSync: AvSyncProbe? = null,
37
42
  ) {
38
43
  private val running = AtomicBoolean(false)
39
44
  private val stream = AtomicReference<RawOutgoingVideoStream?>(null)
@@ -42,6 +47,7 @@ class AcsFrameSender(
42
47
  Thread(runnable, "acs-i420-send").apply { isDaemon = true }
43
48
  }
44
49
  private val gate = SendGate()
50
+ private val pacer = FramePacer()
45
51
  private val pool = ConcurrentHashMap<Int, ConcurrentLinkedQueue<ByteBuffer>>()
46
52
  private val sendSeq = AtomicInteger(0)
47
53
  private val lastTicks = AtomicReference(0L)
@@ -57,6 +63,7 @@ class AcsFrameSender(
57
63
 
58
64
  fun attach(outgoing: VirtualOutgoingVideoStream, onFormat: ((TargetSize) -> Unit)? = null) {
59
65
  detach()
66
+ pacer.reset()
60
67
  this.onFormat = onFormat
61
68
  stream.set(outgoing)
62
69
  attachedStream = outgoing
@@ -111,9 +118,14 @@ class AcsFrameSender(
111
118
  )
112
119
  return
113
120
  }
121
+ if (!pacer.tryAdmit(FramePacer.intervalNs(negotiated.framesPerSecond))) {
122
+ stats.onDropPaced()
123
+ return
124
+ }
114
125
 
115
126
  val prepared = prepareSend(planes, negotiated)
116
127
  stats.setChroma(prepared.chroma)
128
+ avSync?.onVideoLuma(prepared.chroma.y)
117
129
 
118
130
  if (!gate.tryAcquire()) {
119
131
  stats.onDropBusy()
@@ -123,6 +135,7 @@ class AcsFrameSender(
123
135
 
124
136
  val seq = sendSeq.incrementAndGet()
125
137
  stats.onQueued()
138
+ val enqueuedAt = System.nanoTime()
126
139
  if (seq <= TRACE_SENDS) {
127
140
  Log.i(
128
141
  TAG,
@@ -135,6 +148,7 @@ class AcsFrameSender(
135
148
 
136
149
  val submitted = try {
137
150
  sender.execute {
151
+ stats.queue.record(System.nanoTime() - enqueuedAt)
138
152
  var result = "ok"
139
153
  // A timed-out future is NOT cancelled: ACS still reads these direct buffers.
140
154
  var reclaimable = true
@@ -157,7 +171,11 @@ class AcsFrameSender(
157
171
  frame.timestampInTicks = ticks
158
172
  val sendStart = System.nanoTime()
159
173
  out.sendRawVideoFrame(frame).get(SEND_TIMEOUT_MS, TimeUnit.MILLISECONDS)
160
- stats.send.record(System.nanoTime() - sendStart)
174
+ val sendDone = System.nanoTime()
175
+ stats.send.record(sendDone - sendStart)
176
+ if (planes.timestampNs > 0) {
177
+ stats.e2e.record(sendDone - planes.timestampNs)
178
+ }
161
179
  stats.onSub()
162
180
  if (seq <= TRACE_SENDS) {
163
181
  Log.i(
@@ -342,6 +360,7 @@ class AcsFrameSender(
342
360
  format.set(null)
343
361
  sendSeq.set(0)
344
362
  lastTicks.set(0L)
363
+ pacer.reset()
345
364
  pool.clear()
346
365
  onFormat = null
347
366
  }
@@ -0,0 +1,70 @@
1
+ package com.mentra.acsmeeting.video
2
+
3
+ import java.util.concurrent.atomic.AtomicLong
4
+
5
+ /**
6
+ * Drops frames that arrive sooner than the rate we advertised to ACS.
7
+ *
8
+ * [SendGate] only serializes in-flight sends. `sendRawVideoFrame` returns in about
9
+ * a millisecond, so without a time gate we hand ACS fifteen frames a second and
10
+ * its software encoder — which holds ~8.8 fps — is the one that has to drop them.
11
+ * That is exactly the over-reservation that collapses the wire bitrate.
12
+ *
13
+ * The gate keeps an absolute schedule rather than measuring from the last admit.
14
+ * That distinction is the whole rate: a "has [minIntervalNs] elapsed since the
15
+ * last admit" test turns a 14.6 fps source into exact 2:1 decimation, because the
16
+ * frame 68 ms after an admit is inside the 125 ms window and the next one lands at
17
+ * 137 ms. Admits then run at 7.3 fps against an advertised 8 — the same
18
+ * under-delivery this class exists to prevent, just one ladder rung down. Device
19
+ * logs showed it: `sub` sat at 7.0 with `wire=960x540@7.0`, and every arrival
20
+ * hiccup cost another 50% of the ACS bitrate budget. Advancing a deadline by whole
21
+ * intervals lets a frame that lands late still fill its own slot, so the average
22
+ * locks to the advertised rate for any source faster than it.
23
+ */
24
+ class FramePacer {
25
+ private val nextDueNs = AtomicLong(UNSET)
26
+
27
+ fun reset() {
28
+ nextDueNs.set(UNSET)
29
+ }
30
+
31
+ /**
32
+ * True when this frame is due. [minIntervalNs] is derived from the advertised
33
+ * fps so the emit rate cannot drift from the format we declared.
34
+ */
35
+ fun tryAdmit(minIntervalNs: Long, nowNs: Long = System.nanoTime()): Boolean {
36
+ if (minIntervalNs <= 0) return true
37
+ while (true) {
38
+ val due = nextDueNs.get()
39
+ if (due == UNSET) {
40
+ if (nextDueNs.compareAndSet(due, nowNs + minIntervalNs)) return true
41
+ continue
42
+ }
43
+ if (nowNs < due) return false
44
+ // Resync instead of catching up when the source has been away for longer
45
+ // than RESYNC_INTERVALS. Carrying that whole debt forward would admit a
46
+ // burst back-to-back, which is over-delivery against the same advertised
47
+ // rate and hits the ACS encoder the way the ungated path did.
48
+ val next =
49
+ if (nowNs - due > minIntervalNs * RESYNC_INTERVALS) {
50
+ nowNs + minIntervalNs
51
+ } else {
52
+ due + minIntervalNs
53
+ }
54
+ if (nextDueNs.compareAndSet(due, next)) return true
55
+ }
56
+ }
57
+
58
+ companion object {
59
+ /** Not a legal timestamp. 0 is — [System.nanoTime] can start there in tests. */
60
+ private const val UNSET = Long.MIN_VALUE
61
+
62
+ /** How far behind schedule we still treat as catch-up rather than a new start. */
63
+ private const val RESYNC_INTERVALS = 2
64
+
65
+ fun intervalNs(fps: Float): Long {
66
+ if (fps <= 0f) return 0L
67
+ return (1_000_000_000.0 / fps).toLong()
68
+ }
69
+ }
70
+ }
@@ -46,7 +46,7 @@ data class I420FormatSpec(
46
46
  const val MAX_FPS = 30f
47
47
 
48
48
  /** VirtualOutgoingVideoStream join allowlist. P540 is 960×540, never 540×960. */
49
- private val ALLOWED_SIZES = setOf(1280 to 720, 960 to 540)
49
+ private val ALLOWED_SIZES = setOf(1280 to 720, 960 to 540, 858 to 480)
50
50
 
51
51
  fun of(width: Int = 1280, height: Int = 720, fps: Float = 20f): I420FormatSpec {
52
52
  val chroma = I420Packer.chromaStride(width)
@@ -22,31 +22,64 @@ data class VideoProfile(
22
22
  /** Bits available per frame at the ceiling. Below ~20 kbit the encoder starts dropping. */
23
23
  fun bitsPerFrame(): Int = maxBitrateBps / fps
24
24
 
25
+ /**
26
+ * The format ACS's software encoder can actually hold on the phones we ship.
27
+ *
28
+ * Media statistics on SM_S948U (and earlier 720p soaks) report `codecName: "h264 sw"`
29
+ * and a wire rate of ~8.8 fps no matter whether we asked for 15 or 30. Advertising
30
+ * the faster rate makes ACS reserve budget for frames that encoder never produces,
31
+ * and the wire collapses to a few hundred kbps while our own `sub` still reads 15.
32
+ * Size stays; only the declared rate moves, so Teams still gets 540p/720p, just at
33
+ * a cadence the software encoder can keep.
34
+ */
35
+ fun forSoftwareEncoder(): VideoProfile =
36
+ if (fps <= SOFTWARE_ENCODER_FPS) this else copy(fps = SOFTWARE_ENCODER_FPS)
37
+
25
38
  companion object {
26
39
  /**
27
40
  * 720p15. ACS encodes this in software on mid-range hardware (the media
28
41
  * statistics report `codecName: "h264 sw"`), so the pixel count is a real
29
42
  * CPU cost, not just a bitrate cost.
30
43
  */
31
- val HD = VideoProfile(width = 1280, height = 720, fps = 15, maxBitrateBps = 2_500_000)
44
+ val HD = VideoProfile(width = 1280, height = 720, fps = 15, maxBitrateBps = 3_000_000)
32
45
 
33
46
  /** Quarter the pixels of [HD]. Use when the software encoder cannot hold 15 fps at 720p. */
34
47
  val SD = VideoProfile(width = 640, height = 360, fps = 15, maxBitrateBps = 1_000_000)
35
48
 
49
+ /**
50
+ * Fastest rate the ACS software H.264 encoder has held on device.
51
+ *
52
+ * Every soak that reported `h264 sw` also reported wire ≈ 8.8 fps — 720p15,
53
+ * 540p15, I420 and NV12. 8 is just under that so the rate controller is
54
+ * never asked for a frame the encoder cannot produce.
55
+ */
56
+ const val SOFTWARE_ENCODER_FPS = 8
57
+
36
58
  /** Investigation flip. Ships as HD. SD is 360p — a quarter of the encode work. */
37
59
  val DEFAULT = HD
38
60
 
39
61
  /**
40
62
  * ACS P540. VirtualOutgoingVideoStream documents 960×540, not 540×960.
41
63
  * 540p A/B uses this size on both WHIP and ACS.
64
+ *
65
+ * The ceiling sits above the glasses→phone WHIP default (2.5 Mbps) so the
66
+ * Teams hop is not the quality bottleneck. Mentra Call Auto sends this
67
+ * number as `maxBitrateBps`; an explicit picker value still overrides it.
42
68
  */
43
- val P540 = VideoProfile(width = 960, height = 540, fps = 30, maxBitrateBps = 1_500_000)
69
+ val P540 = VideoProfile(width = 960, height = 540, fps = 30, maxBitrateBps = 3_000_000)
44
70
 
45
71
  /**
46
72
  * Same P540 geometry at 15 fps. Selectable from Mentra Call; not the default.
47
73
  * Isolates pixel-count savings without doubling frame work versus [P540].
74
+ *
75
+ * The ceiling is sized from the picture rather than copied from an older
76
+ * 1.5 Mbps grant. By the Kush gauge a high-motion 960x540 at 15 fps needs
77
+ * `960·540·15·0.28` ≈ 2.18 Mbps to avoid visible blocking, and a
78
+ * head-mounted camera is high motion by definition. It also has to clear
79
+ * the 2.5 Mbps glasses→phone WHIP default, so the grant is 3 Mbps — the
80
+ * next discrete step above that link.
48
81
  */
49
- val P540_15 = VideoProfile(width = 960, height = 540, fps = 15, maxBitrateBps = 1_500_000)
82
+ val P540_15 = VideoProfile(width = 960, height = 540, fps = 15, maxBitrateBps = 3_000_000)
50
83
 
51
84
  fun parse(width: Int, height: Int, fps: Int, maxBitrateBps: Int): VideoProfile? {
52
85
  if (maxBitrateBps <= 0) return null
@@ -0,0 +1,74 @@
1
+ package com.mentra.acsmeeting
2
+
3
+ import java.util.concurrent.CompletableFuture
4
+ import java.util.concurrent.Future
5
+ import org.junit.Assert.assertFalse
6
+ import org.junit.Assert.assertNull
7
+ import org.junit.Assert.assertSame
8
+ import org.junit.Assert.assertTrue
9
+ import org.junit.Test
10
+
11
+ class AbandonedCallAgentTest {
12
+
13
+ @Test
14
+ fun `a completed future still yields the agent after we stop waiting`() {
15
+ val agent = Any()
16
+ val pending: Future<Any> = CompletableFuture.completedFuture(agent)
17
+
18
+ assertSame(agent, AbandonedCallAgent.takeIfDone(pending))
19
+ }
20
+
21
+ @Test
22
+ fun `an incomplete future is left for a later sweep`() {
23
+ val pending = CompletableFuture<Any>()
24
+
25
+ assertNull(AbandonedCallAgent.takeIfDone(pending))
26
+ assertFalse(pending.isCancelled)
27
+ }
28
+
29
+ @Test
30
+ fun `a cancelled future cannot hand back the native agent`() {
31
+ val pending = CompletableFuture<Any>()
32
+ pending.cancel(true)
33
+
34
+ assertTrue(pending.isCancelled)
35
+ assertNull(AbandonedCallAgent.takeIfDone(pending))
36
+ }
37
+
38
+ @Test
39
+ fun `a short wait returns the agent once the future completes`() {
40
+ val agent = Any()
41
+ val pending = CompletableFuture<Any>()
42
+ pending.complete(agent)
43
+
44
+ assertSame(agent, AbandonedCallAgent.take(pending, 50))
45
+ }
46
+
47
+ @Test
48
+ fun `a short wait does not cancel an agent that is still signing in`() {
49
+ val pending = CompletableFuture<Any>()
50
+
51
+ assertNull(AbandonedCallAgent.take(pending, 20))
52
+ assertFalse(pending.isDone)
53
+ assertFalse(pending.isCancelled)
54
+ }
55
+
56
+ @Test
57
+ fun `already-exists is recognized through wrapped execution exceptions`() {
58
+ val native = IllegalStateException(
59
+ "Failed to create CallAgent, an instance of CallAgent associated with this identity already exists.",
60
+ )
61
+ val wrapped = java.util.concurrent.ExecutionException(native)
62
+
63
+ assertTrue(AbandonedCallAgent.isExistingAgentError(wrapped))
64
+ assertFalse(AbandonedCallAgent.isExistingAgentError(IllegalStateException("ACS_AGENT_TIMEOUT")))
65
+ }
66
+
67
+ @Test
68
+ fun `take with no wait is the same as takeIfDone`() {
69
+ val pending = CompletableFuture<Any>()
70
+ assertNull(AbandonedCallAgent.take(pending, 0))
71
+ pending.complete("agent")
72
+ assertSame("agent", AbandonedCallAgent.take(pending, 0))
73
+ }
74
+ }
@@ -0,0 +1,47 @@
1
+ package com.mentra.acsmeeting
2
+
3
+ import org.junit.Assert.assertEquals
4
+ import org.junit.Assert.assertNull
5
+ import org.junit.Test
6
+
7
+ class CallCapabilitiesTest {
8
+
9
+ @Test
10
+ fun `an allowed capability proceeds`() {
11
+ assertNull(EndForEveryonePolicy.refusalFor(CapabilityStatus(allowed = true, reason = "capable")))
12
+ }
13
+
14
+ @Test
15
+ fun `a denied capability is refused with its reason`() {
16
+ assertEquals(
17
+ "hang_up_for_everyone_not_allowed:role_restricted",
18
+ EndForEveryonePolicy.refusalFor(CapabilityStatus(allowed = false, reason = "role_restricted")),
19
+ )
20
+ }
21
+
22
+ @Test
23
+ fun `a denied capability with no reason still names itself`() {
24
+ assertEquals(
25
+ "hang_up_for_everyone_not_allowed:unknown",
26
+ EndForEveryonePolicy.refusalFor(CapabilityStatus(allowed = false, reason = " ")),
27
+ )
28
+ }
29
+
30
+ /**
31
+ * Unknown is the ordinary state between joining and the first capabilities event. Treating it as
32
+ * a refusal would hide End on every call that taps it early; letting ACS answer is honest.
33
+ */
34
+ @Test
35
+ fun `an unknown capability is attempted, not refused`() {
36
+ assertNull(EndForEveryonePolicy.refusalFor(CapabilityStatus()))
37
+ assertNull(EndForEveryonePolicy.refusalFor(CapabilityStatus(reason = "capabilities_unavailable")))
38
+ }
39
+
40
+ @Test
41
+ fun `serializes the tri-state for the miniapp`() {
42
+ assertEquals(
43
+ mapOf("allowed" to null, "reason" to "not_reported"),
44
+ CapabilityStatus(reason = "not_reported").toMap(),
45
+ )
46
+ }
47
+ }