@mentra/acs-meeting 3.2.0-dev.235 → 3.2.0-dev.245

package/android/src/main/java/com/mentra/acsmeeting/telemetry/CallDiagnostics.kt

@@ -0,0 +1,315 @@
+package com.mentra.acsmeeting.telemetry
+
+import com.mentra.glassesmedia.telemetry.PipelineStats
+
+/**
+ * The field sets behind the ACS call diagnostics, separated from the session that emits them.
+ *
+ * TEMPORARY DIAGNOSTIC — these feed `scripts/acs-quality-compare.mjs` and go away with it. They
+ * live here because the session cannot be constructed in a JVM test (Android, plus the ACS SDK),
+ * and a trace whose field names drift from the analyzer that parses them is worse than no trace:
+ * the analyzer reports a call with no data rather than an error.
+ */
+object CallDiagnostics {
+
+  /** Dense sampling while a call is still settling. */
+  const val DENSE_INTERVAL_MS = 2_000L
+
+  /** And sparse once it has, so a 20-minute call does not bury the interesting first minute. */
+  const val SPARSE_INTERVAL_MS = 10_000L
+
+  /** How long after `connected` the dense rate lasts. */
+  const val DENSE_WINDOW_MS = 90_000L
+
+  /**
+   * "Visibly bad" and "gone", in bits per second. Same numbers as the analyzer.
+   *
+   * 500k at 540p15 is soft and blocky but legible; 250k is the mush the wearer calls potato. They
+   * are duplicated in `scripts/acs-quality-compare.mjs` on purpose — native decides when to look
+   * harder, the analyzer decides how to score, and neither should silently inherit the other's
+   * threshold. If one moves, move both.
+   */
+  const val LOW_BITRATE_BPS = 500_000L
+  const val VERY_LOW_BITRATE_BPS = 250_000L
+
+  /** Above this the picture is back. Below it a call is still degraded, however far it has climbed. */
+  const val RECOVERED_BITRATE_BPS = 1_000_000L
+
+  /**
+   * How long a dip or a downscale holds the dense cadence after it ends.
+   *
+   * The climb back has been measured at ~170 s against a 90 s dip, so the recovery ramp is the
+   * larger half of what the wearer sits through and the part a sparse sampler renders as three
+   * points. Held open by time rather than by rate so the ramp is sampled all the way up.
+   */
+  const val RECOVERY_WATCH_MS = 30_000L
+
+  /**
+   * What the ACS hop is doing, which decides how often it is worth looking.
+   *
+   * The old cadence was dense for 90 s after `connected` and sparse forever after, on the
+   * assumption that a call which settles low does so early. An 8-minute capture disproved it: the
+   * collapse began at t=150 s, so the interesting 90 s was sampled every 10 s — nine points for
+   * the event the whole investigation was about — while the uneventful first minute got forty-five.
+   *
+   * `SILENT` is deliberately dense and deliberately not `LOW`. ACS publishing a report with unset
+   * fields is a fact about ACS, not a bitrate of zero, and the Start path has been observed doing
+   * it for most of a seven-minute call; that blindness is worth sampling closely for what the
+   * other columns still say (sent fps, subscriber count, glasses hop) even though the rate is
+   * unknown.
+   */
+  enum class WireHealth { SETTLING, SILENT, LOW, RECOVERING, HEALTHY }
+
+  /**
+   * @param sinceConnectedMs negative before `connected`
+   * @param wireBitrateBps null or non-positive when ACS has not published a filled report
+   * @param msSinceLow time since the rate was last under [LOW_BITRATE_BPS], or -1 if never
+   * @param msSinceAdaptation time since the wire size last changed, or -1 if never
+   */
+  fun wireHealth(
+    sinceConnectedMs: Long,
+    wireBitrateBps: Long?,
+    msSinceLow: Long,
+    msSinceAdaptation: Long,
+  ): WireHealth {
+    if (sinceConnectedMs < 0 || sinceConnectedMs <= DENSE_WINDOW_MS) return WireHealth.SETTLING
+    if (wireBitrateBps == null || wireBitrateBps <= 0) return WireHealth.SILENT
+    if (wireBitrateBps < LOW_BITRATE_BPS) return WireHealth.LOW
+    val watching = (msSinceLow in 0..RECOVERY_WATCH_MS) || (msSinceAdaptation in 0..RECOVERY_WATCH_MS)
+    if (wireBitrateBps < RECOVERED_BITRATE_BPS || watching) return WireHealth.RECOVERING
+    return WireHealth.HEALTHY
+  }
+
+  /** Sparse only when there is nothing happening; everything else is worth 2 s. */
+  fun sampleIntervalMs(health: WireHealth): Long =
+    if (health == WireHealth.HEALTHY) SPARSE_INTERVAL_MS else DENSE_INTERVAL_MS
+
+  /**
+   * One sample of both hops at one instant.
+   *
+   * Every rate is nullable or negative-for-unknown rather than defaulted to zero: a call with no
+   * MEDIA_STATISTICS report yet and a call sending nothing look identical once both print 0.
+   */
+  data class BweSample(
+    val state: String,
+    val sinceJoinMs: Long,
+    val sinceConnectedMs: Long,
+    val lobbyDwellMs: Long,
+    val sendQuality: String,
+    val wireBitrateBps: Long?,
+    val wireWidth: Int?,
+    val wireHeight: Int?,
+    val sentFps: Double,
+    val wireFps: Double?,
+    /**
+     * The glasses hop's rate at this instant.
+     *
+     * On the same line as the ACS rate rather than left to `whip_ingest_sample`, because the
+     * comparison between the two is the finding and pairing them by log timestamp was guesswork.
+     * A collapsed wire next to a full-rate source is ACS's rate controller; both low is a starved
+     * pipeline. Those need different fixes and the analyzer should not have to infer which it is.
+     */
+    val inboundBitrateBps: Long?,
+    val inboundFps: Double?,
+    val decodedFps: Double?,
+    val framesGated: Int,
+    val pacerDrops: Int,
+    val budgetBps: Int,
+    val rateArm: String,
+    /** How many MEDIA_STATISTICS reports this call has received so far. Zero with a live `sentFps` is "ACS is silent", not "we are sending nothing". */
+    val mediaStatsReports: Int = 0,
+    val mediaStatsAttached: Boolean = false,
+    /** `none` / ACS `VideoStreamState` name. Independent of whether MEDIA_STATISTICS ever reports. */
+    val videoOut: String = "none",
+    val packetsPerSecond: Double? = null,
+    val subCount: Int = 0,
+    val sinkCount: Int = 0,
+  )
+
+  /**
+   * `origin` and `callId` on the front of every diagnostic.
+   *
+   * Without `origin` the Start-versus-Join comparison cannot be made from the log at all; without
+   * `callId` two calls in one capture merge into one timeline whenever the trace id is reused.
+   */
+  fun stamp(origin: String, callId: String, vararg fields: Pair<String, Any?>): Array<Pair<String, Any?>> =
+    arrayOf("origin" to origin.ifEmpty { "unknown" }, "callId" to callId.ifEmpty { "none" }, *fields)
+
+  fun bweFields(origin: String, callId: String, sample: BweSample): Array<Pair<String, Any?>> = stamp(
+    origin,
+    callId,
+    "state" to sample.state,
+    "sinceJoinMs" to sample.sinceJoinMs,
+    "sinceConnectedMs" to sample.sinceConnectedMs,
+    "lobbyDwellMs" to sample.lobbyDwellMs,
+    "sendQuality" to sample.sendQuality.ifBlank { "na" },
+    // The ACS hop: what MEDIA_STATISTICS says actually left this phone, which is what the far end
+    // sees and what the comparison ranks calls by.
+    "wireBitrateBps" to (sample.wireBitrateBps ?: -1L),
+    "wireWidth" to (sample.wireWidth ?: -1),
+    "wireHeight" to (sample.wireHeight ?: -1),
+    "sentFps" to PipelineStats.formatRate(sample.sentFps),
+    "wireFps" to rate(sample.wireFps),
+    // The glasses hop at the same instant: a phone sending little because it is receiving little
+    // is a different fault from one throttling its own uplink from plenty.
+    "inboundBitrateBps" to (sample.inboundBitrateBps ?: -1L),
+    "inboundFps" to rate(sample.inboundFps),
+    "decodedFps" to rate(sample.decodedFps),
+    "framesGated" to sample.framesGated,
+    "pacerDrops" to sample.pacerDrops,
+    "budgetBps" to sample.budgetBps,
+    "p7RateBound" to sample.rateArm,
+    // Present even when `wireBitrateBps` stays -1, so a Start that never gets MEDIA_STATISTICS
+    // still has a row that can be compared to Join on "is the phone feeding frames" and "did ACS
+    // attach / report".
+    "mediaStatsReports" to sample.mediaStatsReports,
+    "mediaStatsAttached" to sample.mediaStatsAttached,
+    "videoOut" to sample.videoOut.ifBlank { "none" },
+    "packetsPerSecond" to rate(sample.packetsPerSecond),
+    "subCount" to sample.subCount,
+    "sinkCount" to sample.sinkCount,
+  )
+
+  fun mediaStatsAttachFields(
+    origin: String,
+    callId: String,
+    ok: Boolean,
+    error: String = "",
+  ): Array<Pair<String, Any?>> = stamp(
+    origin,
+    callId,
+    "ok" to ok,
+    "error" to error.ifBlank { "none" },
+  )
+
+  fun mediaStatsIntervalFields(
+    origin: String,
+    callId: String,
+    attempt: Int,
+    ok: Boolean,
+    seconds: Int,
+    error: String = "",
+  ): Array<Pair<String, Any?>> = stamp(
+    origin,
+    callId,
+    "attempt" to attempt,
+    "ok" to ok,
+    "seconds" to seconds,
+    "error" to error.ifBlank { "none" },
+  )
+
+  fun mediaStatsReportFields(
+    origin: String,
+    callId: String,
+    n: Int,
+    videos: Int,
+    audios: Int,
+    wireBitrateBps: Long?,
+    width: Int?,
+    height: Int?,
+    fps: Double?,
+    codec: String,
+    packetCount: Int?,
+  ): Array<Pair<String, Any?>> = stamp(
+    origin,
+    callId,
+    "n" to n,
+    "videos" to videos,
+    "audios" to audios,
+    "wireBitrateBps" to (wireBitrateBps ?: -1L),
+    "width" to (width ?: -1),
+    "height" to (height ?: -1),
+    "fps" to rate(fps),
+    "codec" to codec.ifBlank { "na" }.replace(' ', '_'),
+    "packetCount" to (packetCount ?: -1),
+  )
+
+  /**
+   * The wire size ACS actually chose, logged on change.
+   *
+   * A permanent downscale is the single most misleading thing in these captures: ACS trades
+   * resolution for frames inside its budget, so 320x180 at a steady 15 fps prints as a healthy
+   * ladder line and reads like a working call. Until this event existed the drop was a `Log.w` the
+   * analyzer could not see, which is how a call that spent 90 s at 320x180 was scored as fine.
+   *
+   * `ceilingBps` rides along because the whole point of the ceiling A/B is to ask whether a
+   * smaller grant downscales less, and an event that cannot name its own arm cannot answer that.
+   */
+  fun wireAdaptationFields(
+    origin: String,
+    callId: String,
+    width: Int,
+    height: Int,
+    direction: String,
+    sinceConnectedMs: Long,
+    askedWidth: Int,
+    askedHeight: Int,
+    wireBitrateBps: Long?,
+    ceilingBps: Int,
+    fps: Double?,
+  ): Array<Pair<String, Any?>> = stamp(
+    origin,
+    callId,
+    "width" to width,
+    "height" to height,
+    "direction" to direction,
+    "sinceConnectedMs" to sinceConnectedMs,
+    "asked" to "${askedWidth}x$askedHeight",
+    "percentOfAsked" to percentOf(width, height, askedWidth, askedHeight),
+    "wireBitrateBps" to (wireBitrateBps ?: -1L),
+    "ceilingBps" to ceilingBps,
+    "fps" to rate(fps),
+  )
+
+  /**
+   * One low-bitrate episode, as it begins, as it ends, and once it is paid for.
+   *
+   * Emitted as well as being derivable from the samples because the two answer different
+   * questions. The analyzer reconstructs episodes from whatever it can see, and says so when that
+   * is thin; this event is the session's own account, so an episode is on the record even when
+   * MEDIA_STATISTICS went quiet in the middle of it and the reconstruction has a hole.
+   *
+   * `inboundBitrateBps` is the glasses hop measured during the same window. It is the field that
+   * assigns blame, and it belongs on the episode rather than only on the periodic sample: a full
+   * source rate alongside a collapsed wire rate is ACS's rate controller and nothing upstream.
+   */
+  fun episodeFields(
+    origin: String,
+    callId: String,
+    episode: String,
+    startedAtMs: Long,
+    durationMs: Long,
+    minBitrateBps: Long,
+    minResolution: String,
+    recoveryMs: Long,
+    inboundBitrateBps: Double?,
+    sentFps: Double,
+    ceilingBps: Int,
+  ): Array<Pair<String, Any?>> = stamp(
+    origin,
+    callId,
+    "episode" to episode,
+    "startedAtMs" to startedAtMs,
+    "durationMs" to durationMs,
+    "minBitrateBps" to minBitrateBps,
+    "minResolution" to minResolution.ifBlank { "na" },
+    // -1 rather than 0: "has not climbed back yet" and "climbed back instantly" are different
+    // findings, and on the capture that started this the difference was 170 seconds.
+    "recoveryMs" to recoveryMs,
+    "inboundBitrateBps" to rate(inboundBitrateBps),
+    "sentFps" to PipelineStats.formatRate(sentFps),
+    "ceilingBps" to ceilingBps,
+  )
+
+  private fun percentOf(width: Int, height: Int, askedWidth: Int, askedHeight: Int): Int {
+    val asked = askedWidth.toLong() * askedHeight
+    if (asked <= 0) return -1
+    return (width.toLong() * height * 100 / asked).toInt()
+  }
+
+  /** Dense until [DENSE_WINDOW_MS] past `connected`; -1 for "not connected yet" keeps it dense. */
+  fun sampleIntervalMs(sinceConnectedMs: Long): Long =
+    if (sinceConnectedMs in 0..DENSE_WINDOW_MS || sinceConnectedMs < 0) DENSE_INTERVAL_MS else SPARSE_INTERVAL_MS
+
+  private fun rate(value: Double?): String = value?.let { PipelineStats.formatRate(it) } ?: "na"
+}

package/android/src/main/java/com/mentra/acsmeeting/telemetry/WireEpisodeTracker.kt

@@ -0,0 +1,151 @@
+package com.mentra.acsmeeting.telemetry
+
+/**
+ * Watches the ACS uplink rate and calls out each stretch where the picture went bad.
+ *
+ * TEMPORARY DIAGNOSTIC — pairs with `scripts/acs-quality-compare.mjs` and goes away with it.
+ *
+ * This exists because the failure that matters is an interval, not a number. The capture that
+ * motivated it ran eight minutes: the wire sat at ~1.3 Mbps and 960x540, fell to 33 kbps and
+ * 320x180 at t=150 s, stayed there about 90 s, then took another ~170 s to climb back — while the
+ * glasses hop held ~2 Mbps at 14.5 fps throughout and the phone kept handing ACS 15 fps. No single
+ * sample says that. A mean over the call says the opposite.
+ *
+ * It is a separate, dependency-free class so the state machine can be tested without Android or
+ * the ACS SDK, which the session cannot be.
+ *
+ * The one rule that is easy to get wrong: **a missing rate is not a low rate.** ACS publishes
+ * MEDIA_STATISTICS reports with the fields unset — 32 empty against 12 filled across one
+ * seven-minute call — and treating those as zero would invent an episode for every silence and
+ * bury the real ones. Silence leaves the state machine exactly where it was.
+ */
+class WireEpisodeTracker(
+  private val lowBps: Long = CallDiagnostics.LOW_BITRATE_BPS,
+  private val recoveredBps: Long = CallDiagnostics.RECOVERED_BITRATE_BPS,
+) {
+
+  /** What just happened to the episode, if anything. */
+  enum class Phase {
+    /** The rate just fell below the low threshold. */
+    BEGIN,
+
+    /** It just came back above it. The dip is over; the climb is not. */
+    END,
+
+    /** It reached [recoveredBps]. This is when the wearer has their picture back. */
+    RECOVERED,
+  }
+
+  /**
+   * @param durationMs time under the threshold; 0 on [Phase.BEGIN], final on [Phase.END]
+   * @param recoveryMs time from the end of the dip to [recoveredBps]; -1 until [Phase.RECOVERED]
+   * @param inboundBitrateBps the glasses hop during the episode — the field that assigns blame
+   */
+  data class Event(
+    val phase: Phase,
+    val startedAtMs: Long,
+    val durationMs: Long,
+    val minBitrateBps: Long,
+    val minResolution: String,
+    val recoveryMs: Long,
+    val inboundBitrateBps: Double?,
+    val sentFps: Double,
+  )
+
+  private var startedAtMs = -1L
+  private var endedAtMs = -1L
+  private var minBitrateBps = Long.MAX_VALUE
+  private var minPixels = Long.MAX_VALUE
+  private var minResolution = ""
+  private var inboundSum = 0.0
+  private var inboundCount = 0
+  private var inEpisode = false
+  private var awaitingRecovery = false
+
+  /** True while the wire is under the threshold, so the caller can hold its dense cadence. */
+  fun isInEpisode(): Boolean = inEpisode
+
+  /** True after a dip has ended but before the rate has climbed back to [recoveredBps]. */
+  fun isAwaitingRecovery(): Boolean = awaitingRecovery
+
+  /**
+   * Feed one observation. Returns the transition it caused, or null for "carry on".
+   *
+   * At most one transition per observation: a rate that jumps from 33 kbps straight past 1 Mbps
+   * reports [Phase.END] now and [Phase.RECOVERED] on the next observation, so no caller has to
+   * unpack two events from one return.
+   */
+  fun observe(
+    atMs: Long,
+    bitrateBps: Long?,
+    width: Int?,
+    height: Int?,
+    inboundBitrateBps: Double?,
+    sentFps: Double,
+  ): Event? {
+    // Silence is not evidence. See the class comment: this is the branch that keeps ACS's empty
+    // reports from manufacturing an episode per gap.
+    if (bitrateBps == null || bitrateBps <= 0) return null
+
+    if (inEpisode) {
+      if (bitrateBps < lowBps) {
+        accumulate(atMs, bitrateBps, width, height, inboundBitrateBps)
+        return null
+      }
+      inEpisode = false
+      awaitingRecovery = true
+      endedAtMs = atMs
+      return event(Phase.END, atMs, recoveryMs = -1L, sentFps = sentFps)
+    }
+
+    if (bitrateBps < lowBps) {
+      // A dip that returns before recovering is a new episode rather than a continuation: the
+      // wearer saw the picture come back and go again, and averaging the two would hide that.
+      startedAtMs = atMs
+      endedAtMs = -1L
+      minBitrateBps = Long.MAX_VALUE
+      minPixels = Long.MAX_VALUE
+      minResolution = ""
+      inboundSum = 0.0
+      inboundCount = 0
+      inEpisode = true
+      awaitingRecovery = false
+      accumulate(atMs, bitrateBps, width, height, inboundBitrateBps)
+      return event(Phase.BEGIN, atMs, recoveryMs = -1L, sentFps = sentFps)
+    }
+
+    if (awaitingRecovery && bitrateBps >= recoveredBps) {
+      awaitingRecovery = false
+      return event(Phase.RECOVERED, endedAtMs, recoveryMs = atMs - endedAtMs, sentFps = sentFps)
+    }
+
+    return null
+  }
+
+  private fun accumulate(atMs: Long, bitrateBps: Long, width: Int?, height: Int?, inboundBitrateBps: Double?) {
+    endedAtMs = atMs
+    if (bitrateBps < minBitrateBps) minBitrateBps = bitrateBps
+    if (width != null && height != null && width > 0 && height > 0) {
+      val pixels = width.toLong() * height
+      if (pixels < minPixels) {
+        minPixels = pixels
+        minResolution = "${width}x$height"
+      }
+    }
+    if (inboundBitrateBps != null && inboundBitrateBps > 0) {
+      inboundSum += inboundBitrateBps
+      inboundCount += 1
+    }
+  }
+
+  private fun event(phase: Phase, atMs: Long, recoveryMs: Long, sentFps: Double): Event = Event(
+    phase = phase,
+    startedAtMs = startedAtMs,
+    durationMs = if (phase == Phase.BEGIN) 0L else (atMs - startedAtMs).coerceAtLeast(0L),
+    minBitrateBps = if (minBitrateBps == Long.MAX_VALUE) -1L else minBitrateBps,
+    minResolution = minResolution,
+    recoveryMs = recoveryMs,
+    inboundBitrateBps = if (inboundCount > 0) inboundSum / inboundCount else null,
+    sentFps = sentFps,
+  )
+}

package/android/src/main/java/com/mentra/acsmeeting/video/AcsFrameSender.kt

@@ -59,14 +59,21 @@ class AcsFrameSender(
   }
   // Set on the session thread, read from ACS state/format listener threads.
   @Volatile private var onFormat: ((TargetSize) -> Unit)? = null
+  @Volatile private var onNegotiatedFormat: ((VideoStreamFormat) -> Unit)? = null
   private var attachedStream: VirtualOutgoingVideoStream? = null
   private var stateListener: VideoStreamStateChangedListener? = null
   private var formatListener: VideoStreamFormatChangedListener? = null
 
-  fun attach(outgoing: VirtualOutgoingVideoStream, onFormat: ((TargetSize) -> Unit)? = null) {
+  fun attach(
+    outgoing: VirtualOutgoingVideoStream,
+    onFormat: ((TargetSize) -> Unit)? = null,
+    /** The whole negotiated format, for callers that log it. [onFormat] only carries the size. */
+    onNegotiatedFormat: ((VideoStreamFormat) -> Unit)? = null,
+  ) {
     detach()
     pacer.reset()
     this.onFormat = onFormat
+    this.onNegotiatedFormat = onNegotiatedFormat
     stream.set(outgoing)
     attachedStream = outgoing
     val onState = VideoStreamStateChangedListener {
@@ -365,6 +372,7 @@ class AcsFrameSender(
     pacer.reset()
     pool.clear()
     onFormat = null
+    onNegotiatedFormat = null
   }
 
   private fun pushTarget(fmt: VideoStreamFormat?) {
@@ -393,6 +401,7 @@ class AcsFrameSender(
 
   private fun logFormat(fmt: VideoStreamFormat?) {
     if (fmt == null) return
+    onNegotiatedFormat?.invoke(fmt)
     Log.i(
       TAG,
       "P5 negotiated format pixel=${fmt.pixelFormat} ${fmt.width}x${fmt.height} fps=${fmt.framesPerSecond} " +