@mentra/glasses-media 3.2.0-dev.235 → 3.2.0-dev.245

package/android/src/main/java/com/mentra/glassesmedia/source/FrameStallGate.kt

@@ -0,0 +1,61 @@
+package com.mentra.glassesmedia.source
+
+/**
+ * Watches a *live* ingest for a decoder that has stopped advancing.
+ *
+ * [FirstFrameGate] only ever guards the first frame; once promoted it never
+ * fires again. An ingest that dies mid-call therefore reads as healthy from
+ * every angle above this layer — ICE stays connected, audio keeps flowing, and
+ * the wearer sits behind a frozen tile while the call still reports live video.
+ * Two "video frozen, audio still works" reports arrived with nothing in the
+ * traces marking the moment frames stopped.
+ *
+ * Kept free of `org.webrtc` types so the decision is unit testable; the source
+ * feeds it one stats sample at a time and acts on [Verdict.Stalled].
+ */
+internal class FrameStallGate(private val stallSamples: Int) {
+  sealed interface Verdict {
+    /** Frames are advancing, or this sample cannot say. */
+    data object Healthy : Verdict
+
+    /** Frames have stopped but not yet for long enough to call it. */
+    data class Suspected(val samples: Int) : Verdict
+
+    /** Report once, then stay quiet until the next arm. */
+    data class Stalled(val samples: Int) : Verdict
+  }
+
+  private var stalled = 0
+  private var reported = false
+
+  val samples: Int
+    get() = stalled
+
+  /** A new peer generation, or a source that re-earned LIVE. */
+  fun arm() {
+    stalled = 0
+    reported = false
+  }
+
+  /**
+   * @param live the source has painted at least one frame; before that the first-frame gate owns
+   *   the verdict.
+   * @param iceConnected a disconnected transport is its own failure, already handled upstream.
+   * @param fps decoded frames per second, negative when the sample has no prior read to rate
+   *   against. A rate needs two reads, so the first sample is never evidence.
+   */
+  fun sample(live: Boolean, iceConnected: Boolean, fps: Double): Verdict {
+    if (!live || !iceConnected || fps < 0.0) {
+      stalled = 0
+      return Verdict.Healthy
+    }
+    if (fps > 0.0) {
+      stalled = 0
+      return Verdict.Healthy
+    }
+    stalled++
+    if (stalled < stallSamples || reported) return Verdict.Suspected(stalled)
+    reported = true
+    return Verdict.Stalled(stalled)
+  }
+}

package/android/src/main/java/com/mentra/glassesmedia/source/GlassesMediaSource.kt

@@ -100,6 +100,15 @@ class GlassesMediaController(
   private var source: GlassesMediaSource? = null
   private var stateListener: SourceStateListener? = null
 
+  /**
+   * The SoftAP listener this controller has let go of, kept only until its port is released.
+   *
+   * [stop] drops the source, but a WHIP listener answers `410` for a few seconds after that and
+   * still holds its port. Losing the reference here is what made the port's release unobservable
+   * from above, so the next call bound into it.
+   */
+  private var retiringIngest: LocalWhipIngestSource? = null
+
   val state: SourceState
     get() = source?.state ?: SourceState.IDLE
 
@@ -108,7 +117,7 @@ class GlassesMediaController(
     get() = source?.ingestUrl
 
   fun attach(video: VideoFrameListener, pcm: PcmListener, config: SourceConfig) {
-    source?.stop()
+    retire()
     source = factory.create(video, pcm, config).also {
       it.setStateListener(stateListener)
       it.start(config)
@@ -129,10 +138,32 @@ class GlassesMediaController(
   }
 
   fun stop() {
-    source?.stop()
+    retire()
+  }
+
+  private fun retire() {
+    val previous = source
+    previous?.stop()
+    if (previous is LocalWhipIngestSource) retiringIngest = previous
     source = null
   }
 
+  /**
+   * Has the SoftAP listener actually released its port? `true` when there is nothing to wait for.
+   *
+   * A `false` here is the teardown barrier's signal to force the close rather than to give up:
+   * the next call's bind fails on exactly this port.
+   */
+  fun awaitIngestClosed(timeoutMs: Long): Boolean = retiringIngest?.awaitIngestClosed(timeoutMs) ?: true
+
+  /** Close the retiring listener now, skipping the tombstone. Only for a barrier that timed out. */
+  fun forceCloseIngest() {
+    // Do not drop the handle here: the host re-asks awaitIngestClosed after this to verify the port
+    // was released, and a null reference answers `true` unconditionally, hiding a failed close. The
+    // next retire() reassigns retiringIngest, so keeping the closed one until then is harmless.
+    retiringIngest?.forceCloseIngest()
+  }
+
   fun setPcmDeliveryEnabled(enabled: Boolean) {
     source?.setPcmDeliveryEnabled(enabled)
   }

package/android/src/main/java/com/mentra/glassesmedia/source/LocalWhipIngestSource.kt

@@ -68,6 +68,7 @@ class LocalWhipIngestSource(
 
   private val relay = DecodedTrackRelay(videoListener, pcmListener, stats) { notePromotableFrame() }
   private val firstFrame = FirstFrameGate()
+  private val frameStall = FrameStallGate(INGEST_STALL_SAMPLES)
   private val videoIds = TrackRegistry()
   private val audioIds = TrackRegistry()
   private val audioTracks = CopyOnWriteArrayList<AudioTrack>()
@@ -81,8 +82,14 @@ class LocalWhipIngestSource(
   @Volatile private var attachedVideo: VideoTrack? = null
   @Volatile private var boundUrl: String? = null
   @Volatile private var stateListener: SourceStateListener? = null
+  /** The listener handed to the tombstone thread by [stop]. See [awaitIngestClosed]. */
+  @Volatile private var retiring: WhipIngestServer? = null
   @Volatile private var firstFrameDeadline: Runnable? = null
   @Volatile private var selectedPairTask: Runnable? = null
+  @Volatile private var ingestSampleTask: Runnable? = null
+  private var lastIngestBytes = -1L
+  private var lastIngestFrames = -1L
+  private var lastIngestSampleAtMs = 0L
 
   /**
    * Invalidates callbacks from a peer we are disposing. A negotiation can be mid-gather when the
@@ -148,6 +155,7 @@ class LocalWhipIngestSource(
     boundUrl = null
     cancelFirstFrameDeadline()
     cancelSelectedPairProof()
+    cancelIngestSampling()
     firstFrame.reset()
     detachTracks()
     relay.resetRotationLog()
@@ -158,17 +166,46 @@ class LocalWhipIngestSource(
 
     // stop() leaves the listener answering 410 for a few seconds, so a POST the glasses already
     // sent gets an answer it can act on instead of a reset it would retry.
-    server?.let { runCatching { it.stop() } }
+    server?.let {
+      // Held past the field being cleared: this source is done with it, but the port is not free
+      // until the tombstone thread closes it, and the next call needs that port.
+      retiring = it
+      runCatching { it.stop() }
+    }
     server = null
     disposePeer()
     scopedNetwork?.let { ScopedNetworkChangeDetector.releaseReceiverNetwork(it) }
   }
 
+  /**
+   * Wait out the tombstone. `false` means the port is still held and the caller must force it.
+   *
+   * Trivially true when nothing was ever bound, so a teardown after a join that failed before the
+   * listener existed does not spend the whole bound discovering there is nothing to wait for.
+   */
+  fun awaitIngestClosed(timeoutMs: Long): Boolean = (retiring ?: server)?.awaitClosed(timeoutMs) ?: true
+
+  /**
+   * Drop the listener now, tombstone or not.
+   *
+   * Only for the barrier's forced path: closing early means an in-flight request from the glasses
+   * gets a connection reset instead of `410`, which is a worse answer — but it is a better outcome
+   * than a next call that cannot bind its port.
+   */
+  fun forceCloseIngest() {
+    // Keep the handle: forceSoftapCleanup re-asks awaitIngestClosed to confirm the port is really
+    // free. Nulling here would make that check read `null -> true` and mask a closeNow that threw,
+    // so the next Start binds a port this listener still holds. closeNow/awaitClosed are idempotent,
+    // and a fresh stop() reassigns `retiring`, so leaving it set is safe.
+    (retiring ?: server)?.let { runCatching { it.closeNow() } }
+  }
+
   /** Terminal teardown for owners that discard this receiver instead of reusing its factory. */
   fun close() {
     generation++
     check(server?.closeAndAwait() != false) { "Local WHIP requests are still draining" }
     stop()
+    forceCloseIngest()
     factory?.dispose()
     factory = null
     egl?.release()
@@ -360,6 +397,7 @@ class LocalWhipIngestSource(
     }
 
     armFirstFrame(gen)
+    armIngestSampling(gen)
     return Result.success(local)
   }
 
@@ -370,6 +408,7 @@ class LocalWhipIngestSource(
     generation++
     cancelFirstFrameDeadline()
     cancelSelectedPairProof()
+    cancelIngestSampling()
     detachTracks()
     disposePeer()
     if (state != SourceState.IDLE) transition(SourceState.FAILED, "publisher_terminated")
@@ -623,6 +662,115 @@ class LocalWhipIngestSource(
     selectedPairTask = null
   }
 
+  /**
+   * Sample the glasses→phone leg for as long as this peer lives.
+   *
+   * The ACS side already reports what leaves the phone; without this the two hops are impossible
+   * to tell apart, and "the joined call looked compressed" has two completely different causes —
+   * the glasses encoder sending little, or the phone throttling its own uplink from plenty. Same
+   * 2 s cadence as `acs_bwe_sample` so the two series line up without interpolation.
+   */
+  private fun armIngestSampling(gen: Int) {
+    cancelIngestSampling()
+    lastIngestBytes = -1L
+    lastIngestFrames = -1L
+    lastIngestSampleAtMs = 0L
+    stats.inboundBitrateBps = null
+    frameStall.arm()
+    scheduleIngestSample(gen)
+  }
+
+  private fun scheduleIngestSample(gen: Int) {
+    val task = Runnable {
+      if (gen != generation) return@Runnable
+      sampleIngest(gen)
+      scheduleIngestSample(gen)
+    }
+    ingestSampleTask = task
+    mainHandler.postDelayed(task, INGEST_SAMPLE_INTERVAL_MS)
+  }
+
+  private fun cancelIngestSampling() {
+    ingestSampleTask?.let { mainHandler.removeCallbacks(it) }
+    ingestSampleTask = null
+  }
+
+  private fun sampleIngest(gen: Int) {
+    val peer = pc ?: return
+    val iceState = runCatching { peer.iceConnectionState()?.name?.lowercase() }.getOrNull() ?: "unknown"
+    runCatching {
+      peer.getStats { report ->
+        if (gen != generation) return@getStats
+        val inbound = report.statsMap.values.firstOrNull {
+          it.type == "inbound-rtp" && (it.members["kind"] ?: it.members["mediaType"]) == "video"
+        }
+        val now = android.os.SystemClock.elapsedRealtime()
+        val bytes = (inbound?.members?.get("bytesReceived") as? Number)?.toLong() ?: -1L
+        val frames = (inbound?.members?.get("framesDecoded") as? Number)?.toLong() ?: -1L
+        val elapsedMs = if (lastIngestSampleAtMs == 0L) 0L else now - lastIngestSampleAtMs
+        // Rates need two reads. The first sample reports -1 rather than dividing by the time since
+        // the epoch, which would print a plausible and meaningless number.
+        val bitrate = if (lastIngestBytes < 0 || bytes < lastIngestBytes || elapsedMs <= 0) -1L
+        else (bytes - lastIngestBytes) * 8_000L / elapsedMs
+        val fps = if (lastIngestFrames < 0 || frames < lastIngestFrames || elapsedMs <= 0) -1.0
+        else (frames - lastIngestFrames) * 1000.0 / elapsedMs
+        lastIngestBytes = bytes
+        lastIngestFrames = frames
+        lastIngestSampleAtMs = now
+        // Mirrored onto the shared stats so the ACS side can name this hop's rate inside a
+        // low-bitrate episode. Null rather than -1 there: "no reading yet" must not average in.
+        stats.inboundBitrateBps = if (bitrate >= 0) bitrate else null
+        SoftApTrace.stage(
+          "whip_ingest_sample",
+          "iceState" to iceState,
+          "sourceState" to state.name.lowercase(),
+          "inboundBitrateBps" to bitrate,
+          "inboundFps" to PipelineStats.formatRate(fps),
+          "decodedFps" to (stats.decodedFps?.let { PipelineStats.formatRate(it) } ?: "na"),
+          "bytesReceived" to bytes,
+          "framesDecoded" to frames,
+          "stalledSamples" to frameStall.samples,
+        )
+        noteIngestStall(gen, iceState, fps, frames)
+      }
+    }.onFailure { Log.w(TAG, "SoftAP ingest sample failed", it) }
+  }
+
+  /**
+   * Fail a live session whose decoder has stopped advancing. See [FrameStallGate].
+   *
+   * Keep the peer and listener: `AcsMeetingSession` suppresses session-level rebuilds
+   * for SoftAP because rebinding would strand the glasses on a port they were never told about.
+   * `FAILED` surfaces to the host as `mediaSource: failed`, which moves the call out of "video
+   * live". A resumed frame can restore LIVE on this peer, or the glasses can reconnect on the
+   * unchanged URL. A decoder pause need not cause an ICE state change or a new WHIP offer.
+   */
+  @Synchronized
+  private fun noteIngestStall(gen: Int, iceState: String, fps: Double, frames: Long) {
+    if (gen != generation) return
+    val verdict = frameStall.sample(
+      live = state == SourceState.LIVE,
+      // Host-only SoftAP settles on COMPLETED, not CONNECTED, and stays there for the whole call.
+      // Treating only "connected" as live would reset the stall count every sample — the same
+      // steady state onIceConnectionChange and WHEP both count as healthy — and never fail a freeze.
+      iceConnected = iceState == "connected" || iceState == "completed",
+      fps = fps,
+    )
+    if (verdict !is FrameStallGate.Verdict.Stalled) return
+    Log.w(TAG, "SoftAP ingest decoded no frames across ${verdict.samples} samples; failing source")
+    SoftApTrace.failure(
+      "ingest_frames_stalled",
+      "samples" to verdict.samples,
+      "intervalMs" to INGEST_SAMPLE_INTERVAL_MS,
+      "framesDecoded" to frames,
+    )
+    // Serialize re-arming and failure against the decode callback so a resumed frame cannot
+    // promote LIVE just before this sample overwrites it with FAILED.
+    if (gen != generation) return
+    firstFrame.arm(gen)
+    transition(SourceState.FAILED, "frames_stalled")
+  }
+
   /** Reads the prefix per sample: the phone can lose and rejoin the hotspot mid-call. */
   private fun sampleIcePath(gen: Int, onSample: (IcePathVerdict) -> Unit) {
     val peer = pc ?: return
@@ -738,9 +886,14 @@ class LocalWhipIngestSource(
         )
       }
 
+  @Synchronized
   private fun notePromotableFrame() {
     if (!firstFrame.onFrame(generation)) return
     cancelFirstFrameDeadline()
+    // Re-armed on every promotion, not just per peer: ICE recovery re-earns LIVE on the same peer
+    // without restarting the sampler, and a gate still holding its last verdict would let the
+    // second freeze of a call go unreported.
+    frameStall.arm()
     SoftApTrace.stage("ingest_first_frame")
     transition(SourceState.LIVE, "first_frame")
   }
@@ -781,5 +934,11 @@ class LocalWhipIngestSource(
 
     /** Long enough that a healthy 15 fps feed cannot show a flat byte counter across the two reads. */
     private const val SELECTED_PAIR_SAMPLE_GAP_MS = 1_200L
+
+    /** Matches the ACS-side `acs_bwe_sample` cadence so the two hops can be read side by side. */
+    private const val INGEST_SAMPLE_INTERVAL_MS = 2_000L
+
+    /** Matches [FIRST_FRAME_TIMEOUT_MS] at the sample cadence: 6s of a frozen decoder. */
+    private const val INGEST_STALL_SAMPLES = 3
   }
 }

package/android/src/main/java/com/mentra/glassesmedia/source/WhipIngestServer.kt

@@ -11,6 +11,7 @@ import java.net.ServerSocket
 import java.net.Socket
 import java.net.SocketException
 import java.util.UUID
+import java.util.concurrent.CountDownLatch
 import java.util.concurrent.Executors
 import java.util.concurrent.TimeUnit
 import java.util.concurrent.atomic.AtomicInteger
@@ -72,6 +73,16 @@ class WhipIngestServer(
   private val activeSockets = java.util.concurrent.ConcurrentHashMap.newKeySet<Socket>()
   private val accepted = AtomicInteger()
 
+  /**
+   * Open while a listener is bound; counted down once the socket is actually closed.
+   *
+   * [stop] returns immediately and finishes on a tombstone thread [TOMBSTONE_MS] later, so nothing
+   * above this class could tell the port had been released. The next call would bind before this
+   * one let go and fail with a port that was still ours. Starts already-counted-down so a server
+   * that never bound is trivially "closed".
+   */
+  @Volatile private var closed = CountDownLatch(0)
+
   /** Bound endpoint, or null before [start]. This is the URL the glasses must be told to POST to. */
   val boundEndpoint: WhipIngestProtocol.Endpoint?
     get() = synchronized(lock) { endpoint }
@@ -93,6 +104,7 @@ class WhipIngestServer(
       val bound = WhipIngestProtocol.Endpoint(address.hostAddress ?: "127.0.0.1", socket.localPort)
       server = socket
       endpoint = bound
+      closed = CountDownLatch(1)
       state = WhipIngestProtocol.State()
       acceptThread = Thread({ acceptLoop(socket) }, "whip-ingest-accept").apply {
         isDaemon = true
@@ -149,13 +161,27 @@ class WhipIngestServer(
     return connections.awaitTermination(timeoutMs, TimeUnit.MILLISECONDS)
   }
 
+  /**
+   * Block until the listener is really gone, or the bound expires.
+   *
+   * The teardown barrier's whole job is to answer "can the next call bind this port", and only
+   * this can answer it: [stop] hands the close to a thread that runs [TOMBSTONE_MS] later. `false`
+   * means the caller must force the close rather than proceed — a timeout here is not consent.
+   */
+  fun awaitClosed(timeoutMs: Long = TOMBSTONE_MS + 500): Boolean =
+    closed.await(timeoutMs, TimeUnit.MILLISECONDS)
+
   private fun closeListener() {
     val socket = synchronized(lock) {
       val current = server
       server = null
       endpoint = null
       current
-    } ?: return
+    }
+    // Counted down even on the early return: a second close must not leave a waiter parked on a
+    // latch that nothing will ever open again.
+    closed.countDown()
+    if (socket == null) return
     runCatching { socket.close() }
     activeSockets.forEach { runCatching { it.close() } }
     connections.shutdownNow()

package/android/src/main/java/com/mentra/glassesmedia/telemetry/PipelineStats.kt

@@ -108,6 +108,15 @@ class PipelineStats(
   @Volatile var lastSubFps: Double = 0.0
   @Volatile var decodedFps: Double? = null
   @Volatile var recvFps: Double? = null
+  /**
+   * Glasses→phone WHIP rate, mirrored here from the ingest source's own sampler.
+   *
+   * Published on the shared stats rather than kept local to the ingest source because it is the
+   * field that assigns blame for a bad picture, and the thing that needs it is the ACS side: a
+   * collapsed uplink alongside a full-rate source is ACS's rate controller, while both low is a
+   * starved pipeline. Null until two reads have been taken — a rate needs a delta.
+   */
+  @Volatile var inboundBitrateBps: Long? = null
   @Volatile var wireFps: Double? = null
   @Volatile var wireWidth: Int? = null
   @Volatile var wireHeight: Int? = null
@@ -282,6 +291,9 @@ class PipelineStats(
     return sink.get() - settled
   }
 
+  /** Frames refused by the outgoing pacer alone, separated from the other drop reasons. */
+  fun dropPacedCount(): Int = dropPaced.get()
+
   fun dupCount(): Int = dup.get()
   fun rotCount(): Int = rot.get()
 

package/android/src/main/java/com/mentra/glassesmedia/telemetry/PipelineTicker.kt

@@ -11,6 +11,13 @@ class PipelineTicker(
   looper: Looper = Looper.getMainLooper(),
   private val elapsedCpuMs: () -> Long = { Process.getElapsedCpuTime() },
   private val cores: Int = Runtime.getRuntime().availableProcessors(),
+  /**
+   * Run once per tick, after the ladder lines.
+   *
+   * A second timer for sampling would drift against this one, and two sets of numbers a second
+   * apart cannot be compared. Whoever samples here decides its own cadence by counting ticks.
+   */
+  private val onTick: (() -> Unit)? = null,
   private val emit: (String) -> Unit,
 ) {
   private val handler = Handler(looper)
@@ -47,6 +54,7 @@ class PipelineTicker(
         ),
       )
       avSync?.let { emit(it.tick(stats.e2e.p50(), stats.age.p50())) }
+      onTick?.invoke()
       handler.postDelayed(this, INTERVAL_MS)
     }
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mentra/glasses-media",
-  "version": "3.2.0-dev.235",
+  "version": "3.2.0-dev.245",
   "description": "Shared native glasses media receivers and local network transport",
   "license": "MIT",
   "author": "Mentra Community",