@mentra/acs-meeting 3.2.0-dev.221 → 3.2.0-dev.223

package/android/src/main/java/com/mentra/acsmeeting/source/CloudflareWhepSource.kt

@@ -4,14 +4,12 @@ import android.content.Context
 import android.media.AudioAttributes
 import android.util.Log
 import com.mentra.acsmeeting.telemetry.PipelineStats
-import com.mentra.acsmeeting.video.FrameGeometry
 import com.mentra.acsmeeting.video.I420Packer
 import okhttp3.MediaType.Companion.toMediaType
 import okhttp3.OkHttpClient
 import okhttp3.Request
 import okhttp3.RequestBody.Companion.toRequestBody
 import org.webrtc.AudioTrack
-import org.webrtc.AudioTrackSink
 import org.webrtc.DefaultVideoDecoderFactory
 import org.webrtc.DefaultVideoEncoderFactory
 import org.webrtc.EglBase
@@ -23,15 +21,11 @@ import org.webrtc.PeerConnectionFactory
 import org.webrtc.RtpTransceiver
 import org.webrtc.SdpObserver
 import org.webrtc.SessionDescription
-import org.webrtc.VideoFrame
-import org.webrtc.VideoSink
 import org.webrtc.VideoTrack
 import org.webrtc.audio.JavaAudioDeviceModule
 import java.nio.ByteBuffer
 import java.util.concurrent.CopyOnWriteArrayList
 import java.util.concurrent.TimeUnit
-import java.util.concurrent.atomic.AtomicBoolean
-import java.util.concurrent.atomic.AtomicReference
 import java.util.concurrent.atomic.AtomicLong
 
 /**
@@ -51,10 +45,6 @@ class CloudflareWhepSource(
   private var pc: PeerConnection? = null
   private var currentUrl: String? = null
   @Volatile private var offerPosted = false
-  // Fail closed: the audio policy turns delivery on once the ACS virtual
-  // stream is live. Survives restart() so a WHEP rebuild cannot silently
-  // re-open the uplink (or the local playout) against the current decision.
-  @Volatile private var pcmEnabled = false
   @Volatile override var state: SourceState = SourceState.IDLE
     private set
   @Volatile private var stateListener: SourceStateListener? = null
@@ -67,8 +57,9 @@ class CloudflareWhepSource(
   private val videoIds = TrackRegistry()
   private val audioIds = TrackRegistry()
   @Volatile private var attachedVideo: VideoTrack? = null
-  private val targetSize = AtomicReference<TargetSize?>(null)
-  private val rotationLogged = AtomicBoolean(false)
+  // Shared with the SoftAP path: past the decoder, a Cloudflare hop and a hotspot hop are the
+  // same I420 planes going to the same ACS sender.
+  private val relay = DecodedTrackRelay(videoListener, pcmListener, stats) { notePromotableFrame() }
   private val mainHandler = android.os.Handler(android.os.Looper.getMainLooper())
   private var pendingOfferPost: Runnable? = null
   // LIVE means "ACS is being handed frames", not "the WHEP endpoint answered".
@@ -122,8 +113,8 @@ class CloudflareWhepSource(
     // Reuse the existing subscriber only when the URL is unchanged AND the peer is
     // still healthy. After ICE DISCONNECTED/FAILED (e.g. a Wi-Fi drop that
     // reuses the same Cloudflare WHEP URL) we must rebuild, or glasses video and
-    // mic never recover. See canReuseSource for why CONNECTING is reusable.
-    if (canReuseSource(currentUrl, state, config)) return
+    // mic never recover. See SourceReusePolicy for why CONNECTING is reusable.
+    if (SourceReusePolicy.canReuse(currentUrl, state, config)) return
     start(config)
   }
 
@@ -150,12 +141,12 @@ class CloudflareWhepSource(
   }
 
   override fun setPcmDeliveryEnabled(enabled: Boolean) {
-    pcmEnabled = enabled
+    relay.setPcmDeliveryEnabled(enabled)
     Log.i(TAG, "WHEP audio PCM delivery enabled=$enabled")
   }
 
   override fun setTargetSize(size: TargetSize?) {
-    targetSize.set(size)
+    relay.setTargetSize(size)
   }
 
   override fun stop() {
@@ -183,7 +174,7 @@ class CloudflareWhepSource(
     lastDecodedFrames = -1L
     lastDecodedAtMs = 0L
     lastReceivedFrames = -1L
-    rotationLogged.set(false)
+    relay.resetRotationLog()
     // close() stops media; dispose() is what frees the native peer. Without it every
     // WHEP restart leaks a PeerConnection. dispose() also blocks until observer and
     // sink callbacks quiesce, so it must run after removeSink and before dest is reused.
@@ -393,75 +384,7 @@ class CloudflareWhepSource(
     transition(SourceState.LIVE, "first_frame")
   }
 
-  private val videoSink = VideoSink { frame ->
-    val sinkStart = System.nanoTime()
-    notePromotableFrame()
-    stats.onSink()
-    stats.recordGap()
-    val buffer = frame.buffer
-    stats.onFrameBuffer(classifyBuffer(buffer))
-    val geometry = FrameGeometry.packSize(buffer.width, buffer.height, frame.rotation)
-    if (geometry.rotationNonZero) {
-      stats.onRotation()
-      if (rotationLogged.compareAndSet(false, true)) {
-        Log.w(
-          TAG,
-          "P3 rotation=${frame.rotation} using buffer ${geometry.width}x${geometry.height} " +
-            "(rotatedWidth=${frame.rotatedWidth}x${frame.rotatedHeight} would overrun I420 planes)",
-        )
-      }
-    }
-    val target = targetSize.get()
-    var scaled: VideoFrame.Buffer? = null
-    val source = if (target != null && (target.width != geometry.width || target.height != geometry.height)) {
-      val scaleStart = System.nanoTime()
-      scaled = buffer.cropAndScale(0, 0, geometry.width, geometry.height, target.width, target.height)
-      stats.scale.record(System.nanoTime() - scaleStart)
-      scaled
-    } else {
-      buffer
-    }
-    val i420Start = System.nanoTime()
-    val i420 = source.toI420()
-    stats.toI420.record(System.nanoTime() - i420Start)
-    if (i420 == null) {
-      stats.onDropNullI420()
-      scaled?.release()
-      stats.sinkCb.record(System.nanoTime() - sinkStart)
-      return@VideoSink
-    }
-    try {
-      val w = i420.width
-      val h = i420.height
-      stats.onStrides(i420.strideY, i420.strideU, i420.strideV, w)
-      stats.setSize(w, h)
-      videoListener.onVideoFrame(
-        I420Planes(
-          y = i420.dataY,
-          strideY = i420.strideY,
-          u = i420.dataU,
-          strideU = i420.strideU,
-          v = i420.dataV,
-          strideV = i420.strideV,
-          width = w,
-          height = h,
-          timestampNs = frame.timestampNs,
-          retain = { i420.retain() },
-          release = { i420.release() },
-        ),
-      )
-    } finally {
-      i420.release()
-      scaled?.release()
-      stats.sinkCb.record(System.nanoTime() - sinkStart)
-    }
-  }
-
-  private fun classifyBuffer(buffer: VideoFrame.Buffer): String = when (buffer) {
-    is VideoFrame.TextureBuffer -> "tex"
-    is VideoFrame.I420Buffer -> "i420"
-    else -> "other"
-  }
+  private val videoSink get() = relay.videoSink
 
   private fun attachAudio(track: AudioTrack) {
     val id = track.id()
@@ -475,17 +398,10 @@ class CloudflareWhepSource(
     // volume 0 silences local playout while the sink still gets full-scale PCM.
     track.setVolume(0.0)
     track.setEnabled(true)
-    track.addSink(audioSink)
+    track.addSink(relay.audioSink)
     Log.i(TAG, "P3 attach kind=audio id=$id attached=${audioIds.size()} skipped=${audioIds.skipped()} playoutVolume=0")
   }
 
-  private val audioSink = AudioTrackSink { audioData, bitsPerSample, sampleRate, numberOfChannels, numberOfFrames, _ ->
-    if (!pcmEnabled || bitsPerSample != 16) return@AudioTrackSink
-    val bytes = ByteArray(audioData.remaining())
-    audioData.get(bytes)
-    pcmListener.onPcm(bytes, sampleRate, numberOfChannels)
-  }
-
   private fun pollDecodedStats() {
     val peer = pc ?: return
     peer.getStats { report ->
@@ -569,10 +485,3 @@ class CloudflareWhepSource(
 }
 
 typealias WhepVideoSource = CloudflareWhepSource
-
-private open class SdpAdapter : SdpObserver {
-  override fun onCreateSuccess(sdp: SessionDescription) {}
-  override fun onSetSuccess() {}
-  override fun onCreateFailure(error: String) {}
-  override fun onSetFailure(error: String) {}
-}

package/android/src/main/java/com/mentra/acsmeeting/source/DecodedTrackRelay.kt

@@ -0,0 +1,147 @@
+package com.mentra.acsmeeting.source
+
+import android.util.Log
+import com.mentra.acsmeeting.telemetry.PipelineStats
+import com.mentra.acsmeeting.video.FrameGeometry
+import org.webrtc.AudioTrackSink
+import org.webrtc.VideoFrame
+import org.webrtc.VideoSink
+import java.util.concurrent.atomic.AtomicBoolean
+import java.util.concurrent.atomic.AtomicReference
+
+/**
+ * Hands decoded WebRTC tracks to ACS.
+ *
+ * Extracted from [CloudflareWhepSource] so the SoftAP path shares it rather than copying it. The
+ * scaling and geometry handling here is subtle enough that two copies would drift: buffer
+ * coordinates rather than display coordinates, crop-and-scale before `toI420` so libyuv does the
+ * work, and a retain/release pair the ACS sender owns. Whether the bitstream arrived from Cloudflare
+ * or straight off the hotspot makes no difference past the decoder, so neither should the code.
+ *
+ * Thread confinement: [videoSink] and [audioSink] are called on libwebrtc's decode and audio
+ * threads. Nothing here blocks, and the mutable settings are atomics because they are written from
+ * the session thread.
+ */
+class DecodedTrackRelay(
+  private val videoListener: VideoFrameListener,
+  private val pcmListener: PcmListener,
+  private val stats: PipelineStats,
+  /** Called on the first frame of each generation so the owner can promote itself to LIVE. */
+  private val onPromotableFrame: () -> Unit,
+) {
+  private val targetSize = AtomicReference<TargetSize?>(null)
+  private val rotationLogged = AtomicBoolean(false)
+
+  /**
+   * Fail closed. The audio policy turns delivery on once the ACS virtual stream is live, and this
+   * survives a transport rebuild so a reconnect cannot silently re-open the uplink against a
+   * decision that has since changed.
+   */
+  @Volatile
+  private var pcmEnabled = false
+
+  fun setTargetSize(size: TargetSize?) = targetSize.set(size)
+
+  fun setPcmDeliveryEnabled(enabled: Boolean) {
+    pcmEnabled = enabled
+  }
+
+  fun resetRotationLog() = rotationLogged.set(false)
+
+  val videoSink = VideoSink { frame ->
+    val sinkStart = System.nanoTime()
+    onPromotableFrame()
+    stats.onSink()
+    stats.recordGap()
+    // Guarded because a synthetic or malformed frame can carry 0, and an "age" measured from the
+    // epoch would swamp the percentile ring with one absurd sample.
+    if (frame.timestampNs > 0) {
+      stats.age.record(sinkStart - frame.timestampNs)
+    }
+    val buffer = frame.buffer
+    stats.onFrameBuffer(classifyBuffer(buffer))
+    val geometry = FrameGeometry.packSize(buffer.width, buffer.height, frame.rotation)
+    if (geometry.rotationNonZero) {
+      stats.onRotation()
+      if (rotationLogged.compareAndSet(false, true)) {
+        Log.w(
+          TAG,
+          "P3 rotation=${frame.rotation} using buffer ${geometry.width}x${geometry.height} " +
+            "(rotatedWidth=${frame.rotatedWidth}x${frame.rotatedHeight} would overrun I420 planes)",
+        )
+      }
+    }
+    val target = targetSize.get()
+    var scaled: VideoFrame.Buffer? = null
+    val source = if (needsScale(target, geometry.width, geometry.height)) {
+      val scaleStart = System.nanoTime()
+      scaled = buffer.cropAndScale(0, 0, geometry.width, geometry.height, target!!.width, target.height)
+      stats.scale.record(System.nanoTime() - scaleStart)
+      scaled
+    } else {
+      buffer
+    }
+    val i420Start = System.nanoTime()
+    val i420 = source.toI420()
+    stats.toI420.record(System.nanoTime() - i420Start)
+    if (i420 == null) {
+      stats.onDropNullI420()
+      scaled?.release()
+      stats.sinkCb.record(System.nanoTime() - sinkStart)
+      return@VideoSink
+    }
+    try {
+      val w = i420.width
+      val h = i420.height
+      stats.onStrides(i420.strideY, i420.strideU, i420.strideV, w)
+      stats.setSize(w, h)
+      videoListener.onVideoFrame(
+        I420Planes(
+          y = i420.dataY,
+          strideY = i420.strideY,
+          u = i420.dataU,
+          strideU = i420.strideU,
+          v = i420.dataV,
+          strideV = i420.strideV,
+          width = w,
+          height = h,
+          timestampNs = frame.timestampNs,
+          retain = { i420.retain() },
+          release = { i420.release() },
+        ),
+      )
+    } finally {
+      i420.release()
+      scaled?.release()
+      stats.sinkCb.record(System.nanoTime() - sinkStart)
+    }
+  }
+
+  val audioSink = AudioTrackSink { audioData, bitsPerSample, sampleRate, numberOfChannels, _, _ ->
+    if (!pcmEnabled || bitsPerSample != 16) return@AudioTrackSink
+    val bytes = ByteArray(audioData.remaining())
+    audioData.get(bytes)
+    pcmListener.onPcm(bytes, sampleRate, numberOfChannels)
+  }
+
+  companion object {
+    private const val TAG = "ACS-SPIKE"
+
+    /**
+     * Whether a frame needs resizing before ACS sees it.
+     *
+     * Split out and made pure because it decides whether libyuv runs on every single frame, and
+     * because getting it backwards is invisible: an unnecessary scale is a silent CPU cost, and a
+     * skipped one is a plane-size mismatch at the sender. A null target means ACS has not
+     * negotiated a size yet, so the buffer passes through untouched.
+     */
+    fun needsScale(target: TargetSize?, width: Int, height: Int): Boolean =
+      target != null && (target.width != width || target.height != height)
+
+    fun classifyBuffer(buffer: VideoFrame.Buffer): String = when (buffer) {
+      is VideoFrame.TextureBuffer -> "tex"
+      is VideoFrame.I420Buffer -> "i420"
+      else -> "other"
+    }
+  }
+}

package/android/src/main/java/com/mentra/acsmeeting/source/GlassesMediaSource.kt

@@ -1,12 +1,29 @@
 package com.mentra.acsmeeting.source
 
-enum class SourceKind { WHEP, DIRECT }
+enum class SourceKind {
+  /** Subscribe to a Cloudflare WHEP endpoint. The phone is the offerer. */
+  WHEP,
+  DIRECT,
+
+  /**
+   * Serve a WHIP endpoint on the glasses hotspot and let the glasses publish into it. The phone is
+   * the answerer, and no traffic leaves the local link. See [LocalWhipIngestSource].
+   */
+  SOFTAP,
+}
 
 enum class SourceState { IDLE, CONNECTING, LIVE, FAILED }
 
 data class SourceConfig(
+  /**
+   * WHEP: the endpoint to subscribe to. SOFTAP: unused, because the URL is not known until the
+   * listener has bound a port — the source reports it back through
+   * [GlassesMediaSource.ingestUrl] instead.
+   */
   val url: String,
   val kind: SourceKind = SourceKind.WHEP,
+  /** SOFTAP only: the phone's own IPv4 address on the hotspot, to bind the listener to. */
+  val bindAddress: String? = null,
 )
 
 /** ACS-negotiated output size. One object so a frame cannot observe a torn width/height. */
@@ -22,9 +39,18 @@ data class TargetSize(val width: Int, val height: Int)
  * spans the answer → first frame window too. It cannot strand a caller, because
  * every `CONNECTING` is bounded: the offer post fails, or the answer's
  * first-frame deadline expires into `FAILED` and this returns false.
+ *
+ * SoftAP never reuses: the listener, port and peer belong to one publish attempt,
+ * and the ingest URL is an output of binding, not an input that could stay valid.
+ *
+ * Owned by an object rather than a file-level function so Kotlin incremental
+ * compile cannot see two [GlassesMediaSourceKt] facades and report a phantom
+ * overload of the same signature.
  */
-fun canReuseSource(currentUrl: String?, state: SourceState, next: SourceConfig): Boolean =
-  currentUrl == next.url && next.kind == SourceKind.WHEP && state != SourceState.FAILED
+object SourceReusePolicy {
+  fun canReuse(currentUrl: String?, state: SourceState, next: SourceConfig): Boolean =
+    currentUrl == next.url && next.kind == SourceKind.WHEP && state != SourceState.FAILED
+}
 
 /**
  * Observes source health transitions. `FAILED` is the one that matters: ICE
@@ -47,13 +73,25 @@ interface GlassesMediaSource {
   fun forceRestart() {}
   fun stop()
   val state: SourceState
+
+  /**
+   * SOFTAP only: the WHIP URL the glasses must POST their offer to, known only after [start] has
+   * bound a listener. Null for every other kind, where the URL is an input rather than an output.
+   */
+  val ingestUrl: String? get() = null
+
   fun setPcmDeliveryEnabled(enabled: Boolean)
   fun setTargetSize(size: TargetSize?) {}
   fun setStateListener(listener: SourceStateListener?) {}
 }
 
+/**
+ * Builds the transport for one attempt. Takes [SourceConfig] because the kind decides the class:
+ * a Cloudflare subscription and a SoftAP listener are different objects, not one object with a
+ * mode flag, and the session must not have to know which.
+ */
 fun interface GlassesMediaSourceFactory {
-  fun create(video: VideoFrameListener, pcm: PcmListener): GlassesMediaSource
+  fun create(video: VideoFrameListener, pcm: PcmListener, config: SourceConfig): GlassesMediaSource
 }
 
 class GlassesMediaController(
@@ -65,9 +103,13 @@ class GlassesMediaController(
   val state: SourceState
     get() = source?.state ?: SourceState.IDLE
 
+  /** SOFTAP only: the URL the glasses must publish to, once a listener has bound. */
+  val ingestUrl: String?
+    get() = source?.ingestUrl
+
   fun attach(video: VideoFrameListener, pcm: PcmListener, config: SourceConfig) {
     source?.stop()
-    source = factory.create(video, pcm).also {
+    source = factory.create(video, pcm, config).also {
       it.setStateListener(stateListener)
       it.start(config)
     }