@mentra/acs-meeting 3.2.0-dev.222 → 3.2.0-dev.225

package/android/src/main/java/com/mentra/acsmeeting/source/SyntheticFrameFactory.kt

@@ -1,205 +0,0 @@
-package com.mentra.acsmeeting.source
-
-import com.mentra.acsmeeting.video.I420Packer
-import java.nio.ByteBuffer
-import kotlin.math.max
-
-/**
- * Packed I420 generator. No Android / ACS / WebRTC types so JVM tests stay cheap.
- *
- * CHEAP paints a flat field, a moving bar, and a Recall-style stamp (6-digit
- * frame id + 1s FPS). Low motion: enough change that ACS cannot skip, still
- * trivially compressible.
- *
- * MOTION pans a tiled scene across the frame. That is the glasses-turn proxy:
- * temporal prediction dies (almost every pixel changes) but spatial structure
- * stays, so an I-frame still compresses. Use this to ask whether ACS holds
- * 15 fps under high motion.
- *
- * NOISE rewrites the whole Y plane from an LCG. That is not camera motion.
- * Neighbors are uncorrelated, so even I-frames stay huge. Measured on ACS at
- * 720p it pinned ~1.9 Mbps and collapsed the wire rate to 1.4 fps while we
- * submitted 15. Use it to stress the encoder, never to model glasses movement.
- *
- * Chroma stays at 128 in every mode.
- */
-class SyntheticFrameFactory(
-  private val entropy: SyntheticEntropy = SyntheticEntropy.CHEAP,
-) {
-  private var lastDest: ByteBuffer? = null
-  private var lastWidth = 0
-  private var lastHeight = 0
-  private var background: ByteArray? = null
-  private var scene: ByteArray? = null
-  private var sceneWidth = 0
-  private var sceneHeight = 0
-
-  fun write(dest: ByteBuffer, width: Int, height: Int, frameIndex: Int, emitFps: Double? = null) {
-    require(width > 0 && height > 0) { "size must be positive" }
-    val needed = I420Packer.packedSize(width, height)
-    require(dest.capacity() >= needed) { "dest capacity ${dest.capacity()} < $needed" }
-    when (entropy) {
-      SyntheticEntropy.NOISE -> writeNoise(dest, width, height, frameIndex)
-      SyntheticEntropy.MOTION -> writeMotion(dest, width, height, frameIndex)
-      SyntheticEntropy.CHEAP -> writeCheap(dest, width, height, frameIndex)
-    }
-    writeStamp(dest, width, height, frameIndex, emitFps)
-    dest.position(0)
-    dest.limit(needed)
-  }
-
-  private fun writeCheap(dest: ByteBuffer, width: Int, height: Int, frameIndex: Int) {
-    // Bulk-restore the field instead of a per-pixel loop: this runs on the emit
-    // thread every frame, and 921600 single-byte puts is exactly the Kotlin
-    // per-pixel cost the rest of this module exists to avoid.
-    val ySize = width * height
-    val template = background.takeIf { it != null && it.size == ySize }
-      ?: ByteArray(ySize) { BG_Y }.also { background = it }
-    dest.clear()
-    dest.put(template, 0, ySize)
-    if (dest !== lastDest || width != lastWidth || height != lastHeight) {
-      fillChroma(dest, width, height)
-      lastDest = dest
-      lastWidth = width
-      lastHeight = height
-    }
-    val barWidth = max(8, width / 80)
-    val travel = max(1, width - barWidth)
-    val barX = ((frameIndex.toLong() * barWidth) % travel).toInt()
-    fillYRect(dest, width, height, barX, 0, barWidth, height, BAR_Y)
-  }
-
-  /**
-   * Copy a pre-painted tiled scene with a wrap-safe offset. One bulk put per
-   * row, not one put per pixel — the scene is the glasses-turn stand-in, and
-   * generating it in Kotlin every frame would be the cost we are trying to
-   * isolate out of this measurement.
-   */
-  private fun writeMotion(dest: ByteBuffer, width: Int, height: Int, frameIndex: Int) {
-    val field = ensureScene(width, height)
-    val ox = ((frameIndex.toLong() * PAN_X) % TILE).toInt()
-    val oy = ((frameIndex.toLong() * PAN_Y) % TILE).toInt()
-    for (row in 0 until height) {
-      dest.position(row * width)
-      dest.put(field, (row + oy) * sceneWidth + ox, width)
-    }
-    if (dest !== lastDest || width != lastWidth || height != lastHeight) {
-      fillChroma(dest, width, height)
-      lastDest = dest
-      lastWidth = width
-      lastHeight = height
-    }
-  }
-
-  private fun ensureScene(width: Int, height: Int): ByteArray {
-    val sw = width + TILE
-    val sh = height + TILE
-    val existing = scene
-    if (existing != null && sceneWidth == sw && sceneHeight == sh) return existing
-    val field = ByteArray(sw * sh)
-    for (y in 0 until sh) {
-      val rowOff = y * sw
-      val tileY = y / TILE
-      val gy = y % TILE
-      for (x in 0 until sw) {
-        val tile = (x / TILE + tileY) and 1
-        val gx = x % TILE
-        val base = if (tile == 0) DARK_TILE else LIGHT_TILE
-        field[rowOff + x] = (base + gx - TILE / 2 + gy / 4).toByte()
-      }
-    }
-    scene = field
-    sceneWidth = sw
-    sceneHeight = sh
-    return field
-  }
-
-  private fun writeNoise(dest: ByteBuffer, width: Int, height: Int, frameIndex: Int) {
-    writeNoiseY(dest, width, height, frameIndex)
-    fillChroma(dest, width, height)
-    lastDest = dest
-    lastWidth = width
-    lastHeight = height
-  }
-
-  private fun writeNoiseY(dest: ByteBuffer, width: Int, height: Int, frameIndex: Int) {
-    val ySize = width * height
-    var seed = (frameIndex + 1) * LCG_SEED
-    for (i in 0 until ySize) {
-      seed = seed * LCG_MUL + LCG_ADD
-      dest.put(i, ((seed ushr 24) and 0xFF).toByte())
-    }
-  }
-
-  /**
-   * Recall burns `F 000123` plus live rates into the captured page. Same idea here:
-   * a 6-digit unique-frame id and the last 1s emit rate, readable on the Teams tile.
-   */
-  private fun writeStamp(
-    dest: ByteBuffer,
-    width: Int,
-    height: Int,
-    frameIndex: Int,
-    emitFps: Double?,
-  ) {
-    val scale = (height / 60).coerceIn(8, 16)
-    val line1 = "F ${frameIndex.coerceAtLeast(0).toString().padStart(6, '0')}"
-    val fpsLabel = emitFps?.let { ((it * 10).toInt() / 10.0).toString() } ?: "--"
-    val line2 = "FPS $fpsLabel"
-    val pad = scale * 2
-    val lineH = BitmapFont.textHeight(scale)
-    val gap = scale
-    val boxW = max(BitmapFont.textWidth(line1.length, scale), BitmapFont.textWidth(line2.length, scale)) + pad * 2
-    val boxH = lineH * 2 + gap + pad * 2
-    val boxX = (width / 16).coerceAtLeast(16)
-    val boxY = (height / 2 - boxH / 2).coerceAtLeast(0)
-    fillYRect(dest, width, height, boxX, boxY, boxW, boxH, OFF_Y)
-    BitmapFont.draw(dest, width, height, line1, boxX + pad, boxY + pad, scale, BAR_Y)
-    BitmapFont.draw(dest, width, height, line2, boxX + pad, boxY + pad + lineH + gap, scale, BAR_Y)
-  }
-
-  private fun fillChroma(dest: ByteBuffer, width: Int, height: Int) {
-    val ySize = width * height
-    val uvSize = I420Packer.chromaStride(width) * I420Packer.chromaStride(height)
-    val end = ySize + 2 * uvSize
-    for (i in ySize until end) dest.put(i, CHROMA)
-  }
-
-  private fun fillYRect(
-    dest: ByteBuffer,
-    width: Int,
-    height: Int,
-    x0: Int,
-    y0: Int,
-    rectW: Int,
-    rectH: Int,
-    value: Byte,
-  ) {
-    val xStart = x0.coerceIn(0, width)
-    val yStart = y0.coerceIn(0, height)
-    val xEnd = (x0 + rectW).coerceIn(0, width)
-    val yEnd = (y0 + rectH).coerceIn(0, height)
-    if (xStart >= xEnd || yStart >= yEnd) return
-    for (row in yStart until yEnd) {
-      val rowOff = row * width
-      for (col in xStart until xEnd) {
-        dest.put(rowOff + col, value)
-      }
-    }
-  }
-
-  companion object {
-    const val BG_Y: Byte = 40
-    const val OFF_Y: Byte = 16
-    const val BAR_Y: Byte = 235.toByte()
-    const val CHROMA: Byte = 128.toByte()
-    const val TILE = 32
-    const val PAN_X = 8
-    const val PAN_Y = 4
-    private const val DARK_TILE = 70
-    private const val LIGHT_TILE = 170
-    private const val LCG_SEED = -1640531527
-    private const val LCG_MUL = 1664525
-    private const val LCG_ADD = 1013904223
-  }
-}

package/android/src/main/java/com/mentra/acsmeeting/source/SyntheticI420Source.kt

@@ -1,214 +0,0 @@
-package com.mentra.acsmeeting.source
-
-import android.util.Log
-import com.mentra.acsmeeting.telemetry.PipelineStats
-import com.mentra.acsmeeting.video.I420Packer
-import java.nio.ByteBuffer
-import java.util.concurrent.Executors
-import java.util.concurrent.ScheduledExecutorService
-import java.util.concurrent.TimeUnit
-import java.util.concurrent.atomic.AtomicBoolean
-import java.util.concurrent.atomic.AtomicInteger
-import java.util.concurrent.atomic.AtomicLong
-import java.util.concurrent.atomic.AtomicReference
-
-/**
- * Fixed-rate packed-I420 source. Bypasses glasses, Cloudflare, WHEP and decode.
- * Ticks that fail [isReady] or lack a target size are withheld and uncounted.
- */
-class SyntheticI420Source(
-  private val video: VideoFrameListener,
-  private val stats: PipelineStats,
-  private val isReady: () -> Boolean,
-  private val config: SyntheticConfig = SyntheticConfig(),
-) : GlassesMediaSource {
-  private val factory = SyntheticFrameFactory(config.entropy)
-  private val targetSize = AtomicReference<TargetSize?>(null)
-  private val pcmEnabled = AtomicBoolean(false)
-  private val frameIndex = AtomicInteger(0)
-  private val withheld = AtomicInteger(0)
-  private val withheldThisGap = AtomicInteger(0)
-  private val emitted = AtomicInteger(0)
-  private val emitting = AtomicBoolean(false)
-  private val firstLogged = AtomicBoolean(false)
-  private val fpsWindowStartNs = AtomicLong(0)
-  private val fpsWindowCount = AtomicInteger(0)
-  @Volatile private var stampFps: Double? = null
-  @Volatile override var state: SourceState = SourceState.IDLE
-    private set
-  @Volatile private var dest: ByteBuffer? = null
-  @Volatile private var destWidth = 0
-  @Volatile private var destHeight = 0
-  @Volatile private var destY: ByteBuffer? = null
-  @Volatile private var destU: ByteBuffer? = null
-  @Volatile private var destV: ByteBuffer? = null
-  private var scheduler: ScheduledExecutorService? = null
-
-  /** Test-visible gate transitions. Production also writes them to ACS-SPIKE. */
-  internal val events = mutableListOf<String>()
-
-  /** Puts the source in LIVE without starting the scheduler. For emitOnce tests. */
-  internal fun becomeLive() {
-    state = SourceState.LIVE
-  }
-
-  override fun start(config: SourceConfig) {
-    stop()
-    becomeLive()
-    val fps = this.config.fps.coerceAtLeast(1)
-    val periodUs = 1_000_000L / fps
-    val exec = Executors.newSingleThreadScheduledExecutor { runnable ->
-      Thread(runnable, "acs-synthetic-i420").apply { isDaemon = true }
-    }
-    scheduler = exec
-    exec.scheduleAtFixedRate({ emitOnce() }, 0, periodUs, TimeUnit.MICROSECONDS)
-  }
-
-  override fun restart(config: SourceConfig) {
-    stop()
-    start(config)
-  }
-
-  override fun stop() {
-    scheduler?.shutdownNow()
-    scheduler = null
-    frameIndex.set(0)
-    withheld.set(0)
-    withheldThisGap.set(0)
-    emitted.set(0)
-    emitting.set(false)
-    firstLogged.set(false)
-    fpsWindowStartNs.set(0)
-    fpsWindowCount.set(0)
-    stampFps = null
-    dest = null
-    destWidth = 0
-    destHeight = 0
-    destY = null
-    destU = null
-    destV = null
-    state = SourceState.IDLE
-  }
-
-  override fun setPcmDeliveryEnabled(enabled: Boolean) {
-    pcmEnabled.set(enabled)
-  }
-
-  override fun setTargetSize(size: TargetSize?) {
-    targetSize.set(size)
-  }
-
-  internal fun pcmDeliveryEnabled(): Boolean = pcmEnabled.get()
-
-  /**
-   * Generate-and-emit one frame if the sender is ready and a target size exists.
-   * Returns true only when a frame was handed to [video].
-   */
-  internal fun emitOnce(): Boolean {
-    if (state != SourceState.LIVE) return false
-    val size = targetSize.get()
-    val ready = isReady() && size != null
-    if (!ready) {
-      withheld.incrementAndGet()
-      withheldThisGap.incrementAndGet()
-      if (emitting.compareAndSet(true, false)) {
-        logEvent("P6 synthetic paused reason=not-ready afterFrames=${emitted.get()}")
-      }
-      return false
-    }
-    val gap = withheldThisGap.getAndSet(0)
-    if (!emitting.get() && emitted.get() > 0) {
-      logEvent("P6 synthetic resumed withheld=$gap")
-    }
-    emitting.set(true)
-
-    val w = size!!.width
-    val h = size.height
-    val buffer = ensureDest(w, h)
-    val index = frameIndex.getAndIncrement()
-    stampFps = sampleStampFps()
-    factory.write(buffer, w, h, index, stampFps)
-    stats.onSink()
-    stats.recordGap()
-    stats.setSize(w, h)
-    val chroma = I420Packer.chromaStride(w)
-    video.onVideoFrame(
-      I420Planes(
-        y = destY!!,
-        strideY = w,
-        u = destU!!,
-        strideU = chroma,
-        v = destV!!,
-        strideV = chroma,
-        width = w,
-        height = h,
-        timestampNs = System.nanoTime(),
-      ),
-    )
-    val count = emitted.incrementAndGet()
-    if (firstLogged.compareAndSet(false, true)) {
-      logEvent(
-        "P6 synthetic first-frame withheld=${withheld.get()} size=${w}x${h} " +
-          "fps=${config.fps} entropy=${config.entropy}",
-      )
-    }
-    return count > 0
-  }
-
-  /** 1-second emit rate, same window Recall uses for the P column on the stamp. */
-  private fun sampleStampFps(): Double? {
-    val now = System.nanoTime()
-    val started = fpsWindowStartNs.get()
-    if (started == 0L) {
-      fpsWindowStartNs.set(now)
-      fpsWindowCount.set(1)
-      return stampFps
-    }
-    fpsWindowCount.incrementAndGet()
-    val elapsedNs = now - started
-    if (elapsedNs >= 1_000_000_000L) {
-      val rate = fpsWindowCount.get() * 1_000_000_000.0 / elapsedNs
-      fpsWindowCount.set(0)
-      fpsWindowStartNs.set(now)
-      return rate
-    }
-    return stampFps
-  }
-
-  private fun ensureDest(width: Int, height: Int): ByteBuffer {
-    val needed = I420Packer.packedSize(width, height)
-    val existing = dest
-    if (existing != null && destWidth == width && destHeight == height && existing.capacity() >= needed) {
-      existing.clear()
-      return existing
-    }
-    val next = ByteBuffer.allocateDirect(needed)
-    dest = next
-    destWidth = width
-    destHeight = height
-    slicePlanes(next, width, height)
-    return next
-  }
-
-  private fun slicePlanes(packed: ByteBuffer, width: Int, height: Int) {
-    val ySize = width * height
-    val uvSize = I420Packer.chromaStride(width) * I420Packer.chromaStride(height)
-    val view = packed.duplicate()
-    view.clear()
-    destY = view.duplicate().apply { position(0); limit(ySize) }.slice()
-    destU = view.duplicate().apply { position(ySize); limit(ySize + uvSize) }.slice()
-    destV = view.duplicate().apply { position(ySize + uvSize); limit(ySize + 2 * uvSize) }.slice()
-  }
-
-  private fun logEvent(line: String) {
-    synchronized(events) { events.add(line) }
-    try {
-      Log.i(TAG, line)
-    } catch (_: Throwable) {
-    }
-  }
-
-  companion object {
-    private const val TAG = "ACS-SPIKE"
-  }
-}

package/android/src/main/java/com/mentra/acsmeeting/source/TrackRegistry.kt

@@ -1,33 +0,0 @@
-package com.mentra.acsmeeting.source
-
-/**
- * Dedupe WebRTC track attachments by native track id.
- * Three observer callbacks can deliver distinct Java wrappers of one track;
- * object identity cannot be used. [reset] is load-bearing: [CloudflareWhepSource.start]
- * calls stop first, and a stale claim set makes reconnect never attach a sink. SoftAP
- * WHIP renegotiate reuses the same track id (`video0`); without a reset the new peer
- * decodes frames that never reach the ACS sink.
- */
-class TrackRegistry {
-  private val ids = mutableSetOf<String>()
-  private var skipped = 0
-
-  @Synchronized
-  fun claim(id: String): Boolean {
-    if (ids.add(id)) return true
-    skipped += 1
-    return false
-  }
-
-  @Synchronized
-  fun reset() {
-    ids.clear()
-    skipped = 0
-  }
-
-  @Synchronized
-  fun size(): Int = ids.size
-
-  @Synchronized
-  fun skipped(): Int = skipped
-}

package/android/src/main/java/com/mentra/acsmeeting/source/VideoSourceArm.kt

@@ -1,110 +0,0 @@
-package com.mentra.acsmeeting.source
-
-enum class VideoSourceArm { WHEP, SYNTHETIC }
-
-enum class SyntheticEntropy { CHEAP, MOTION, NOISE }
-
-enum class DecoderMode { TEXTURE, BYTE_BUFFER }
-
-enum class PixelFormatArm { I420, NV12 }
-
-enum class OutgoingRateArm { ADVERTISE_REQUESTED, CLAMP_TO_SOFTWARE_CEILING }
-
-/**
- * Investigation arm. SYNTHETIC bypasses glasses, Cloudflare, WHEP and decode.
- * Ships as WHEP. Flip locally to run the experiment; do not commit SYNTHETIC.
- *
- * [decoderMode] TEXTURE is the production default (shared EGL, MediaCodec to
- * Surface). BYTE_BUFFER is the A/B: no shared context, CPU planes, no
- * glReadPixels. 720p BYTE_BUFFER on SM_S948U decoded on hardware with
- * i420P95=0, but wire/codec never attached and busy drops climbed. Do not
- * commit BYTE_BUFFER until a same-phone 540p A/B clears the campaign gates.
- *
- * [zeroCopy] hands WebRTC I420 planes straight to ACS when they are tight and
- * retainable. Ships off. Do not commit true.
- *
- * [pixelFormat] I420 is the production default. NV12 is the encoder-flip A/B:
- * advertise and send biplanar NV12 in case ACS picks a hardware H.264
- * encoder. Revert unless `codecName` leaves `h264 sw`. Do not commit NV12
- * on a failed flip.
- */
-object AcsInvestigation {
-  val videoArm = VideoSourceArm.WHEP
-  val syntheticFps = 15
-  val syntheticEntropy = SyntheticEntropy.MOTION
-  val decoderMode = DecoderMode.TEXTURE
-  val zeroCopy = false
-  val pixelFormat = PixelFormatArm.I420
-
-  /**
-   * The rate we declare to ACS.
-   *
-   * CLAMP_TO_SOFTWARE_CEILING pins it to [VideoProfile.SOFTWARE_ENCODER_FPS]
-   * whenever the profile asks for more. It was the default on the strength of
-   * soaks that reported `codecName: "h264 sw"` alongside wire ≈ 8.8 fps, read as
-   * an encoder ceiling. Two things in the device logs say it was never the
-   * encoder.
-   *
-   * 720p15 and 540p15 both reported ~8.8 fps. 540p is 52% of the pixels of 720p,
-   * so a CPU-bound encoder would have been close to twice as fast at the smaller
-   * size; a limit that does not move with pixel count is not a pixel cost. And
-   * once [FramePacer] delivered a steady cadence, `wire` tracked what we handed
-   * ACS — 6.9 against 7.1 admitted — which is an encoder keeping up, not one
-   * saturating. Both of those soaks ran through a send path that could not hold a
-   * rate, so what they measured was our delivery.
-   *
-   * ADVERTISE_REQUESTED runs the configuration that was never actually tried:
-   * declare the profile's own fps and let the corrected pacer deliver it. Read
-   * the `P7 rate` verdict for what binds; it distinguishes an encoder that cannot
-   * keep up from a rate controller starving one that can, and those two want
-   * opposite fixes.
-   */
-  val outgoingRate = OutgoingRateArm.ADVERTISE_REQUESTED
-
-  /**
-   * libwebrtc's own `LS_INFO` stream during a SoftAP call.
-   *
-   * The one place that says which interfaces `BasicNetworkManager` kept, what adapter type it gave
-   * each one, and every `BindSocketToNetwork` result — the evidence that separated a hotspot the
-   * kernel had from one libwebrtc had erased. It is also loud, and it is the native stack's whole
-   * log, not ours.
-   *
-   * Flip to false once the SoftAP path is green on device. Keep the `NetworkFacts` stages either
-   * way: they are ours, bounded, and the thing that makes the next regression readable.
-   */
-  const val LIBWEBRTC_VERBOSE = true
-
-  /**
-   * How long to hold outgoing microphone PCM before the resampler, to align the wearer's voice
-   * with their video at the far end.
-   *
-   * Ships at 0, and 0 is not a placeholder: the audio and the video reach ACS over different
-   * transports (BLE LC3 versus SoftAP WebRTC) with different latencies, but *how* different is a
-   * receiver-side measurement, not something either side can infer. Deriving it from first-frame
-   * arrival times measures startup skew and would be wrong by however long the camera took to boot.
-   *
-   * Calibrate the ingest path with a clap in frame and read `AVSYNC clap audioLeadMs`. That is
-   * the offset the delay line can cancel. ACS/Teams jitter after ingest still needs a Teams
-   * recording. It is logged as `configuredDelayMs` and deliberately never reported as a measured
-   * offset.
-   */
-  const val acsAudioDelayMs = 0
-
-  /**
-   * Whether outgoing audio buffers carry a presentation timestamp.
-   *
-   * Whether ACS honours these for lip-sync on *raw* audio is unanswered — the SDK exposes the
-   * setter and documents nothing about the receiver. The only way to find out is to record the
-   * Teams receiver with this on and with it off and compare, so it has to be switchable; a constant
-   * true would leave two of the four measurement configurations unreachable.
-   *
-   * Defaults to true because the fallback is worse for an unrelated reason: a run of zero
-   * timestamps is what makes the Teams jitter buffer hold.
-   */
-  const val acsAudioTimestamps = true
-}
-
-data class SyntheticConfig(
-  val fps: Int = AcsInvestigation.syntheticFps,
-  val entropy: SyntheticEntropy = AcsInvestigation.syntheticEntropy,
-)