@mentra/acs-meeting 3.2.0-dev.120

package/android/src/main/java/com/mentra/acsmeeting/audio/PcmResampler.kt

@@ -0,0 +1,159 @@
+package com.mentra.acsmeeting.audio
+
+import java.nio.ByteBuffer
+import java.nio.ByteOrder
+import kotlin.math.PI
+import kotlin.math.cos
+import kotlin.math.roundToInt
+import kotlin.math.sin
+
+/**
+ * Stateful PCM16 downmix + sample-rate converter.
+ *
+ * WHEP Opus decodes at 48 kHz; ACS raw outgoing is also 48 kHz now, so the
+ * common path is downmix-only. If a side still crosses a rate boundary,
+ * picking samples without a low-pass filter aliases into the speech band.
+ * This filters first (windowed sinc at the input rate, cutoff just under
+ * the output Nyquist) and keeps filter history plus the fractional phase
+ * across calls, so chunked input produces the same samples as one
+ * contiguous buffer.
+ */
+class PcmResampler(
+  private val targetRate: Int,
+) {
+  private var inRate = 0
+  private var inChannels = 0
+  private var taps = FloatArray(0)
+  /** Raw (unfiltered) input tail so the FIR sees taps.size - 1 samples of context. */
+  private var rawTail = FloatArray(0)
+  /** Filtered samples not yet consumed by the interpolator. */
+  private var history = FloatArray(0)
+  /** Position of the next output sample, in input-sample units, relative to the start of [history]. */
+  private var phase = 0.0
+
+  fun reset() {
+    inRate = 0
+    inChannels = 0
+    taps = FloatArray(0)
+    rawTail = FloatArray(0)
+    history = FloatArray(0)
+    phase = 0.0
+  }
+
+  fun process(pcm16Le: ByteArray, sampleRate: Int, channels: Int): ByteArray {
+    require(sampleRate > 0) { "sampleRate must be positive" }
+    require(channels > 0) { "channels must be positive" }
+    if (sampleRate != inRate || channels != inChannels) configure(sampleRate, channels)
+
+    val mono = downmix(pcm16Le, channels)
+    if (sampleRate == targetRate) return toBytes(mono)
+
+    // Filtered input signal = history tail + newly filtered samples. The
+    // history keeps taps.size - 1 raw samples so the FIR has full context,
+    // plus one filtered sample so linear interpolation can straddle the seam.
+    val filtered = if (taps.isEmpty()) mono else lowPass(mono)
+    val signal = FloatArray(history.size + filtered.size)
+    System.arraycopy(history, 0, signal, 0, history.size)
+    System.arraycopy(filtered, 0, signal, history.size, filtered.size)
+
+    val step = sampleRate.toDouble() / targetRate
+    val capacity = maxOf(0, ((signal.size - phase) / step).toInt() + 2)
+    val out = ShortArray(capacity)
+    var count = 0
+    var pos = phase
+    while (pos + 1 < signal.size) {
+      val i = pos.toInt()
+      val frac = (pos - i).toFloat()
+      val sample = signal[i] + (signal[i + 1] - signal[i]) * frac
+      out[count++] = clamp(sample)
+      pos += step
+    }
+    // Keep everything from the next output position onward so the seam interpolates.
+    val keepFrom = maxOf(0, minOf(pos.toInt(), signal.size - 1))
+    history = signal.copyOfRange(keepFrom, signal.size)
+    phase = pos - keepFrom
+
+    val result = ByteArray(count * 2)
+    val bb = ByteBuffer.wrap(result).order(ByteOrder.LITTLE_ENDIAN)
+    for (i in 0 until count) bb.putShort(out[i])
+    return result
+  }
+
+  private fun configure(sampleRate: Int, channels: Int) {
+    inRate = sampleRate
+    inChannels = channels
+    history = FloatArray(0)
+    rawTail = FloatArray(0)
+    phase = 0.0
+    taps = if (sampleRate > targetRate) designLowPass(sampleRate, targetRate) else FloatArray(0)
+  }
+
+  private fun lowPass(mono: FloatArray): FloatArray {
+    val n = taps.size
+    val input = FloatArray(rawTail.size + mono.size)
+    System.arraycopy(rawTail, 0, input, 0, rawTail.size)
+    System.arraycopy(mono, 0, input, rawTail.size, mono.size)
+    val outCount = input.size - (n - 1)
+    if (outCount <= 0) {
+      rawTail = input
+      return FloatArray(0)
+    }
+    val out = FloatArray(outCount)
+    for (i in 0 until outCount) {
+      var acc = 0f
+      for (k in 0 until n) acc += input[i + k] * taps[k]
+      out[i] = acc
+    }
+    rawTail = input.copyOfRange(input.size - (n - 1), input.size)
+    return out
+  }
+
+  companion object {
+    /** Odd tap count keeps the group delay an integer number of samples. */
+    private const val TAPS = 63
+
+    fun designLowPass(sampleRate: Int, targetRate: Int): FloatArray {
+      // Cutoff at 90% of the output Nyquist: full speech band through, the
+      // transition band finished before anything can alias.
+      val cutoff = 0.45 * targetRate / sampleRate
+      val m = (TAPS - 1) / 2
+      val taps = FloatArray(TAPS)
+      var sum = 0.0
+      for (i in 0 until TAPS) {
+        val x = (i - m).toDouble()
+        val sinc = if (x == 0.0) 2 * cutoff else sin(2 * PI * cutoff * x) / (PI * x)
+        val window = 0.54 - 0.46 * cos(2 * PI * i / (TAPS - 1))
+        val h = sinc * window
+        taps[i] = h.toFloat()
+        sum += h
+      }
+      for (i in 0 until TAPS) taps[i] = (taps[i] / sum).toFloat()
+      return taps
+    }
+
+    fun downmix(pcm16Le: ByteArray, channels: Int): FloatArray {
+      val frames = pcm16Le.size / (2 * channels)
+      val shorts = ByteBuffer.wrap(pcm16Le, 0, frames * 2 * channels).order(ByteOrder.LITTLE_ENDIAN).asShortBuffer()
+      val mono = FloatArray(frames)
+      if (channels == 1) {
+        for (i in 0 until frames) mono[i] = shorts.get(i).toFloat()
+      } else {
+        for (i in 0 until frames) {
+          var acc = 0f
+          for (c in 0 until channels) acc += shorts.get(i * channels + c)
+          mono[i] = acc / channels
+        }
+      }
+      return mono
+    }
+
+    private fun clamp(v: Float): Short = v.roundToInt().coerceIn(Short.MIN_VALUE.toInt(), Short.MAX_VALUE.toInt()).toShort()
+
+    private fun toBytes(mono: FloatArray): ByteArray {
+      val out = ByteArray(mono.size * 2)
+      val bb = ByteBuffer.wrap(out).order(ByteOrder.LITTLE_ENDIAN)
+      for (v in mono) bb.putShort(clamp(v))
+      return out
+    }
+  }
+}

package/android/src/main/java/com/mentra/acsmeeting/audio/PhoneMicCapturer.kt

@@ -0,0 +1,115 @@
+package com.mentra.acsmeeting.audio
+
+import android.annotation.SuppressLint
+import android.media.AudioFormat
+import android.media.AudioRecord
+import android.media.MediaRecorder
+import android.os.Process
+import android.util.Log
+
+/**
+ * Phone-mic uplink for ACS virtual outgoing. Uses [AudioRecord] MIC at 48 kHz
+ * mono PCM16 and never touches [android.media.AudioManager] mode — ACS
+ * communication mode stays off so A2DP playback remains on the glasses.
+ */
+class PhoneMicCapturer(
+  private val onPcm: (ByteArray, Int, Int) -> Unit,
+) {
+  @Volatile private var record: AudioRecord? = null
+  private var thread: Thread? = null
+
+  @Synchronized
+  fun setEnabled(enabled: Boolean) {
+    if (enabled) start() else stop()
+  }
+
+  @SuppressLint("MissingPermission")
+  private fun start() {
+    if (record != null) return
+    val min = AudioRecord.getMinBufferSize(SAMPLE_RATE, CHANNEL, ENCODING)
+    if (min == AudioRecord.ERROR || min == AudioRecord.ERROR_BAD_VALUE) {
+      Log.w(TAG, "PhoneMicCapturer: bad min buffer $min")
+      return
+    }
+    val bufferSize = maxOf(min, FRAME_BYTES * 4)
+    val rec = try {
+      AudioRecord.Builder()
+        .setAudioSource(MediaRecorder.AudioSource.MIC)
+        .setAudioFormat(
+          AudioFormat.Builder()
+            .setSampleRate(SAMPLE_RATE)
+            .setChannelMask(CHANNEL)
+            .setEncoding(ENCODING)
+            .build(),
+        )
+        .setBufferSizeInBytes(bufferSize)
+        .build()
+    } catch (error: Exception) {
+      Log.w(TAG, "PhoneMicCapturer: create failed", error)
+      return
+    }
+    if (rec.state != AudioRecord.STATE_INITIALIZED) {
+      rec.release()
+      Log.w(TAG, "PhoneMicCapturer: AudioRecord failed to initialize")
+      return
+    }
+    try {
+      rec.startRecording()
+    } catch (error: Exception) {
+      rec.release()
+      Log.w(TAG, "PhoneMicCapturer: startRecording failed", error)
+      return
+    }
+    if (rec.recordingState != AudioRecord.RECORDSTATE_RECORDING) {
+      rec.release()
+      Log.w(TAG, "PhoneMicCapturer: did not enter RECORDING")
+      return
+    }
+    record = rec
+    thread = Thread({
+      Process.setThreadPriority(Process.THREAD_PRIORITY_AUDIO)
+      val buf = ByteArray(FRAME_BYTES)
+      while (true) {
+        val current = record ?: break
+        val n = try {
+          current.read(buf, 0, buf.size)
+        } catch (_: Exception) {
+          break
+        }
+        if (n <= 0) {
+          if (n < 0) break
+          continue
+        }
+        val copy = if (n == buf.size) buf.copyOf() else buf.copyOf(n)
+        onPcm(copy, SAMPLE_RATE, CHANNELS)
+      }
+    }, "acs-phone-mic").also { it.start() }
+    Log.i(TAG, "PhoneMicCapturer started 48kHz mono PCM16")
+  }
+
+  @Synchronized
+  fun stop() {
+    val rec = record ?: return
+    record = null
+    thread = null
+    try {
+      rec.stop()
+    } catch (_: Exception) {
+    }
+    try {
+      rec.release()
+    } catch (_: Exception) {
+    }
+    Log.i(TAG, "PhoneMicCapturer stopped")
+  }
+
+  companion object {
+    private const val TAG = "ACS-SPIKE"
+    const val SAMPLE_RATE = 48_000
+    const val CHANNELS = 1
+    private const val FRAME_MS = 20
+    private const val FRAME_BYTES = SAMPLE_RATE * CHANNELS * 2 * FRAME_MS / 1000
+    private const val CHANNEL = AudioFormat.CHANNEL_IN_MONO
+    private const val ENCODING = AudioFormat.ENCODING_PCM_16BIT
+  }
+}

package/android/src/main/java/com/mentra/acsmeeting/audio/UplinkPacer.kt

@@ -0,0 +1,302 @@
+package com.mentra.acsmeeting.audio
+
+import kotlin.math.min
+
+/**
+ * Clock-domain adapter in front of ACS's raw outgoing audio stream.
+ *
+ * The producer (glasses → WHEP → [PcmBridge]) runs on the glasses' audio
+ * clock; the consumer runs on the phone's monotonic clock. Those clocks never
+ * agree exactly, so a plain FIFO drained by a 20 ms timer walks to empty or to
+ * the cap over a long call even when there is no jitter at all. This class
+ * locks the buffer depth to [TARGET_MS] instead of merely bounding it:
+ *
+ * - sustained depth above [HIGH_MS] (producer fast) discards a [CORRECTION_MS] slice
+ * - sustained depth below [LOW_MS] (producer slow) inserts a [CORRECTION_MS] slice of silence
+ * - depth above [EMERGENCY_CAP_MS] drops oldest down to target immediately
+ *
+ * Corrections read an EMA over roughly two seconds and are rate limited to one
+ * per [CORRECTION_INTERVAL_MS], so ordinary jitter never triggers one; only
+ * sustained error does. A cleaner follow-up is to nudge the WHEP → ACS
+ * resampling ratio by the depth error instead of dropping and inserting
+ * slices, which removes the discontinuity entirely.
+ *
+ * [tick] returns exactly one frame per call and never a burst. The pacing
+ * deadline and the ACS I/O live in [UplinkSender]; this class is pure logic
+ * driven by an injected clock so tests can run an hour in milliseconds.
+ *
+ * [push] runs on the WebRTC audio thread and [tick] on the pacing thread, so
+ * every entry point is synchronized on this instance.
+ */
+class UplinkPacer(private val targetMs: Int = TARGET_MS) {
+  enum class State { PREROLLING, RUNNING, STARVED }
+
+  /** One [FRAME_MS] frame. [silence] marks a frame the pacer manufactured. */
+  class Frame(val bytes: ByteArray, val silence: Boolean)
+
+  data class Stats(
+    val state: State,
+    val depthMs: Int,
+    val targetMs: Int,
+    val sentFps: Double,
+    val framesEmitted: Long,
+    val silenceFrames: Long,
+    val overflowDroppedMs: Long,
+    val driftCorrections: Long,
+    val driftDroppedMs: Long,
+    val driftInsertedMs: Long,
+  )
+
+  private val ring = ByteArray(CAPACITY_BYTES)
+  private var head = 0
+  private var used = 0
+
+  private var state = State.PREROLLING
+  /** Negative until seeded on the first RUNNING tick. */
+  private var depthEmaBytes = -1.0
+  /**
+   * Depth seen by the last tick's control decision, before it consumed a
+   * frame. This — not the post-pop depth a between-ticks reader would see — is
+   * the headroom a late producer actually gets, so it is what the envelope and
+   * the log are stated in.
+   */
+  private var controlDepthBytes = 0
+  private var starvingSinceNanos: Long? = null
+  private var lastCorrectionNanos: Long? = null
+  private var windowStartNanos: Long? = null
+  private var windowFrames = 0L
+
+  private var framesEmitted = 0L
+  private var silenceFrames = 0L
+  private var overflowDroppedBytes = 0L
+  private var driftCorrections = 0L
+  private var driftDroppedBytes = 0L
+  private var driftInsertedBytes = 0L
+
+  /** Appends 48 kHz mono PCM16 from the producer thread. */
+  @Synchronized
+  fun push(pcm16Le: ByteArray) {
+    if (pcm16Le.isEmpty()) return
+    var offset = 0
+    var len = pcm16Le.size
+    if (len > CAPACITY_BYTES) {
+      // A single burst larger than the whole ring: keep only the newest audio.
+      offset = len - CAPACITY_BYTES
+      len = CAPACITY_BYTES
+      overflowDroppedBytes += offset
+    }
+    val overflow = used + len - CAPACITY_BYTES
+    if (overflow > 0) {
+      dropOldest(overflow)
+      overflowDroppedBytes += overflow
+    }
+    write(pcm16Le, offset, len)
+  }
+
+  /**
+   * Advances one pacing period. Returns exactly one frame — real audio when
+   * the state machine allows it, otherwise silence to keep the ACS cadence
+   * unbroken.
+   */
+  @Synchronized
+  fun tick(nowNanos: Long): Frame {
+    if (windowStartNanos == null) windowStartNanos = nowNanos
+    enforceCap()
+    if (state == State.RUNNING) {
+      depthEmaBytes =
+        if (depthEmaBytes < 0) used.toDouble() else depthEmaBytes + EMA_ALPHA * (used - depthEmaBytes)
+      correctDrift(nowNanos)
+    }
+    controlDepthBytes = used
+    val frame = when (state) {
+      // Hold the line until there is real headroom, so the first wobble of a
+      // call does not underrun immediately.
+      State.PREROLLING -> if (depthMs() >= targetMs) enterRunning() else silence()
+      State.RUNNING -> if (used >= FRAME_BYTES) audio() else starve(nowNanos)
+      // A WHEP outage would otherwise resume with zero headroom and underrun
+      // again on the next wobble; rebuild the target first.
+      State.STARVED -> if (depthMs() >= targetMs) enterRunning() else silence()
+    }
+    framesEmitted += 1
+    windowFrames += 1
+    return frame
+  }
+
+  /**
+   * Stats for the 1 Hz ladder. `sentFps` covers the window since the previous
+   * call, so exactly one caller (the sender's logger) may invoke this.
+   */
+  @Synchronized
+  fun snapshot(nowNanos: Long): Stats {
+    val start = windowStartNanos
+    val elapsedNanos = if (start == null) 0L else nowNanos - start
+    val fps = if (elapsedNanos > 0) windowFrames * 1_000_000_000.0 / elapsedNanos else 0.0
+    windowStartNanos = nowNanos
+    windowFrames = 0
+    return Stats(
+      state = state,
+      depthMs = controlDepthBytes / BYTES_PER_MS,
+      targetMs = targetMs,
+      sentFps = fps,
+      framesEmitted = framesEmitted,
+      silenceFrames = silenceFrames,
+      overflowDroppedMs = overflowDroppedBytes / BYTES_PER_MS,
+      driftCorrections = driftCorrections,
+      driftDroppedMs = driftDroppedBytes / BYTES_PER_MS,
+      driftInsertedMs = driftInsertedBytes / BYTES_PER_MS,
+    )
+  }
+
+  @Synchronized
+  fun reset() {
+    head = 0
+    used = 0
+    state = State.PREROLLING
+    depthEmaBytes = -1.0
+    controlDepthBytes = 0
+    starvingSinceNanos = null
+    lastCorrectionNanos = null
+    windowStartNanos = null
+    windowFrames = 0
+    framesEmitted = 0
+    silenceFrames = 0
+    overflowDroppedBytes = 0
+    driftCorrections = 0
+    driftDroppedBytes = 0
+    driftInsertedBytes = 0
+  }
+
+  @Synchronized
+  fun state(): State = state
+
+  /** Buffered audio right now, i.e. after the last tick consumed its frame. */
+  @Synchronized
+  fun depthMs(): Int = used / BYTES_PER_MS
+
+  /** Headroom the last tick decided on, before it consumed its frame. */
+  @Synchronized
+  fun controlDepthMs(): Int = controlDepthBytes / BYTES_PER_MS
+
+  private fun enterRunning(): Frame {
+    state = State.RUNNING
+    starvingSinceNanos = null
+    // Seed at target, not at the observed depth: preroll just guaranteed the
+    // target, and seeding from one sample would let the very first period of a
+    // stalled producer look like sustained drift.
+    depthEmaBytes = (targetMs * BYTES_PER_MS).toDouble()
+    return audio()
+  }
+
+  private fun audio(): Frame {
+    starvingSinceNanos = null
+    return Frame(readFrame(), silence = false)
+  }
+
+  private fun silence(): Frame {
+    silenceFrames += 1
+    return Frame(ByteArray(FRAME_BYTES), silence = true)
+  }
+
+  private fun starve(nowNanos: Long): Frame {
+    val since = starvingSinceNanos
+    if (since == null) {
+      starvingSinceNanos = nowNanos
+    } else if (nowNanos - since >= STARVE_MS * 1_000_000L) {
+      state = State.STARVED
+      depthEmaBytes = -1.0
+    }
+    return silence()
+  }
+
+  private fun enforceCap() {
+    val cap = EMERGENCY_CAP_MS * BYTES_PER_MS
+    if (used <= cap) return
+    val excess = used - targetMs * BYTES_PER_MS
+    dropOldest(excess)
+    overflowDroppedBytes += excess
+    depthEmaBytes = used.toDouble()
+  }
+
+  private fun correctDrift(nowNanos: Long) {
+    if (depthEmaBytes < 0) return
+    val last = lastCorrectionNanos
+    if (last != null && nowNanos - last < CORRECTION_INTERVAL_MS * 1_000_000L) return
+    val emaMs = depthEmaBytes / BYTES_PER_MS
+    val slice = CORRECTION_MS * BYTES_PER_MS
+    when {
+      emaMs > HIGH_MS -> {
+        dropOldest(slice)
+        driftDroppedBytes += slice
+      }
+      emaMs < LOW_MS -> {
+        if (used + slice > CAPACITY_BYTES) return
+        write(SILENCE_SLICE, 0, slice)
+        driftInsertedBytes += slice
+      }
+      else -> return
+    }
+    driftCorrections += 1
+    lastCorrectionNanos = nowNanos
+    depthEmaBytes = used.toDouble()
+  }
+
+  private fun dropOldest(bytes: Int) {
+    val n = min(bytes, used)
+    head = (head + n) % CAPACITY_BYTES
+    used -= n
+  }
+
+  private fun write(src: ByteArray, srcOffset: Int, len: Int) {
+    var tail = (head + used) % CAPACITY_BYTES
+    var from = srcOffset
+    var remaining = len
+    while (remaining > 0) {
+      val chunk = min(remaining, CAPACITY_BYTES - tail)
+      System.arraycopy(src, from, ring, tail, chunk)
+      tail = (tail + chunk) % CAPACITY_BYTES
+      from += chunk
+      remaining -= chunk
+    }
+    used += len
+  }
+
+  private fun readFrame(): ByteArray {
+    val out = ByteArray(FRAME_BYTES)
+    var from = head
+    var to = 0
+    var remaining = FRAME_BYTES
+    while (remaining > 0) {
+      val chunk = min(remaining, CAPACITY_BYTES - from)
+      System.arraycopy(ring, from, out, to, chunk)
+      from = (from + chunk) % CAPACITY_BYTES
+      to += chunk
+      remaining -= chunk
+    }
+    head = from
+    used -= FRAME_BYTES
+    return out
+  }
+
+  companion object {
+    const val RATE = 48_000
+    const val CHANNELS = 1
+    const val FRAME_MS = 20
+    const val BYTES_PER_MS = RATE * CHANNELS * 2 / 1000
+    const val FRAME_BYTES = FRAME_MS * BYTES_PER_MS
+
+    const val TARGET_MS = 60
+    const val LOW_MS = 40
+    const val HIGH_MS = 100
+    const val EMERGENCY_CAP_MS = 200
+
+    const val CORRECTION_MS = 10
+    const val CORRECTION_INTERVAL_MS = 500L
+    const val STARVE_MS = 200L
+
+    private const val CAPACITY_MS = 2_000
+    private const val CAPACITY_BYTES = CAPACITY_MS * BYTES_PER_MS
+    /** ~2 s at 50 ticks/s: long enough that jitter does not move it. */
+    private const val EMA_ALPHA = 0.02
+    private val SILENCE_SLICE = ByteArray(CORRECTION_MS * BYTES_PER_MS)
+  }
+}