@mentra/glasses-media 3.2.1-dev.288 → 3.2.1-dev.291

package/README.md

@@ -37,15 +37,20 @@ Before merging/releasing, exercise iPhone + glasses: first-use local-network per
 `PhoneStreamCoordinator` provisions Cloudflare and owns subscribers for the entire route:
 
 ```text
-Glasses -- WHIP over SoftAP --> shared receiver -- decoded video/PCM --> phone WHIP publisher --> Cloudflare
+Glasses -- video-only WHIP over SoftAP --> shared receiver --> phone WHIP publisher --> Cloudflare
+Glasses MCU -- BLE LC3 --> phone PCM decoder ----------------------^
 ```
 
 Only `ingest: "whip"` selects this route. Default SRT and explicit RTMP continue publishing directly from the glasses. WHIP never falls back to SRT/RTMP when Cloudflare omits its WHIP URL. The phone needs mobile data; the glasses need no internet Wi-Fi credentials. Livestreamer's Stream Here path skips its glasses Wi-Fi setup gate accordingly.
 
 `MentraGlassesMediaRelay.prepare` joins the persistent hotspot, prepares both native peers, and returns a local WHIP URL for the BLE start command. Cloudflare credentials stay on the phone. Video queues hold at most one frame; decoded PCM is downmixed/resampled into a bounded 200ms, 48kHz mono queue. WebRTC re-encodes both tracks. No phone microphone is captured. Android uses the WebRTC library's external AudioRecord buffer mode; iOS uses a custom `RTCAudioDevice`.
 
+On Android hosts with microphone pinning and native PCM input, managed streams use the same BLE LC3 capture path as ACS SoftAP calls. An engine-owned `livestream` microphone session requests continuous glasses PCM; it leaves the configured gain in force and disables the call-specific loudness gate. The glasses receive `captureAudio: false`, avoiding SoC microphone capture and audio encoding, while the phone publishes video plus LC3-derived audio to Cloudflare. Attempt-scoped callbacks and native PCM gating reject late audio after stop/retry. Explicit `captureAudio: false` disables audio entirely. iOS and older hosts retain WHIP audio, matching ACS capability gating. ACS's measured 170ms audio delay is specific to its outgoing pipeline and is not applied to the Cloudflare publisher without calibration.
+
 Android's process-wide network inventory exposes both the scoped hotspot and real cellular handles during a relay. Per-factory masks keep the receiver on Wi-Fi and the publisher off Wi-Fi; signaling also uses the cellular network's socket factory and DNS. ACS retains its existing hotspot-only inventory. iOS uses corresponding per-factory masks after validating the cellular default route.
 
+Android temporarily lifts the cellular process pin while creating the local WHIP listener, then restores it before starting the outgoing publisher. Otherwise the listener inherits the cellular socket mark and cannot reply to the glasses. The outgoing audio callback supplies its own PCM frame clock: WebRTC's external-recording mode does not pace callbacks and reports zero captured bytes, so the callback fills the entire buffer and waits for the next frame interval without opening the phone microphone.
+
 An attempt-specific ID isolates late callbacks and glasses status. Reconnect tears down the glasses publisher, both phone peers and hotspot before rebuilding against the same provisioned input. Three retries bound consecutive failures; a minute of stable streaming renews the retry budget. Last-subscriber stop cancels startup/backoff, awaits native cleanup and disables the hotspot. Failed cleanup retains ownership and can be retried by stop. BLE-unreachable stop/hotspot commands are deferred until reconnection. ACS and managed streaming share a hotspot reservation. Native media queues keep running when the UI is backgrounded; Android holds a partial wake lock and uses the engine's background timers for retry/readiness.
 
 Device qualification for this outgoing pipeline must include Android and iPhone: audio/video playback on the provisioned WHEP URL, mute/video-only capture, last-subscriber stop, cancellation at each setup step, cellular/hotspot/BLE loss and recovery, repeated starts, screen-off/background operation and a sustained thermal/audio-sync run. Unit tests and platform compilation cover lifecycle/format contracts; they do not establish those hardware results.

package/android/src/main/java/com/mentra/glassesmedia/GlassesMediaRelayModule.kt

@@ -5,8 +5,11 @@ import android.os.Handler
 import android.os.Looper
 import android.os.PowerManager
 import android.provider.Settings
+import android.util.Base64
+import android.util.Log
 import com.mentra.glassesmedia.network.*
 import com.mentra.glassesmedia.publisher.PhoneWhipPublisher
+import com.mentra.glassesmedia.publisher.RelayPcmInput
 import com.mentra.glassesmedia.source.*
 import expo.modules.kotlin.Promise
 import expo.modules.kotlin.modules.Module
@@ -22,6 +25,7 @@ class GlassesMediaRelayModule : Module() {
   private var source: LocalWhipIngestSource? = null
   private var publisher: PhoneWhipPublisher? = null
   private var wakeLock: PowerManager.WakeLock? = null
+  private val pcmInput = RelayPcmInput()
 
   override fun definition() = ModuleDefinition {
     Name("MentraGlassesMediaRelay")
@@ -64,14 +68,21 @@ class GlassesMediaRelayModule : Module() {
           val outgoing = PhoneWhipPublisher(context, endpoint, options["captureAudio"] != false,
             (options["bitrate"] as? Number)?.toInt() ?: 2_000_000) { state, reason -> emit(id, state, reason) }
           publisher = outgoing
+          val bleAudio = options["captureAudio"] != false && options["audioTransport"] == "ble-lc3"
+          if (bleAudio) pcmInput.attach(id, outgoing::onPcm)
+          Log.i("ManagedRelay", "Audio transport=${options["audioTransport"] ?: "whip"} attempt=$id")
           val incoming = LocalWhipIngestSource(context, VideoFrameListener(outgoing::onVideoFrame),
             PcmListener(outgoing::onPcm), scopedNetwork = scoped)
           source = incoming
           incoming.setStateListener { state, reason ->
             if (state == SourceState.FAILED) emit(id, "failed", "Glasses receiver: $reason")
           }
-          incoming.setPcmDeliveryEnabled(options["captureAudio"] != false)
-          incoming.start(SourceConfig("", SourceKind.SOFTAP, scoped.localIpv4()))
+          incoming.setPcmDeliveryEnabled(options["captureAudio"] != false && !bleAudio)
+          // A listener created while pinned to cellular cannot reply to the glasses over Wi-Fi.
+          // Restore cellular before opening the outgoing publisher, including when the bind fails.
+          hold.withProcessUnpinned {
+            incoming.start(SourceConfig("", SourceKind.SOFTAP, scoped.localIpv4()))
+          }
           wakeLock = context.getSystemService(PowerManager::class.java)
             .newWakeLock(PowerManager.PARTIAL_WAKE_LOCK, "Mentra:managed-relay").apply { acquire() }
           outgoing.start()
@@ -83,6 +94,11 @@ class GlassesMediaRelayModule : Module() {
       }
     }
 
+    Function("pushOutgoingPcm") { attemptId: String, base64: String, sampleRate: Int, channels: Int ->
+      val bytes = try { Base64.decode(base64, Base64.DEFAULT) } catch (_: IllegalArgumentException) { null }
+      bytes != null && pcmInput.push(attemptId, bytes, sampleRate, channels)
+    }
+
     AsyncFunction("stop") { attemptId: String, promise: Promise ->
       worker.execute {
         try {
@@ -100,6 +116,7 @@ class GlassesMediaRelayModule : Module() {
   }
 
   private fun cleanup() {
+    pcmInput.detach()
     source?.setStateListener(null)
     // Try every cleanup even if one fails. Keep the slot occupied on any failure.
     var failure: Exception? = null

package/android/src/main/java/com/mentra/glassesmedia/network/InternetHold.kt

@@ -239,6 +239,21 @@ class InternetHold(private val context: Context) {
         return@synchronized ok
     }
 
+    /**
+     * Create a SoftAP listener without inheriting the cellular socket mark, then restore the pin.
+     * Holding the lock prevents a network callback from re-pinning during the bind. The previous
+     * routing state is restored even when creating the listener fails.
+     */
+    fun <T> withProcessUnpinned(bind: () -> T): T = synchronized(lock) {
+        val wasPinned = processPinned
+        unbindProcess()
+        try {
+            bind()
+        } finally {
+            if (wasPinned) bindProcessToCellular()
+        }
+    }
+
     /** Undo [bindProcessToCellular]. Safe to call when nothing is pinned, and safe to call twice. */
     fun unbindProcess() = synchronized(lock) {
         if (closed) return@synchronized

package/android/src/main/java/com/mentra/glassesmedia/publisher/PhoneWhipPublisher.kt

@@ -40,6 +40,7 @@ class PhoneWhipPublisher(
   private val closeDone = CountDownLatch(1)
   @Volatile private var cleanupFailure: Throwable? = null
   private val pcm = RelayPcmBuffer()
+  private val audioInput = RelayAudioInput(pcm)
   private var factory: PeerConnectionFactory? = null
   private var egl: EglBase? = null
   private var adm: JavaAudioDeviceModule? = null
@@ -78,12 +79,10 @@ class PhoneWhipPublisher(
         adm = JavaAudioDeviceModule.builder(context)
           .setInputSampleRate(48_000).setUseStereoInput(false)
           .setUseHardwareAcousticEchoCanceler(false).setUseHardwareNoiseSuppressor(false)
-          .setAudioBufferCallback { buffer, _, channels, sampleRate, bytes, _ ->
-            if (channels == 1 && sampleRate == 48_000) pcm.read(buffer, bytes)
-            else for (i in 0 until minOf(bytes, buffer.capacity())) buffer.put(i, 0)
-            System.nanoTime()
+          .setAudioBufferCallback { buffer, _, channels, sampleRate, _, _ ->
+            audioInput.read(buffer, channels, sampleRate)
           }.createAudioDeviceModule().also {
-            // WebRTC's external-buffer mode: paced callbacks without AudioRecord/microphone.
+            // External mode has no blocking AudioRecord.read; our callback supplies the clock.
             it.setAudioRecordEnabled(false)
           }
         factory = PeerConnectionFactory.builder()

package/android/src/main/java/com/mentra/glassesmedia/publisher/RelayAudioInput.kt

@@ -0,0 +1,33 @@
+package com.mentra.glassesmedia.publisher
+
+import java.nio.ByteBuffer
+import java.util.concurrent.TimeUnit
+
+/** Supplies the clock normally provided by a blocking AudioRecord.read, without using the mic. */
+class RelayAudioInput(
+  private val pcm: RelayPcmBuffer,
+  private val nowNs: () -> Long = System::nanoTime,
+  private val sleepNs: (Long) -> Unit = { TimeUnit.NANOSECONDS.sleep(it) },
+) {
+  private var nextFrameAtNs: Long? = null
+
+  /** Called only by WebRTC's audio recording thread, with a complete PCM16 frame to fill. */
+  fun read(buffer: ByteBuffer, channels: Int, sampleRate: Int): Long {
+    require(channels > 0 && sampleRate > 0 && buffer.capacity() >= channels * 2)
+    val durationNs = (buffer.capacity() / (channels * 2)).toLong() * 1_000_000_000L / sampleRate
+    val deadline = nextFrameAtNs ?: nowNs()
+    var now = nowNs()
+    while (now < deadline) {
+      sleepNs(deadline - now)
+      now = nowNs()
+    }
+    // Preserve cadence across small scheduling delays, but never replay a backlog after a stall.
+    nextFrameAtNs = if (now - deadline >= durationNs) now + durationNs else deadline + durationNs
+
+    // External-recording mode reports bytesRead=0, then sends buffer.capacity() bytes to native.
+    // Fill that entire frame; using bytesRead would send only the library's prefilled silence.
+    if (channels == 1 && sampleRate == 48_000) pcm.read(buffer, buffer.capacity())
+    else for (i in 0 until buffer.capacity()) buffer.put(i, 0)
+    return now
+  }
+}

package/android/src/main/java/com/mentra/glassesmedia/publisher/RelayPcmInput.kt

@@ -0,0 +1,25 @@
+package com.mentra.glassesmedia.publisher
+
+/** Gates JS microphone callbacks by attempt, without queueing audio behind network operations. */
+class RelayPcmInput {
+  private var attemptId: String? = null
+  private var sink: ((ByteArray, Int, Int) -> Unit)? = null
+
+  @Synchronized fun attach(id: String, target: (ByteArray, Int, Int) -> Unit) {
+    attemptId = id
+    sink = target
+  }
+
+  @Synchronized fun push(id: String, bytes: ByteArray, sampleRate: Int, channels: Int): Boolean {
+    if (id != attemptId || bytes.isEmpty() || sampleRate !in 8_000..192_000 || channels !in 1..8) return false
+    val target = sink ?: return false
+    target(bytes, sampleRate, channels)
+    return true
+  }
+
+  /** Wait for any in-flight push before the owner closes the publisher. */
+  @Synchronized fun detach() {
+    attemptId = null
+    sink = null
+  }
+}

package/android/src/main/java/com/mentra/glassesmedia/source/WhipIngestServer.kt

@@ -213,6 +213,7 @@ class WhipIngestServer(
       }
       if (!registered) { runCatching { connection.close() }; return }
       accepted.incrementAndGet()
+      SoftApTrace.stage("whip_socket_accepted")
       // Each connection on its own thread: a negotiation blocks for the length of an ICE gather,
       // and a DELETE arriving during one must not queue behind it.
       try {
@@ -371,6 +372,7 @@ class WhipIngestServer(
       val (method, target) = WhipIngestProtocol.parseRequestLine(
         reader.readLine() ?: return null,
       ) ?: return null
+      SoftApTrace.stage("whip_request_line_read", "method" to method)
 
       val headers = mutableMapOf<String, String>()
       while (true) {
@@ -383,6 +385,7 @@ class WhipIngestServer(
 
       val contentType = WhipIngestProtocol.headerValue(headers, "Content-Type")
       val contentLength = WhipIngestProtocol.headerValue(headers, "Content-Length")?.toLongOrNull()
+      SoftApTrace.stage("whip_request_headers_read", "contentLength" to contentLength)
 
       // Do not read a body we have already decided to refuse. decide() checks the length first for
       // exactly this reason.
@@ -398,6 +401,7 @@ class WhipIngestServer(
         val count = reader.read(body, read, body.size - read)
         if (count < 0) break
         read += count
+        SoftApTrace.stage("whip_request_body_read", "read" to read, "expected" to body.size)
       }
       return WhipIngestProtocol.Request(method, target, contentType, contentLength, String(body, 0, read))
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mentra/glasses-media",
-  "version": "3.2.1-dev.288",
+  "version": "3.2.1-dev.291",
   "description": "Shared native glasses media receivers and local network transport",
   "license": "MIT",
   "author": "Mentra Community",