@mentra/glasses-media 3.2.1-dev.307 → 3.2.1-dev.308

package/README.md

@@ -49,6 +49,8 @@ On Android hosts with microphone pinning and native PCM input, managed streams u
 
 Android's process-wide network inventory exposes both the scoped hotspot and real cellular handles during a relay. Per-factory masks keep the receiver on Wi-Fi and the publisher off Wi-Fi; signaling also uses the cellular network's socket factory and DNS. ACS retains its existing hotspot-only inventory. iOS uses corresponding per-factory masks after validating the cellular default route.
 
+The iOS internet publisher posts its offer on the first server-reflexive candidate, complete ICE gathering, or a 1.5-second gather deadline, matching the glasses publisher's STUN policy. It posts once and ignores late triggers after stop. The local SoftAP receiver still waits for complete host-only gathering. Incident logs record the outgoing negotiation stage, candidate counts, HTTP result and first decoded video frame without SDP or publish credentials.
+
 Android temporarily lifts the cellular process pin while creating the local WHIP listener, then restores it before starting the outgoing publisher. Otherwise the listener inherits the cellular socket mark and cannot reply to the glasses. The outgoing audio callback supplies its own PCM frame clock: WebRTC's external-recording mode does not pace callbacks and reports zero captured bytes, so the callback fills the entire buffer and waits for the next frame interval without opening the phone microphone.
 
 An attempt-specific ID isolates late callbacks and glasses status. Reconnect tears down the glasses publisher, both phone peers and hotspot before rebuilding against the same provisioned input. Three retries bound consecutive failures; a minute of stable streaming renews the retry budget. Last-subscriber stop cancels startup/backoff, awaits native cleanup and disables the hotspot. Failed cleanup retains ownership and can be retried by stop. BLE-unreachable stop/hotspot commands are deferred until reconnection. ACS and managed streaming share a hotspot reservation. Native media queues keep running when the UI is backgrounded; Android holds a partial wake lock and uses the engine's background timers for retry/readiness.

package/ios/CoreKit/Sources/GlassesMediaCore/WhipOfferGate.swift

@@ -0,0 +1,24 @@
+/// Internet WHIP uses the same bounded STUN gather as the glasses publisher.
+/// The local, host-only SoftAP receiver must still wait for complete gathering.
+public struct WhipOfferGate {
+    public enum Signal { case localDescriptionSet, serverReflexiveCandidate, gatheringComplete, deadline, stop }
+
+    private var localSet = false
+    private var ready = false
+    private var posted = false
+    private var stopped = false
+
+    public init() {}
+
+    /// True once per publisher, after both the description and a POST trigger exist.
+    public mutating func observe(_ signal: Signal) -> Bool {
+        switch signal {
+        case .localDescriptionSet: localSet = true
+        case .serverReflexiveCandidate, .gatheringComplete, .deadline: ready = true
+        case .stop: stopped = true
+        }
+        guard localSet, ready, !posted, !stopped else { return false }
+        posted = true
+        return true
+    }
+}

package/ios/PhoneWhipPublisher.swift

@@ -2,7 +2,7 @@ import CoreVideo
 import Foundation
 import WebRTC
 
-/// ASG's full-ICE WHIP negotiation, fed from the shared decoded receiver. One peer per attempt.
+/// ASG's bounded STUN-gather WHIP negotiation, fed from the shared decoded receiver.
 final class PhoneWhipPublisher: NSObject {
     private let queue = DispatchQueue(label: "com.mentra.glassesmedia.whip.publish")
     private let frameSlot = DispatchSemaphore(value: 1)
@@ -17,7 +17,10 @@ final class PhoneWhipPublisher: NSObject {
     private var capturer: RTCVideoCapturer?
     private var resource: URL?
     private var posted = false
-    private var localSet = false
+    private var offerGate = WhipOfferGate()
+    private var candidateCount = 0
+    private var phase = "creating offer"
+    private var firstFrameSeen = false
     private var stopped = false
     private var failed = false
     private var connected = false
@@ -81,13 +84,19 @@ final class PhoneWhipPublisher: NSObject {
                         self.queue.async {
                             guard !self.stopped else { return }
                             guard error == nil else { self.fail("WHIP local description failed"); return }
-                            self.localSet = true; self.maybePost()
+                            self.phase = "gathering ICE"
+                            self.maybePost(.localDescriptionSet)
+                            if peer.iceGatheringState == .complete { self.maybePost(.gatheringComplete) }
+                            // STUN gathering can stall longer than the entire start request.
+                            // Match the glasses publisher's 1.5s cap; the remote ICE-lite
+                            // candidates in the answer let this peer initiate connectivity.
+                            self.queue.asyncAfter(deadline: .now() + 1.5) { self.maybePost(.deadline) }
                         }
                     }
                 }
             }
             self.queue.asyncAfter(deadline: .now() + 35) {
-                if !self.connected { self.fail("WHIP connection timed out") }
+                if !self.connected { self.fail("WHIP connection timed out while \(self.phase) (\(self.candidateCount) candidates)") }
             }
         }
     }
@@ -97,6 +106,10 @@ final class PhoneWhipPublisher: NSObject {
         queue.async {
             defer { self.frameSlot.signal() }
             guard !self.stopped, let source = self.videoSource, let capturer = self.capturer else { return }
+            if !self.firstFrameSeen {
+                self.firstFrameSeen = true
+                self.onState("diagnostic", "First decoded glasses video frame reached the phone publisher")
+            }
             self.lastTimestamp = max(Int64(ProcessInfo.processInfo.systemUptime * 1_000_000_000), self.lastTimestamp + 1)
             let frame = RTCVideoFrame(buffer: RTCCVPixelBuffer(pixelBuffer: buffer), rotation: ._0, timeStampNs: self.lastTimestamp)
             source.capturer(capturer, didCapture: frame)
@@ -107,9 +120,12 @@ final class PhoneWhipPublisher: NSObject {
         if captureAudio { audioDevice.pcm.push(data, sampleRate: rate, channels: channels) }
     }
 
-    private func maybePost() {
-        guard !stopped, !posted, localSet, let peer, peer.iceGatheringState == .complete, let offer = peer.localDescription else { return }
+    private func maybePost(_ signal: WhipOfferGate.Signal) {
+        guard !stopped, !failed, let peer, offerGate.observe(signal) else { return }
+        guard let offer = peer.localDescription else { fail("WHIP local description unavailable"); return }
         posted = true
+        phase = "sending offer"
+        onState("diagnostic", "Posting WHIP offer (\(candidateCount) candidates)")
         var request = URLRequest(url: endpoint)
         request.httpMethod = "POST"
         request.setValue("application/sdp", forHTTPHeaderField: "Content-Type")
@@ -128,9 +144,16 @@ final class PhoneWhipPublisher: NSObject {
                       let data, let answer = String(data: data, encoding: .utf8), !answer.isEmpty
                 else {
                     if let location { self.delete(location) }
-                    self.fail("WHIP server rejected publish (HTTP \(response?.statusCode ?? 0))"); return
+                    if let error = error as NSError? {
+                        self.fail("WHIP signaling failed (\(error.domain) \(error.code))")
+                    } else {
+                        self.fail("WHIP server rejected publish (HTTP \(response?.statusCode ?? 0))")
+                    }
+                    return
                 }
                 self.resource = location
+                self.phase = "connecting ICE"
+                self.onState("diagnostic", "WHIP server accepted publish (HTTP 201)")
                 peer.setRemoteDescription(RTCSessionDescription(type: .answer, sdp: answer)) { error in
                     self.queue.async { if error != nil { self.fail("WHIP answer failed") } }
                 }
@@ -154,6 +177,7 @@ final class PhoneWhipPublisher: NSObject {
         queue.async {
             if !self.stopped {
                 self.stopped = true
+                _ = self.offerGate.observe(.stop)
                 self.peer?.close()
                 // Drain the external audio callback while its native delegate/factory is still alive.
                 _ = self.audioDevice.terminateDevice()
@@ -176,6 +200,7 @@ extension PhoneWhipPublisher: RTCPeerConnectionDelegate {
     func peerConnection(_ peerConnection: RTCPeerConnection, didChange state: RTCIceConnectionState) {
         queue.async {
             guard self.peer === peerConnection, !self.stopped else { return }
+            self.onState("diagnostic", "WHIP ICE state \(state.rawValue)")
             switch state {
             case .connected, .completed:
                 self.connected = true; self.disconnectGeneration += 1
@@ -192,11 +217,22 @@ extension PhoneWhipPublisher: RTCPeerConnectionDelegate {
         }
     }
 
-    func peerConnection(_ peerConnection: RTCPeerConnection, didChange _: RTCIceGatheringState) {
-        queue.async { if self.peer === peerConnection { self.maybePost() } }
+    func peerConnection(_ peerConnection: RTCPeerConnection, didChange state: RTCIceGatheringState) {
+        queue.async {
+            guard self.peer === peerConnection, !self.stopped else { return }
+            self.onState("diagnostic", "WHIP ICE gathering state \(state.rawValue) (\(self.candidateCount) candidates)")
+            if state == .complete { self.maybePost(.gatheringComplete) }
+        }
+    }
+
+    func peerConnection(_ peerConnection: RTCPeerConnection, didGenerate candidate: RTCIceCandidate) {
+        queue.async {
+            guard self.peer === peerConnection, !self.stopped else { return }
+            self.candidateCount += 1
+            if candidate.sdp.contains(" typ srflx ") { self.maybePost(.serverReflexiveCandidate) }
+        }
     }
 
-    func peerConnection(_: RTCPeerConnection, didGenerate _: RTCIceCandidate) {}
     func peerConnection(_: RTCPeerConnection, didRemove _: [RTCIceCandidate]) {}
     func peerConnection(_: RTCPeerConnection, didOpen _: RTCDataChannel) {}
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mentra/glasses-media",
-  "version": "3.2.1-dev.307",
+  "version": "3.2.1-dev.308",
   "description": "Shared native glasses media receivers and local network transport",
   "license": "MIT",
   "author": "Mentra Community",