@mentra/glasses-media 3.2.0-dev.226 → 3.2.0-dev.232

package/android/src/main/java/com/mentra/glassesmedia/source/WhipIngestServer.kt

@@ -69,6 +69,7 @@ class WhipIngestServer(
   private val connections = Executors.newCachedThreadPool { runnable ->
     Thread(runnable, "whip-ingest-conn").apply { isDaemon = true }
   }
+  private val activeSockets = java.util.concurrent.ConcurrentHashMap.newKeySet<Socket>()
   private val accepted = AtomicInteger()
 
   /** Bound endpoint, or null before [start]. This is the URL the glasses must be told to POST to. */
@@ -141,6 +142,13 @@ class WhipIngestServer(
     closeListener()
   }
 
+  /** Hard teardown barrier: no request may still create a peer after this returns. */
+  fun closeAndAwait(timeoutMs: Long = 15_000): Boolean {
+    closeNow()
+    connections.shutdownNow()
+    return connections.awaitTermination(timeoutMs, TimeUnit.MILLISECONDS)
+  }
+
   private fun closeListener() {
     val socket = synchronized(lock) {
       val current = server
@@ -149,6 +157,7 @@ class WhipIngestServer(
       current
     } ?: return
     runCatching { socket.close() }
+    activeSockets.forEach { runCatching { it.close() } }
     connections.shutdownNow()
     SoftApTrace.stage("whip_listener_closed", "accepted" to accepted.get())
   }
@@ -163,6 +172,10 @@ class WhipIngestServer(
         Log.w(TAG, "accept failed", error)
         return
       }
+      val registered = synchronized(lock) {
+        if (server !== socket) false else { activeSockets.add(connection); true }
+      }
+      if (!registered) { runCatching { connection.close() }; return }
       accepted.incrementAndGet()
       // Each connection on its own thread: a negotiation blocks for the length of an ICE gather,
       // and a DELETE arriving during one must not queue behind it.
@@ -170,28 +183,31 @@ class WhipIngestServer(
         connections.execute { serve(connection) }
       } catch (_: java.util.concurrent.RejectedExecutionException) {
         runCatching { connection.close() }
+        activeSockets.remove(connection)
       }
     }
   }
 
   private fun serve(connection: Socket) {
-    connection.use { socket ->
-      socket.soTimeout = READ_TIMEOUT_MS
-      val output = BufferedOutputStream(socket.getOutputStream())
-      try {
-        val request = readRequest(socket.getInputStream())
-        if (request == null) {
-          write(output, WhipIngestProtocol.Response(400, "Bad Request", body = "malformed request"))
-          return
-        }
-        write(output, handle(request))
-      } catch (error: Exception) {
-        Log.w(TAG, "connection failed", error)
-        runCatching {
-          write(output, WhipIngestProtocol.Response(400, "Bad Request", body = "read failed"))
+    try {
+      connection.use { socket ->
+        socket.soTimeout = READ_TIMEOUT_MS
+        val output = BufferedOutputStream(socket.getOutputStream())
+        try {
+          val request = readRequest(socket.getInputStream())
+          if (request == null) {
+            write(output, WhipIngestProtocol.Response(400, "Bad Request", body = "malformed request"))
+            return
+          }
+          write(output, handle(request))
+        } catch (error: Exception) {
+          Log.w(TAG, "connection failed", error)
+          runCatching {
+            write(output, WhipIngestProtocol.Response(400, "Bad Request", body = "read failed"))
+          }
         }
       }
-    }
+    } finally { activeSockets.remove(connection) }
   }
 
   private fun handle(request: WhipIngestProtocol.Request): WhipIngestProtocol.Response {

package/expo-module.config.json

@@ -4,9 +4,13 @@
     "android"
   ],
   "apple": {
-    "modules": []
+    "modules": [
+      "GlassesMediaRelayModule"
+    ]
   },
   "android": {
-    "modules": []
+    "modules": [
+      "com.mentra.glassesmedia.GlassesMediaRelayModule"
+    ]
   }
 }

package/ios/CoreKit/Sources/GlassesMediaCore/LocalMediaPolicy.swift

@@ -31,6 +31,14 @@ public enum LocalMediaPolicy {
             (bytes[0] == 192 && bytes[1] == 168)
     }
 
+    /// SSID association can finish before DHCP replaces the previous Wi-Fi address.
+    /// Accept only a client address on the /24 advertised by the glasses over BLE.
+    public static func isHotspotClientAddress(_ address: String, gateway: String) -> Bool {
+        guard isPrivate(gateway), sameSubnet(address, gateway), address != gateway,
+              let bytes = ipv4(address) else { return false }
+        return bytes[3] > 0 && bytes[3] < 255
+    }
+
     /// Keep only concrete host candidates on the local link. Never rewrite a cellular socket's
     /// advertised address: a candidate must already belong to the interface it claims.
     public static func localSdp(_ sdp: String, address: String, answer: Bool) throws -> String {
@@ -45,7 +53,11 @@ public enum LocalMediaPolicy {
             candidates += 1
             return true
         }
-        guard candidates > 0 else { throw LocalMediaError("No host ICE candidate on the glasses hotspot") }
+        guard candidates > 0 else {
+            throw LocalMediaError(answer
+                ? "Phone answer has no host ICE candidate on the glasses hotspot"
+                : "Glasses offer has no host ICE candidate on the glasses hotspot")
+        }
         return lines.joined(separator: "\r\n") + "\r\n"
     }
 }

package/ios/CoreKit/Sources/GlassesMediaCore/RelayAudioClock.swift

@@ -0,0 +1,56 @@
+import Foundation
+
+/// A paced audio worker whose stop is a barrier for the final render callback and its resources.
+public final class RelayAudioClock {
+    private let condition = NSCondition()
+    private var quitting = false
+    private var exited = true
+    private var enabled = false
+
+    public init() {}
+
+    public func start(render: @escaping (Double) -> Void, finish: @escaping () -> Void = {}) {
+        condition.lock()
+        precondition(exited, "Previous audio worker must stop before reinitialization")
+        quitting = false; enabled = false; exited = false
+        condition.unlock()
+        let thread = Thread { [self] in
+            defer {
+                finish()
+                condition.lock(); exited = true; condition.broadcast(); condition.unlock()
+            }
+            var sampleTime: Double = 0
+            var next = ProcessInfo.processInfo.systemUptime
+            while true {
+                condition.lock()
+                while !enabled && !quitting {
+                    condition.wait(); next = ProcessInfo.processInfo.systemUptime
+                }
+                let stop = quitting
+                condition.unlock()
+                if stop { return }
+                render(sampleTime)
+                sampleTime += 480
+                next += 0.01
+                let now = ProcessInfo.processInfo.systemUptime
+                if next > now { Thread.sleep(forTimeInterval: next - now) }
+                else if now - next > 0.1 { next = now }
+            }
+        }
+        thread.name = "Mentra glasses audio publish"
+        thread.qualityOfService = .userInitiated
+        thread.start()
+    }
+
+    public func setEnabled(_ value: Bool) {
+        condition.lock(); enabled = value; condition.broadcast(); condition.unlock()
+    }
+
+    public func stop() {
+        condition.lock(); quitting = true; enabled = false; condition.broadcast()
+        while !exited {
+            condition.wait()
+        }
+        condition.unlock()
+    }
+}

package/ios/CoreKit/Sources/GlassesMediaCore/RelayPcmBuffer.swift

@@ -0,0 +1,57 @@
+import Foundation
+
+/// Bounded decoded audio queue. Converts PCM16 to 48kHz mono without touching phone capture.
+public final class RelayPcmBuffer {
+    private let lock = NSLock()
+    private var samples: [Int16]
+    private var head = 0
+    private var count = 0
+    private var phase = 0.0
+    private var rate = 0
+    private var channels = 0
+    private var previous: Double?
+
+    public init(capacity: Int = 9600) {
+        samples = Array(repeating: 0, count: capacity)
+    }
+
+    public func push(_ data: Data, sampleRate: Int, channels channelCount: Int) {
+        guard (8000 ... 192_000).contains(sampleRate), (1 ... 8).contains(channelCount) else { return }
+        lock.lock(); defer { lock.unlock() }
+        if rate != sampleRate || channels != channelCount {
+            rate = sampleRate; channels = channelCount; phase = 0; previous = nil
+        }
+        let step = Double(sampleRate) / 48000
+        data.withUnsafeBytes { raw in
+            let bytes = raw.bindMemory(to: UInt8.self)
+            for frame in 0 ..< data.count / (2 * channelCount) {
+                var sum = 0.0
+                for channel in 0 ..< channelCount {
+                    let offset = (frame * channelCount + channel) * 2
+                    sum += Double(Int16(bitPattern: UInt16(bytes[offset]) | UInt16(bytes[offset + 1]) << 8))
+                }
+                let value = sum / Double(channelCount)
+                if let prior = previous {
+                    while phase < 1 {
+                        if count == samples.count { head = (head + 1) % samples.count; count -= 1 }
+                        samples[(head + count) % samples.count] = Int16(prior + (value - prior) * phase)
+                        count += 1
+                        phase += step
+                    }
+                    phase -= 1
+                }
+                previous = value
+            }
+        }
+    }
+
+    /// Pads underruns with silence. Drops oldest samples on overrun to bound live latency.
+    public func read(into output: UnsafeMutableBufferPointer<Int16>) {
+        lock.lock(); defer { lock.unlock() }
+        for i in output.indices {
+            if count > 0 {
+                output[i] = samples[head]; head = (head + 1) % samples.count; count -= 1
+            } else { output[i] = 0 }
+        }
+    }
+}

package/ios/GlassesHotspotNetwork.swift

@@ -7,7 +7,9 @@ import NetworkExtension
 public final class GlassesHotspotNetwork {
     private let queue = DispatchQueue(label: "com.mentra.glassesmedia.hotspot")
     private var ssid: String?
+    private var lastHotspotSSID: String?
     private var localAddress: String?
+    private var gatewayAddress: String?
     private var generation = 0
     private var applying = false
     private var cancelled = false
@@ -17,12 +19,17 @@ public final class GlassesHotspotNetwork {
     public var onLost: ((String) -> Void)?
     public init() {}
 
-    public func join(ssid: String, passphrase: String, completion: @escaping (Result<String, Error>) -> Void) {
+    public func join(ssid: String, passphrase: String, gateway: String? = nil, completion: @escaping (Result<String, Error>) -> Void) {
         queue.async {
             guard self.ssid == nil, !self.applying else { completion(.failure(LocalMediaError("Previous hotspot session has not finished cleaning up"))); return }
+            if let gateway, !LocalMediaPolicy.isPrivate(gateway) {
+                completion(.failure(LocalMediaError("The glasses reported an invalid hotspot gateway"))); return
+            }
             self.generation += 1
             let gen = self.generation
             self.ssid = ssid
+            self.lastHotspotSSID = ssid
+            self.gatewayAddress = gateway
             self.cancelled = false
             self.applying = true
             self.joinReply = completion
@@ -75,7 +82,7 @@ public final class GlassesHotspotNetwork {
     public func probeGateway(completion: @escaping (Bool, String) -> Void) {
         queue.async {
             guard let address = self.localAddress else { completion(false, "No joined hotspot"); return }
-            let gateway = address.split(separator: ".").prefix(3).joined(separator: ".") + ".1"
+            let gateway = self.gatewayAddress ?? address.split(separator: ".").prefix(3).joined(separator: ".") + ".1"
             let parameters = NWParameters.tcp
             parameters.requiredInterfaceType = .wifi
             let connection = NWConnection(host: NWEndpoint.Host(gateway), port: 8089, using: parameters)
@@ -99,7 +106,7 @@ public final class GlassesHotspotNetwork {
         }
     }
 
-    public func awaitInternet(completion: @escaping (Bool, String) -> Void) {
+    public func awaitInternet(requireCellular: Bool = true, completion: @escaping (Bool, String) -> Void) {
         queue.async {
             let monitor = NWPathMonitor()
             var finished = false
@@ -111,9 +118,21 @@ public final class GlassesHotspotNetwork {
             }
             monitor.pathUpdateHandler = { path in
                 if path.status == .satisfied, path.usesInterfaceType(.cellular) { finish(true, "cellular") }
+                // Once the hotspot is released, a return to the user's Wi-Fi is also valid.
+                // Do not mistake the departing glasses AP's local-only path for restored internet.
+                if !requireCellular, path.status == .satisfied, path.usesInterfaceType(.wifi) {
+                    NEHotspotNetwork.fetchCurrent { network in
+                        self.queue.async {
+                            guard let network, !network.ssid.isEmpty, network.ssid != self.lastHotspotSSID else { return }
+                            finish(true, "wifi")
+                        }
+                    }
+                }
             }
             monitor.start(queue: self.queue)
-            self.queue.asyncAfter(deadline: .now() + 15) { finish(false, "Cellular internet did not become the default route") }
+            self.queue.asyncAfter(deadline: .now() + 15) {
+                finish(false, requireCellular ? "Cellular internet did not become the default route" : "Internet did not return after leaving the glasses hotspot")
+            }
         }
     }
 
@@ -122,14 +141,18 @@ public final class GlassesHotspotNetwork {
         NEHotspotNetwork.fetchCurrent { [weak self] network in
             self?.queue.async {
                 guard let self, gen == self.generation, !self.cancelled else { return }
-                if network?.ssid == ssid, let address = Self.wifiAddress() {
+                if network?.ssid == ssid, let address = Self.wifiAddress(),
+                   self.gatewayAddress.map({ LocalMediaPolicy.isHotspotClientAddress(address, gateway: $0) }) ?? true
+                {
                     self.localAddress = address
                     self.startMonitor(generation: gen)
                     self.finishJoin(.success(address))
                 } else if remaining > 0 {
                     self.queue.asyncAfter(deadline: .now() + 0.5) { self.waitForAddress(ssid: ssid, generation: gen, remaining: remaining - 1) }
                 } else {
-                    self.finishJoin(.failure(LocalMediaError("Glasses hotspot has no verified Wi-Fi address")))
+                    let association = network == nil ? "unavailable" : (network?.ssid == ssid ? "matched" : "different")
+                    let address = Self.wifiAddress() ?? "none"
+                    self.finishJoin(.failure(LocalMediaError("Glasses hotspot has no verified Wi-Fi address (SSID=\(association), Wi-Fi IPv4=\(address))")))
                     self.finishLeave()
                 }
             }
@@ -160,6 +183,7 @@ public final class GlassesHotspotNetwork {
         if let ssid { NEHotspotConfigurationManager.shared.removeConfiguration(forSSID: ssid) }
         ssid = nil
         localAddress = nil
+        gatewayAddress = nil
         let replies = leaveReplies
         leaveReplies.removeAll()
         replies.forEach { $0() }

package/ios/GlassesMedia.podspec

@@ -11,7 +11,9 @@ Pod::Spec.new do |s|
   s.platforms = { :ios => '15.1' }
   s.swift_version = '5.9'
   s.static_framework = true
-  s.dependency 'WebRTC-SDK', '137.7151.09'
+  s.dependency 'ExpoModulesCore'
+  # Keep in sync with the iOS-only @livekit/react-native-webrtc podspec patch.
+  s.dependency 'WebRTC-SDK', '144.7559.15'
   s.frameworks = 'AVFoundation', 'CoreMedia', 'CoreVideo', 'Network', 'NetworkExtension'
   s.source_files = '*.{swift,h,m,mm}', 'CoreKit/Sources/GlassesMediaCore/*.swift'
   s.pod_target_xcconfig = { 'DEFINES_MODULE' => 'YES' }

package/ios/GlassesMediaRelayModule.swift

@@ -0,0 +1,133 @@
+import ExpoModulesCore
+import Foundation
+
+public final class GlassesMediaRelayModule: Module {
+    private let queue = DispatchQueue(label: "com.mentra.glassesmedia.relay")
+    private var session: ManagedRelaySession?
+
+    public func definition() -> ModuleDefinition {
+        Name("MentraGlassesMediaRelay")
+        Events("onRelayState")
+
+        AsyncFunction("prepare") { (options: [String: Any], promise: Promise) in
+            guard let id = options["attemptId"] as? String,
+                  let ingest = options["ingestUrl"] as? String, let url = URL(string: ingest), url.scheme == "https",
+                  let ssid = options["ssid"] as? String, let password = options["password"] as? String,
+                  let gateway = options["gatewayAddress"] as? String
+            else {
+                throw LocalMediaError("Invalid relay options")
+            }
+            self.queue.async {
+                guard self.session == nil else { promise.reject(LocalMediaError("Previous relay has not stopped")); return }
+                let session = ManagedRelaySession(id: id, queue: self.queue)
+                self.session = session
+                session.start(ssid: ssid, password: password, gateway: gateway, endpoint: url,
+                              captureAudio: options["captureAudio"] as? Bool ?? true,
+                              bitrate: options["bitrate"] as? Int ?? 2_000_000,
+                              onState: { [weak self] state, reason in
+                                  self?.sendEvent("onRelayState", ["attemptId": id, "state": state, "reason": reason])
+                              }) { result in
+                    switch result {
+                    case let .success(url): promise.resolve(url)
+                    case let .failure(error): promise.reject(error)
+                    }
+                }
+            }
+        }
+
+        AsyncFunction("stop") { (id: String, promise: Promise) in
+            self.queue.async {
+                guard let session = self.session, session.id == id else { promise.resolve(nil); return }
+                session.stop {
+                    if self.session === session { self.session = nil }
+                    promise.resolve(nil)
+                }
+            }
+        }
+
+        OnDestroy {
+            self.queue.async {
+                self.session?.stop { self.session = nil }
+            }
+        }
+    }
+}
+
+/// Serialized owner of both native peers and the persistent hotspot join; no JS media callbacks.
+private final class ManagedRelaySession {
+    let id: String
+    private let queue: DispatchQueue
+    private let hotspot = GlassesHotspotNetwork()
+    private var receiver: LocalWhipIngestSource?
+    private var publisher: PhoneWhipPublisher?
+    private var stopped = false
+    private var stopFinished = false
+    private var stopReplies: [() -> Void] = []
+    private var ready: ((Result<String, Error>) -> Void)?
+
+    init(id: String, queue: DispatchQueue) {
+        self.id = id; self.queue = queue
+    }
+
+    func start(ssid: String, password: String, gateway: String, endpoint: URL, captureAudio: Bool, bitrate: Int,
+               onState: @escaping (String, String) -> Void, completion: @escaping (Result<String, Error>) -> Void)
+    {
+        ready = completion
+        hotspot.onLost = { reason in onState("failed", reason) }
+        hotspot.join(ssid: ssid, passphrase: password, gateway: gateway) { result in
+            self.queue.async {
+                guard !self.stopped else { return }
+                switch result {
+                case let .failure(error): self.finish(.failure(error))
+                case let .success(address):
+                    self.hotspot.awaitInternet { usable, _ in
+                        self.queue.async {
+                            guard !self.stopped else { return }
+                            guard usable else { self.finish(.failure(LocalMediaError("Turn on phone mobile data to stream through the glasses hotspot"))); return }
+                            let publisher = PhoneWhipPublisher(endpoint: endpoint, captureAudio: captureAudio, bitrate: bitrate, onState: onState)
+                            self.publisher = publisher
+                            let receiver = LocalWhipIngestSource()
+                            self.receiver = receiver
+                            receiver.onFrame = { [weak publisher] in publisher?.onFrame($0) }
+                            receiver.onPcm = { [weak publisher] in publisher?.onPcm($0, rate: $1, channels: $2) }
+                            receiver.onStateChange = { state, reason in if state == .failed { onState("failed", "Glasses receiver: \(reason)") } }
+                            receiver.setPcmDeliveryEnabled(captureAudio)
+                            receiver.prepare(config: SourceConfig(url: "", kind: .softap, bindAddress: address)) { result in
+                                self.queue.async {
+                                    guard !self.stopped else { return }
+                                    if case .success = result { publisher.start() }
+                                    self.finish(result)
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    private func finish(_ result: Result<String, Error>) {
+        let callback = ready; ready = nil; callback?(result)
+    }
+
+    func stop(completion: @escaping () -> Void) {
+        if stopFinished { completion(); return }
+        stopReplies.append(completion)
+        guard !stopped else { return }
+        stopped = true
+        finish(.failure(LocalMediaError("Relay cancelled")))
+        hotspot.onLost = nil
+        let group = DispatchGroup()
+        if let receiver {
+            group.enter(); receiver.stop { group.leave() }
+        }
+        if let publisher {
+            group.enter(); publisher.stop { group.leave() }
+        }
+        group.enter(); hotspot.leave { group.leave() }
+        group.notify(queue: queue) {
+            self.receiver = nil; self.publisher = nil; self.stopFinished = true
+            let replies = self.stopReplies; self.stopReplies.removeAll(); replies.forEach { $0() }
+        }
+    }
+}

package/ios/GlassesPeerFactory.swift

@@ -0,0 +1,23 @@
+import WebRTC
+
+/// Both legs need real interface enumeration: local Wi-Fi can be absent from the default
+/// internet NWPath while cellular carries the uplink. Each factory's adapter mask and the
+/// receiver's SDP policy still restrict candidates to the appropriate leg.
+enum GlassesPeerFactory {
+    /// M144 honors this field trial for factories with custom audio devices; M137 installed
+    /// NWPathMonitor unconditionally (including when disableNetworkMonitor was set).
+    /// The SDK setting is process-wide. Set it once and never toggle it around factory creation:
+    /// receiver, relay, and React Native WebRTC factories may initialize concurrently.
+    private static let configure: Void = {
+        RTCPeerConnectionFactory.configureFieldTrials("WebRTC-Network-UseNWPathMonitor/Disabled/")
+    }()
+
+    static func make(audioDevice: RTCAudioDevice) -> RTCPeerConnectionFactory {
+        _ = configure
+        return RTCPeerConnectionFactory(
+            encoderFactory: RTCDefaultVideoEncoderFactory(),
+            decoderFactory: RTCDefaultVideoDecoderFactory(),
+            audioDevice: audioDevice
+        )
+    }
+}

package/ios/LocalWhipIngestSource.swift

@@ -67,7 +67,7 @@ public final class LocalWhipIngestSource: NSObject, DecodedGlassesMediaSource {
             let gen = self.generation
             self.transition(.connecting, "listening")
             RTCInitializeSSL()
-            let factory = RTCPeerConnectionFactory(encoderFactory: RTCDefaultVideoEncoderFactory(), decoderFactory: RTCDefaultVideoDecoderFactory(), audioDevice: ReceiveOnlyAudioDevice())
+            let factory = GlassesPeerFactory.make(audioDevice: ReceiveOnlyAudioDevice())
             let options = RTCPeerConnectionFactoryOptions()
             options.ignoreCellularNetworkAdapter = true
             options.ignoreVPNNetworkAdapter = true