@mentra/glasses-media 3.2.0-dev.225 → 3.2.0-dev.227
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -3
- package/android/src/main/AndroidManifest.xml +1 -0
- package/android/src/main/java/com/mentra/glassesmedia/GlassesMediaRelayModule.kt +113 -0
- package/android/src/main/java/com/mentra/glassesmedia/network/ScopedNetworkChangeDetector.kt +34 -4
- package/android/src/main/java/com/mentra/glassesmedia/network/ScopedNetworkObserver.kt +11 -4
- package/android/src/main/java/com/mentra/glassesmedia/network/ScopedNetworkRegistry.kt +16 -0
- package/android/src/main/java/com/mentra/glassesmedia/publisher/PhoneWhipPublisher.kt +263 -0
- package/android/src/main/java/com/mentra/glassesmedia/publisher/RelayPcmBuffer.kt +54 -0
- package/android/src/main/java/com/mentra/glassesmedia/source/LocalWhipIngestSource.kt +16 -3
- package/android/src/main/java/com/mentra/glassesmedia/source/WhipIngestServer.kt +31 -15
- package/expo-module.config.json +6 -2
- package/ios/CoreKit/Sources/GlassesMediaCore/LocalMediaPolicy.swift +8 -0
- package/ios/CoreKit/Sources/GlassesMediaCore/RelayAudioClock.swift +56 -0
- package/ios/CoreKit/Sources/GlassesMediaCore/RelayPcmBuffer.swift +57 -0
- package/ios/GlassesHotspotNetwork.swift +11 -3
- package/ios/GlassesMedia.podspec +1 -0
- package/ios/GlassesMediaRelayModule.swift +133 -0
- package/ios/PhoneWhipPublisher.swift +202 -0
- package/ios/RelayAudioDevice.swift +70 -0
- package/package.json +1 -1
|
@@ -69,6 +69,7 @@ class WhipIngestServer(
|
|
|
69
69
|
private val connections = Executors.newCachedThreadPool { runnable ->
|
|
70
70
|
Thread(runnable, "whip-ingest-conn").apply { isDaemon = true }
|
|
71
71
|
}
|
|
72
|
+
private val activeSockets = java.util.concurrent.ConcurrentHashMap.newKeySet<Socket>()
|
|
72
73
|
private val accepted = AtomicInteger()
|
|
73
74
|
|
|
74
75
|
/** Bound endpoint, or null before [start]. This is the URL the glasses must be told to POST to. */
|
|
@@ -141,6 +142,13 @@ class WhipIngestServer(
|
|
|
141
142
|
closeListener()
|
|
142
143
|
}
|
|
143
144
|
|
|
145
|
+
/** Hard teardown barrier: no request may still create a peer after this returns. */
|
|
146
|
+
fun closeAndAwait(timeoutMs: Long = 15_000): Boolean {
|
|
147
|
+
closeNow()
|
|
148
|
+
connections.shutdownNow()
|
|
149
|
+
return connections.awaitTermination(timeoutMs, TimeUnit.MILLISECONDS)
|
|
150
|
+
}
|
|
151
|
+
|
|
144
152
|
private fun closeListener() {
|
|
145
153
|
val socket = synchronized(lock) {
|
|
146
154
|
val current = server
|
|
@@ -149,6 +157,7 @@ class WhipIngestServer(
|
|
|
149
157
|
current
|
|
150
158
|
} ?: return
|
|
151
159
|
runCatching { socket.close() }
|
|
160
|
+
activeSockets.forEach { runCatching { it.close() } }
|
|
152
161
|
connections.shutdownNow()
|
|
153
162
|
SoftApTrace.stage("whip_listener_closed", "accepted" to accepted.get())
|
|
154
163
|
}
|
|
@@ -163,6 +172,10 @@ class WhipIngestServer(
|
|
|
163
172
|
Log.w(TAG, "accept failed", error)
|
|
164
173
|
return
|
|
165
174
|
}
|
|
175
|
+
val registered = synchronized(lock) {
|
|
176
|
+
if (server !== socket) false else { activeSockets.add(connection); true }
|
|
177
|
+
}
|
|
178
|
+
if (!registered) { runCatching { connection.close() }; return }
|
|
166
179
|
accepted.incrementAndGet()
|
|
167
180
|
// Each connection on its own thread: a negotiation blocks for the length of an ICE gather,
|
|
168
181
|
// and a DELETE arriving during one must not queue behind it.
|
|
@@ -170,28 +183,31 @@ class WhipIngestServer(
|
|
|
170
183
|
connections.execute { serve(connection) }
|
|
171
184
|
} catch (_: java.util.concurrent.RejectedExecutionException) {
|
|
172
185
|
runCatching { connection.close() }
|
|
186
|
+
activeSockets.remove(connection)
|
|
173
187
|
}
|
|
174
188
|
}
|
|
175
189
|
}
|
|
176
190
|
|
|
177
191
|
private fun serve(connection: Socket) {
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
+
try {
|
|
193
|
+
connection.use { socket ->
|
|
194
|
+
socket.soTimeout = READ_TIMEOUT_MS
|
|
195
|
+
val output = BufferedOutputStream(socket.getOutputStream())
|
|
196
|
+
try {
|
|
197
|
+
val request = readRequest(socket.getInputStream())
|
|
198
|
+
if (request == null) {
|
|
199
|
+
write(output, WhipIngestProtocol.Response(400, "Bad Request", body = "malformed request"))
|
|
200
|
+
return
|
|
201
|
+
}
|
|
202
|
+
write(output, handle(request))
|
|
203
|
+
} catch (error: Exception) {
|
|
204
|
+
Log.w(TAG, "connection failed", error)
|
|
205
|
+
runCatching {
|
|
206
|
+
write(output, WhipIngestProtocol.Response(400, "Bad Request", body = "read failed"))
|
|
207
|
+
}
|
|
192
208
|
}
|
|
193
209
|
}
|
|
194
|
-
}
|
|
210
|
+
} finally { activeSockets.remove(connection) }
|
|
195
211
|
}
|
|
196
212
|
|
|
197
213
|
private fun handle(request: WhipIngestProtocol.Request): WhipIngestProtocol.Response {
|
package/expo-module.config.json
CHANGED
|
@@ -31,6 +31,14 @@ public enum LocalMediaPolicy {
|
|
|
31
31
|
(bytes[0] == 192 && bytes[1] == 168)
|
|
32
32
|
}
|
|
33
33
|
|
|
34
|
+
/// SSID association can finish before DHCP replaces the previous Wi-Fi address.
|
|
35
|
+
/// Accept only a client address on the /24 advertised by the glasses over BLE.
|
|
36
|
+
public static func isHotspotClientAddress(_ address: String, gateway: String) -> Bool {
|
|
37
|
+
guard isPrivate(gateway), sameSubnet(address, gateway), address != gateway,
|
|
38
|
+
let bytes = ipv4(address) else { return false }
|
|
39
|
+
return bytes[3] > 0 && bytes[3] < 255
|
|
40
|
+
}
|
|
41
|
+
|
|
34
42
|
/// Keep only concrete host candidates on the local link. Never rewrite a cellular socket's
|
|
35
43
|
/// advertised address: a candidate must already belong to the interface it claims.
|
|
36
44
|
public static func localSdp(_ sdp: String, address: String, answer: Bool) throws -> String {
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import Foundation
|
|
2
|
+
|
|
3
|
+
/// A paced audio worker whose stop is a barrier for the final render callback and its resources.
|
|
4
|
+
public final class RelayAudioClock {
|
|
5
|
+
private let condition = NSCondition()
|
|
6
|
+
private var quitting = false
|
|
7
|
+
private var exited = true
|
|
8
|
+
private var enabled = false
|
|
9
|
+
|
|
10
|
+
public init() {}
|
|
11
|
+
|
|
12
|
+
public func start(render: @escaping (Double) -> Void, finish: @escaping () -> Void = {}) {
|
|
13
|
+
condition.lock()
|
|
14
|
+
precondition(exited, "Previous audio worker must stop before reinitialization")
|
|
15
|
+
quitting = false; enabled = false; exited = false
|
|
16
|
+
condition.unlock()
|
|
17
|
+
let thread = Thread { [self] in
|
|
18
|
+
defer {
|
|
19
|
+
finish()
|
|
20
|
+
condition.lock(); exited = true; condition.broadcast(); condition.unlock()
|
|
21
|
+
}
|
|
22
|
+
var sampleTime: Double = 0
|
|
23
|
+
var next = ProcessInfo.processInfo.systemUptime
|
|
24
|
+
while true {
|
|
25
|
+
condition.lock()
|
|
26
|
+
while !enabled && !quitting {
|
|
27
|
+
condition.wait(); next = ProcessInfo.processInfo.systemUptime
|
|
28
|
+
}
|
|
29
|
+
let stop = quitting
|
|
30
|
+
condition.unlock()
|
|
31
|
+
if stop { return }
|
|
32
|
+
render(sampleTime)
|
|
33
|
+
sampleTime += 480
|
|
34
|
+
next += 0.01
|
|
35
|
+
let now = ProcessInfo.processInfo.systemUptime
|
|
36
|
+
if next > now { Thread.sleep(forTimeInterval: next - now) }
|
|
37
|
+
else if now - next > 0.1 { next = now }
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
thread.name = "Mentra glasses audio publish"
|
|
41
|
+
thread.qualityOfService = .userInitiated
|
|
42
|
+
thread.start()
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
public func setEnabled(_ value: Bool) {
|
|
46
|
+
condition.lock(); enabled = value; condition.broadcast(); condition.unlock()
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
public func stop() {
|
|
50
|
+
condition.lock(); quitting = true; enabled = false; condition.broadcast()
|
|
51
|
+
while !exited {
|
|
52
|
+
condition.wait()
|
|
53
|
+
}
|
|
54
|
+
condition.unlock()
|
|
55
|
+
}
|
|
56
|
+
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import Foundation
|
|
2
|
+
|
|
3
|
+
/// Bounded decoded audio queue. Converts PCM16 to 48kHz mono without touching phone capture.
|
|
4
|
+
public final class RelayPcmBuffer {
|
|
5
|
+
private let lock = NSLock()
|
|
6
|
+
private var samples: [Int16]
|
|
7
|
+
private var head = 0
|
|
8
|
+
private var count = 0
|
|
9
|
+
private var phase = 0.0
|
|
10
|
+
private var rate = 0
|
|
11
|
+
private var channels = 0
|
|
12
|
+
private var previous: Double?
|
|
13
|
+
|
|
14
|
+
public init(capacity: Int = 9600) {
|
|
15
|
+
samples = Array(repeating: 0, count: capacity)
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
public func push(_ data: Data, sampleRate: Int, channels channelCount: Int) {
|
|
19
|
+
guard (8000 ... 192_000).contains(sampleRate), (1 ... 8).contains(channelCount) else { return }
|
|
20
|
+
lock.lock(); defer { lock.unlock() }
|
|
21
|
+
if rate != sampleRate || channels != channelCount {
|
|
22
|
+
rate = sampleRate; channels = channelCount; phase = 0; previous = nil
|
|
23
|
+
}
|
|
24
|
+
let step = Double(sampleRate) / 48000
|
|
25
|
+
data.withUnsafeBytes { raw in
|
|
26
|
+
let bytes = raw.bindMemory(to: UInt8.self)
|
|
27
|
+
for frame in 0 ..< data.count / (2 * channelCount) {
|
|
28
|
+
var sum = 0.0
|
|
29
|
+
for channel in 0 ..< channelCount {
|
|
30
|
+
let offset = (frame * channelCount + channel) * 2
|
|
31
|
+
sum += Double(Int16(bitPattern: UInt16(bytes[offset]) | UInt16(bytes[offset + 1]) << 8))
|
|
32
|
+
}
|
|
33
|
+
let value = sum / Double(channelCount)
|
|
34
|
+
if let prior = previous {
|
|
35
|
+
while phase < 1 {
|
|
36
|
+
if count == samples.count { head = (head + 1) % samples.count; count -= 1 }
|
|
37
|
+
samples[(head + count) % samples.count] = Int16(prior + (value - prior) * phase)
|
|
38
|
+
count += 1
|
|
39
|
+
phase += step
|
|
40
|
+
}
|
|
41
|
+
phase -= 1
|
|
42
|
+
}
|
|
43
|
+
previous = value
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/// Pads underruns with silence. Drops oldest samples on overrun to bound live latency.
|
|
49
|
+
public func read(into output: UnsafeMutableBufferPointer<Int16>) {
|
|
50
|
+
lock.lock(); defer { lock.unlock() }
|
|
51
|
+
for i in output.indices {
|
|
52
|
+
if count > 0 {
|
|
53
|
+
output[i] = samples[head]; head = (head + 1) % samples.count; count -= 1
|
|
54
|
+
} else { output[i] = 0 }
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
@@ -8,6 +8,7 @@ public final class GlassesHotspotNetwork {
|
|
|
8
8
|
private let queue = DispatchQueue(label: "com.mentra.glassesmedia.hotspot")
|
|
9
9
|
private var ssid: String?
|
|
10
10
|
private var localAddress: String?
|
|
11
|
+
private var gatewayAddress: String?
|
|
11
12
|
private var generation = 0
|
|
12
13
|
private var applying = false
|
|
13
14
|
private var cancelled = false
|
|
@@ -17,12 +18,16 @@ public final class GlassesHotspotNetwork {
|
|
|
17
18
|
public var onLost: ((String) -> Void)?
|
|
18
19
|
public init() {}
|
|
19
20
|
|
|
20
|
-
public func join(ssid: String, passphrase: String, completion: @escaping (Result<String, Error>) -> Void) {
|
|
21
|
+
public func join(ssid: String, passphrase: String, gateway: String? = nil, completion: @escaping (Result<String, Error>) -> Void) {
|
|
21
22
|
queue.async {
|
|
22
23
|
guard self.ssid == nil, !self.applying else { completion(.failure(LocalMediaError("Previous hotspot session has not finished cleaning up"))); return }
|
|
24
|
+
if let gateway, !LocalMediaPolicy.isPrivate(gateway) {
|
|
25
|
+
completion(.failure(LocalMediaError("The glasses reported an invalid hotspot gateway"))); return
|
|
26
|
+
}
|
|
23
27
|
self.generation += 1
|
|
24
28
|
let gen = self.generation
|
|
25
29
|
self.ssid = ssid
|
|
30
|
+
self.gatewayAddress = gateway
|
|
26
31
|
self.cancelled = false
|
|
27
32
|
self.applying = true
|
|
28
33
|
self.joinReply = completion
|
|
@@ -75,7 +80,7 @@ public final class GlassesHotspotNetwork {
|
|
|
75
80
|
public func probeGateway(completion: @escaping (Bool, String) -> Void) {
|
|
76
81
|
queue.async {
|
|
77
82
|
guard let address = self.localAddress else { completion(false, "No joined hotspot"); return }
|
|
78
|
-
let gateway = address.split(separator: ".").prefix(3).joined(separator: ".") + ".1"
|
|
83
|
+
let gateway = self.gatewayAddress ?? address.split(separator: ".").prefix(3).joined(separator: ".") + ".1"
|
|
79
84
|
let parameters = NWParameters.tcp
|
|
80
85
|
parameters.requiredInterfaceType = .wifi
|
|
81
86
|
let connection = NWConnection(host: NWEndpoint.Host(gateway), port: 8089, using: parameters)
|
|
@@ -122,7 +127,9 @@ public final class GlassesHotspotNetwork {
|
|
|
122
127
|
NEHotspotNetwork.fetchCurrent { [weak self] network in
|
|
123
128
|
self?.queue.async {
|
|
124
129
|
guard let self, gen == self.generation, !self.cancelled else { return }
|
|
125
|
-
if network?.ssid == ssid, let address = Self.wifiAddress()
|
|
130
|
+
if network?.ssid == ssid, let address = Self.wifiAddress(),
|
|
131
|
+
self.gatewayAddress.map({ LocalMediaPolicy.isHotspotClientAddress(address, gateway: $0) }) ?? true
|
|
132
|
+
{
|
|
126
133
|
self.localAddress = address
|
|
127
134
|
self.startMonitor(generation: gen)
|
|
128
135
|
self.finishJoin(.success(address))
|
|
@@ -160,6 +167,7 @@ public final class GlassesHotspotNetwork {
|
|
|
160
167
|
if let ssid { NEHotspotConfigurationManager.shared.removeConfiguration(forSSID: ssid) }
|
|
161
168
|
ssid = nil
|
|
162
169
|
localAddress = nil
|
|
170
|
+
gatewayAddress = nil
|
|
163
171
|
let replies = leaveReplies
|
|
164
172
|
leaveReplies.removeAll()
|
|
165
173
|
replies.forEach { $0() }
|
package/ios/GlassesMedia.podspec
CHANGED
|
@@ -11,6 +11,7 @@ Pod::Spec.new do |s|
|
|
|
11
11
|
s.platforms = { :ios => '15.1' }
|
|
12
12
|
s.swift_version = '5.9'
|
|
13
13
|
s.static_framework = true
|
|
14
|
+
s.dependency 'ExpoModulesCore'
|
|
14
15
|
s.dependency 'WebRTC-SDK', '137.7151.09'
|
|
15
16
|
s.frameworks = 'AVFoundation', 'CoreMedia', 'CoreVideo', 'Network', 'NetworkExtension'
|
|
16
17
|
s.source_files = '*.{swift,h,m,mm}', 'CoreKit/Sources/GlassesMediaCore/*.swift'
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import ExpoModulesCore
|
|
2
|
+
import Foundation
|
|
3
|
+
|
|
4
|
+
public final class GlassesMediaRelayModule: Module {
|
|
5
|
+
private let queue = DispatchQueue(label: "com.mentra.glassesmedia.relay")
|
|
6
|
+
private var session: ManagedRelaySession?
|
|
7
|
+
|
|
8
|
+
public func definition() -> ModuleDefinition {
|
|
9
|
+
Name("MentraGlassesMediaRelay")
|
|
10
|
+
Events("onRelayState")
|
|
11
|
+
|
|
12
|
+
AsyncFunction("prepare") { (options: [String: Any], promise: Promise) in
|
|
13
|
+
guard let id = options["attemptId"] as? String,
|
|
14
|
+
let ingest = options["ingestUrl"] as? String, let url = URL(string: ingest), url.scheme == "https",
|
|
15
|
+
let ssid = options["ssid"] as? String, let password = options["password"] as? String,
|
|
16
|
+
let gateway = options["gatewayAddress"] as? String
|
|
17
|
+
else {
|
|
18
|
+
throw LocalMediaError("Invalid relay options")
|
|
19
|
+
}
|
|
20
|
+
self.queue.async {
|
|
21
|
+
guard self.session == nil else { promise.reject(LocalMediaError("Previous relay has not stopped")); return }
|
|
22
|
+
let session = ManagedRelaySession(id: id, queue: self.queue)
|
|
23
|
+
self.session = session
|
|
24
|
+
session.start(ssid: ssid, password: password, gateway: gateway, endpoint: url,
|
|
25
|
+
captureAudio: options["captureAudio"] as? Bool ?? true,
|
|
26
|
+
bitrate: options["bitrate"] as? Int ?? 2_000_000,
|
|
27
|
+
onState: { [weak self] state, reason in
|
|
28
|
+
self?.sendEvent("onRelayState", ["attemptId": id, "state": state, "reason": reason])
|
|
29
|
+
}) { result in
|
|
30
|
+
switch result {
|
|
31
|
+
case let .success(url): promise.resolve(url)
|
|
32
|
+
case let .failure(error): promise.reject(error)
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
AsyncFunction("stop") { (id: String, promise: Promise) in
|
|
39
|
+
self.queue.async {
|
|
40
|
+
guard let session = self.session, session.id == id else { promise.resolve(nil); return }
|
|
41
|
+
session.stop {
|
|
42
|
+
if self.session === session { self.session = nil }
|
|
43
|
+
promise.resolve(nil)
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
OnDestroy {
|
|
49
|
+
self.queue.async {
|
|
50
|
+
self.session?.stop { self.session = nil }
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/// Serialized owner of both native peers and the persistent hotspot join; no JS media callbacks.
|
|
57
|
+
private final class ManagedRelaySession {
|
|
58
|
+
let id: String
|
|
59
|
+
private let queue: DispatchQueue
|
|
60
|
+
private let hotspot = GlassesHotspotNetwork()
|
|
61
|
+
private var receiver: LocalWhipIngestSource?
|
|
62
|
+
private var publisher: PhoneWhipPublisher?
|
|
63
|
+
private var stopped = false
|
|
64
|
+
private var stopFinished = false
|
|
65
|
+
private var stopReplies: [() -> Void] = []
|
|
66
|
+
private var ready: ((Result<String, Error>) -> Void)?
|
|
67
|
+
|
|
68
|
+
init(id: String, queue: DispatchQueue) {
|
|
69
|
+
self.id = id; self.queue = queue
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
func start(ssid: String, password: String, gateway: String, endpoint: URL, captureAudio: Bool, bitrate: Int,
|
|
73
|
+
onState: @escaping (String, String) -> Void, completion: @escaping (Result<String, Error>) -> Void)
|
|
74
|
+
{
|
|
75
|
+
ready = completion
|
|
76
|
+
hotspot.onLost = { reason in onState("failed", reason) }
|
|
77
|
+
hotspot.join(ssid: ssid, passphrase: password, gateway: gateway) { result in
|
|
78
|
+
self.queue.async {
|
|
79
|
+
guard !self.stopped else { return }
|
|
80
|
+
switch result {
|
|
81
|
+
case let .failure(error): self.finish(.failure(error))
|
|
82
|
+
case let .success(address):
|
|
83
|
+
self.hotspot.awaitInternet { usable, _ in
|
|
84
|
+
self.queue.async {
|
|
85
|
+
guard !self.stopped else { return }
|
|
86
|
+
guard usable else { self.finish(.failure(LocalMediaError("Turn on phone mobile data to stream through the glasses hotspot"))); return }
|
|
87
|
+
let publisher = PhoneWhipPublisher(endpoint: endpoint, captureAudio: captureAudio, bitrate: bitrate, onState: onState)
|
|
88
|
+
self.publisher = publisher
|
|
89
|
+
let receiver = LocalWhipIngestSource()
|
|
90
|
+
self.receiver = receiver
|
|
91
|
+
receiver.onFrame = { [weak publisher] in publisher?.onFrame($0) }
|
|
92
|
+
receiver.onPcm = { [weak publisher] in publisher?.onPcm($0, rate: $1, channels: $2) }
|
|
93
|
+
receiver.onStateChange = { state, reason in if state == .failed { onState("failed", "Glasses receiver: \(reason)") } }
|
|
94
|
+
receiver.setPcmDeliveryEnabled(captureAudio)
|
|
95
|
+
receiver.prepare(config: SourceConfig(url: "", kind: .softap, bindAddress: address)) { result in
|
|
96
|
+
self.queue.async {
|
|
97
|
+
guard !self.stopped else { return }
|
|
98
|
+
if case .success = result { publisher.start() }
|
|
99
|
+
self.finish(result)
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
private func finish(_ result: Result<String, Error>) {
|
|
110
|
+
let callback = ready; ready = nil; callback?(result)
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
func stop(completion: @escaping () -> Void) {
|
|
114
|
+
if stopFinished { completion(); return }
|
|
115
|
+
stopReplies.append(completion)
|
|
116
|
+
guard !stopped else { return }
|
|
117
|
+
stopped = true
|
|
118
|
+
finish(.failure(LocalMediaError("Relay cancelled")))
|
|
119
|
+
hotspot.onLost = nil
|
|
120
|
+
let group = DispatchGroup()
|
|
121
|
+
if let receiver {
|
|
122
|
+
group.enter(); receiver.stop { group.leave() }
|
|
123
|
+
}
|
|
124
|
+
if let publisher {
|
|
125
|
+
group.enter(); publisher.stop { group.leave() }
|
|
126
|
+
}
|
|
127
|
+
group.enter(); hotspot.leave { group.leave() }
|
|
128
|
+
group.notify(queue: queue) {
|
|
129
|
+
self.receiver = nil; self.publisher = nil; self.stopFinished = true
|
|
130
|
+
let replies = self.stopReplies; self.stopReplies.removeAll(); replies.forEach { $0() }
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
}
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
import CoreVideo
|
|
2
|
+
import Foundation
|
|
3
|
+
import WebRTC
|
|
4
|
+
|
|
5
|
+
/// ASG's full-ICE WHIP negotiation, fed from the shared decoded receiver. One peer per attempt.
|
|
6
|
+
final class PhoneWhipPublisher: NSObject {
|
|
7
|
+
private let queue = DispatchQueue(label: "com.mentra.glassesmedia.whip.publish")
|
|
8
|
+
private let frameSlot = DispatchSemaphore(value: 1)
|
|
9
|
+
private let endpoint: URL
|
|
10
|
+
private let captureAudio: Bool
|
|
11
|
+
private let bitrate: Int
|
|
12
|
+
private let onState: (String, String) -> Void
|
|
13
|
+
private let audioDevice = RelayAudioDevice()
|
|
14
|
+
private var factory: RTCPeerConnectionFactory?
|
|
15
|
+
private var peer: RTCPeerConnection?
|
|
16
|
+
private var videoSource: RTCVideoSource?
|
|
17
|
+
private var capturer: RTCVideoCapturer?
|
|
18
|
+
private var resource: URL?
|
|
19
|
+
private var posted = false
|
|
20
|
+
private var localSet = false
|
|
21
|
+
private var stopped = false
|
|
22
|
+
private var failed = false
|
|
23
|
+
private var connected = false
|
|
24
|
+
private var disconnectGeneration = 0
|
|
25
|
+
private var lastTimestamp: Int64 = 0
|
|
26
|
+
private let http: URLSession
|
|
27
|
+
|
|
28
|
+
init(endpoint: URL, captureAudio: Bool, bitrate: Int, onState: @escaping (String, String) -> Void) {
|
|
29
|
+
self.endpoint = endpoint; self.captureAudio = captureAudio; self.bitrate = bitrate; self.onState = onState
|
|
30
|
+
let config = URLSessionConfiguration.ephemeral
|
|
31
|
+
config.allowsCellularAccess = true
|
|
32
|
+
config.timeoutIntervalForRequest = 20
|
|
33
|
+
config.timeoutIntervalForResource = 25
|
|
34
|
+
http = URLSession(configuration: config)
|
|
35
|
+
super.init()
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
func start() {
|
|
39
|
+
queue.async {
|
|
40
|
+
guard !self.stopped else { return }
|
|
41
|
+
RTCInitializeSSL()
|
|
42
|
+
let factory = RTCPeerConnectionFactory(encoderFactory: RTCDefaultVideoEncoderFactory(), decoderFactory: RTCDefaultVideoDecoderFactory(), audioDevice: self.audioDevice)
|
|
43
|
+
let options = RTCPeerConnectionFactoryOptions()
|
|
44
|
+
options.ignoreWiFiNetworkAdapter = true
|
|
45
|
+
options.ignoreVPNNetworkAdapter = true
|
|
46
|
+
options.ignoreLoopbackNetworkAdapter = true
|
|
47
|
+
factory.setOptions(options)
|
|
48
|
+
self.factory = factory
|
|
49
|
+
let config = RTCConfiguration()
|
|
50
|
+
config.sdpSemantics = .unifiedPlan
|
|
51
|
+
config.iceServers = [RTCIceServer(urlStrings: ["stun:stun.cloudflare.com:3478"])]
|
|
52
|
+
let constraints = RTCMediaConstraints(mandatoryConstraints: nil, optionalConstraints: nil)
|
|
53
|
+
guard let peer = factory.peerConnection(with: config, constraints: constraints, delegate: self) else {
|
|
54
|
+
self.fail("Could not create WHIP peer"); return
|
|
55
|
+
}
|
|
56
|
+
self.peer = peer
|
|
57
|
+
let source = factory.videoSource()
|
|
58
|
+
self.videoSource = source
|
|
59
|
+
self.capturer = RTCVideoCapturer(delegate: source)
|
|
60
|
+
let video = factory.videoTrack(with: source, trackId: "glasses-video")
|
|
61
|
+
let sendOnly = RTCRtpTransceiverInit()
|
|
62
|
+
sendOnly.direction = .sendOnly
|
|
63
|
+
let transceiver = peer.addTransceiver(with: video, init: sendOnly)
|
|
64
|
+
let codecs = factory.rtpSenderCapabilities(forKind: kRTCMediaStreamTrackKindVideo).codecs.filter { $0.name.lowercased() == "h264" }
|
|
65
|
+
if !codecs.isEmpty { try? transceiver?.setCodecPreferences(codecs, error: ()) }
|
|
66
|
+
if let sender = transceiver?.sender {
|
|
67
|
+
let parameters = sender.parameters
|
|
68
|
+
parameters.encodings.forEach { $0.maxBitrateBps = NSNumber(value: min(12_000_000, max(250_000, self.bitrate))) }
|
|
69
|
+
sender.parameters = parameters
|
|
70
|
+
}
|
|
71
|
+
if self.captureAudio {
|
|
72
|
+
let audioConstraints = RTCMediaConstraints(mandatoryConstraints: ["googEchoCancellation": "false", "googNoiseSuppression": "false", "googAutoGainControl": "false"], optionalConstraints: nil)
|
|
73
|
+
let audio = factory.audioTrack(with: factory.audioSource(with: audioConstraints), trackId: "glasses-audio")
|
|
74
|
+
peer.addTransceiver(with: audio, init: sendOnly)
|
|
75
|
+
}
|
|
76
|
+
peer.offer(for: constraints) { [weak self] offer, error in
|
|
77
|
+
self?.queue.async {
|
|
78
|
+
guard let self, !self.stopped else { return }
|
|
79
|
+
guard let offer, error == nil else { self.fail("WHIP offer failed"); return }
|
|
80
|
+
peer.setLocalDescription(offer) { error in
|
|
81
|
+
self.queue.async {
|
|
82
|
+
guard !self.stopped else { return }
|
|
83
|
+
guard error == nil else { self.fail("WHIP local description failed"); return }
|
|
84
|
+
self.localSet = true; self.maybePost()
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
self.queue.asyncAfter(deadline: .now() + 35) {
|
|
90
|
+
if !self.connected { self.fail("WHIP connection timed out") }
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
func onFrame(_ buffer: CVPixelBuffer) {
|
|
96
|
+
guard frameSlot.wait(timeout: .now()) == .success else { return }
|
|
97
|
+
queue.async {
|
|
98
|
+
defer { self.frameSlot.signal() }
|
|
99
|
+
guard !self.stopped, let source = self.videoSource, let capturer = self.capturer else { return }
|
|
100
|
+
self.lastTimestamp = max(Int64(ProcessInfo.processInfo.systemUptime * 1_000_000_000), self.lastTimestamp + 1)
|
|
101
|
+
let frame = RTCVideoFrame(buffer: RTCCVPixelBuffer(pixelBuffer: buffer), rotation: ._0, timeStampNs: self.lastTimestamp)
|
|
102
|
+
source.capturer(capturer, didCapture: frame)
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
func onPcm(_ data: Data, rate: Int, channels: Int) {
|
|
107
|
+
if captureAudio { audioDevice.pcm.push(data, sampleRate: rate, channels: channels) }
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
private func maybePost() {
|
|
111
|
+
guard !stopped, !posted, localSet, let peer, peer.iceGatheringState == .complete, let offer = peer.localDescription else { return }
|
|
112
|
+
posted = true
|
|
113
|
+
var request = URLRequest(url: endpoint)
|
|
114
|
+
request.httpMethod = "POST"
|
|
115
|
+
request.setValue("application/sdp", forHTTPHeaderField: "Content-Type")
|
|
116
|
+
request.httpBody = Data(offer.sdp.utf8)
|
|
117
|
+
// Retain this attempt through a late response, so stop cannot orphan a created WHIP resource.
|
|
118
|
+
http.dataTask(with: request) { data, response, error in
|
|
119
|
+
self.queue.async {
|
|
120
|
+
let response = response as? HTTPURLResponse
|
|
121
|
+
let location = response?.value(forHTTPHeaderField: "Location").flatMap { URL(string: $0, relativeTo: self.endpoint)?.absoluteURL }
|
|
122
|
+
if self.stopped {
|
|
123
|
+
if let location { self.delete(location) }
|
|
124
|
+
self.http.finishTasksAndInvalidate()
|
|
125
|
+
return
|
|
126
|
+
}
|
|
127
|
+
guard error == nil, response?.statusCode == 201, let location, location.scheme == "https",
|
|
128
|
+
let data, let answer = String(data: data, encoding: .utf8), !answer.isEmpty
|
|
129
|
+
else {
|
|
130
|
+
if let location { self.delete(location) }
|
|
131
|
+
self.fail("WHIP server rejected publish (HTTP \(response?.statusCode ?? 0))"); return
|
|
132
|
+
}
|
|
133
|
+
self.resource = location
|
|
134
|
+
peer.setRemoteDescription(RTCSessionDescription(type: .answer, sdp: answer)) { error in
|
|
135
|
+
self.queue.async { if error != nil { self.fail("WHIP answer failed") } }
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}.resume()
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
private func delete(_ url: URL) {
|
|
142
|
+
guard url.scheme == "https" else { return }
|
|
143
|
+
var request = URLRequest(url: url)
|
|
144
|
+
request.httpMethod = "DELETE"
|
|
145
|
+
http.dataTask(with: request) { _, _, _ in }.resume()
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
private func fail(_ reason: String) {
|
|
149
|
+
guard !stopped, !failed else { return }
|
|
150
|
+
failed = true; onState("failed", reason)
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
func stop(completion: @escaping () -> Void) {
|
|
154
|
+
queue.async {
|
|
155
|
+
if !self.stopped {
|
|
156
|
+
self.stopped = true
|
|
157
|
+
self.peer?.close()
|
|
158
|
+
// Drain the external audio callback while its native delegate/factory is still alive.
|
|
159
|
+
_ = self.audioDevice.terminateDevice()
|
|
160
|
+
self.peer = nil
|
|
161
|
+
self.videoSource = nil; self.capturer = nil; self.factory = nil
|
|
162
|
+
if let resource = self.resource { self.delete(resource); self.resource = nil; self.http.finishTasksAndInvalidate() }
|
|
163
|
+
else if !self.posted { self.http.finishTasksAndInvalidate() }
|
|
164
|
+
// An in-flight POST completes cleanup in its callback above.
|
|
165
|
+
}
|
|
166
|
+
completion()
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
extension PhoneWhipPublisher: RTCPeerConnectionDelegate {
|
|
172
|
+
func peerConnection(_: RTCPeerConnection, didChange _: RTCSignalingState) {}
|
|
173
|
+
func peerConnection(_: RTCPeerConnection, didAdd _: RTCMediaStream) {}
|
|
174
|
+
func peerConnection(_: RTCPeerConnection, didRemove _: RTCMediaStream) {}
|
|
175
|
+
func peerConnectionShouldNegotiate(_: RTCPeerConnection) {}
|
|
176
|
+
func peerConnection(_ peerConnection: RTCPeerConnection, didChange state: RTCIceConnectionState) {
|
|
177
|
+
queue.async {
|
|
178
|
+
guard self.peer === peerConnection, !self.stopped else { return }
|
|
179
|
+
switch state {
|
|
180
|
+
case .connected, .completed:
|
|
181
|
+
self.connected = true; self.disconnectGeneration += 1
|
|
182
|
+
self.onState("connected", "Phone publisher connected")
|
|
183
|
+
case .disconnected:
|
|
184
|
+
self.disconnectGeneration += 1
|
|
185
|
+
let gen = self.disconnectGeneration
|
|
186
|
+
self.queue.asyncAfter(deadline: .now() + 10) {
|
|
187
|
+
if gen == self.disconnectGeneration { self.fail("Phone internet connection was lost") }
|
|
188
|
+
}
|
|
189
|
+
case .failed: self.fail("WHIP ICE failed")
|
|
190
|
+
default: break
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
func peerConnection(_ peerConnection: RTCPeerConnection, didChange _: RTCIceGatheringState) {
|
|
196
|
+
queue.async { if self.peer === peerConnection { self.maybePost() } }
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
func peerConnection(_: RTCPeerConnection, didGenerate _: RTCIceCandidate) {}
|
|
200
|
+
func peerConnection(_: RTCPeerConnection, didRemove _: [RTCIceCandidate]) {}
|
|
201
|
+
func peerConnection(_: RTCPeerConnection, didOpen _: RTCDataChannel) {}
|
|
202
|
+
}
|