@mentra/acs-meeting 3.2.0-dev.223 → 3.2.0-dev.225

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. package/README.md +14 -3
  2. package/android/build.gradle +1 -0
  3. package/android/src/main/java/com/mentra/acsmeeting/AcsMeetingModule.kt +10 -10
  4. package/android/src/main/java/com/mentra/acsmeeting/AcsMeetingSession.kt +37 -37
  5. package/android/src/main/java/com/mentra/acsmeeting/audio/UplinkSender.kt +4 -4
  6. package/android/src/main/java/com/mentra/acsmeeting/source/MeetingVideoSourceSpec.kt +3 -0
  7. package/android/src/main/java/com/mentra/acsmeeting/video/AcsFrameSender.kt +12 -10
  8. package/android/src/main/java/com/mentra/acsmeeting/video/I420FormatSpec.kt +2 -0
  9. package/android/src/main/java/com/mentra/acsmeeting/video/Nv12Packer.kt +2 -0
  10. package/android/src/test/java/com/mentra/acsmeeting/source/{AcsInvestigationTest.kt → MediaDiagnosticsTest.kt} +10 -10
  11. package/android/src/test/java/com/mentra/acsmeeting/source/MeetingVideoSourceSpecTest.kt +3 -0
  12. package/android/src/test/java/com/mentra/acsmeeting/telemetry/VideoQualityTest.kt +2 -2
  13. package/android/src/test/java/com/mentra/acsmeeting/video/I420FormatSpecTest.kt +2 -0
  14. package/android/src/test/java/com/mentra/acsmeeting/video/VideoProfileTest.kt +2 -2
  15. package/build/AcsMeeting.types.d.ts +10 -1
  16. package/build/AcsMeeting.types.d.ts.map +1 -1
  17. package/build/AcsMeeting.types.js.map +1 -1
  18. package/build/AcsMeetingModule.d.ts +2 -2
  19. package/build/AcsMeetingModule.d.ts.map +1 -1
  20. package/build/AcsMeetingModule.js.map +1 -1
  21. package/ios/AcsMeeting.podspec +1 -1
  22. package/ios/AcsMeetingModule.swift +284 -148
  23. package/ios/PolicyKit/Package.swift +2 -1
  24. package/ios/PolicyKit/Sources/AcsAudioPolicy/CallbackOperation.swift +60 -0
  25. package/ios/PolicyKit/Sources/AcsAudioPolicy/CapturePolicy.swift +5 -0
  26. package/ios/PolicyKit/Tests/AcsAudioPolicyTests/AcsAudioPolicyTests.swift +1 -0
  27. package/ios/PolicyKit/Tests/AcsAudioPolicyTests/CallAgentRetirementTests.swift +113 -0
  28. package/package.json +3 -2
  29. package/src/AcsMeeting.types.ts +4 -1
  30. package/src/AcsMeetingModule.ts +2 -2
  31. package/android/src/androidTest/java/com/mentra/acsmeeting/network/SoftApFeasibilityGateTest.kt +0 -291
  32. package/android/src/main/java/com/mentra/acsmeeting/network/InternetHold.kt +0 -501
  33. package/android/src/main/java/com/mentra/acsmeeting/network/Ipv4Prefix.kt +0 -51
  34. package/android/src/main/java/com/mentra/acsmeeting/network/NetworkFacts.kt +0 -81
  35. package/android/src/main/java/com/mentra/acsmeeting/network/ScopedNetworkChangeDetector.kt +0 -355
  36. package/android/src/main/java/com/mentra/acsmeeting/network/ScopedNetworkError.kt +0 -109
  37. package/android/src/main/java/com/mentra/acsmeeting/network/ScopedNetworkObserver.kt +0 -160
  38. package/android/src/main/java/com/mentra/acsmeeting/network/ScopedNetworkReadiness.kt +0 -103
  39. package/android/src/main/java/com/mentra/acsmeeting/network/ScopedNetworkRequestSpec.kt +0 -53
  40. package/android/src/main/java/com/mentra/acsmeeting/network/ScopedNetworkState.kt +0 -127
  41. package/android/src/main/java/com/mentra/acsmeeting/network/ScopedSoftApNetwork.kt +0 -573
  42. package/android/src/main/java/com/mentra/acsmeeting/source/BitmapFont.kt +0 -99
  43. package/android/src/main/java/com/mentra/acsmeeting/source/CloudflareWhepSource.kt +0 -487
  44. package/android/src/main/java/com/mentra/acsmeeting/source/DecodedTrackRelay.kt +0 -147
  45. package/android/src/main/java/com/mentra/acsmeeting/source/FirstFrameGate.kt +0 -45
  46. package/android/src/main/java/com/mentra/acsmeeting/source/GlassesMediaSource.kt +0 -143
  47. package/android/src/main/java/com/mentra/acsmeeting/source/LocalWhipIngestSource.kt +0 -772
  48. package/android/src/main/java/com/mentra/acsmeeting/source/MediaListeners.kt +0 -57
  49. package/android/src/main/java/com/mentra/acsmeeting/source/SdpAdapter.kt +0 -17
  50. package/android/src/main/java/com/mentra/acsmeeting/source/SelectedIcePair.kt +0 -139
  51. package/android/src/main/java/com/mentra/acsmeeting/source/SoftApIceDiagnosis.kt +0 -144
  52. package/android/src/main/java/com/mentra/acsmeeting/source/SoftApIcePolicy.kt +0 -46
  53. package/android/src/main/java/com/mentra/acsmeeting/source/SoftApSdpGuard.kt +0 -160
  54. package/android/src/main/java/com/mentra/acsmeeting/source/SyntheticFrameFactory.kt +0 -205
  55. package/android/src/main/java/com/mentra/acsmeeting/source/SyntheticI420Source.kt +0 -214
  56. package/android/src/main/java/com/mentra/acsmeeting/source/TrackRegistry.kt +0 -33
  57. package/android/src/main/java/com/mentra/acsmeeting/source/VideoSourceArm.kt +0 -110
  58. package/android/src/main/java/com/mentra/acsmeeting/source/WhipIngestProtocol.kt +0 -207
  59. package/android/src/main/java/com/mentra/acsmeeting/source/WhipIngestServer.kt +0 -356
  60. package/android/src/main/java/com/mentra/acsmeeting/telemetry/AvSyncProbe.kt +0 -155
  61. package/android/src/main/java/com/mentra/acsmeeting/telemetry/ChromaProbe.kt +0 -78
  62. package/android/src/main/java/com/mentra/acsmeeting/telemetry/PipelineStats.kt +0 -372
  63. package/android/src/main/java/com/mentra/acsmeeting/telemetry/PipelineTicker.kt +0 -79
  64. package/android/src/main/java/com/mentra/acsmeeting/telemetry/ProcessCpu.kt +0 -15
  65. package/android/src/main/java/com/mentra/acsmeeting/telemetry/RingPercentile.kt +0 -48
  66. package/android/src/main/java/com/mentra/acsmeeting/telemetry/VideoQuality.kt +0 -178
  67. package/android/src/main/java/com/mentra/acsmeeting/telemetry/VideoRateVerdict.kt +0 -145
  68. package/android/src/main/java/com/mentra/acsmeeting/trace/SoftApTrace.kt +0 -143
  69. package/android/src/main/java/com/mentra/acsmeeting/video/FrameGeometry.kt +0 -22
  70. package/android/src/main/java/com/mentra/acsmeeting/video/I420Packer.kt +0 -72
  71. package/android/src/test/java/com/mentra/acsmeeting/network/InternetHoldLifecycleTest.kt +0 -102
  72. package/android/src/test/java/com/mentra/acsmeeting/network/InternetHoldTest.kt +0 -148
  73. package/android/src/test/java/com/mentra/acsmeeting/network/Ipv4PrefixTest.kt +0 -59
  74. package/android/src/test/java/com/mentra/acsmeeting/network/NetworkFactsTest.kt +0 -59
  75. package/android/src/test/java/com/mentra/acsmeeting/network/ScopedLocalNetworkPermissionTest.kt +0 -35
  76. package/android/src/test/java/com/mentra/acsmeeting/network/ScopedNetworkChangeDetectorTest.kt +0 -165
  77. package/android/src/test/java/com/mentra/acsmeeting/network/ScopedNetworkErrorTest.kt +0 -96
  78. package/android/src/test/java/com/mentra/acsmeeting/network/ScopedNetworkLifecycleTest.kt +0 -187
  79. package/android/src/test/java/com/mentra/acsmeeting/network/ScopedNetworkObserverTest.kt +0 -172
  80. package/android/src/test/java/com/mentra/acsmeeting/network/ScopedNetworkReadinessTest.kt +0 -141
  81. package/android/src/test/java/com/mentra/acsmeeting/network/ScopedNetworkRequestSpecTest.kt +0 -69
  82. package/android/src/test/java/com/mentra/acsmeeting/source/BitmapFontTest.kt +0 -22
  83. package/android/src/test/java/com/mentra/acsmeeting/source/DecodedTrackRelayTest.kt +0 -41
  84. package/android/src/test/java/com/mentra/acsmeeting/source/FirstFrameGateTest.kt +0 -60
  85. package/android/src/test/java/com/mentra/acsmeeting/source/GlassesMediaSourceTest.kt +0 -151
  86. package/android/src/test/java/com/mentra/acsmeeting/source/I420PlanesTest.kt +0 -67
  87. package/android/src/test/java/com/mentra/acsmeeting/source/SelectedIcePairTest.kt +0 -165
  88. package/android/src/test/java/com/mentra/acsmeeting/source/SoftApIceDiagnosisTest.kt +0 -148
  89. package/android/src/test/java/com/mentra/acsmeeting/source/SoftApIcePolicyTest.kt +0 -76
  90. package/android/src/test/java/com/mentra/acsmeeting/source/SoftApSdpGuardTest.kt +0 -275
  91. package/android/src/test/java/com/mentra/acsmeeting/source/SyntheticFrameFactoryTest.kt +0 -193
  92. package/android/src/test/java/com/mentra/acsmeeting/source/SyntheticI420SourceTest.kt +0 -176
  93. package/android/src/test/java/com/mentra/acsmeeting/source/TrackRegistryTest.kt +0 -45
  94. package/android/src/test/java/com/mentra/acsmeeting/source/WhipIngestProtocolTest.kt +0 -356
  95. package/android/src/test/java/com/mentra/acsmeeting/source/WhipIngestServerLoopbackTest.kt +0 -411
  96. package/android/src/test/java/com/mentra/acsmeeting/telemetry/AvSyncProbeTest.kt +0 -83
  97. package/android/src/test/java/com/mentra/acsmeeting/telemetry/ChromaProbeTest.kt +0 -61
  98. package/android/src/test/java/com/mentra/acsmeeting/telemetry/PipelineStatsTest.kt +0 -322
  99. package/android/src/test/java/com/mentra/acsmeeting/telemetry/ProcessCpuTest.kt +0 -18
  100. package/android/src/test/java/com/mentra/acsmeeting/telemetry/RingPercentileTest.kt +0 -49
  101. package/android/src/test/java/com/mentra/acsmeeting/telemetry/VideoRateVerdictTest.kt +0 -149
  102. package/android/src/test/java/com/mentra/acsmeeting/trace/SoftApTraceTest.kt +0 -97
  103. package/android/src/test/java/com/mentra/acsmeeting/video/FrameGeometryTest.kt +0 -33
  104. package/android/src/test/java/com/mentra/acsmeeting/video/I420PackerTest.kt +0 -126
  105. package/ios/PolicyKit/Sources/AcsAudioPolicy/GlassesMediaSource.swift +0 -68
  106. package/ios/WhepVideoSource.swift +0 -356
@@ -3,60 +3,104 @@ import AzureCommunicationCalling
3
3
  import AzureCommunicationCommon
4
4
  import ExpoModulesCore
5
5
  import Foundation
6
+ import GlassesMedia
6
7
 
7
8
  public class AcsMeetingModule: Module {
8
9
  private var session: AcsMeetingSession?
10
+ private let hotspot = GlassesHotspotNetwork()
11
+
12
+ private func meetingSession() -> AcsMeetingSession {
13
+ if let session { return session }
14
+ let session = AcsMeetingSession(
15
+ onState: { [weak self] state in self?.sendEvent("onState", state) },
16
+ onIncomingPcm: { [weak self] base64, rate, channels in
17
+ self?.sendEvent("onIncomingPcm", ["base64": base64, "sampleRate": rate, "channels": channels])
18
+ }
19
+ )
20
+ self.session = session
21
+ return session
22
+ }
9
23
 
10
24
  public func definition() -> ModuleDefinition {
11
25
  Name("MentraAcsMeeting")
12
- Events("onState", "onIncomingPcm")
26
+ Events("onState", "onIncomingPcm", "onScopedNetworkLost")
27
+
28
+ AsyncFunction("prepareAgent") { (options: [String: Any], promise: Promise) in
29
+ let token = try requireString(options, "token")
30
+ self.meetingSession().prepareAgent(token: token, displayName: options["displayName"] as? String) { result in
31
+ switch result {
32
+ case let .success(state): promise.resolve(state)
33
+ case let .failure(error): promise.reject(error)
34
+ }
35
+ }
36
+ }
13
37
 
14
- AsyncFunction("join") { (options: [String: Any]) in
38
+ AsyncFunction("join") { (options: [String: Any], promise: Promise) in
15
39
  let token = try requireString(options, "token")
16
40
  let meetingUrl = try requireString(options, "meetingUrl")
17
- let whepUrl = try requireWhepUrl(options)
18
- let displayName = options["displayName"] as? String
19
- let dumpWav = options["dumpPcmWav"] as? Bool ?? false
20
- let audioSource = options["audioSource"] as? String ?? "glasses"
41
+ let source = try parseMediaSource(options)
42
+ if source.kind == .softap, source.bindAddress != GlassesHotspotNetwork.wifiAddress() {
43
+ throw AcsMeetingError("The glasses hotspot address changed before join")
44
+ }
21
45
  let video = try parseAcsOutgoingVideo(options["video"])
22
- let meeting = self.session ?? AcsMeetingSession(
23
- onState: { [weak self] state in self?.sendEvent("onState", state) },
24
- onIncomingPcm: { [weak self] base64, rate, channels in
25
- self?.sendEvent("onIncomingPcm", ["base64": base64, "sampleRate": rate, "channels": channels])
46
+ self.meetingSession().join(
47
+ token: token, meetingUrl: meetingUrl, sourceConfig: source,
48
+ displayName: options["displayName"] as? String,
49
+ dumpWav: options["dumpPcmWav"] as? Bool ?? false,
50
+ audioSource: options["audioSource"] as? String ?? "glasses", video: video
51
+ ) { result in
52
+ switch result {
53
+ case let .success(state): promise.resolve(state)
54
+ case let .failure(error): promise.reject(error)
26
55
  }
27
- )
28
- self.session = meeting
29
- try meeting.join(token: token, meetingUrl: meetingUrl, whepUrl: whepUrl, displayName: displayName, dumpWav: dumpWav, audioSource: audioSource, video: video)
30
- return meeting.snapshot()
56
+ }
31
57
  }
32
58
 
33
- /// Refuses rather than no-ops. A silent success here would let the orchestrator go on to an ACS
34
- /// join and a glasses publish that have no network to meet on, which surfaces as a black tile
35
- /// several steps later instead of as the unsupported transport it is.
36
- AsyncFunction("joinScopedNetwork") { (_: String, _: String) -> String in
37
- throw NSError(
38
- domain: "MentraAcsMeeting",
39
- code: 2,
40
- userInfo: [
41
- NSLocalizedDescriptionKey:
42
- "NOT_IMPLEMENTED: SoftAP calling is Android-only; iOS cannot join the glasses hotspot as a scoped network",
43
- ]
44
- )
59
+ AsyncFunction("joinScopedNetwork") { (ssid: String, passphrase: String, promise: Promise) in
60
+ self.hotspot.onLost = { [weak self] reason in
61
+ self?.sendEvent("onScopedNetworkLost", ["code": "SOFTAP_LOST", "message": reason])
62
+ }
63
+ self.hotspot.join(ssid: ssid, passphrase: passphrase) { result in
64
+ switch result {
65
+ case let .success(address): promise.resolve(address)
66
+ case let .failure(error): promise.reject(error)
67
+ }
68
+ }
45
69
  }
46
70
 
47
- AsyncFunction("leaveScopedNetwork") {
48
- // Nothing was ever joined, so releasing is a no-op rather than an error: teardown must stay
49
- // safe to run after a failed start.
71
+ AsyncFunction("leaveScopedNetwork") { (promise: Promise) in
72
+ self.hotspot.leave { promise.resolve(nil) }
50
73
  }
51
74
 
52
- AsyncFunction("probeScopedGateway") { () -> [String: Any] in
53
- // No scoped network exists on iOS, so there is nothing to probe. Report it as such rather
54
- // than as unreachable, which the orchestrator would read as a live network failure.
55
- ["reachable": false, "detail": "no scoped network on iOS"]
75
+ AsyncFunction("beginTrace") { (traceId: String) in
76
+ NSLog("SOFTAP_TRACE trace=\(traceId) stage=ios_begin")
56
77
  }
57
78
 
58
- AsyncFunction("leave") {
59
- self.session?.leave()
79
+ AsyncFunction("probeScopedGateway") { (promise: Promise) in
80
+ self.hotspot.probeGateway { reachable, detail in
81
+ promise.resolve(["reachable": reachable, "detail": detail])
82
+ }
83
+ }
84
+
85
+ AsyncFunction("awaitValidatedDefaultNetwork") { (promise: Promise) in
86
+ self.hotspot.awaitInternet { usable, detail in
87
+ promise.resolve(["usable": usable, "detail": detail, "transport": usable ? "cellular" : "unknown", "validated": usable, "present": usable])
88
+ }
89
+ }
90
+
91
+ AsyncFunction("leave") { (promise: Promise) in
92
+ guard let session = self.session else { promise.resolve(nil); return }
93
+ session.leaveAndAwait(timeout: 30) { completed in
94
+ if completed { promise.resolve(nil) }
95
+ else { promise.reject(AcsMeetingError("Previous call cleanup is still pending")) }
96
+ }
97
+ }
98
+
99
+ AsyncFunction("leaveAndAwait") { (options: [String: Any], promise: Promise) in
100
+ guard let session = self.session else { promise.resolve(["completed": true]); return }
101
+ session.leaveAndAwait(timeout: Double(options["timeoutMs"] as? Int ?? 30000) / 1000) { completed in
102
+ promise.resolve(["completed": completed])
103
+ }
60
104
  }
61
105
 
62
106
  AsyncFunction("endForEveryone") { (promise: Promise) in
@@ -79,11 +123,8 @@ public class AcsMeetingModule: Module {
79
123
  }
80
124
  }
81
125
 
82
- AsyncFunction("scopedNetworkInfo") { () -> [String: Any?] in
83
- // No scoped network on iOS, so there is no prefix for the ICE-path proof to check
84
- // against. Reported as unavailable rather than as an empty prefix, which the proof
85
- // would read as "could not describe the network" on a platform that has none.
86
- ["available": false, "localIpv4": nil, "prefix": nil]
126
+ AsyncFunction("scopedNetworkInfo") { (promise: Promise) in
127
+ self.hotspot.info { promise.resolve($0) }
87
128
  }
88
129
 
89
130
  AsyncFunction("setMuted") { (muted: Bool) in
@@ -109,6 +150,8 @@ public class AcsMeetingModule: Module {
109
150
  OnDestroy {
110
151
  self.session?.leave()
111
152
  self.session = nil
153
+ self.hotspot.onLost = nil
154
+ self.hotspot.leave {}
112
155
  }
113
156
  }
114
157
  }
@@ -148,7 +191,16 @@ final class AcsMeetingSession {
148
191
  private var callClient: CallClient?
149
192
  private var callAgent: CallAgent?
150
193
  private var call: Call?
151
- private var whep: WhepVideoSource?
194
+ private var media: DecodedGlassesMediaSource?
195
+ private var sourceConfig = SourceConfig(url: "")
196
+ private var preparedAgent: CallAgent?
197
+ private var preparedClient: CallClient?
198
+ private var preparedToken: String?
199
+ private var pendingJoin: ((Result<[String: Any], Error>) -> Void)?
200
+ private var pendingPrepare: ((Result<[String: Any], Error>) -> Void)?
201
+ private var cancelPendingCallJoin: (() -> Void)?
202
+ private let cleanup = DispatchGroup()
203
+ private let pcmSlots = DispatchSemaphore(value: 8)
152
204
  private var frameSender = AcsFrameSender()
153
205
  private var pcmBridge: PcmBridge?
154
206
  private var audioOut: RawOutgoingAudioStream?
@@ -201,68 +253,103 @@ final class AcsMeetingSession {
201
253
  hangUp["reason"] = capability.reason
202
254
  }
203
255
  result["capabilities"] = ["hangUpForEveryone": hangUp]
256
+ if let ingestUrl = media?.ingestUrl { result["ingestUrl"] = ingestUrl }
204
257
  if let meetingUrl { result["meetingUrl"] = meetingUrl }
205
258
  if let lastError { result["error"] = lastError }
206
259
  return result
207
260
  }
208
261
 
209
- func join(token: String, meetingUrl: String, whepUrl: String, displayName: String?, dumpWav: Bool, audioSource: String = "glasses", video: AcsOutgoingVideo = .hd) throws {
210
- // Both glasses and phone feed RawOutgoingAudioStream so ACS never owns the
211
- // phone audio route. Phone PCM comes from AVAudioEngine; glasses via WHEP.
212
- let parsed = AcsAudioPolicy.parseSource(audioSource) ?? .glasses
213
- if parsed == .phone {
214
- NSLog("ACS-SPIKE audioSource=phone: input tap → virtual outgoing; voice-chat session off")
215
- }
216
- self.audioSource = parsed == .phone ? "phone" : "glasses"
217
- // The ACS work below is queued; reflect the intent synchronously so the
218
- // resolved snapshot reads "connecting" rather than a stale idle.
219
- phase = "connecting"
220
- lastError = nil
221
- self.meetingUrl = meetingUrl
222
- queue.async { [weak self] in
223
- guard let self else { return }
262
+ func prepareAgent(token: String, displayName: String?, completion: @escaping (Result<[String: Any], Error>) -> Void) {
263
+ queue.async {
264
+ guard self.cleanup.wait(timeout: .now()) == .success, self.pendingPrepare == nil, self.pendingJoin == nil, self.call == nil, self.media == nil else {
265
+ completion(.failure(AcsMeetingError("Previous call cleanup is still pending"))); return
266
+ }
267
+ self.leaveLocked()
268
+ let generation = self.joinGeneration
269
+ self.pendingPrepare = completion
224
270
  do {
225
- self.leaveLocked(emitIdle: false)
226
- let generation = self.joinGeneration
227
- self.audioSource = parsed == .phone ? "phone" : "glasses"
228
- self.meetingUrl = meetingUrl
229
- self.lastError = nil
230
- self.emit("connecting")
231
- let credential = try CommunicationTokenCredential(token: token)
232
271
  let client = CallClient()
233
- self.callClient = client
272
+ self.preparedClient = client
273
+ let credential = try CommunicationTokenCredential(token: token)
234
274
  let options = CallAgentOptions()
235
275
  options.displayName = displayName ?? "Mentra Call"
236
- client.createCallAgent(userCredential: credential, options: options) { [weak self, weak client] agent, error in
237
- self?.queue.async {
238
- guard let self, let client, self.callClient === client, self.joinGeneration == generation else {
276
+ client.createCallAgent(userCredential: credential, options: options) { agent, error in
277
+ self.queue.async {
278
+ guard self.joinGeneration == generation, self.pendingPrepare != nil else { agent?.dispose(); return }
279
+ let reply = self.pendingPrepare
280
+ self.pendingPrepare = nil
281
+ if let agent, error == nil {
282
+ self.preparedAgent = agent
283
+ self.preparedToken = token
284
+ reply?(.success(self.snapshot()))
285
+ } else {
239
286
  agent?.dispose()
240
- return
241
- }
242
- if let error {
243
- self.failJoinLocked(error, generation: generation)
244
- return
245
- }
246
- guard let agent else {
247
- self.failJoinLocked(AcsMeetingError("ACS returned no call agent"), generation: generation)
248
- return
249
- }
250
- do {
251
- try self.joinWithAgentLocked(
252
- agent,
253
- generation: generation,
254
- meetingUrl: meetingUrl,
255
- whepUrl: whepUrl,
256
- dumpWav: dumpWav,
257
- video: video
258
- )
259
- } catch {
260
- self.failJoinLocked(error, generation: generation)
287
+ self.preparedClient = nil
288
+ reply?(.failure(error ?? AcsMeetingError("ACS returned no call agent")))
261
289
  }
262
290
  }
263
291
  }
292
+ self.queue.asyncAfter(deadline: .now() + 30) {
293
+ guard self.joinGeneration == generation, self.pendingPrepare != nil else { return }
294
+ self.leaveLocked()
295
+ }
264
296
  } catch {
265
- self.failJoinLocked(error, generation: self.joinGeneration)
297
+ self.pendingPrepare = nil
298
+ self.preparedClient = nil
299
+ completion(.failure(error))
300
+ }
301
+ }
302
+ }
303
+
304
+ func join(token: String, meetingUrl: String, sourceConfig: SourceConfig, displayName: String?, dumpWav: Bool,
305
+ audioSource: String = "glasses", video: AcsOutgoingVideo = .hd,
306
+ completion: @escaping (Result<[String: Any], Error>) -> Void)
307
+ {
308
+ queue.async {
309
+ guard self.cleanup.wait(timeout: .now()) == .success, self.pendingPrepare == nil, self.pendingJoin == nil, self.call == nil, self.media == nil else {
310
+ completion(.failure(AcsMeetingError("Previous call cleanup is still pending"))); return
311
+ }
312
+ let prepared = self.preparedToken == token ? self.preparedAgent : nil
313
+ let preparedClient = self.preparedClient
314
+ if prepared != nil { self.preparedAgent = nil; self.preparedClient = nil; self.preparedToken = nil }
315
+ self.leaveLocked(emitIdle: false)
316
+ let generation = self.joinGeneration
317
+ self.pendingJoin = completion
318
+ self.sourceConfig = sourceConfig
319
+ self.audioSource = AcsAudioPolicy.parseSource(audioSource) == .phone ? "phone" : "glasses"
320
+ self.meetingUrl = meetingUrl
321
+ self.lastError = nil
322
+ self.emit("connecting")
323
+ let useAgent: (CallAgent) -> Void = { agent in
324
+ do { try self.joinWithAgentLocked(agent, generation: generation, meetingUrl: meetingUrl,
325
+ sourceConfig: sourceConfig, dumpWav: dumpWav, video: video) } catch { self.failJoinLocked(error, generation: generation) }
326
+ }
327
+ if let prepared {
328
+ self.callClient = preparedClient
329
+ useAgent(prepared)
330
+ } else {
331
+ do {
332
+ let credential = try CommunicationTokenCredential(token: token)
333
+ let client = CallClient()
334
+ self.callClient = client
335
+ let options = CallAgentOptions()
336
+ options.displayName = displayName ?? "Mentra Call"
337
+ client.createCallAgent(userCredential: credential, options: options) { agent, error in
338
+ self.queue.async {
339
+ guard self.joinGeneration == generation else { agent?.dispose(); return }
340
+ guard let agent, error == nil else {
341
+ agent?.dispose()
342
+ self.failJoinLocked(error ?? AcsMeetingError("ACS returned no call agent"), generation: generation)
343
+ return
344
+ }
345
+ useAgent(agent)
346
+ }
347
+ }
348
+ } catch { self.failJoinLocked(error, generation: generation) }
349
+ }
350
+ self.queue.asyncAfter(deadline: .now() + 40) {
351
+ guard self.joinGeneration == generation, self.pendingJoin != nil else { return }
352
+ self.failJoinLocked(AcsMeetingError("ACS join timed out"), generation: generation)
266
353
  }
267
354
  }
268
355
  }
@@ -271,7 +358,7 @@ final class AcsMeetingSession {
271
358
  _ agent: CallAgent,
272
359
  generation: UInt64,
273
360
  meetingUrl: String,
274
- whepUrl: String,
361
+ sourceConfig: SourceConfig,
275
362
  dumpWav: Bool,
276
363
  video: AcsOutgoingVideo
277
364
  ) throws {
@@ -285,6 +372,7 @@ final class AcsMeetingSession {
285
372
  let videoOptions = RawOutgoingVideoStreamOptions()
286
373
  videoOptions.formats = [videoFormat]
287
374
  let videoStream = VirtualOutgoingVideoStream(videoStreamOptions: videoOptions)
375
+ frameSender = AcsFrameSender()
288
376
  frameSender.attach(videoStream)
289
377
 
290
378
  let outAudioProperties = RawOutgoingAudioStreamProperties()
@@ -341,12 +429,23 @@ final class AcsMeetingSession {
341
429
  joinOptions.incomingAudioOptions = incomingAudio
342
430
 
343
431
  let locator = TeamsMeetingLinkLocator(meetingLink: meetingUrl)
344
- agent.join(with: locator, joinCallOptions: joinOptions) { [weak self, weak agent] call, error in
345
- self?.queue.async {
346
- guard let self, let agent, self.callAgent === agent, self.joinGeneration == generation else {
432
+ // Cancellation reserves cleanup and retains the agent until the late join result
433
+ // can be hung up. The retirement deadline still handles an SDK callback that is lost.
434
+ let joinRetirement = CallJoinRetirement<Call>(group: cleanup, queue: queue, dispose: { agent.dispose() }) { call, finished in
435
+ call.hangUp(options: nil) { error in
436
+ if let error { NSLog("ACS-SPIKE cancelled join hangUp failed: \(error)") }
437
+ finished()
438
+ }
439
+ }
440
+ cancelPendingCallJoin = { joinRetirement.cancel() }
441
+ agent.join(with: locator, joinCallOptions: joinOptions) { call, error in
442
+ self.queue.async {
443
+ guard joinRetirement.receive(call) else { return }
444
+ guard self.callAgent === agent, self.joinGeneration == generation else {
347
445
  call?.hangUp(options: nil) { _ in }
348
446
  return
349
447
  }
448
+ self.cancelPendingCallJoin = nil
350
449
  if let error {
351
450
  self.failJoinLocked(error, generation: generation)
352
451
  return
@@ -358,7 +457,7 @@ final class AcsMeetingSession {
358
457
  self.finishJoinLocked(
359
458
  call,
360
459
  generation: generation,
361
- whepUrl: whepUrl,
460
+ sourceConfig: sourceConfig,
362
461
  dumpWav: dumpWav,
363
462
  video: video,
364
463
  plan: plan
@@ -370,7 +469,7 @@ final class AcsMeetingSession {
370
469
  private func finishJoinLocked(
371
470
  _ call: Call,
372
471
  generation: UInt64,
373
- whepUrl: String,
472
+ sourceConfig: SourceConfig,
374
473
  dumpWav: Bool,
375
474
  video: AcsOutgoingVideo,
376
475
  plan: JoinAudioPlan
@@ -386,25 +485,42 @@ final class AcsMeetingSession {
386
485
  let bridge = PcmBridge(dumpWav: dumpWav)
387
486
  pcmBridge = bridge
388
487
  phoneMic.onPcm = { [weak self] pcm, rate, channels in
389
- self?.feedOutgoingPcm(pcm, sampleRate: rate, channels: channels)
488
+ self?.feedOutgoingPcm(pcm, sampleRate: rate, channels: channels, generation: generation)
390
489
  }
391
- let source = WhepVideoSource()
392
- source.onFrame = { [weak self] buffer in self?.frameSender.send(buffer) }
490
+ let source: DecodedGlassesMediaSource = sourceConfig.kind == .softap ? LocalWhipIngestSource() : WhepVideoSource()
491
+ source.onFrame = { [frameSender] buffer in frameSender.send(buffer) }
393
492
  source.onPcm = { [weak self] pcm, rate, channels in
394
- self?.feedOutgoingPcm(pcm, sampleRate: rate, channels: channels)
493
+ self?.feedOutgoingPcm(pcm, sampleRate: rate, channels: channels, generation: generation)
395
494
  }
396
495
  mediaRestartAttempts = 0
397
496
  source.onStateChange = { [weak self, weak source] state, reason in
398
497
  // Fired from WebRTC/URLSession threads; hop to the session queue so it
399
498
  // serializes with join/leave/policy like everything else.
400
499
  self?.queue.async {
401
- guard let self, let source, self.whep === source else { return }
500
+ guard let self, let source, self.media === source else { return }
402
501
  self.onMediaSourceState(state, reason: reason)
403
502
  }
404
503
  }
405
- source.start(config: SourceConfig(url: whepUrl))
406
- whep = source
504
+ media = source
407
505
  applyAudioPolicyOnQueue("join")
506
+ let ready: (Result<String, Error>) -> Void = { result in
507
+ self.queue.async {
508
+ guard self.joinGeneration == generation, self.media === source else { return }
509
+ switch result {
510
+ case .success:
511
+ let reply = self.pendingJoin
512
+ self.pendingJoin = nil
513
+ reply?(.success(self.snapshot()))
514
+ case let .failure(error): self.failJoinLocked(error, generation: generation)
515
+ }
516
+ }
517
+ }
518
+ if let local = source as? LocalWhipIngestSource {
519
+ local.prepare(config: sourceConfig, completion: ready)
520
+ } else {
521
+ source.start(config: sourceConfig)
522
+ ready(.success(""))
523
+ }
408
524
  NSLog("ACS-SPIKE iOS ACS join started source=\(audioSource) profile=\(video.width)x\(video.height)@\(video.fps) armVirtual=\(plan.armVirtual) transportMuted=\(plan.transportMuted)")
409
525
  }
410
526
 
@@ -412,17 +528,21 @@ final class AcsMeetingSession {
412
528
  guard joinGeneration == generation else { return }
413
529
  // Record failure before teardown: lastError makes late call callbacks no-ops,
414
530
  // and emitIdle=false preserves the terminal error rather than resetting idle.
531
+ let reply = pendingJoin
532
+ pendingJoin = nil
415
533
  lastError = error.localizedDescription
416
534
  leaveLocked(emitIdle: false)
417
535
  emit("error")
536
+ reply?(.failure(error))
418
537
  }
419
538
 
420
539
  func updateVideoSource(_ whepUrl: String) {
421
540
  queue.async {
422
541
  // The host has a fresher opinion about where the glasses publish; drop any
423
542
  // automatic retry against the old URL.
543
+ guard self.sourceConfig.kind == .whep else { return }
424
544
  self.cancelMediaRestart()
425
- self.whep?.restart(config: SourceConfig(url: whepUrl))
545
+ self.media?.restart(config: SourceConfig(url: whepUrl))
426
546
  }
427
547
  }
428
548
 
@@ -430,8 +550,9 @@ final class AcsMeetingSession {
430
550
  /// The host calls this when the phone changed networks.
431
551
  func restartVideoSource() {
432
552
  queue.async {
553
+ guard self.sourceConfig.kind == .whep else { return }
433
554
  self.cancelMediaRestart()
434
- self.whep?.forceRestart()
555
+ self.media?.forceRestart()
435
556
  }
436
557
  }
437
558
 
@@ -448,7 +569,7 @@ final class AcsMeetingSession {
448
569
  /// Teams call with a frozen last frame looks healthy from every other angle.
449
570
  /// Exponential backoff capped at mediaRestartMaxMs, for as long as the call is alive.
450
571
  private func scheduleMediaRestart(reason: String) {
451
- guard call != nil, !["idle", "disconnected", "error"].contains(phase) else { return }
572
+ guard sourceConfig.kind == .whep, call != nil, !["idle", "disconnected", "error"].contains(phase) else { return }
452
573
  guard mediaRestartTask == nil else { return }
453
574
  let attempt = mediaRestartAttempts
454
575
  mediaRestartAttempts += 1
@@ -458,7 +579,7 @@ final class AcsMeetingSession {
458
579
  guard let self else { return }
459
580
  self.mediaRestartTask = nil
460
581
  guard self.call != nil, self.mediaSource == .failed else { return }
461
- self.whep?.forceRestart()
582
+ self.media?.forceRestart()
462
583
  }
463
584
  mediaRestartTask = task
464
585
  queue.asyncAfter(deadline: .now() + .milliseconds(delayMs), execute: task)
@@ -491,6 +612,15 @@ final class AcsMeetingSession {
491
612
  queue.async { self.leaveLocked() }
492
613
  }
493
614
 
615
+ func leaveAndAwait(timeout: TimeInterval, completion: @escaping (Bool) -> Void) {
616
+ queue.async {
617
+ self.leaveLocked()
618
+ DispatchQueue.global(qos: .userInitiated).async {
619
+ completion(self.cleanup.wait(timeout: .now() + max(0, min(timeout, 60))) == .success)
620
+ }
621
+ }
622
+ }
623
+
494
624
  /**
495
625
  End the meeting for everyone, then tear this device down.
496
626
 
@@ -600,7 +730,16 @@ final class AcsMeetingSession {
600
730
  onState(snapshot())
601
731
  }
602
732
 
603
- private func feedOutgoingPcm(_ pcm: Data, sampleRate: Int, channels: Int) {
733
+ private func feedOutgoingPcm(_ pcm: Data, sampleRate: Int, channels: Int, generation: UInt64) {
734
+ guard pcmSlots.wait(timeout: .now()) == .success else { return }
735
+ queue.async {
736
+ defer { self.pcmSlots.signal() }
737
+ guard self.joinGeneration == generation else { return }
738
+ self.feedOutgoingPcmLocked(pcm, sampleRate: sampleRate, channels: channels)
739
+ }
740
+ }
741
+
742
+ private func feedOutgoingPcmLocked(_ pcm: Data, sampleRate: Int, channels: Int) {
604
743
  guard !muted, outgoingReady, let stream = audioOut else { return }
605
744
  for frame in pcmBridge?.ingest(pcm16Le: pcm, sampleRate: sampleRate, channels: channels) ?? [] {
606
745
  guard let pcmBuffer = PcmBridge.audioBuffer(pcm16Le: frame, sampleRate: PcmBridge.targetRate, channels: 1) else {
@@ -625,6 +764,16 @@ final class AcsMeetingSession {
625
764
 
626
765
  private func leaveLocked(emitIdle: Bool = true) {
627
766
  joinGeneration &+= 1
767
+ let joinReply = pendingJoin
768
+ pendingJoin = nil
769
+ joinReply?(.failure(AcsMeetingError("Meeting join cancelled")))
770
+ let prepareReply = pendingPrepare
771
+ pendingPrepare = nil
772
+ prepareReply?(.failure(AcsMeetingError("Meeting preparation cancelled or timed out")))
773
+ preparedAgent?.dispose()
774
+ preparedAgent = nil
775
+ preparedClient = nil
776
+ preparedToken = nil
628
777
  phoneMic.setEnabled(false)
629
778
  phoneMic.onPcm = nil
630
779
  applier.reset()
@@ -634,27 +783,41 @@ final class AcsMeetingSession {
634
783
  // snapshot (or schedule a rebuild) for a call that is going away.
635
784
  cancelMediaRestart()
636
785
  detachCapabilities()
637
- whep?.onStateChange = nil
786
+ media?.onStateChange = nil
787
+ media?.onFrame = nil
788
+ media?.onPcm = nil
638
789
  mediaSource = .idle
639
- whep?.stop()
790
+ if let local = media as? LocalWhipIngestSource {
791
+ cleanup.enter()
792
+ local.stop { self.cleanup.leave() }
793
+ } else { media?.stop() }
640
794
  frameSender.detach()
641
795
  let leavingCall = call
642
796
  let leavingAgent = callAgent
797
+ let cancelJoin = cancelPendingCallJoin
798
+ cancelPendingCallJoin = nil
643
799
  leavingCall?.delegate = nil
644
- leavingCall?.hangUp(options: nil) { error in
645
- if let error {
646
- NSLog("ACS-SPIKE leave hangUp failed: \(error)")
800
+ if let leavingCall {
801
+ // CallAgent.dispose releases all local SDK resources. The retirement deadline
802
+ // also disposes the agent if hangUp loses its callback, before opening the barrier.
803
+ let retirement = CallAgentRetirement(group: cleanup, queue: queue) { leavingAgent?.dispose() }
804
+ leavingCall.hangUp(options: nil) { error in
805
+ self.queue.async {
806
+ if let error { NSLog("ACS-SPIKE leave hangUp failed: \(error)") }
807
+ retirement.finish()
808
+ }
647
809
  }
648
- leavingAgent?.dispose()
649
- }
650
- // A join that never produced a call still owns an agent that must be released.
651
- if leavingCall == nil {
810
+ } else if let cancelJoin {
811
+ // Keep the agent alive while awaiting the cancelled join and its hang-up.
812
+ cancelJoin()
813
+ } else {
814
+ // Agent creation has no call to retire; its callback disposes any late agent.
652
815
  leavingAgent?.dispose()
653
816
  }
654
817
  callAgent = nil
655
818
  callClient = nil
656
819
  call = nil
657
- whep = nil
820
+ media = nil
658
821
  audioOut = nil
659
822
  audioIn = nil
660
823
  localOut = nil
@@ -677,8 +840,8 @@ final class AcsMeetingSession {
677
840
  call
678
841
  }
679
842
 
680
- fileprivate func currentWhep() -> WhepVideoSource? {
681
- whep
843
+ fileprivate func currentWhep() -> DecodedGlassesMediaSource? {
844
+ media
682
845
  }
683
846
 
684
847
  fileprivate func setPhonePcmEnabled(_ enabled: Bool) {
@@ -866,35 +1029,8 @@ private func requireString(_ options: [String: Any], _ key: String) throws -> St
866
1029
  return value
867
1030
  }
868
1031
 
869
- /// Resolves the WHEP URL from either the `videoSource` union or a legacy bare `whepUrl`.
870
- ///
871
- /// SoftAP is refused here with `NOT_IMPLEMENTED` rather than allowed to fall through. Joining
872
- /// without a usable video source would produce a call that connects and shows the remote
873
- /// participant a black tile, with nothing in the logs to explain it. iOS cannot serve the local
874
- /// WHIP endpoint yet — see the SoftAP plan's iOS spike — so saying so plainly is the correct
875
- /// behaviour until it can.
876
- private func requireWhepUrl(_ options: [String: Any]) throws -> String {
877
- guard let source = options["videoSource"] as? [String: Any] else {
878
- return try requireString(options, "whepUrl")
879
- }
880
- let type = (source["type"] as? String)?.lowercased() ?? ""
881
- switch type {
882
- case "whep":
883
- if let url = source["url"] as? String, !url.isEmpty { return url }
884
- return try requireString(options, "whepUrl")
885
- case "softap":
886
- throw NSError(
887
- domain: "MentraAcsMeeting",
888
- code: 2,
889
- userInfo: [NSLocalizedDescriptionKey: "NOT_IMPLEMENTED: SoftAP calling is Android-only"]
890
- )
891
- default:
892
- throw NSError(
893
- domain: "MentraAcsMeeting",
894
- code: 1,
895
- userInfo: [NSLocalizedDescriptionKey: "unsupported videoSource.type: \(type)"]
896
- )
897
- }
1032
+ private func parseMediaSource(_ options: [String: Any]) throws -> SourceConfig {
1033
+ try SourceConfig.fromBridge(options["videoSource"] as? [String: Any], legacyWhepUrl: options["whepUrl"] as? String)
898
1034
  }
899
1035
 
900
1036
  private func parseAcsOutgoingVideo(_ raw: Any?) throws -> AcsOutgoingVideo {
@@ -10,8 +10,9 @@ let package = Package(
10
10
  products: [
11
11
  .library(name: "AcsAudioPolicy", targets: ["AcsAudioPolicy"]),
12
12
  ],
13
+ dependencies: [.package(path: "../../../glasses-media/ios/CoreKit")],
13
14
  targets: [
14
- .target(name: "AcsAudioPolicy"),
15
+ .target(name: "AcsAudioPolicy", dependencies: [.product(name: "GlassesMediaCore", package: "CoreKit")]),
15
16
  .testTarget(name: "AcsAudioPolicyTests", dependencies: ["AcsAudioPolicy"]),
16
17
  ]
17
18
  )