@mentra/acs-meeting 3.2.0-dev.120

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. package/NOTICE +20 -0
  2. package/README.md +412 -0
  3. package/THIRD_PARTY +7 -0
  4. package/android/build.gradle +64 -0
  5. package/android/src/main/AndroidManifest.xml +5 -0
  6. package/android/src/main/java/com/mentra/acsmeeting/AcsMeetingModule.kt +80 -0
  7. package/android/src/main/java/com/mentra/acsmeeting/AcsMeetingSession.kt +846 -0
  8. package/android/src/main/java/com/mentra/acsmeeting/RemoteRoster.kt +123 -0
  9. package/android/src/main/java/com/mentra/acsmeeting/audio/AcsAudioPolicy.kt +90 -0
  10. package/android/src/main/java/com/mentra/acsmeeting/audio/AudioPolicyApplier.kt +176 -0
  11. package/android/src/main/java/com/mentra/acsmeeting/audio/IncomingAudioPump.kt +112 -0
  12. package/android/src/main/java/com/mentra/acsmeeting/audio/IncomingRateProbe.kt +65 -0
  13. package/android/src/main/java/com/mentra/acsmeeting/audio/PcmBridge.kt +108 -0
  14. package/android/src/main/java/com/mentra/acsmeeting/audio/PcmResampler.kt +159 -0
  15. package/android/src/main/java/com/mentra/acsmeeting/audio/PhoneMicCapturer.kt +115 -0
  16. package/android/src/main/java/com/mentra/acsmeeting/audio/UplinkPacer.kt +302 -0
  17. package/android/src/main/java/com/mentra/acsmeeting/audio/UplinkSender.kt +230 -0
  18. package/android/src/main/java/com/mentra/acsmeeting/source/BitmapFont.kt +99 -0
  19. package/android/src/main/java/com/mentra/acsmeeting/source/CloudflareWhepSource.kt +578 -0
  20. package/android/src/main/java/com/mentra/acsmeeting/source/FirstFrameGate.kt +45 -0
  21. package/android/src/main/java/com/mentra/acsmeeting/source/GlassesMediaSource.kt +101 -0
  22. package/android/src/main/java/com/mentra/acsmeeting/source/MediaListeners.kt +57 -0
  23. package/android/src/main/java/com/mentra/acsmeeting/source/SyntheticFrameFactory.kt +205 -0
  24. package/android/src/main/java/com/mentra/acsmeeting/source/SyntheticI420Source.kt +214 -0
  25. package/android/src/main/java/com/mentra/acsmeeting/source/TrackRegistry.kt +31 -0
  26. package/android/src/main/java/com/mentra/acsmeeting/source/VideoSourceArm.kt +41 -0
  27. package/android/src/main/java/com/mentra/acsmeeting/telemetry/ChromaProbe.kt +78 -0
  28. package/android/src/main/java/com/mentra/acsmeeting/telemetry/PipelineStats.kt +281 -0
  29. package/android/src/main/java/com/mentra/acsmeeting/telemetry/PipelineTicker.kt +51 -0
  30. package/android/src/main/java/com/mentra/acsmeeting/telemetry/ProcessCpu.kt +15 -0
  31. package/android/src/main/java/com/mentra/acsmeeting/telemetry/RingPercentile.kt +33 -0
  32. package/android/src/main/java/com/mentra/acsmeeting/video/AcsFrameSender.kt +475 -0
  33. package/android/src/main/java/com/mentra/acsmeeting/video/AcsTimestamp.kt +26 -0
  34. package/android/src/main/java/com/mentra/acsmeeting/video/FrameGeometry.kt +22 -0
  35. package/android/src/main/java/com/mentra/acsmeeting/video/I420FormatSpec.kt +90 -0
  36. package/android/src/main/java/com/mentra/acsmeeting/video/I420Packer.kt +72 -0
  37. package/android/src/main/java/com/mentra/acsmeeting/video/Nv12FormatSpec.kt +36 -0
  38. package/android/src/main/java/com/mentra/acsmeeting/video/Nv12Packer.kt +51 -0
  39. package/android/src/main/java/com/mentra/acsmeeting/video/SendGate.kt +27 -0
  40. package/android/src/main/java/com/mentra/acsmeeting/video/VideoProfile.kt +57 -0
  41. package/android/src/test/java/com/mentra/acsmeeting/RemoteRosterTest.kt +40 -0
  42. package/android/src/test/java/com/mentra/acsmeeting/audio/AcsAudioPolicyTest.kt +204 -0
  43. package/android/src/test/java/com/mentra/acsmeeting/audio/AudioPolicyApplierTest.kt +223 -0
  44. package/android/src/test/java/com/mentra/acsmeeting/audio/CapturePolicyTest.kt +12 -0
  45. package/android/src/test/java/com/mentra/acsmeeting/audio/IncomingAudioPumpTest.kt +97 -0
  46. package/android/src/test/java/com/mentra/acsmeeting/audio/IncomingRateProbeTest.kt +78 -0
  47. package/android/src/test/java/com/mentra/acsmeeting/audio/PcmResamplerTest.kt +125 -0
  48. package/android/src/test/java/com/mentra/acsmeeting/audio/UplinkPacerTest.kt +201 -0
  49. package/android/src/test/java/com/mentra/acsmeeting/audio/UplinkSenderTest.kt +166 -0
  50. package/android/src/test/java/com/mentra/acsmeeting/source/AcsInvestigationTest.kt +26 -0
  51. package/android/src/test/java/com/mentra/acsmeeting/source/BitmapFontTest.kt +22 -0
  52. package/android/src/test/java/com/mentra/acsmeeting/source/FirstFrameGateTest.kt +60 -0
  53. package/android/src/test/java/com/mentra/acsmeeting/source/GlassesMediaSourceTest.kt +133 -0
  54. package/android/src/test/java/com/mentra/acsmeeting/source/I420PlanesTest.kt +67 -0
  55. package/android/src/test/java/com/mentra/acsmeeting/source/SyntheticFrameFactoryTest.kt +193 -0
  56. package/android/src/test/java/com/mentra/acsmeeting/source/SyntheticI420SourceTest.kt +176 -0
  57. package/android/src/test/java/com/mentra/acsmeeting/source/TrackRegistryTest.kt +36 -0
  58. package/android/src/test/java/com/mentra/acsmeeting/telemetry/ChromaProbeTest.kt +61 -0
  59. package/android/src/test/java/com/mentra/acsmeeting/telemetry/PipelineStatsTest.kt +256 -0
  60. package/android/src/test/java/com/mentra/acsmeeting/telemetry/ProcessCpuTest.kt +18 -0
  61. package/android/src/test/java/com/mentra/acsmeeting/telemetry/RingPercentileTest.kt +49 -0
  62. package/android/src/test/java/com/mentra/acsmeeting/video/AcsTimestampTest.kt +43 -0
  63. package/android/src/test/java/com/mentra/acsmeeting/video/FrameGeometryTest.kt +33 -0
  64. package/android/src/test/java/com/mentra/acsmeeting/video/I420FormatSpecTest.kt +77 -0
  65. package/android/src/test/java/com/mentra/acsmeeting/video/I420PackerTest.kt +126 -0
  66. package/android/src/test/java/com/mentra/acsmeeting/video/Nv12FormatSpecTest.kt +36 -0
  67. package/android/src/test/java/com/mentra/acsmeeting/video/Nv12PackerTest.kt +58 -0
  68. package/android/src/test/java/com/mentra/acsmeeting/video/SendGateTest.kt +79 -0
  69. package/android/src/test/java/com/mentra/acsmeeting/video/VideoProfileTest.kt +68 -0
  70. package/build/AcsMeeting.types.d.ts +59 -0
  71. package/build/AcsMeeting.types.d.ts.map +1 -0
  72. package/build/AcsMeeting.types.js +2 -0
  73. package/build/AcsMeeting.types.js.map +1 -0
  74. package/build/AcsMeetingModule.d.ts +15 -0
  75. package/build/AcsMeetingModule.d.ts.map +1 -0
  76. package/build/AcsMeetingModule.js +3 -0
  77. package/build/AcsMeetingModule.js.map +1 -0
  78. package/build/AcsMeetingModule.web.d.ts +16 -0
  79. package/build/AcsMeetingModule.web.d.ts.map +1 -0
  80. package/build/AcsMeetingModule.web.js +31 -0
  81. package/build/AcsMeetingModule.web.js.map +1 -0
  82. package/build/index.d.ts +3 -0
  83. package/build/index.d.ts.map +1 -0
  84. package/build/index.js +3 -0
  85. package/build/index.js.map +1 -0
  86. package/expo-module.config.json +9 -0
  87. package/ios/AcsFrameSender.swift +47 -0
  88. package/ios/AcsMeeting.podspec +27 -0
  89. package/ios/AcsMeetingModule.swift +558 -0
  90. package/ios/PcmBridge.swift +117 -0
  91. package/ios/PhoneMicCapturer.swift +64 -0
  92. package/ios/PolicyKit/Package.swift +17 -0
  93. package/ios/PolicyKit/Sources/AcsAudioPolicy/AcsAudioPolicy.swift +102 -0
  94. package/ios/PolicyKit/Sources/AcsAudioPolicy/AudioPolicyApplier.swift +162 -0
  95. package/ios/PolicyKit/Sources/AcsAudioPolicy/GlassesMediaSource.swift +68 -0
  96. package/ios/PolicyKit/Tests/AcsAudioPolicyTests/AcsAudioPolicyTests.swift +273 -0
  97. package/ios/WhepVideoSource.swift +356 -0
  98. package/package.json +56 -0
  99. package/src/AcsMeeting.types.ts +66 -0
  100. package/src/AcsMeetingModule.ts +16 -0
  101. package/src/AcsMeetingModule.web.ts +33 -0
  102. package/src/index.ts +2 -0
@@ -0,0 +1,558 @@
1
+ import AzureCommunicationCalling
2
+ import AzureCommunicationCommon
3
+ import ExpoModulesCore
4
+ import Foundation
5
+
6
+ public class AcsMeetingModule: Module {
7
+ private var session: AcsMeetingSession?
8
+
9
+ public func definition() -> ModuleDefinition {
10
+ Name("MentraAcsMeeting")
11
+ Events("onState", "onIncomingPcm")
12
+
13
+ AsyncFunction("join") { (options: [String: Any]) in
14
+ let token = try requireString(options, "token")
15
+ let meetingUrl = try requireString(options, "meetingUrl")
16
+ let whepUrl = try requireString(options, "whepUrl")
17
+ let displayName = options["displayName"] as? String
18
+ let dumpWav = options["dumpPcmWav"] as? Bool ?? false
19
+ let audioSource = options["audioSource"] as? String ?? "glasses"
20
+ let video = try parseAcsOutgoingVideo(options["video"])
21
+ let meeting = self.session ?? AcsMeetingSession(
22
+ onState: { [weak self] state in self?.sendEvent("onState", state) },
23
+ onIncomingPcm: { [weak self] base64, rate, channels in
24
+ self?.sendEvent("onIncomingPcm", ["base64": base64, "sampleRate": rate, "channels": channels])
25
+ }
26
+ )
27
+ self.session = meeting
28
+ try meeting.join(token: token, meetingUrl: meetingUrl, whepUrl: whepUrl, displayName: displayName, dumpWav: dumpWav, audioSource: audioSource, video: video)
29
+ return meeting.snapshot()
30
+ }
31
+
32
+ AsyncFunction("leave") {
33
+ self.session?.leave()
34
+ }
35
+
36
+ AsyncFunction("setMuted") { (muted: Bool) in
37
+ self.session?.setMuted(muted) ?? ["state": "idle", "muted": muted]
38
+ }
39
+
40
+ AsyncFunction("setAudioSource") { (source: String) in
41
+ self.session?.setAudioSource(source) ?? ["state": "idle", "muted": false, "audioSource": source]
42
+ }
43
+
44
+ AsyncFunction("updateVideoSource") { (whepUrl: String) in
45
+ self.session?.updateVideoSource(whepUrl)
46
+ }
47
+
48
+ AsyncFunction("restartVideoSource") {
49
+ self.session?.restartVideoSource()
50
+ }
51
+
52
+ AsyncFunction("getState") {
53
+ self.session?.snapshot() ?? ["state": "idle", "muted": false]
54
+ }
55
+
56
+ OnDestroy {
57
+ self.session?.leave()
58
+ self.session = nil
59
+ }
60
+ }
61
+ }
62
+
63
+ final class QueuePolicyScheduler: PolicyScheduler {
64
+ private let queue: DispatchQueue
65
+ private var pending: [DispatchWorkItem] = []
66
+
67
+ init(queue: DispatchQueue) {
68
+ self.queue = queue
69
+ }
70
+
71
+ func schedule(delayMs: Int, task: @escaping () -> Void) {
72
+ let item = DispatchWorkItem(block: task)
73
+ pending.append(item)
74
+ queue.asyncAfter(deadline: .now() + .milliseconds(delayMs), execute: item)
75
+ }
76
+
77
+ func cancelPending() {
78
+ pending.forEach { $0.cancel() }
79
+ pending.removeAll()
80
+ }
81
+ }
82
+
83
+ final class AcsMeetingSession {
84
+ private static let glassesRequiresUnmutedTransport = true
85
+ private let onState: ([String: Any]) -> Void
86
+ private let onIncomingPcm: (String, Int, Int) -> Void
87
+ private let queue = DispatchQueue(label: "com.mentra.acsmeeting")
88
+ private lazy var scheduler = QueuePolicyScheduler(queue: queue)
89
+ private lazy var controller = SessionAudioController(session: self)
90
+ private lazy var applier = AudioPolicyApplier(controller: controller, scheduler: scheduler) { NSLog("ACS-SPIKE \($0)") }
91
+ private var phase = "idle"
92
+ private var muted = false
93
+ private var meetingUrl: String?
94
+ private var lastError: String?
95
+ private var callClient: CallClient?
96
+ private var callAgent: CallAgent?
97
+ private var call: Call?
98
+ private var whep: WhepVideoSource?
99
+ private var frameSender = AcsFrameSender()
100
+ private var pcmBridge: PcmBridge?
101
+ private var audioOut: RawOutgoingAudioStream?
102
+ private var localOut: LocalOutgoingAudioStream?
103
+ private let phoneMic = PhoneMicCapturer()
104
+ private var outgoingReady = false
105
+ private var audioSource = "glasses"
106
+ private var lastSafety: AudioSafety = .degraded
107
+ // Health of the glasses WHEP feed, reported alongside the ACS phase so the host
108
+ // can tell "call is up, glasses video is dead" from a healthy call.
109
+ private var mediaSource: SourceState = .idle
110
+ private var mediaRestartAttempts = 0
111
+ private var mediaRestartTask: DispatchWorkItem?
112
+ private static let mediaRestartBaseMs = 1_000
113
+ private static let mediaRestartMaxMs = 10_000
114
+
115
+ init(onState: @escaping ([String: Any]) -> Void, onIncomingPcm: @escaping (String, Int, Int) -> Void) {
116
+ self.onState = onState
117
+ self.onIncomingPcm = onIncomingPcm
118
+ }
119
+
120
+ func snapshot() -> [String: Any] {
121
+ var result: [String: Any] = [
122
+ "state": phase,
123
+ "muted": muted,
124
+ "provider": "acs-teams",
125
+ "audioSource": audioSource,
126
+ "activeStream": controller.readActive().rawValue,
127
+ "audioSafety": lastSafety.rawValue,
128
+ "mediaSource": mediaSource.rawValue,
129
+ ]
130
+ if let meetingUrl { result["meetingUrl"] = meetingUrl }
131
+ if let lastError { result["error"] = lastError }
132
+ return result
133
+ }
134
+
135
+ func join(token: String, meetingUrl: String, whepUrl: String, displayName: String?, dumpWav: Bool, audioSource: String = "glasses", video: AcsOutgoingVideo = .hd) throws {
136
+ // Both glasses and phone feed RawOutgoingAudioStream so ACS never owns the
137
+ // phone audio route. Phone PCM comes from AVAudioEngine; glasses via WHEP.
138
+ let parsed = AcsAudioPolicy.parseSource(audioSource) ?? .glasses
139
+ if parsed == .phone {
140
+ NSLog("ACS-SPIKE audioSource=phone: input tap → virtual outgoing; voice-chat session off")
141
+ }
142
+ self.audioSource = parsed == .phone ? "phone" : "glasses"
143
+ // The ACS work below is queued; reflect the intent synchronously so the
144
+ // resolved snapshot reads "connecting" rather than a stale idle.
145
+ phase = "connecting"
146
+ lastError = nil
147
+ self.meetingUrl = meetingUrl
148
+ queue.async { [weak self] in
149
+ guard let self else { return }
150
+ do {
151
+ self.leaveLocked(emitIdle: false)
152
+ self.audioSource = parsed == .phone ? "phone" : "glasses"
153
+ self.meetingUrl = meetingUrl
154
+ self.lastError = nil
155
+ self.emit("connecting")
156
+ let credential = try CommunicationTokenCredential(token: token)
157
+ let client = CallClient()
158
+ self.callClient = client
159
+ let options = CallAgentOptions()
160
+ options.displayName = displayName ?? "Mentra Call"
161
+ let agent = try client.createCallAgent(userCredential: credential, options: options).get()
162
+ self.callAgent = agent
163
+
164
+ let videoFormat = VideoStreamFormat()
165
+ videoFormat.pixelFormat = .nv12
166
+ videoFormat.width = Int32(video.width)
167
+ videoFormat.height = Int32(video.height)
168
+ videoFormat.framesPerSecond = Float(video.fps)
169
+ let videoOptions = RawOutgoingVideoStreamOptions()
170
+ videoOptions.formats = [videoFormat]
171
+ let videoStream = VirtualOutgoingVideoStream(videoStreamOptions: videoOptions)
172
+ self.frameSender.attach(videoStream)
173
+
174
+ let outAudioFormat = AudioStreamFormat()
175
+ outAudioFormat.sampleRate = .sampleRate48000
176
+ outAudioFormat.channelMode = .channelModeMono
177
+ outAudioFormat.encodedAudioFormat = .pcm
178
+
179
+ let outAudioOptions = RawOutgoingAudioStreamOptions()
180
+ outAudioOptions.format = outAudioFormat
181
+ let outgoing = RawOutgoingAudioStream(audioStreamOptions: outAudioOptions)
182
+ outgoing.delegate = self
183
+ self.audioOut = outgoing
184
+
185
+ let desired: AudioSourceKind = self.audioSource == "phone" ? .phone : .glasses
186
+ let plan = AcsAudioPolicy.planJoin(
187
+ desired: desired,
188
+ userMuted: self.muted,
189
+ glassesRequiresUnmutedTransport: Self.glassesRequiresUnmutedTransport
190
+ )
191
+
192
+ // Virtual outgoing stays armed for phone and glasses. A LocalOutgoing
193
+ // stream would make ACS own the phone route and open an echo loop.
194
+ let local: LocalOutgoingAudioStream? = plan.armVirtual ? nil : LocalOutgoingAudioStream()
195
+ self.localOut = local
196
+
197
+ let inAudioFormat = AudioStreamFormat()
198
+ inAudioFormat.sampleRate = .sampleRate16000
199
+ inAudioFormat.channelMode = .channelModeMono
200
+ inAudioFormat.encodedAudioFormat = .pcm
201
+ let inAudioOptions = RawIncomingAudioStreamOptions()
202
+ inAudioOptions.format = inAudioFormat
203
+ let incoming = RawIncomingAudioStream(audioStreamOptions: inAudioOptions)
204
+ incoming.delegate = self
205
+
206
+ let joinOptions = JoinCallOptions()
207
+ let outgoingVideo = OutgoingVideoOptions()
208
+ outgoingVideo.streams = [videoStream]
209
+ joinOptions.outgoingVideoOptions = outgoingVideo
210
+ let outgoingAudio = OutgoingAudioOptions()
211
+ outgoingAudio.stream = plan.armVirtual ? outgoing : local!
212
+ outgoingAudio.muted = plan.transportMuted
213
+ joinOptions.outgoingAudioOptions = outgoingAudio
214
+ let incomingAudio = IncomingAudioOptions()
215
+ incomingAudio.stream = incoming
216
+ // Return audio is ours to route (base64 → host → A2DP on the glasses), so
217
+ // the raw incoming stream must actually deliver buffers.
218
+ incomingAudio.muted = false
219
+ joinOptions.incomingAudioOptions = incomingAudio
220
+
221
+ let locator = TeamsMeetingLinkLocator(meetingLink: meetingUrl)
222
+ let call = try agent.join(with: locator, joinCallOptions: joinOptions)
223
+ self.call = call
224
+ call.delegate = self
225
+
226
+ let bridge = PcmBridge(dumpWav: dumpWav)
227
+ self.pcmBridge = bridge
228
+ self.phoneMic.onPcm = { [weak self] pcm, rate, channels in
229
+ self?.feedOutgoingPcm(pcm, sampleRate: rate, channels: channels)
230
+ }
231
+ let source = WhepVideoSource()
232
+ source.onFrame = { buffer in self.frameSender.send(buffer) }
233
+ source.onPcm = { pcm, rate, channels in
234
+ self.feedOutgoingPcm(pcm, sampleRate: rate, channels: channels)
235
+ }
236
+ self.mediaRestartAttempts = 0
237
+ source.onStateChange = { [weak self, weak source] state, reason in
238
+ // Fired from WebRTC/URLSession threads; hop to the session queue so it
239
+ // serializes with join/leave/policy like everything else.
240
+ self?.queue.async {
241
+ guard let self, let source, self.whep === source else { return }
242
+ self.onMediaSourceState(state, reason: reason)
243
+ }
244
+ }
245
+ source.start(config: SourceConfig(url: whepUrl))
246
+ self.whep = source
247
+ self.applyAudioPolicyOnQueue("join")
248
+ NSLog("ACS-SPIKE iOS ACS join started source=\(self.audioSource) profile=\(video.width)x\(video.height)@\(video.fps) armVirtual=\(plan.armVirtual) transportMuted=\(plan.transportMuted)")
249
+ } catch {
250
+ let message = error.localizedDescription
251
+ // A step after a successful ACS join (e.g. WHEP start) can throw. Record
252
+ // the failure before teardown: lastError makes the call delegate ignore
253
+ // the hang-up's async disconnected callback, and emitIdle=false keeps the
254
+ // terminal state as error instead of resetting to idle. Either would
255
+ // otherwise let Mentra Call treat the failed join as a clean end.
256
+ self.lastError = message
257
+ self.leaveLocked(emitIdle: false)
258
+ self.emit("error")
259
+ }
260
+ }
261
+ }
262
+
263
+ func updateVideoSource(_ whepUrl: String) {
264
+ queue.async {
265
+ // The host has a fresher opinion about where the glasses publish; drop any
266
+ // automatic retry against the old URL.
267
+ self.cancelMediaRestart()
268
+ self.whep?.restart(config: SourceConfig(url: whepUrl))
269
+ }
270
+ }
271
+
272
+ /// Rebuild the WHEP subscription on the current URL even when it looks healthy.
273
+ /// The host calls this when the phone changed networks.
274
+ func restartVideoSource() {
275
+ queue.async {
276
+ self.cancelMediaRestart()
277
+ self.whep?.forceRestart()
278
+ }
279
+ }
280
+
281
+ private func onMediaSourceState(_ state: SourceState, reason: String) {
282
+ let previous = mediaSource
283
+ mediaSource = state
284
+ if state == .live { mediaRestartAttempts = 0 }
285
+ if state == .failed { scheduleMediaRestart(reason: reason) }
286
+ // start() emits idle then connecting back to back; one snapshot per real change.
287
+ if previous != state, call != nil, phase != "idle" { onState(snapshot()) }
288
+ }
289
+
290
+ /// Native owns first-line recovery: nothing above this layer can see ICE fail, and a
291
+ /// Teams call with a frozen last frame looks healthy from every other angle.
292
+ /// Exponential backoff capped at mediaRestartMaxMs, for as long as the call is alive.
293
+ private func scheduleMediaRestart(reason: String) {
294
+ guard call != nil, !["idle", "disconnected", "error"].contains(phase) else { return }
295
+ guard mediaRestartTask == nil else { return }
296
+ let attempt = mediaRestartAttempts
297
+ mediaRestartAttempts += 1
298
+ let delayMs = min(Self.mediaRestartBaseMs << min(attempt, 4), Self.mediaRestartMaxMs)
299
+ NSLog("ACS-SPIKE glasses media source failed (\(reason)); WHEP rebuild #\(attempt + 1) in \(delayMs)ms")
300
+ let task = DispatchWorkItem { [weak self] in
301
+ guard let self else { return }
302
+ self.mediaRestartTask = nil
303
+ guard self.call != nil, self.mediaSource == .failed else { return }
304
+ self.whep?.forceRestart()
305
+ }
306
+ mediaRestartTask = task
307
+ queue.asyncAfter(deadline: .now() + .milliseconds(delayMs), execute: task)
308
+ }
309
+
310
+ private func cancelMediaRestart() {
311
+ mediaRestartTask?.cancel()
312
+ mediaRestartTask = nil
313
+ mediaRestartAttempts = 0
314
+ }
315
+
316
+ func setMuted(_ next: Bool) -> [String: Any] {
317
+ muted = next
318
+ queue.async { self.applyAudioPolicyOnQueue("set-muted") }
319
+ let snap = snapshot()
320
+ onState(snap)
321
+ return snap
322
+ }
323
+
324
+ func setAudioSource(_ source: String) -> [String: Any] {
325
+ if AcsAudioPolicy.parseSource(source) == nil {
326
+ NSLog("ACS-SPIKE unknown audioSource=\(source) ignored; source is locked for this call")
327
+ } else {
328
+ NSLog("ACS-SPIKE setAudioSource=\(source) ignored; audio source is locked for this call at \(audioSource)")
329
+ }
330
+ return snapshot()
331
+ }
332
+
333
+ func leave() {
334
+ queue.async { self.leaveLocked() }
335
+ }
336
+
337
+ fileprivate func applyAudioPolicy(_ reason: String) {
338
+ queue.async { self.applyAudioPolicyOnQueue(reason) }
339
+ }
340
+
341
+ private func applyAudioPolicyOnQueue(_ reason: String) {
342
+ let desired: AudioSourceKind = audioSource == "phone" ? .phone : .glasses
343
+ lastSafety = applier.apply(desired: desired, userMuted: muted, reason: reason)
344
+ if lastSafety == .unsafe {
345
+ NSLog("ACS-SPIKE audioSafety=unsafe — mute and stopAudio both failed; unintended mic may be live")
346
+ }
347
+ onState(snapshot())
348
+ }
349
+
350
+ private func feedOutgoingPcm(_ pcm: Data, sampleRate: Int, channels: Int) {
351
+ guard !muted, outgoingReady, let stream = audioOut else { return }
352
+ for frame in pcmBridge?.ingest(pcm16Le: pcm, sampleRate: sampleRate, channels: channels) ?? [] {
353
+ do {
354
+ let buffer = RawAudioBuffer()
355
+ buffer.data = frame
356
+ try stream.send(buffer)
357
+ } catch {
358
+ NSLog("ACS-SPIKE sendRawAudioBuffer failed: \(error)")
359
+ break
360
+ }
361
+ }
362
+ }
363
+
364
+ private func emit(_ next: String) {
365
+ phase = next
366
+ onState(snapshot())
367
+ }
368
+
369
+ private func leaveLocked(emitIdle: Bool = true) {
370
+ phoneMic.setEnabled(false)
371
+ phoneMic.onPcm = nil
372
+ applier.reset()
373
+ scheduler.cancelPending()
374
+ pcmBridge?.finishDump()
375
+ // Detach before stop so the teardown's own idle transition does not emit a
376
+ // snapshot (or schedule a rebuild) for a call that is going away.
377
+ cancelMediaRestart()
378
+ whep?.onStateChange = nil
379
+ mediaSource = .idle
380
+ whep?.stop()
381
+ frameSender.detach()
382
+ do {
383
+ try call?.hangUp().get()
384
+ } catch {
385
+ NSLog("ACS-SPIKE leave hangUp failed: \(error)")
386
+ }
387
+ // Dispose independently of hang-up: a failed hang-up must not leak the ACS
388
+ // agent, and each join/leave cycle must release the previous agent.
389
+ callAgent?.dispose()
390
+ callAgent = nil
391
+ callClient = nil
392
+ call = nil
393
+ whep = nil
394
+ audioOut = nil
395
+ localOut = nil
396
+ pcmBridge = nil
397
+ outgoingReady = false
398
+ muted = false
399
+ audioSource = "glasses"
400
+ lastSafety = .degraded
401
+ meetingUrl = nil
402
+ // Clearing lastError is scoped to the clean idle reset. A failed join tears down
403
+ // with emitIdle=false and relies on lastError staying set so emit("error") still
404
+ // carries it and the call delegate keeps ignoring late disconnected callbacks.
405
+ if emitIdle {
406
+ lastError = nil
407
+ emit("idle")
408
+ }
409
+ }
410
+
411
+ fileprivate func currentCall() -> Call? { call }
412
+ fileprivate func currentWhep() -> WhepVideoSource? { whep }
413
+ fileprivate func setPhonePcmEnabled(_ enabled: Bool) { phoneMic.setEnabled(enabled) }
414
+ }
415
+
416
+ final class SessionAudioController: AudioStreamController {
417
+ private weak var session: AcsMeetingSession?
418
+
419
+ init(session: AcsMeetingSession) {
420
+ self.session = session
421
+ }
422
+
423
+ func readActive() -> ActiveStreamKind {
424
+ guard let stream = session?.currentCall()?.activeOutgoingAudioStream else { return .none }
425
+ let started = String(describing: stream.state).localizedCaseInsensitiveContains("started")
426
+ guard started else { return .none }
427
+ let type = String(describing: stream.type).lowercased()
428
+ if type.contains("virtual") { return .virtual }
429
+ if type.contains("local") { return .local }
430
+ return .none
431
+ }
432
+
433
+ func isPhysicallyMuted() -> Bool? {
434
+ session?.currentCall()?.isMuted
435
+ }
436
+
437
+ func setGlassesPcmEnabled(_ enabled: Bool) {
438
+ session?.currentWhep()?.setPcmDeliveryEnabled(enabled)
439
+ }
440
+
441
+ func setPhonePcmEnabled(_ enabled: Bool) {
442
+ session?.setPhonePcmEnabled(enabled)
443
+ }
444
+
445
+ func mutePhysical() -> Result<Void, Error> {
446
+ switch CallGuard.require(session?.currentCall()) {
447
+ case .failure(let error): return .failure(error)
448
+ case .success(let call):
449
+ do {
450
+ try call.mute().get()
451
+ return .success(())
452
+ } catch {
453
+ return .failure(error)
454
+ }
455
+ }
456
+ }
457
+
458
+ func unmutePhysical() -> Result<Void, Error> {
459
+ switch CallGuard.require(session?.currentCall()) {
460
+ case .failure(let error): return .failure(error)
461
+ case .success(let call):
462
+ do {
463
+ try call.unmute().get()
464
+ return .success(())
465
+ } catch {
466
+ return .failure(error)
467
+ }
468
+ }
469
+ }
470
+
471
+ func stopActive() -> Result<Void, Error> {
472
+ switch CallGuard.require(session?.currentCall()) {
473
+ case .failure(let error): return .failure(error)
474
+ case .success(let call):
475
+ guard let stream = call.activeOutgoingAudioStream else {
476
+ return .failure(CallMissingError())
477
+ }
478
+ do {
479
+ try call.stopAudio(stream: stream).get()
480
+ return .success(())
481
+ } catch {
482
+ return .failure(error)
483
+ }
484
+ }
485
+ }
486
+ }
487
+
488
+ extension AcsMeetingSession: CallDelegate {
489
+ func call(_ call: Call, didChangeState args: PropertyChangedEventArgs) {
490
+ // A failed join has already reported a terminal error and torn the call
491
+ // down; ignore any late ACS state callback so it cannot overwrite error.
492
+ if lastError != nil { return }
493
+ switch call.state {
494
+ case .connecting: emit("connecting")
495
+ case .inLobby: emit("lobby")
496
+ case .connected:
497
+ emit("connected")
498
+ applyAudioPolicy("call-connected")
499
+ case .disconnecting, .disconnected: emit("disconnected")
500
+ default: break
501
+ }
502
+ }
503
+
504
+ func call(_ call: Call, didChangeMuteState args: PropertyChangedEventArgs) {
505
+ applyAudioPolicy("outgoing-audio-state")
506
+ }
507
+ }
508
+
509
+ extension AcsMeetingSession: RawOutgoingAudioStreamDelegate {
510
+ func rawOutgoingAudioStream(_ rawOutgoingAudioStream: RawOutgoingAudioStream, didChangeState args: AudioStreamStateChangedEventArgs) {
511
+ outgoingReady = String(describing: rawOutgoingAudioStream.state).localizedCaseInsensitiveContains("started")
512
+ NSLog("ACS-SPIKE iOS raw outgoing audio state=\(rawOutgoingAudioStream.state)")
513
+ applyAudioPolicy("virtual-stream-state")
514
+ }
515
+ }
516
+
517
+ extension AcsMeetingSession: RawIncomingAudioStreamDelegate {
518
+ func rawIncomingAudioStream(_ rawIncomingAudioStream: RawIncomingAudioStream, didReceiveRawAudioBuffer args: IncomingAudioStreamRawBufferReceivedEventArgs) {
519
+ guard let data = args.audioBuffer?.data else { return }
520
+ onIncomingPcm(data.base64EncodedString(), 16000, 1)
521
+ }
522
+ }
523
+
524
+ struct AcsOutgoingVideo {
525
+ let width: Int
526
+ let height: Int
527
+ let fps: Int
528
+ let maxBitrateBps: Int
529
+
530
+ static let hd = AcsOutgoingVideo(width: 1280, height: 720, fps: 15, maxBitrateBps: 2_500_000)
531
+ static let allowedSizes: Set<String> = ["1280x720", "960x540"]
532
+ }
533
+
534
+ private func requireString(_ options: [String: Any], _ key: String) throws -> String {
535
+ guard let value = options[key] as? String, !value.isEmpty else {
536
+ throw NSError(domain: "MentraAcsMeeting", code: 1, userInfo: [NSLocalizedDescriptionKey: "\(key) is required"])
537
+ }
538
+ return value
539
+ }
540
+
541
+ private func parseAcsOutgoingVideo(_ raw: Any?) throws -> AcsOutgoingVideo {
542
+ guard let raw else { return .hd }
543
+ guard let map = raw as? [String: Any] else {
544
+ throw NSError(domain: "MentraAcsMeeting", code: 1, userInfo: [NSLocalizedDescriptionKey: "video must be an object"])
545
+ }
546
+ guard
547
+ let width = (map["width"] as? NSNumber)?.intValue,
548
+ let height = (map["height"] as? NSNumber)?.intValue,
549
+ let fps = (map["fps"] as? NSNumber)?.intValue,
550
+ let bitrate = (map["maxBitrateBps"] as? NSNumber)?.intValue
551
+ else {
552
+ throw NSError(domain: "MentraAcsMeeting", code: 1, userInfo: [NSLocalizedDescriptionKey: "video requires width, height, fps, and maxBitrateBps"])
553
+ }
554
+ guard AcsOutgoingVideo.allowedSizes.contains("\(width)x\(height)"), fps >= 1, fps <= 30, bitrate > 0 else {
555
+ throw NSError(domain: "MentraAcsMeeting", code: 1, userInfo: [NSLocalizedDescriptionKey: "unsupported ACS video \(width)x\(height)@\(fps)"])
556
+ }
557
+ return AcsOutgoingVideo(width: width, height: height, fps: fps, maxBitrateBps: bitrate)
558
+ }
@@ -0,0 +1,117 @@
1
+ import Foundation
2
+
3
+ /// Downmix/resample PCM16 to 48 kHz mono ~20 ms blocks for ACS raw outgoing audio.
4
+ final class PcmBridge {
5
+ private var dump = Data()
6
+ private var dumpRate = 0
7
+ private var dumpChannels = 0
8
+ private var dumped = false
9
+ private var pending = Data()
10
+ private let dumpWav: Bool
11
+ private var lastRmsLog = Date.distantPast
12
+
13
+ init(dumpWav: Bool) {
14
+ self.dumpWav = dumpWav
15
+ }
16
+
17
+ func ingest(pcm16Le: Data, sampleRate: Int, channels: Int) -> [Data] {
18
+ if dumpWav, !dumped {
19
+ dump.append(pcm16Le)
20
+ dumpRate = sampleRate
21
+ dumpChannels = channels
22
+ if dump.count >= 30 * sampleRate * channels * 2 {
23
+ finishDump()
24
+ }
25
+ }
26
+ logLevel(pcm16Le, sampleRate: sampleRate, channels: channels)
27
+ let mono = Self.toMono(pcm16Le, sampleRate: sampleRate, channels: channels)
28
+ pending.append(mono)
29
+ let frameBytes = Self.targetRate * 2 / 50
30
+ var frames: [Data] = []
31
+ while pending.count >= frameBytes {
32
+ frames.append(pending.prefix(frameBytes))
33
+ pending.removeFirst(frameBytes)
34
+ }
35
+ return frames
36
+ }
37
+
38
+ func finishDump() {
39
+ guard dumpWav, !dumped else { return }
40
+ dumped = true
41
+ let url = FileManager.default.temporaryDirectory.appendingPathComponent("acs-whep-p4.wav")
42
+ try? Self.wav(pcm: dump, sampleRate: dumpRate, channels: dumpChannels).write(to: url)
43
+ NSLog("ACS-SPIKE P4 wrote \(url.path) bytes=\(dump.count) rate=\(dumpRate) ch=\(dumpChannels)")
44
+ dump.removeAll()
45
+ }
46
+
47
+ private func logLevel(_ pcm: Data, sampleRate: Int, channels: Int) {
48
+ let now = Date()
49
+ guard now.timeIntervalSince(lastRmsLog) >= 1 else { return }
50
+ lastRmsLog = now
51
+ var acc: Int64 = 0
52
+ var count = 0
53
+ pcm.withUnsafeBytes { raw in
54
+ let samples = raw.bindMemory(to: Int16.self)
55
+ for sample in samples {
56
+ acc += Int64(abs(Int32(sample)))
57
+ count += 1
58
+ }
59
+ }
60
+ let mean = count == 0 ? 0 : acc / Int64(count)
61
+ NSLog("ACS-SPIKE P4 pcm rate=\(sampleRate) ch=\(channels) meanAbs=\(mean) bytes=\(pcm.count)")
62
+ }
63
+
64
+ static let targetRate = 48_000
65
+
66
+ static func toMono(_ pcm16Le: Data, sampleRate: Int, channels: Int) -> Data {
67
+ let samples = pcm16Le.withUnsafeBytes { Array($0.bindMemory(to: Int16.self)) }
68
+ let mono: [Int16]
69
+ if channels <= 1 {
70
+ mono = samples
71
+ } else {
72
+ mono = stride(from: 0, to: samples.count, by: channels).map { index in
73
+ var acc = 0
74
+ for channel in 0 ..< channels where index + channel < samples.count {
75
+ acc += Int(samples[index + channel])
76
+ }
77
+ return Int16(acc / channels)
78
+ }
79
+ }
80
+ if sampleRate == targetRate {
81
+ return mono.withUnsafeBufferPointer { Data(buffer: $0) }
82
+ }
83
+ let outCount = max(1, mono.count * targetRate / sampleRate)
84
+ var resampled = [Int16](repeating: 0, count: outCount)
85
+ for i in 0 ..< outCount {
86
+ let src = min(mono.count - 1, i * sampleRate / targetRate)
87
+ resampled[i] = mono[src]
88
+ }
89
+ return resampled.withUnsafeBufferPointer { Data(buffer: $0) }
90
+ }
91
+
92
+ static func wav(pcm: Data, sampleRate: Int, channels: Int) -> Data {
93
+ var header = Data()
94
+ func put(_ value: UInt32) {
95
+ var le = value.littleEndian
96
+ header.append(Data(bytes: &le, count: 4))
97
+ }
98
+ func put16(_ value: UInt16) {
99
+ var le = value.littleEndian
100
+ header.append(Data(bytes: &le, count: 2))
101
+ }
102
+ header.append(contentsOf: [UInt8]("RIFF".utf8))
103
+ put(UInt32(36 + pcm.count))
104
+ header.append(contentsOf: [UInt8]("WAVE".utf8))
105
+ header.append(contentsOf: [UInt8]("fmt ".utf8))
106
+ put(16)
107
+ put16(1)
108
+ put16(UInt16(channels))
109
+ put(UInt32(sampleRate))
110
+ put(UInt32(sampleRate * channels * 2))
111
+ put16(UInt16(channels * 2))
112
+ put16(16)
113
+ header.append(contentsOf: [UInt8]("data".utf8))
114
+ put(UInt32(pcm.count))
115
+ return header + pcm
116
+ }
117
+ }