@mentra/acs-meeting 3.2.0-dev.136 → 3.2.0-dev.153
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/app.plugin.js +31 -2
- package/ios/AcsFrameSender.swift +59 -59
- package/ios/AcsMeeting.podspec +60 -0
- package/ios/AcsMeetingModule.swift +602 -470
- package/ios/PcmBridge.swift +49 -0
- package/package.json +1 -1
|
@@ -5,164 +5,210 @@ import ExpoModulesCore
|
|
|
5
5
|
import Foundation
|
|
6
6
|
|
|
7
7
|
public class AcsMeetingModule: Module {
|
|
8
|
-
|
|
8
|
+
private var session: AcsMeetingSession?
|
|
9
|
+
|
|
10
|
+
public func definition() -> ModuleDefinition {
|
|
11
|
+
Name("MentraAcsMeeting")
|
|
12
|
+
Events("onState", "onIncomingPcm")
|
|
13
|
+
|
|
14
|
+
AsyncFunction("join") { (options: [String: Any]) in
|
|
15
|
+
let token = try requireString(options, "token")
|
|
16
|
+
let meetingUrl = try requireString(options, "meetingUrl")
|
|
17
|
+
let whepUrl = try requireString(options, "whepUrl")
|
|
18
|
+
let displayName = options["displayName"] as? String
|
|
19
|
+
let dumpWav = options["dumpPcmWav"] as? Bool ?? false
|
|
20
|
+
let audioSource = options["audioSource"] as? String ?? "glasses"
|
|
21
|
+
let video = try parseAcsOutgoingVideo(options["video"])
|
|
22
|
+
let meeting = self.session ?? AcsMeetingSession(
|
|
23
|
+
onState: { [weak self] state in self?.sendEvent("onState", state) },
|
|
24
|
+
onIncomingPcm: { [weak self] base64, rate, channels in
|
|
25
|
+
self?.sendEvent("onIncomingPcm", ["base64": base64, "sampleRate": rate, "channels": channels])
|
|
26
|
+
}
|
|
27
|
+
)
|
|
28
|
+
self.session = meeting
|
|
29
|
+
try meeting.join(token: token, meetingUrl: meetingUrl, whepUrl: whepUrl, displayName: displayName, dumpWav: dumpWav, audioSource: audioSource, video: video)
|
|
30
|
+
return meeting.snapshot()
|
|
31
|
+
}
|
|
9
32
|
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
33
|
+
AsyncFunction("leave") {
|
|
34
|
+
self.session?.leave()
|
|
35
|
+
}
|
|
13
36
|
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
let meetingUrl = try requireString(options, "meetingUrl")
|
|
17
|
-
let whepUrl = try requireString(options, "whepUrl")
|
|
18
|
-
let displayName = options["displayName"] as? String
|
|
19
|
-
let dumpWav = options["dumpPcmWav"] as? Bool ?? false
|
|
20
|
-
let audioSource = options["audioSource"] as? String ?? "glasses"
|
|
21
|
-
let video = try parseAcsOutgoingVideo(options["video"])
|
|
22
|
-
let meeting = self.session ?? AcsMeetingSession(
|
|
23
|
-
onState: { [weak self] state in self?.sendEvent("onState", state) },
|
|
24
|
-
onIncomingPcm: { [weak self] base64, rate, channels in
|
|
25
|
-
self?.sendEvent("onIncomingPcm", ["base64": base64, "sampleRate": rate, "channels": channels])
|
|
37
|
+
AsyncFunction("setMuted") { (muted: Bool) in
|
|
38
|
+
self.session?.setMuted(muted) ?? ["state": "idle", "muted": muted]
|
|
26
39
|
}
|
|
27
|
-
)
|
|
28
|
-
self.session = meeting
|
|
29
|
-
try meeting.join(token: token, meetingUrl: meetingUrl, whepUrl: whepUrl, displayName: displayName, dumpWav: dumpWav, audioSource: audioSource, video: video)
|
|
30
|
-
return meeting.snapshot()
|
|
31
|
-
}
|
|
32
40
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
41
|
+
AsyncFunction("setAudioSource") { (source: String) in
|
|
42
|
+
self.session?.setAudioSource(source) ?? ["state": "idle", "muted": false, "audioSource": source]
|
|
43
|
+
}
|
|
36
44
|
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
45
|
+
AsyncFunction("updateVideoSource") { (whepUrl: String) in
|
|
46
|
+
self.session?.updateVideoSource(whepUrl)
|
|
47
|
+
}
|
|
40
48
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
49
|
+
AsyncFunction("restartVideoSource") {
|
|
50
|
+
self.session?.restartVideoSource()
|
|
51
|
+
}
|
|
44
52
|
|
|
45
|
-
|
|
46
|
-
|
|
53
|
+
AsyncFunction("getState") {
|
|
54
|
+
self.session?.snapshot() ?? ["state": "idle", "muted": false]
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
OnDestroy {
|
|
58
|
+
self.session?.leave()
|
|
59
|
+
self.session = nil
|
|
60
|
+
}
|
|
47
61
|
}
|
|
62
|
+
}
|
|
48
63
|
|
|
49
|
-
|
|
50
|
-
|
|
64
|
+
final class QueuePolicyScheduler: PolicyScheduler {
|
|
65
|
+
private let queue: DispatchQueue
|
|
66
|
+
private var pending: [DispatchWorkItem] = []
|
|
67
|
+
|
|
68
|
+
init(queue: DispatchQueue) {
|
|
69
|
+
self.queue = queue
|
|
51
70
|
}
|
|
52
71
|
|
|
53
|
-
|
|
54
|
-
|
|
72
|
+
func schedule(delayMs: Int, task: @escaping () -> Void) {
|
|
73
|
+
let item = DispatchWorkItem(block: task)
|
|
74
|
+
pending.append(item)
|
|
75
|
+
queue.asyncAfter(deadline: .now() + .milliseconds(delayMs), execute: item)
|
|
55
76
|
}
|
|
56
77
|
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
78
|
+
func cancelPending() {
|
|
79
|
+
pending.forEach { $0.cancel() }
|
|
80
|
+
pending.removeAll()
|
|
60
81
|
}
|
|
61
|
-
}
|
|
62
82
|
}
|
|
63
83
|
|
|
64
|
-
final class
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
84
|
+
final class AcsMeetingSession {
|
|
85
|
+
private static let glassesRequiresUnmutedTransport = true
|
|
86
|
+
private let onState: ([String: Any]) -> Void
|
|
87
|
+
private let onIncomingPcm: (String, Int, Int) -> Void
|
|
88
|
+
private let queue = DispatchQueue(label: "com.mentra.acsmeeting")
|
|
89
|
+
private lazy var scheduler = QueuePolicyScheduler(queue: queue)
|
|
90
|
+
private lazy var controller = SessionAudioController(session: self)
|
|
91
|
+
private lazy var applier = AudioPolicyApplier(controller: controller, scheduler: scheduler) { NSLog("ACS-SPIKE \($0)") }
|
|
92
|
+
private var phase = "idle"
|
|
93
|
+
private var muted = false
|
|
94
|
+
private var meetingUrl: String?
|
|
95
|
+
private var lastError: String?
|
|
96
|
+
private var callClient: CallClient?
|
|
97
|
+
private var callAgent: CallAgent?
|
|
98
|
+
private var call: Call?
|
|
99
|
+
private var whep: WhepVideoSource?
|
|
100
|
+
private var frameSender = AcsFrameSender()
|
|
101
|
+
private var pcmBridge: PcmBridge?
|
|
102
|
+
private var audioOut: RawOutgoingAudioStream?
|
|
103
|
+
private var audioIn: RawIncomingAudioStream?
|
|
104
|
+
private var localOut: LocalOutgoingAudioStream?
|
|
105
|
+
private let phoneMic = PhoneMicCapturer()
|
|
106
|
+
private var outgoingReady = false
|
|
107
|
+
private var audioSource = "glasses"
|
|
108
|
+
private var lastSafety: AudioSafety = .degraded
|
|
109
|
+
// Health of the glasses WHEP feed, reported alongside the ACS phase so the host
|
|
110
|
+
// can tell "call is up, glasses video is dead" from a healthy call.
|
|
111
|
+
private var mediaSource: SourceState = .idle
|
|
112
|
+
private var mediaRestartAttempts = 0
|
|
113
|
+
private var mediaRestartTask: DispatchWorkItem?
|
|
114
|
+
private static let mediaRestartBaseMs = 1000
|
|
115
|
+
private static let mediaRestartMaxMs = 10000
|
|
116
|
+
private var joinGeneration: UInt64 = 0
|
|
117
|
+
private lazy var callDelegateProxy = AcsCallDelegateProxy(
|
|
118
|
+
onStateChange: { [weak self] call in self?.handleCallStateChange(call) },
|
|
119
|
+
onMuteChange: { [weak self] call in self?.handleCallMuteChange(call) }
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
init(onState: @escaping ([String: Any]) -> Void, onIncomingPcm: @escaping (String, Int, Int) -> Void) {
|
|
123
|
+
self.onState = onState
|
|
124
|
+
self.onIncomingPcm = onIncomingPcm
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
func snapshot() -> [String: Any] {
|
|
128
|
+
var result: [String: Any] = [
|
|
129
|
+
"state": phase,
|
|
130
|
+
"muted": muted,
|
|
131
|
+
"provider": "acs-teams",
|
|
132
|
+
"audioSource": audioSource,
|
|
133
|
+
"activeStream": controller.readActive().rawValue,
|
|
134
|
+
"audioSafety": lastSafety.rawValue,
|
|
135
|
+
"mediaSource": mediaSource.rawValue,
|
|
136
|
+
]
|
|
137
|
+
if let meetingUrl { result["meetingUrl"] = meetingUrl }
|
|
138
|
+
if let lastError { result["error"] = lastError }
|
|
139
|
+
return result
|
|
140
|
+
}
|
|
83
141
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
private lazy var applier = AudioPolicyApplier(controller: controller, scheduler: scheduler) { NSLog("ACS-SPIKE \($0)") }
|
|
92
|
-
private var phase = "idle"
|
|
93
|
-
private var muted = false
|
|
94
|
-
private var meetingUrl: String?
|
|
95
|
-
private var lastError: String?
|
|
96
|
-
private var callClient: CallClient?
|
|
97
|
-
private var callAgent: CallAgent?
|
|
98
|
-
private var call: Call?
|
|
99
|
-
private var whep: WhepVideoSource?
|
|
100
|
-
private var frameSender = AcsFrameSender()
|
|
101
|
-
private var pcmBridge: PcmBridge?
|
|
102
|
-
private var audioOut: RawOutgoingAudioStream?
|
|
103
|
-
private var localOut: LocalOutgoingAudioStream?
|
|
104
|
-
private let phoneMic = PhoneMicCapturer()
|
|
105
|
-
private var outgoingReady = false
|
|
106
|
-
private var audioSource = "glasses"
|
|
107
|
-
private var lastSafety: AudioSafety = .degraded
|
|
108
|
-
// Health of the glasses WHEP feed, reported alongside the ACS phase so the host
|
|
109
|
-
// can tell "call is up, glasses video is dead" from a healthy call.
|
|
110
|
-
private var mediaSource: SourceState = .idle
|
|
111
|
-
private var mediaRestartAttempts = 0
|
|
112
|
-
private var mediaRestartTask: DispatchWorkItem?
|
|
113
|
-
private static let mediaRestartBaseMs = 1_000
|
|
114
|
-
private static let mediaRestartMaxMs = 10_000
|
|
115
|
-
|
|
116
|
-
init(onState: @escaping ([String: Any]) -> Void, onIncomingPcm: @escaping (String, Int, Int) -> Void) {
|
|
117
|
-
self.onState = onState
|
|
118
|
-
self.onIncomingPcm = onIncomingPcm
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
func snapshot() -> [String: Any] {
|
|
122
|
-
var result: [String: Any] = [
|
|
123
|
-
"state": phase,
|
|
124
|
-
"muted": muted,
|
|
125
|
-
"provider": "acs-teams",
|
|
126
|
-
"audioSource": audioSource,
|
|
127
|
-
"activeStream": controller.readActive().rawValue,
|
|
128
|
-
"audioSafety": lastSafety.rawValue,
|
|
129
|
-
"mediaSource": mediaSource.rawValue,
|
|
130
|
-
]
|
|
131
|
-
if let meetingUrl { result["meetingUrl"] = meetingUrl }
|
|
132
|
-
if let lastError { result["error"] = lastError }
|
|
133
|
-
return result
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
func join(token: String, meetingUrl: String, whepUrl: String, displayName: String?, dumpWav: Bool, audioSource: String = "glasses", video: AcsOutgoingVideo = .hd) throws {
|
|
137
|
-
// Both glasses and phone feed RawOutgoingAudioStream so ACS never owns the
|
|
138
|
-
// phone audio route. Phone PCM comes from AVAudioEngine; glasses via WHEP.
|
|
139
|
-
let parsed = AcsAudioPolicy.parseSource(audioSource) ?? .glasses
|
|
140
|
-
if parsed == .phone {
|
|
141
|
-
NSLog("ACS-SPIKE audioSource=phone: input tap → virtual outgoing; voice-chat session off")
|
|
142
|
-
}
|
|
143
|
-
self.audioSource = parsed == .phone ? "phone" : "glasses"
|
|
144
|
-
// The ACS work below is queued; reflect the intent synchronously so the
|
|
145
|
-
// resolved snapshot reads "connecting" rather than a stale idle.
|
|
146
|
-
phase = "connecting"
|
|
147
|
-
lastError = nil
|
|
148
|
-
self.meetingUrl = meetingUrl
|
|
149
|
-
queue.async { [weak self] in
|
|
150
|
-
guard let self else { return }
|
|
151
|
-
do {
|
|
152
|
-
self.leaveLocked(emitIdle: false)
|
|
142
|
+
func join(token: String, meetingUrl: String, whepUrl: String, displayName: String?, dumpWav: Bool, audioSource: String = "glasses", video: AcsOutgoingVideo = .hd) throws {
|
|
143
|
+
// Both glasses and phone feed RawOutgoingAudioStream so ACS never owns the
|
|
144
|
+
// phone audio route. Phone PCM comes from AVAudioEngine; glasses via WHEP.
|
|
145
|
+
let parsed = AcsAudioPolicy.parseSource(audioSource) ?? .glasses
|
|
146
|
+
if parsed == .phone {
|
|
147
|
+
NSLog("ACS-SPIKE audioSource=phone: input tap → virtual outgoing; voice-chat session off")
|
|
148
|
+
}
|
|
153
149
|
self.audioSource = parsed == .phone ? "phone" : "glasses"
|
|
150
|
+
// The ACS work below is queued; reflect the intent synchronously so the
|
|
151
|
+
// resolved snapshot reads "connecting" rather than a stale idle.
|
|
152
|
+
phase = "connecting"
|
|
153
|
+
lastError = nil
|
|
154
154
|
self.meetingUrl = meetingUrl
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
155
|
+
queue.async { [weak self] in
|
|
156
|
+
guard let self else { return }
|
|
157
|
+
do {
|
|
158
|
+
self.leaveLocked(emitIdle: false)
|
|
159
|
+
let generation = self.joinGeneration
|
|
160
|
+
self.audioSource = parsed == .phone ? "phone" : "glasses"
|
|
161
|
+
self.meetingUrl = meetingUrl
|
|
162
|
+
self.lastError = nil
|
|
163
|
+
self.emit("connecting")
|
|
164
|
+
let credential = try CommunicationTokenCredential(token: token)
|
|
165
|
+
let client = CallClient()
|
|
166
|
+
self.callClient = client
|
|
167
|
+
let options = CallAgentOptions()
|
|
168
|
+
options.displayName = displayName ?? "Mentra Call"
|
|
169
|
+
client.createCallAgent(userCredential: credential, options: options) { [weak self, weak client] agent, error in
|
|
170
|
+
self?.queue.async {
|
|
171
|
+
guard let self, let client, self.callClient === client, self.joinGeneration == generation else {
|
|
172
|
+
agent?.dispose()
|
|
173
|
+
return
|
|
174
|
+
}
|
|
175
|
+
if let error {
|
|
176
|
+
self.failJoinLocked(error, generation: generation)
|
|
177
|
+
return
|
|
178
|
+
}
|
|
179
|
+
guard let agent else {
|
|
180
|
+
self.failJoinLocked(AcsMeetingError("ACS returned no call agent"), generation: generation)
|
|
181
|
+
return
|
|
182
|
+
}
|
|
183
|
+
do {
|
|
184
|
+
try self.joinWithAgentLocked(
|
|
185
|
+
agent,
|
|
186
|
+
generation: generation,
|
|
187
|
+
meetingUrl: meetingUrl,
|
|
188
|
+
whepUrl: whepUrl,
|
|
189
|
+
dumpWav: dumpWav,
|
|
190
|
+
video: video
|
|
191
|
+
)
|
|
192
|
+
} catch {
|
|
193
|
+
self.failJoinLocked(error, generation: generation)
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
} catch {
|
|
198
|
+
self.failJoinLocked(error, generation: self.joinGeneration)
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
private func joinWithAgentLocked(
|
|
204
|
+
_ agent: CallAgent,
|
|
205
|
+
generation: UInt64,
|
|
206
|
+
meetingUrl: String,
|
|
207
|
+
whepUrl: String,
|
|
208
|
+
dumpWav: Bool,
|
|
209
|
+
video: AcsOutgoingVideo
|
|
210
|
+
) throws {
|
|
211
|
+
callAgent = agent
|
|
166
212
|
|
|
167
213
|
let videoFormat = VideoStreamFormat()
|
|
168
214
|
videoFormat.pixelFormat = .nv12
|
|
@@ -172,52 +218,45 @@ final class AcsMeetingSession: NSObject {
|
|
|
172
218
|
let videoOptions = RawOutgoingVideoStreamOptions()
|
|
173
219
|
videoOptions.formats = [videoFormat]
|
|
174
220
|
let videoStream = VirtualOutgoingVideoStream(videoStreamOptions: videoOptions)
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
let outAudioFormat = RawOutgoingAudioStreamProperties()
|
|
178
|
-
outAudioFormat.sampleRate = .hz48000
|
|
179
|
-
outAudioFormat.channelMode = .mono
|
|
180
|
-
outAudioFormat.format = .pcm16Bit
|
|
221
|
+
frameSender.attach(videoStream)
|
|
181
222
|
|
|
223
|
+
let outAudioProperties = RawOutgoingAudioStreamProperties()
|
|
224
|
+
outAudioProperties.sampleRate = .hz48000
|
|
225
|
+
outAudioProperties.channelMode = .mono
|
|
226
|
+
outAudioProperties.format = .pcm16Bit
|
|
227
|
+
outAudioProperties.bufferDuration = .ms20
|
|
182
228
|
let outAudioOptions = RawOutgoingAudioStreamOptions()
|
|
183
|
-
|
|
184
|
-
outAudioOptions.properties = outAudioFormat
|
|
229
|
+
outAudioOptions.properties = outAudioProperties
|
|
185
230
|
let outgoing = RawOutgoingAudioStream(options: outAudioOptions)
|
|
186
231
|
outgoing.events.onStateChanged = { [weak self, weak outgoing] _ in
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
guard self.audioOut === outgoing else { return }
|
|
190
|
-
self.outgoingReady = outgoing.state == .started
|
|
191
|
-
self.applyAudioPolicyOnQueue("virtual-stream-state")
|
|
192
|
-
}
|
|
232
|
+
guard let outgoing else { return }
|
|
233
|
+
self?.handleOutgoingAudioStateChange(outgoing)
|
|
193
234
|
}
|
|
194
|
-
|
|
235
|
+
audioOut = outgoing
|
|
195
236
|
|
|
196
|
-
let desired: AudioSourceKind =
|
|
237
|
+
let desired: AudioSourceKind = audioSource == "phone" ? .phone : .glasses
|
|
197
238
|
let plan = AcsAudioPolicy.planJoin(
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
239
|
+
desired: desired,
|
|
240
|
+
userMuted: muted,
|
|
241
|
+
glassesRequiresUnmutedTransport: Self.glassesRequiresUnmutedTransport
|
|
201
242
|
)
|
|
202
243
|
|
|
203
244
|
// Virtual outgoing stays armed for phone and glasses. A LocalOutgoing
|
|
204
245
|
// stream would make ACS own the phone route and open an echo loop.
|
|
205
246
|
let local: LocalOutgoingAudioStream? = plan.armVirtual ? nil : LocalOutgoingAudioStream()
|
|
206
|
-
|
|
247
|
+
localOut = local
|
|
207
248
|
|
|
208
|
-
let
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
249
|
+
let inAudioProperties = RawIncomingAudioStreamProperties()
|
|
250
|
+
inAudioProperties.sampleRate = .hz16000
|
|
251
|
+
inAudioProperties.channelMode = .mono
|
|
252
|
+
inAudioProperties.format = .pcm16Bit
|
|
212
253
|
let inAudioOptions = RawIncomingAudioStreamOptions()
|
|
213
|
-
inAudioOptions.properties =
|
|
254
|
+
inAudioOptions.properties = inAudioProperties
|
|
214
255
|
let incoming = RawIncomingAudioStream(options: inAudioOptions)
|
|
215
256
|
incoming.events.onMixedAudioBufferReceived = { [weak self] args in
|
|
216
|
-
|
|
217
|
-
let samples = pcm.int16ChannelData else { return }
|
|
218
|
-
let data = Data(bytes: samples[0], count: Int(pcm.frameLength) * Int(pcm.format.channelCount) * 2)
|
|
219
|
-
self?.onIncomingPcm(data.base64EncodedString(), Int(pcm.format.sampleRate), Int(pcm.format.channelCount))
|
|
257
|
+
self?.handleIncomingAudio(args)
|
|
220
258
|
}
|
|
259
|
+
audioIn = incoming
|
|
221
260
|
|
|
222
261
|
let joinOptions = JoinCallOptions()
|
|
223
262
|
let outgoingVideo = OutgoingVideoOptions()
|
|
@@ -235,340 +274,433 @@ final class AcsMeetingSession: NSObject {
|
|
|
235
274
|
joinOptions.incomingAudioOptions = incomingAudio
|
|
236
275
|
|
|
237
276
|
let locator = TeamsMeetingLinkLocator(meetingLink: meetingUrl)
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
277
|
+
agent.join(with: locator, joinCallOptions: joinOptions) { [weak self, weak agent] call, error in
|
|
278
|
+
self?.queue.async {
|
|
279
|
+
guard let self, let agent, self.callAgent === agent, self.joinGeneration == generation else {
|
|
280
|
+
call?.hangUp(options: nil) { _ in }
|
|
281
|
+
return
|
|
282
|
+
}
|
|
283
|
+
if let error {
|
|
284
|
+
self.failJoinLocked(error, generation: generation)
|
|
285
|
+
return
|
|
286
|
+
}
|
|
287
|
+
guard let call else {
|
|
288
|
+
self.failJoinLocked(AcsMeetingError("ACS returned no call"), generation: generation)
|
|
289
|
+
return
|
|
290
|
+
}
|
|
291
|
+
self.finishJoinLocked(
|
|
292
|
+
call,
|
|
293
|
+
generation: generation,
|
|
294
|
+
whepUrl: whepUrl,
|
|
295
|
+
dumpWav: dumpWav,
|
|
296
|
+
video: video,
|
|
297
|
+
plan: plan
|
|
298
|
+
)
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
private func finishJoinLocked(
|
|
304
|
+
_ call: Call,
|
|
305
|
+
generation: UInt64,
|
|
306
|
+
whepUrl: String,
|
|
307
|
+
dumpWav: Bool,
|
|
308
|
+
video: AcsOutgoingVideo,
|
|
309
|
+
plan: JoinAudioPlan
|
|
310
|
+
) {
|
|
311
|
+
guard joinGeneration == generation else {
|
|
312
|
+
call.hangUp(options: nil) { _ in }
|
|
313
|
+
return
|
|
242
314
|
}
|
|
243
315
|
self.call = call
|
|
244
|
-
call.delegate =
|
|
316
|
+
call.delegate = callDelegateProxy
|
|
245
317
|
|
|
246
318
|
let bridge = PcmBridge(dumpWav: dumpWav)
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
319
|
+
pcmBridge = bridge
|
|
320
|
+
phoneMic.onPcm = { [weak self] pcm, rate, channels in
|
|
321
|
+
self?.feedOutgoingPcm(pcm, sampleRate: rate, channels: channels)
|
|
250
322
|
}
|
|
251
323
|
let source = WhepVideoSource()
|
|
252
|
-
source.onFrame = { buffer in self
|
|
253
|
-
source.onPcm = { pcm, rate, channels in
|
|
254
|
-
|
|
324
|
+
source.onFrame = { [weak self] buffer in self?.frameSender.send(buffer) }
|
|
325
|
+
source.onPcm = { [weak self] pcm, rate, channels in
|
|
326
|
+
self?.feedOutgoingPcm(pcm, sampleRate: rate, channels: channels)
|
|
255
327
|
}
|
|
256
|
-
|
|
328
|
+
mediaRestartAttempts = 0
|
|
257
329
|
source.onStateChange = { [weak self, weak source] state, reason in
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
330
|
+
// Fired from WebRTC/URLSession threads; hop to the session queue so it
|
|
331
|
+
// serializes with join/leave/policy like everything else.
|
|
332
|
+
self?.queue.async {
|
|
333
|
+
guard let self, let source, self.whep === source else { return }
|
|
334
|
+
self.onMediaSourceState(state, reason: reason)
|
|
335
|
+
}
|
|
264
336
|
}
|
|
265
337
|
source.start(config: SourceConfig(url: whepUrl))
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
NSLog("ACS-SPIKE iOS ACS join started source=\(
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
//
|
|
274
|
-
//
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
// automatic retry against the old URL.
|
|
287
|
-
self.cancelMediaRestart()
|
|
288
|
-
self.whep?.restart(config: SourceConfig(url: whepUrl))
|
|
289
|
-
}
|
|
290
|
-
}
|
|
291
|
-
|
|
292
|
-
/// Rebuild the WHEP subscription on the current URL even when it looks healthy.
|
|
293
|
-
/// The host calls this when the phone changed networks.
|
|
294
|
-
func restartVideoSource() {
|
|
295
|
-
queue.async {
|
|
296
|
-
self.cancelMediaRestart()
|
|
297
|
-
self.whep?.forceRestart()
|
|
298
|
-
}
|
|
299
|
-
}
|
|
300
|
-
|
|
301
|
-
private func onMediaSourceState(_ state: SourceState, reason: String) {
|
|
302
|
-
let previous = mediaSource
|
|
303
|
-
mediaSource = state
|
|
304
|
-
if state == .live { mediaRestartAttempts = 0 }
|
|
305
|
-
if state == .failed { scheduleMediaRestart(reason: reason) }
|
|
306
|
-
// start() emits idle then connecting back to back; one snapshot per real change.
|
|
307
|
-
if previous != state, call != nil, phase != "idle" { onState(snapshot()) }
|
|
308
|
-
}
|
|
309
|
-
|
|
310
|
-
/// Native owns first-line recovery: nothing above this layer can see ICE fail, and a
|
|
311
|
-
/// Teams call with a frozen last frame looks healthy from every other angle.
|
|
312
|
-
/// Exponential backoff capped at mediaRestartMaxMs, for as long as the call is alive.
|
|
313
|
-
private func scheduleMediaRestart(reason: String) {
|
|
314
|
-
guard call != nil, !["idle", "disconnected", "error"].contains(phase) else { return }
|
|
315
|
-
guard mediaRestartTask == nil else { return }
|
|
316
|
-
let attempt = mediaRestartAttempts
|
|
317
|
-
mediaRestartAttempts += 1
|
|
318
|
-
let delayMs = min(Self.mediaRestartBaseMs << min(attempt, 4), Self.mediaRestartMaxMs)
|
|
319
|
-
NSLog("ACS-SPIKE glasses media source failed (\(reason)); WHEP rebuild #\(attempt + 1) in \(delayMs)ms")
|
|
320
|
-
let task = DispatchWorkItem { [weak self] in
|
|
321
|
-
guard let self else { return }
|
|
322
|
-
self.mediaRestartTask = nil
|
|
323
|
-
guard self.call != nil, self.mediaSource == .failed else { return }
|
|
324
|
-
self.whep?.forceRestart()
|
|
325
|
-
}
|
|
326
|
-
mediaRestartTask = task
|
|
327
|
-
queue.asyncAfter(deadline: .now() + .milliseconds(delayMs), execute: task)
|
|
328
|
-
}
|
|
329
|
-
|
|
330
|
-
private func cancelMediaRestart() {
|
|
331
|
-
mediaRestartTask?.cancel()
|
|
332
|
-
mediaRestartTask = nil
|
|
333
|
-
mediaRestartAttempts = 0
|
|
334
|
-
}
|
|
335
|
-
|
|
336
|
-
func setMuted(_ next: Bool) -> [String: Any] {
|
|
337
|
-
muted = next
|
|
338
|
-
queue.async { self.applyAudioPolicyOnQueue("set-muted") }
|
|
339
|
-
let snap = snapshot()
|
|
340
|
-
onState(snap)
|
|
341
|
-
return snap
|
|
342
|
-
}
|
|
343
|
-
|
|
344
|
-
func setAudioSource(_ source: String) -> [String: Any] {
|
|
345
|
-
if AcsAudioPolicy.parseSource(source) == nil {
|
|
346
|
-
NSLog("ACS-SPIKE unknown audioSource=\(source) ignored; source is locked for this call")
|
|
347
|
-
} else {
|
|
348
|
-
NSLog("ACS-SPIKE setAudioSource=\(source) ignored; audio source is locked for this call at \(audioSource)")
|
|
349
|
-
}
|
|
350
|
-
return snapshot()
|
|
351
|
-
}
|
|
352
|
-
|
|
353
|
-
func leave() {
|
|
354
|
-
queue.async { self.leaveLocked() }
|
|
355
|
-
}
|
|
356
|
-
|
|
357
|
-
fileprivate func applyAudioPolicy(_ reason: String) {
|
|
358
|
-
queue.async { self.applyAudioPolicyOnQueue(reason) }
|
|
359
|
-
}
|
|
360
|
-
|
|
361
|
-
private func applyAudioPolicyOnQueue(_ reason: String) {
|
|
362
|
-
let desired: AudioSourceKind = audioSource == "phone" ? .phone : .glasses
|
|
363
|
-
lastSafety = applier.apply(desired: desired, userMuted: muted, reason: reason)
|
|
364
|
-
if lastSafety == .unsafe {
|
|
365
|
-
NSLog("ACS-SPIKE audioSafety=unsafe — mute and stopAudio both failed; unintended mic may be live")
|
|
366
|
-
}
|
|
367
|
-
onState(snapshot())
|
|
368
|
-
}
|
|
369
|
-
|
|
370
|
-
private func feedOutgoingPcm(_ pcm: Data, sampleRate: Int, channels: Int) {
|
|
371
|
-
guard !muted, outgoingReady, let stream = audioOut else { return }
|
|
372
|
-
for frame in pcmBridge?.ingest(pcm16Le: pcm, sampleRate: sampleRate, channels: channels) ?? [] {
|
|
373
|
-
guard let format = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: 48000, channels: 1, interleaved: true),
|
|
374
|
-
let pcm = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(frame.count / 2)),
|
|
375
|
-
let samples = pcm.int16ChannelData else { continue }
|
|
376
|
-
pcm.frameLength = pcm.frameCapacity
|
|
377
|
-
frame.copyBytes(to: UnsafeMutableRawBufferPointer(start: samples[0], count: frame.count))
|
|
378
|
-
let buffer = RawAudioBuffer()
|
|
379
|
-
buffer.buffer = pcm
|
|
380
|
-
stream.send(buffer: buffer) { error in
|
|
381
|
-
buffer.dispose()
|
|
382
|
-
if let error { NSLog("ACS-SPIKE sendRawAudioBuffer failed: \(error)") }
|
|
383
|
-
}
|
|
384
|
-
}
|
|
385
|
-
}
|
|
386
|
-
|
|
387
|
-
private func emit(_ next: String) {
|
|
388
|
-
phase = next
|
|
389
|
-
onState(snapshot())
|
|
390
|
-
}
|
|
391
|
-
|
|
392
|
-
private func leaveLocked(emitIdle: Bool = true) {
|
|
393
|
-
phoneMic.setEnabled(false)
|
|
394
|
-
phoneMic.onPcm = nil
|
|
395
|
-
applier.reset()
|
|
396
|
-
scheduler.cancelPending()
|
|
397
|
-
pcmBridge?.finishDump()
|
|
398
|
-
// Detach before stop so the teardown's own idle transition does not emit a
|
|
399
|
-
// snapshot (or schedule a rebuild) for a call that is going away.
|
|
400
|
-
cancelMediaRestart()
|
|
401
|
-
whep?.onStateChange = nil
|
|
402
|
-
mediaSource = .idle
|
|
403
|
-
whep?.stop()
|
|
404
|
-
frameSender.detach()
|
|
405
|
-
do {
|
|
406
|
-
if let call {
|
|
407
|
-
try CallbackOperation<Void>().wait { completion in
|
|
408
|
-
call.hangUp(options: nil) { completion((), $0) }
|
|
338
|
+
whep = source
|
|
339
|
+
applyAudioPolicyOnQueue("join")
|
|
340
|
+
NSLog("ACS-SPIKE iOS ACS join started source=\(audioSource) profile=\(video.width)x\(video.height)@\(video.fps) armVirtual=\(plan.armVirtual) transportMuted=\(plan.transportMuted)")
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
private func failJoinLocked(_ error: Error, generation: UInt64) {
|
|
344
|
+
guard joinGeneration == generation else { return }
|
|
345
|
+
// Record failure before teardown: lastError makes late call callbacks no-ops,
|
|
346
|
+
// and emitIdle=false preserves the terminal error rather than resetting idle.
|
|
347
|
+
lastError = error.localizedDescription
|
|
348
|
+
leaveLocked(emitIdle: false)
|
|
349
|
+
emit("error")
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
func updateVideoSource(_ whepUrl: String) {
|
|
353
|
+
queue.async {
|
|
354
|
+
// The host has a fresher opinion about where the glasses publish; drop any
|
|
355
|
+
// automatic retry against the old URL.
|
|
356
|
+
self.cancelMediaRestart()
|
|
357
|
+
self.whep?.restart(config: SourceConfig(url: whepUrl))
|
|
409
358
|
}
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
/// Rebuild the WHEP subscription on the current URL even when it looks healthy.
|
|
362
|
+
/// The host calls this when the phone changed networks.
|
|
363
|
+
func restartVideoSource() {
|
|
364
|
+
queue.async {
|
|
365
|
+
self.cancelMediaRestart()
|
|
366
|
+
self.whep?.forceRestart()
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
private func onMediaSourceState(_ state: SourceState, reason: String) {
|
|
371
|
+
let previous = mediaSource
|
|
372
|
+
mediaSource = state
|
|
373
|
+
if state == .live { mediaRestartAttempts = 0 }
|
|
374
|
+
if state == .failed { scheduleMediaRestart(reason: reason) }
|
|
375
|
+
// start() emits idle then connecting back to back; one snapshot per real change.
|
|
376
|
+
if previous != state, call != nil, phase != "idle" { onState(snapshot()) }
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
/// Native owns first-line recovery: nothing above this layer can see ICE fail, and a
|
|
380
|
+
/// Teams call with a frozen last frame looks healthy from every other angle.
|
|
381
|
+
/// Exponential backoff capped at mediaRestartMaxMs, for as long as the call is alive.
|
|
382
|
+
private func scheduleMediaRestart(reason: String) {
|
|
383
|
+
guard call != nil, !["idle", "disconnected", "error"].contains(phase) else { return }
|
|
384
|
+
guard mediaRestartTask == nil else { return }
|
|
385
|
+
let attempt = mediaRestartAttempts
|
|
386
|
+
mediaRestartAttempts += 1
|
|
387
|
+
let delayMs = min(Self.mediaRestartBaseMs << min(attempt, 4), Self.mediaRestartMaxMs)
|
|
388
|
+
NSLog("ACS-SPIKE glasses media source failed (\(reason)); WHEP rebuild #\(attempt + 1) in \(delayMs)ms")
|
|
389
|
+
let task = DispatchWorkItem { [weak self] in
|
|
390
|
+
guard let self else { return }
|
|
391
|
+
self.mediaRestartTask = nil
|
|
392
|
+
guard self.call != nil, self.mediaSource == .failed else { return }
|
|
393
|
+
self.whep?.forceRestart()
|
|
394
|
+
}
|
|
395
|
+
mediaRestartTask = task
|
|
396
|
+
queue.asyncAfter(deadline: .now() + .milliseconds(delayMs), execute: task)
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
private func cancelMediaRestart() {
|
|
400
|
+
mediaRestartTask?.cancel()
|
|
401
|
+
mediaRestartTask = nil
|
|
402
|
+
mediaRestartAttempts = 0
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
func setMuted(_ next: Bool) -> [String: Any] {
|
|
406
|
+
muted = next
|
|
407
|
+
queue.async { self.applyAudioPolicyOnQueue("set-muted") }
|
|
408
|
+
let snap = snapshot()
|
|
409
|
+
onState(snap)
|
|
410
|
+
return snap
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
func setAudioSource(_ source: String) -> [String: Any] {
|
|
414
|
+
if AcsAudioPolicy.parseSource(source) == nil {
|
|
415
|
+
NSLog("ACS-SPIKE unknown audioSource=\(source) ignored; source is locked for this call")
|
|
416
|
+
} else {
|
|
417
|
+
NSLog("ACS-SPIKE setAudioSource=\(source) ignored; audio source is locked for this call at \(audioSource)")
|
|
418
|
+
}
|
|
419
|
+
return snapshot()
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
func leave() {
|
|
423
|
+
queue.async { self.leaveLocked() }
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
fileprivate func applyAudioPolicy(_ reason: String) {
|
|
427
|
+
queue.async { self.applyAudioPolicyOnQueue(reason) }
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
private func applyAudioPolicyOnQueue(_ reason: String) {
|
|
431
|
+
let desired: AudioSourceKind = audioSource == "phone" ? .phone : .glasses
|
|
432
|
+
lastSafety = applier.apply(desired: desired, userMuted: muted, reason: reason)
|
|
433
|
+
if lastSafety == .unsafe {
|
|
434
|
+
NSLog("ACS-SPIKE audioSafety=unsafe — mute and stopAudio both failed; unintended mic may be live")
|
|
435
|
+
}
|
|
436
|
+
onState(snapshot())
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
private func feedOutgoingPcm(_ pcm: Data, sampleRate: Int, channels: Int) {
|
|
440
|
+
guard !muted, outgoingReady, let stream = audioOut else { return }
|
|
441
|
+
for frame in pcmBridge?.ingest(pcm16Le: pcm, sampleRate: sampleRate, channels: channels) ?? [] {
|
|
442
|
+
guard let pcmBuffer = PcmBridge.audioBuffer(pcm16Le: frame, sampleRate: PcmBridge.targetRate, channels: 1) else {
|
|
443
|
+
NSLog("ACS-SPIKE could not create outgoing AVAudioPCMBuffer")
|
|
444
|
+
break
|
|
445
|
+
}
|
|
446
|
+
let buffer = RawAudioBuffer()
|
|
447
|
+
buffer.buffer = pcmBuffer
|
|
448
|
+
stream.send(buffer: buffer) { error in
|
|
449
|
+
if let error {
|
|
450
|
+
NSLog("ACS-SPIKE sendRawAudioBuffer failed: \(error)")
|
|
451
|
+
}
|
|
452
|
+
buffer.dispose()
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
private func emit(_ next: String) {
|
|
458
|
+
phase = next
|
|
459
|
+
onState(snapshot())
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
private func leaveLocked(emitIdle: Bool = true) {
|
|
463
|
+
joinGeneration &+= 1
|
|
464
|
+
phoneMic.setEnabled(false)
|
|
465
|
+
phoneMic.onPcm = nil
|
|
466
|
+
applier.reset()
|
|
467
|
+
scheduler.cancelPending()
|
|
468
|
+
pcmBridge?.finishDump()
|
|
469
|
+
// Detach before stop so the teardown's own idle transition does not emit a
|
|
470
|
+
// snapshot (or schedule a rebuild) for a call that is going away.
|
|
471
|
+
cancelMediaRestart()
|
|
472
|
+
whep?.onStateChange = nil
|
|
473
|
+
mediaSource = .idle
|
|
474
|
+
whep?.stop()
|
|
475
|
+
frameSender.detach()
|
|
476
|
+
let leavingCall = call
|
|
477
|
+
let leavingAgent = callAgent
|
|
478
|
+
leavingCall?.delegate = nil
|
|
479
|
+
leavingCall?.hangUp(options: nil) { error in
|
|
480
|
+
if let error {
|
|
481
|
+
NSLog("ACS-SPIKE leave hangUp failed: \(error)")
|
|
482
|
+
}
|
|
483
|
+
leavingAgent?.dispose()
|
|
484
|
+
}
|
|
485
|
+
// A join that never produced a call still owns an agent that must be released.
|
|
486
|
+
if leavingCall == nil {
|
|
487
|
+
leavingAgent?.dispose()
|
|
488
|
+
}
|
|
489
|
+
callAgent = nil
|
|
490
|
+
callClient = nil
|
|
491
|
+
call = nil
|
|
492
|
+
whep = nil
|
|
493
|
+
audioOut = nil
|
|
494
|
+
audioIn = nil
|
|
495
|
+
localOut = nil
|
|
496
|
+
pcmBridge = nil
|
|
497
|
+
outgoingReady = false
|
|
498
|
+
muted = false
|
|
499
|
+
audioSource = "glasses"
|
|
500
|
+
lastSafety = .degraded
|
|
501
|
+
meetingUrl = nil
|
|
502
|
+
// Clearing lastError is scoped to the clean idle reset. A failed join tears down
|
|
503
|
+
// with emitIdle=false and relies on lastError staying set so emit("error") still
|
|
504
|
+
// carries it and the call delegate keeps ignoring late disconnected callbacks.
|
|
505
|
+
if emitIdle {
|
|
506
|
+
lastError = nil
|
|
507
|
+
emit("idle")
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
fileprivate func currentCall() -> Call? {
|
|
512
|
+
call
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
fileprivate func currentWhep() -> WhepVideoSource? {
|
|
516
|
+
whep
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
fileprivate func setPhonePcmEnabled(_ enabled: Bool) {
|
|
520
|
+
phoneMic.setEnabled(enabled)
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
private func handleCallStateChange(_ changedCall: Call) {
|
|
524
|
+
queue.async {
|
|
525
|
+
guard self.call === changedCall, self.lastError == nil else { return }
|
|
526
|
+
switch changedCall.state {
|
|
527
|
+
case .connecting: self.emit("connecting")
|
|
528
|
+
case .inLobby: self.emit("lobby")
|
|
529
|
+
case .connected:
|
|
530
|
+
self.emit("connected")
|
|
531
|
+
self.applyAudioPolicyOnQueue("call-connected")
|
|
532
|
+
case .disconnecting, .disconnected: self.emit("disconnected")
|
|
533
|
+
default: break
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
private func handleOutgoingAudioStateChange(_ stream: RawOutgoingAudioStream) {
|
|
539
|
+
queue.async {
|
|
540
|
+
guard self.audioOut === stream else { return }
|
|
541
|
+
self.outgoingReady = stream.state == .started
|
|
542
|
+
NSLog("ACS-SPIKE iOS raw outgoing audio state=\(stream.state)")
|
|
543
|
+
self.applyAudioPolicyOnQueue("virtual-stream-state")
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
private func handleCallMuteChange(_ changedCall: Call) {
|
|
548
|
+
queue.async {
|
|
549
|
+
guard self.call === changedCall, self.lastError == nil else { return }
|
|
550
|
+
self.applyAudioPolicyOnQueue("outgoing-audio-state")
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
private func handleIncomingAudio(_ args: IncomingMixedAudioEventArgs) {
|
|
555
|
+
let rawBuffer = args.audioBuffer
|
|
556
|
+
defer { rawBuffer.dispose() }
|
|
557
|
+
guard let pcmBuffer = rawBuffer.buffer as? AVAudioPCMBuffer,
|
|
558
|
+
let data = PcmBridge.pcm16Data(from: pcmBuffer) else { return }
|
|
559
|
+
onIncomingPcm(
|
|
560
|
+
data.base64EncodedString(),
|
|
561
|
+
Int(args.streamProperties.sampleRate.valueInHz),
|
|
562
|
+
Int(args.streamProperties.channelMode.channelCount)
|
|
563
|
+
)
|
|
564
|
+
}
|
|
441
565
|
}
|
|
442
566
|
|
|
443
567
|
final class SessionAudioController: AudioStreamController {
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
}
|
|
459
|
-
|
|
460
|
-
func isPhysicallyMuted() -> Bool? {
|
|
461
|
-
session?.currentCall()?.isOutgoingAudioMuted
|
|
462
|
-
}
|
|
463
|
-
|
|
464
|
-
func setGlassesPcmEnabled(_ enabled: Bool) {
|
|
465
|
-
session?.currentWhep()?.setPcmDeliveryEnabled(enabled)
|
|
466
|
-
}
|
|
467
|
-
|
|
468
|
-
func setPhonePcmEnabled(_ enabled: Bool) {
|
|
469
|
-
session?.setPhonePcmEnabled(enabled)
|
|
470
|
-
}
|
|
471
|
-
|
|
472
|
-
func mutePhysical() -> Result<Void, Error> {
|
|
473
|
-
switch CallGuard.require(session?.currentCall()) {
|
|
474
|
-
case .failure(let error): return .failure(error)
|
|
475
|
-
case .success(let call):
|
|
476
|
-
do {
|
|
477
|
-
try CallbackOperation<Void>().wait { completion in
|
|
478
|
-
call.muteOutgoingAudio { completion((), $0) }
|
|
568
|
+
private weak var session: AcsMeetingSession?
|
|
569
|
+
|
|
570
|
+
init(session: AcsMeetingSession) {
|
|
571
|
+
self.session = session
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
func readActive() -> ActiveStreamKind {
|
|
575
|
+
guard let call = session?.currentCall() else { return .none }
|
|
576
|
+
let stream = call.activeOutgoingAudioStream
|
|
577
|
+
guard stream.state == .started else { return .none }
|
|
578
|
+
switch stream.type {
|
|
579
|
+
case .virtualOutgoing: return .virtual
|
|
580
|
+
case .localOutgoing: return .local
|
|
581
|
+
default: return .none
|
|
479
582
|
}
|
|
480
|
-
return .success(())
|
|
481
|
-
} catch {
|
|
482
|
-
return .failure(error)
|
|
483
|
-
}
|
|
484
583
|
}
|
|
485
|
-
}
|
|
486
584
|
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
585
|
+
func isPhysicallyMuted() -> Bool? {
|
|
586
|
+
session?.currentCall()?.isOutgoingAudioMuted
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
func setGlassesPcmEnabled(_ enabled: Bool) {
|
|
590
|
+
session?.currentWhep()?.setPcmDeliveryEnabled(enabled)
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
func setPhonePcmEnabled(_ enabled: Bool) {
|
|
594
|
+
session?.setPhonePcmEnabled(enabled)
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
func mutePhysical() -> Result<Void, Error> {
|
|
598
|
+
switch CallGuard.require(session?.currentCall()) {
|
|
599
|
+
case let .failure(error): return .failure(error)
|
|
600
|
+
case let .success(call):
|
|
601
|
+
return waitForAcsOperation { completion in
|
|
602
|
+
call.muteOutgoingAudio(completionHandler: completion)
|
|
603
|
+
}
|
|
494
604
|
}
|
|
495
|
-
return .success(())
|
|
496
|
-
} catch {
|
|
497
|
-
return .failure(error)
|
|
498
|
-
}
|
|
499
605
|
}
|
|
500
|
-
}
|
|
501
606
|
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
call.stopAudio(stream: stream) { completion((), $0) }
|
|
607
|
+
func unmutePhysical() -> Result<Void, Error> {
|
|
608
|
+
switch CallGuard.require(session?.currentCall()) {
|
|
609
|
+
case let .failure(error): return .failure(error)
|
|
610
|
+
case let .success(call):
|
|
611
|
+
return waitForAcsOperation { completion in
|
|
612
|
+
call.unmuteOutgoingAudio(completionHandler: completion)
|
|
613
|
+
}
|
|
510
614
|
}
|
|
511
|
-
return .success(())
|
|
512
|
-
} catch {
|
|
513
|
-
return .failure(error)
|
|
514
|
-
}
|
|
515
615
|
}
|
|
516
|
-
|
|
616
|
+
|
|
617
|
+
func stopActive() -> Result<Void, Error> {
|
|
618
|
+
switch CallGuard.require(session?.currentCall()) {
|
|
619
|
+
case let .failure(error): return .failure(error)
|
|
620
|
+
case let .success(call):
|
|
621
|
+
let stream = call.activeOutgoingAudioStream
|
|
622
|
+
return waitForAcsOperation { completion in
|
|
623
|
+
call.stopAudio(stream: stream, completionHandler: completion)
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
private final class AcsCallDelegateProxy: NSObject, CallDelegate {
|
|
630
|
+
private let onStateChange: (Call) -> Void
|
|
631
|
+
private let onMuteChange: (Call) -> Void
|
|
632
|
+
|
|
633
|
+
init(onStateChange: @escaping (Call) -> Void, onMuteChange: @escaping (Call) -> Void) {
|
|
634
|
+
self.onStateChange = onStateChange
|
|
635
|
+
self.onMuteChange = onMuteChange
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
func call(_ call: Call, didChangeState _: PropertyChangedEventArgs) {
|
|
639
|
+
onStateChange(call)
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
func call(_ call: Call, didUpdateOutgoingAudioState _: PropertyChangedEventArgs) {
|
|
643
|
+
onMuteChange(call)
|
|
644
|
+
}
|
|
517
645
|
}
|
|
518
646
|
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
647
|
+
private struct AcsMeetingError: LocalizedError {
|
|
648
|
+
let message: String
|
|
649
|
+
|
|
650
|
+
init(_ message: String) {
|
|
651
|
+
self.message = message
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
var errorDescription: String? {
|
|
655
|
+
message
|
|
656
|
+
}
|
|
657
|
+
}
|
|
658
|
+
|
|
659
|
+
private func waitForAcsOperation(
|
|
660
|
+
_ start: (@escaping (Error?) -> Void) -> Void
|
|
661
|
+
) -> Result<Void, Error> {
|
|
662
|
+
do {
|
|
663
|
+
try CallbackOperation<Void>().wait(timeout: 10) { completion in
|
|
664
|
+
start { completion((), $0) }
|
|
665
|
+
}
|
|
666
|
+
return .success(())
|
|
667
|
+
} catch {
|
|
668
|
+
return .failure(error)
|
|
669
|
+
}
|
|
538
670
|
}
|
|
539
671
|
|
|
540
672
|
struct AcsOutgoingVideo {
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
673
|
+
let width: Int
|
|
674
|
+
let height: Int
|
|
675
|
+
let fps: Int
|
|
676
|
+
let maxBitrateBps: Int
|
|
545
677
|
|
|
546
|
-
|
|
547
|
-
|
|
678
|
+
static let hd = AcsOutgoingVideo(width: 1280, height: 720, fps: 15, maxBitrateBps: 2_500_000)
|
|
679
|
+
static let allowedSizes: Set<String> = ["1280x720", "960x540"]
|
|
548
680
|
}
|
|
549
681
|
|
|
550
682
|
private func requireString(_ options: [String: Any], _ key: String) throws -> String {
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
683
|
+
guard let value = options[key] as? String, !value.isEmpty else {
|
|
684
|
+
throw NSError(domain: "MentraAcsMeeting", code: 1, userInfo: [NSLocalizedDescriptionKey: "\(key) is required"])
|
|
685
|
+
}
|
|
686
|
+
return value
|
|
555
687
|
}
|
|
556
688
|
|
|
557
689
|
private func parseAcsOutgoingVideo(_ raw: Any?) throws -> AcsOutgoingVideo {
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
690
|
+
guard let raw else { return .hd }
|
|
691
|
+
guard let map = raw as? [String: Any] else {
|
|
692
|
+
throw NSError(domain: "MentraAcsMeeting", code: 1, userInfo: [NSLocalizedDescriptionKey: "video must be an object"])
|
|
693
|
+
}
|
|
694
|
+
guard
|
|
695
|
+
let width = (map["width"] as? NSNumber)?.intValue,
|
|
696
|
+
let height = (map["height"] as? NSNumber)?.intValue,
|
|
697
|
+
let fps = (map["fps"] as? NSNumber)?.intValue,
|
|
698
|
+
let bitrate = (map["maxBitrateBps"] as? NSNumber)?.intValue
|
|
699
|
+
else {
|
|
700
|
+
throw NSError(domain: "MentraAcsMeeting", code: 1, userInfo: [NSLocalizedDescriptionKey: "video requires width, height, fps, and maxBitrateBps"])
|
|
701
|
+
}
|
|
702
|
+
guard AcsOutgoingVideo.allowedSizes.contains("\(width)x\(height)"), fps >= 1, fps <= 30, bitrate > 0 else {
|
|
703
|
+
throw NSError(domain: "MentraAcsMeeting", code: 1, userInfo: [NSLocalizedDescriptionKey: "unsupported ACS video \(width)x\(height)@\(fps)"])
|
|
704
|
+
}
|
|
705
|
+
return AcsOutgoingVideo(width: width, height: height, fps: fps, maxBitrateBps: bitrate)
|
|
574
706
|
}
|