@mentra/acs-meeting 3.2.0-dev.125 → 3.2.0-dev.136

package/README.md

@@ -381,6 +381,18 @@ rollback; it takes effect on the next join.
 `scripts/acs-ladder.ts` in the Mentra-Call repo parses these lines and prints pass/fail
 over a trailing 10-second window, including the `recv`-vs-`dec` attribution.
 
+## iOS host setup
+
+Add `"@mentra/acs-meeting"` to the host's Expo `plugins` list, then run
+`expo prebuild --platform ios` and `pod install`. The Mentra App and example
+OEM host already include it.
+
+The plugin builds only `AzureCommunicationCommon` as a dynamic framework with
+`BUILD_LIBRARY_FOR_DISTRIBUTION=YES`. Calling's binary requires that framework at
+runtime and imports its generated Swift header and stable module interface at
+build time. CocoaPods' default static-library layout cannot satisfy this contract.
+The host's other pods retain their configured linkage.
+
 ## Tests
 
 ```bash

package/app.plugin.js

@@ -0,0 +1,35 @@
+const {withPodfile} = require("expo/config-plugins")
+const {mergeContents} = require("@expo/config-plugins/build/utils/generateCode")
+
+// Calling is a vendored dynamic framework: its umbrella header and LC_LOAD_DYLIB
+// both require Common.framework. Expo otherwise builds Common as a static library.
+const commonFramework = `  installer.pod_targets.each do |pod|
+    next unless pod.name == 'AzureCommunicationCommon'
+    def pod.build_type
+      Pod::BuildType.dynamic_framework
+    end
+    # Calling's Swift interface must import Common through a stable interface,
+    # including when the two SDKs have different minimum deployment targets.
+    pod.root_spec.pod_target_xcconfig = (pod.root_spec.attributes_hash['pod_target_xcconfig'] || {}).merge(
+      'BUILD_LIBRARY_FOR_DISTRIBUTION' => 'YES'
+    )
+  end`
+
+module.exports = function withAcsMeeting(config) {
+  return withPodfile(config, (config) => {
+    let contents = config.modResults.contents
+    const hook = /^\s*pre_install do \|installer\|\s*$/m
+    if (!hook.test(contents)) {
+      contents += "\npre_install do |installer|\nend\n"
+    }
+    config.modResults.contents = mergeContents({
+      src: contents,
+      newSrc: commonFramework,
+      tag: "acs-common-framework",
+      anchor: hook,
+      offset: 1,
+      comment: "#",
+    }).contents
+    return config
+  })
+}

package/ios/AcsFrameSender.swift

@@ -4,44 +4,71 @@ import Foundation
 
 /// Fresh Swift sender on ACS iOS RawOutgoingVideoStream + CVPixelBuffer.
 /// No RealWear code (they have no iOS equivalent).
-final class AcsFrameSender {
-  private var stream: RawOutgoingVideoStream?
+final class AcsFrameSender: NSObject {
+  private var stream: VirtualOutgoingVideoStream?
   private var running = false
   private var lastSent: CFTimeInterval = 0
+  private let sendGate = VideoSendGate()
+  private let stateLock = NSLock()
 
   func attach(_ stream: VirtualOutgoingVideoStream) {
     detach()
+    stateLock.lock()
     self.stream = stream
+    lastSent = 0
+    stateLock.unlock()
     stream.delegate = self
   }
 
   func send(_ pixelBuffer: CVPixelBuffer) {
-    guard running, let stream else { return }
-    let fps = stream.format?.framesPerSecond ?? 15
+    stateLock.lock()
+    guard running, let stream else {
+      stateLock.unlock()
+      return
+    }
+    let fps = stream.format.framesPerSecond
     let now = CFAbsoluteTimeGetCurrent()
-    if lastSent > 0, now - lastSent < 1.0 / max(fps, 1) { return }
+    if lastSent > 0, now - lastSent < 1.0 / Double(max(fps, 1)) {
+      stateLock.unlock()
+      return
+    }
+    guard let token = sendGate.acquire() else {
+      stateLock.unlock()
+      return
+    }
     lastSent = now
-    do {
-      let frame = RawVideoFrameBuffer(pixelBuffer, streamFormat: stream.format)
-      try stream.send(frame)
-    } catch {
-      NSLog("ACS-SPIKE send frame failed: \(error)")
+    stateLock.unlock()
+    let frame = RawVideoFrameBuffer()
+    frame.buffer = pixelBuffer
+    frame.streamFormat = stream.format
+    frame.timestampInTicks = stream.timestampInTicks
+    stream.send(frame: frame) { [sendGate] error in
+      // Keep the source pixels alive until ACS finishes reading the frame.
+      withExtendedLifetime(pixelBuffer) { frame.dispose() }
+      sendGate.release(token)
+      if let error { NSLog("ACS-SPIKE send frame failed: \(error)") }
     }
   }
 
   func detach() {
+    stateLock.lock()
     running = false
     // Drop the delegate on the previous stream so a late STOPPED from a prior
     // call cannot freeze outgoing video for the current meeting.
-    stream?.delegate = nil
+    let previous = stream
     stream = nil
+    sendGate.reset()
+    stateLock.unlock()
+    previous?.delegate = nil
   }
 }
 
-extension AcsFrameSender: RawOutgoingVideoStreamDelegate {
-  func rawOutgoingVideoStream(_ rawOutgoingVideoStream: RawOutgoingVideoStream, didChangeState args: VideoStreamStateChangedEventArgs) {
-    guard rawOutgoingVideoStream === stream else { return }
-    running = rawOutgoingVideoStream.state == .started
-    NSLog("ACS-SPIKE iOS raw video state=\(rawOutgoingVideoStream.state)")
+extension AcsFrameSender: VirtualOutgoingVideoStreamDelegate {
+  func virtualOutgoingVideoStream(_ virtualOutgoingVideoStream: VirtualOutgoingVideoStream, didChangeState _: VideoStreamStateChangedEventArgs) {
+    stateLock.lock()
+    defer { stateLock.unlock() }
+    guard virtualOutgoingVideoStream === stream else { return }
+    running = virtualOutgoingVideoStream.state == .started
+    NSLog("ACS-SPIKE iOS raw video state=\(virtualOutgoingVideoStream.state)")
   }
 }

package/ios/AcsMeetingModule.swift

@@ -1,3 +1,4 @@
+import AVFoundation
 import AzureCommunicationCalling
 import AzureCommunicationCommon
 import ExpoModulesCore
@@ -80,7 +81,7 @@ final class QueuePolicyScheduler: PolicyScheduler {
   }
 }
 
-final class AcsMeetingSession {
+final class AcsMeetingSession: NSObject {
   private static let glassesRequiresUnmutedTransport = true
   private let onState: ([String: Any]) -> Void
   private let onIncomingPcm: (String, Int, Int) -> Void
@@ -158,7 +159,9 @@ final class AcsMeetingSession {
         self.callClient = client
         let options = CallAgentOptions()
         options.displayName = displayName ?? "Mentra Call"
-        let agent = try client.createCallAgent(userCredential: credential, options: options).get()
+        let agent = try CallbackOperation<CallAgent>().wait(onLateSuccess: { $0.dispose() }) { completion in
+          client.createCallAgent(userCredential: credential, options: options, completionHandler: completion)
+        }
         self.callAgent = agent
 
         let videoFormat = VideoStreamFormat()
@@ -171,15 +174,23 @@ final class AcsMeetingSession {
         let videoStream = VirtualOutgoingVideoStream(videoStreamOptions: videoOptions)
         self.frameSender.attach(videoStream)
 
-        let outAudioFormat = AudioStreamFormat()
-        outAudioFormat.sampleRate = .sampleRate48000
-        outAudioFormat.channelMode = .channelModeMono
-        outAudioFormat.encodedAudioFormat = .pcm
+        let outAudioFormat = RawOutgoingAudioStreamProperties()
+        outAudioFormat.sampleRate = .hz48000
+        outAudioFormat.channelMode = .mono
+        outAudioFormat.format = .pcm16Bit
 
         let outAudioOptions = RawOutgoingAudioStreamOptions()
-        outAudioOptions.format = outAudioFormat
-        let outgoing = RawOutgoingAudioStream(audioStreamOptions: outAudioOptions)
-        outgoing.delegate = self
+        outAudioFormat.bufferDuration = .ms20
+        outAudioOptions.properties = outAudioFormat
+        let outgoing = RawOutgoingAudioStream(options: outAudioOptions)
+        outgoing.events.onStateChanged = { [weak self, weak outgoing] _ in
+          guard let self, let outgoing else { return }
+          self.queue.async {
+            guard self.audioOut === outgoing else { return }
+            self.outgoingReady = outgoing.state == .started
+            self.applyAudioPolicyOnQueue("virtual-stream-state")
+          }
+        }
         self.audioOut = outgoing
 
         let desired: AudioSourceKind = self.audioSource == "phone" ? .phone : .glasses
@@ -194,14 +205,19 @@ final class AcsMeetingSession {
         let local: LocalOutgoingAudioStream? = plan.armVirtual ? nil : LocalOutgoingAudioStream()
         self.localOut = local
 
-        let inAudioFormat = AudioStreamFormat()
-        inAudioFormat.sampleRate = .sampleRate16000
-        inAudioFormat.channelMode = .channelModeMono
-        inAudioFormat.encodedAudioFormat = .pcm
+        let inAudioFormat = RawIncomingAudioStreamProperties()
+        inAudioFormat.sampleRate = .hz16000
+        inAudioFormat.channelMode = .mono
+        inAudioFormat.format = .pcm16Bit
         let inAudioOptions = RawIncomingAudioStreamOptions()
-        inAudioOptions.format = inAudioFormat
-        let incoming = RawIncomingAudioStream(audioStreamOptions: inAudioOptions)
-        incoming.delegate = self
+        inAudioOptions.properties = inAudioFormat
+        let incoming = RawIncomingAudioStream(options: inAudioOptions)
+        incoming.events.onMixedAudioBufferReceived = { [weak self] args in
+          guard let pcm = args.audioBuffer.buffer as? AVAudioPCMBuffer,
+                let samples = pcm.int16ChannelData else { return }
+          let data = Data(bytes: samples[0], count: Int(pcm.frameLength) * Int(pcm.format.channelCount) * 2)
+          self?.onIncomingPcm(data.base64EncodedString(), Int(pcm.format.sampleRate), Int(pcm.format.channelCount))
+        }
 
         let joinOptions = JoinCallOptions()
         let outgoingVideo = OutgoingVideoOptions()
@@ -219,7 +235,11 @@ final class AcsMeetingSession {
         joinOptions.incomingAudioOptions = incomingAudio
 
         let locator = TeamsMeetingLinkLocator(meetingLink: meetingUrl)
-        let call = try agent.join(with: locator, joinCallOptions: joinOptions)
+        let call = try CallbackOperation<Call>().wait(onLateSuccess: { lateCall in
+          lateCall.hangUp(options: nil) { _ in }
+        }) { completion in
+          agent.join(with: locator, joinCallOptions: joinOptions, completionHandler: completion)
+        }
         self.call = call
         call.delegate = self
 
@@ -350,13 +370,16 @@ final class AcsMeetingSession {
   private func feedOutgoingPcm(_ pcm: Data, sampleRate: Int, channels: Int) {
     guard !muted, outgoingReady, let stream = audioOut else { return }
     for frame in pcmBridge?.ingest(pcm16Le: pcm, sampleRate: sampleRate, channels: channels) ?? [] {
-      do {
-        let buffer = RawAudioBuffer()
-        buffer.data = frame
-        try stream.send(buffer)
-      } catch {
-        NSLog("ACS-SPIKE sendRawAudioBuffer failed: \(error)")
-        break
+      guard let format = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: 48000, channels: 1, interleaved: true),
+            let pcm = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(frame.count / 2)),
+            let samples = pcm.int16ChannelData else { continue }
+      pcm.frameLength = pcm.frameCapacity
+      frame.copyBytes(to: UnsafeMutableRawBufferPointer(start: samples[0], count: frame.count))
+      let buffer = RawAudioBuffer()
+      buffer.buffer = pcm
+      stream.send(buffer: buffer) { error in
+        buffer.dispose()
+        if let error { NSLog("ACS-SPIKE sendRawAudioBuffer failed: \(error)") }
       }
     }
   }
@@ -380,7 +403,11 @@ final class AcsMeetingSession {
     whep?.stop()
     frameSender.detach()
     do {
-      try call?.hangUp().get()
+      if let call {
+        try CallbackOperation<Void>().wait { completion in
+          call.hangUp(options: nil) { completion((), $0) }
+        }
+      }
     } catch {
       NSLog("ACS-SPIKE leave hangUp failed: \(error)")
     }
@@ -422,16 +449,16 @@ final class SessionAudioController: AudioStreamController {
 
   func readActive() -> ActiveStreamKind {
     guard let stream = session?.currentCall()?.activeOutgoingAudioStream else { return .none }
-    let started = String(describing: stream.state).localizedCaseInsensitiveContains("started")
-    guard started else { return .none }
-    let type = String(describing: stream.type).lowercased()
-    if type.contains("virtual") { return .virtual }
-    if type.contains("local") { return .local }
-    return .none
+    guard stream.state == .started else { return .none }
+    switch stream.type {
+    case .virtualOutgoing: return .virtual
+    case .localOutgoing: return .local
+    default: return .none
+    }
   }
 
   func isPhysicallyMuted() -> Bool? {
-    session?.currentCall()?.isMuted
+    session?.currentCall()?.isOutgoingAudioMuted
   }
 
   func setGlassesPcmEnabled(_ enabled: Bool) {
@@ -447,7 +474,9 @@ final class SessionAudioController: AudioStreamController {
     case .failure(let error): return .failure(error)
     case .success(let call):
       do {
-        try call.mute().get()
+        try CallbackOperation<Void>().wait { completion in
+          call.muteOutgoingAudio { completion((), $0) }
+        }
         return .success(())
       } catch {
         return .failure(error)
@@ -460,7 +489,9 @@ final class SessionAudioController: AudioStreamController {
     case .failure(let error): return .failure(error)
     case .success(let call):
       do {
-        try call.unmute().get()
+        try CallbackOperation<Void>().wait { completion in
+          call.unmuteOutgoingAudio { completion((), $0) }
+        }
         return .success(())
       } catch {
         return .failure(error)
@@ -472,11 +503,11 @@ final class SessionAudioController: AudioStreamController {
     switch CallGuard.require(session?.currentCall()) {
     case .failure(let error): return .failure(error)
     case .success(let call):
-      guard let stream = call.activeOutgoingAudioStream else {
-        return .failure(CallMissingError())
-      }
+      let stream = call.activeOutgoingAudioStream
       do {
-        try call.stopAudio(stream: stream).get()
+        try CallbackOperation<Void>().wait { completion in
+          call.stopAudio(stream: stream) { completion((), $0) }
+        }
         return .success(())
       } catch {
         return .failure(error)
@@ -501,26 +532,11 @@ extension AcsMeetingSession: CallDelegate {
     }
   }
 
-  func call(_ call: Call, didChangeMuteState args: PropertyChangedEventArgs) {
+  func call(_: Call, didUpdateOutgoingAudioState _: PropertyChangedEventArgs) {
     applyAudioPolicy("outgoing-audio-state")
   }
 }
 
-extension AcsMeetingSession: RawOutgoingAudioStreamDelegate {
-  func rawOutgoingAudioStream(_ rawOutgoingAudioStream: RawOutgoingAudioStream, didChangeState args: AudioStreamStateChangedEventArgs) {
-    outgoingReady = String(describing: rawOutgoingAudioStream.state).localizedCaseInsensitiveContains("started")
-    NSLog("ACS-SPIKE iOS raw outgoing audio state=\(rawOutgoingAudioStream.state)")
-    applyAudioPolicy("virtual-stream-state")
-  }
-}
-
-extension AcsMeetingSession: RawIncomingAudioStreamDelegate {
-  func rawIncomingAudioStream(_ rawIncomingAudioStream: RawIncomingAudioStream, didReceiveRawAudioBuffer args: IncomingAudioStreamRawBufferReceivedEventArgs) {
-    guard let data = args.audioBuffer?.data else { return }
-    onIncomingPcm(data.base64EncodedString(), 16000, 1)
-  }
-}
-
 struct AcsOutgoingVideo {
   let width: Int
   let height: Int

package/ios/PolicyKit/Sources/AcsAudioPolicy/CallbackOperation.swift

@@ -0,0 +1,51 @@
+import Foundation
+
+/// Adapts ACS completion handlers to the session's serial control queue. Never
+/// used on the main thread or the media path. Policy decisions must observe the
+/// completed mute/stop result before deciding whether the microphone is safe.
+public final class CallbackOperation<Value> {
+  private let condition = NSCondition()
+  private var result: Result<Value, Error>?
+  private var expired = false
+
+  public init() {}
+
+  public func wait(
+    timeout: TimeInterval = 30,
+    onLateSuccess: @escaping (Value) -> Void = { _ in },
+    start: (@escaping (Value?, Error?) -> Void) -> Void
+  ) throws -> Value {
+    precondition(!Thread.isMainThread, "ACS control operations must run on the session queue")
+    let deadline = Date().addingTimeInterval(timeout)
+    start { value, error in
+      let outcome: Result<Value, Error> = if let error { .failure(error) }
+      else if let value { .success(value) }
+      else { .failure(CallbackOperationError.missingResult) }
+
+      self.condition.lock()
+      if self.expired {
+        self.condition.unlock()
+        if case let .success(value) = outcome { onLateSuccess(value) }
+        return
+      }
+      if self.result == nil { self.result = outcome }
+      self.condition.broadcast()
+      self.condition.unlock()
+    }
+
+    condition.lock()
+    defer { condition.unlock() }
+    while result == nil {
+      if !condition.wait(until: deadline), result == nil {
+        expired = true
+        throw CallbackOperationError.timedOut
+      }
+    }
+    return try result!.get()
+  }
+}
+
+public enum CallbackOperationError: Error {
+  case missingResult
+  case timedOut
+}

package/ios/PolicyKit/Sources/AcsAudioPolicy/VideoSendGate.swift

@@ -0,0 +1,32 @@
+import Foundation
+
+/// At most one retained video frame per stream generation, even if the SDK
+/// stops completing sends. Old completions cannot release a new stream's frame.
+public final class VideoSendGate {
+  private let lock = NSLock()
+  private var nextToken: UInt64 = 0
+  private var pending: UInt64?
+
+  public init() {}
+
+  public func acquire() -> UInt64? {
+    lock.lock()
+    defer { lock.unlock() }
+    guard pending == nil else { return nil }
+    nextToken &+= 1
+    pending = nextToken
+    return nextToken
+  }
+
+  public func release(_ token: UInt64) {
+    lock.lock()
+    defer { lock.unlock() }
+    if token == pending { pending = nil }
+  }
+
+  public func reset() {
+    lock.lock()
+    defer { lock.unlock() }
+    pending = nil
+  }
+}

package/ios/PolicyKit/Tests/AcsAudioPolicyTests/CallbackOperationTests.swift

@@ -0,0 +1,53 @@
+@testable import AcsAudioPolicy
+import Foundation
+import XCTest
+
+final class CallbackOperationTests: XCTestCase {
+  private enum TestError: Error { case sdkFailure }
+
+  private func onWorker(_ body: @escaping () throws -> Void) {
+    let finished = expectation(description: "control queue operation")
+    DispatchQueue.global().async {
+      do { try body() }
+      catch { XCTFail("Unexpected error: \(error)") }
+      finished.fulfill()
+    }
+    wait(for: [finished], timeout: 2)
+  }
+
+  func testInlineAndDelayedSuccess() {
+    onWorker {
+      XCTAssertEqual(try CallbackOperation<Int>().wait { $0(7, nil) }, 7)
+      let result = try CallbackOperation<Int>().wait { completion in
+        DispatchQueue.global().asyncAfter(deadline: .now() + 0.01) { completion(9, nil) }
+      }
+      XCTAssertEqual(result, 9)
+    }
+  }
+
+  func testPropagatesSdkFailureAndRejectsMissingResult() {
+    onWorker {
+      XCTAssertThrowsError(try CallbackOperation<Int>().wait { $0(nil, TestError.sdkFailure) }) {
+        XCTAssertTrue($0 is TestError)
+      }
+      XCTAssertThrowsError(try CallbackOperation<Int>().wait { $0(nil, nil) }) {
+        guard case CallbackOperationError.missingResult = $0 else { return XCTFail("Wrong error: \($0)") }
+      }
+    }
+  }
+
+  func testTimeoutDisposesAResourceReturnedAfterTheWaitEnds() {
+    let disposed = expectation(description: "late agent or call cleaned up")
+    onWorker {
+      var callback: ((Int?, Error?) -> Void)?
+      XCTAssertThrowsError(try CallbackOperation<Int>().wait(timeout: 0.01, onLateSuccess: { value in
+        XCTAssertEqual(value, 42)
+        disposed.fulfill()
+      }) { callback = $0 }) {
+        guard case CallbackOperationError.timedOut = $0 else { return XCTFail("Wrong error: \($0)") }
+      }
+      callback?(42, nil)
+    }
+    wait(for: [disposed], timeout: 1)
+  }
+}

package/ios/PolicyKit/Tests/AcsAudioPolicyTests/VideoSendGateTests.swift

@@ -0,0 +1,40 @@
+@testable import AcsAudioPolicy
+import Foundation
+import XCTest
+
+final class VideoSendGateTests: XCTestCase {
+  func testDelayedCompletionBoundsOutstandingFrames() throws {
+    let gate = VideoSendGate()
+    let token = try XCTUnwrap(gate.acquire())
+    for _ in 0 ..< 1000 {
+      XCTAssertNil(gate.acquire())
+    }
+    gate.release(token)
+    XCTAssertNotNil(gate.acquire())
+  }
+
+  func testOldCompletionDoesNotReleaseNewStreamFrame() throws {
+    let gate = VideoSendGate()
+    let old = try XCTUnwrap(gate.acquire())
+    gate.reset()
+    let current = try XCTUnwrap(gate.acquire())
+    gate.release(old)
+    XCTAssertNil(gate.acquire())
+    gate.release(current)
+    XCTAssertNotNil(gate.acquire())
+  }
+
+  func testConcurrentFramesAdmitExactlyOneSubmission() {
+    let gate = VideoSendGate()
+    let lock = NSLock()
+    var admitted = 0
+    DispatchQueue.concurrentPerform(iterations: 100) { _ in
+      if gate.acquire() != nil {
+        lock.lock()
+        admitted += 1
+        lock.unlock()
+      }
+    }
+    XCTAssertEqual(admitted, 1)
+  }
+}

package/ios/WhepVideoSource.swift

@@ -305,7 +305,7 @@ extension WhepVideoSource: RTCVideoRenderer {
 }
 
 extension WhepVideoSource: RTCAudioRenderer {
-  func renderPCMBuffer(_ pcmBuffer: AVAudioPCMBuffer) {
+  func render(pcmBuffer: AVAudioPCMBuffer) {
     guard pcmEnabled else { return }
     let channels = Int(pcmBuffer.format.channelCount)
     let rate = Int(pcmBuffer.format.sampleRate)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mentra/acs-meeting",
-  "version": "3.2.0-dev.125",
+  "version": "3.2.0-dev.136",
   "description": "MentraOS native ACS Teams meeting module (WHEP decode → ACS raw media)",
   "main": "build/index.js",
   "react-native": "src/index.ts",
@@ -30,6 +30,9 @@
   "publishConfig": {
     "access": "public"
   },
+  "dependencies": {
+    "@expo/config-plugins": "~55.0.10"
+  },
   "devDependencies": {
     "expo-module-scripts": "^55.0.2",
     "typescript": "~5.9.2"
@@ -40,6 +43,7 @@
     "react-native": "*"
   },
   "files": [
+    "app.plugin.js",
     "android",
     "!android/.gradle",
     "!android/build",