@mentra/acs-meeting 3.2.0-dev.222 → 3.2.0-dev.225

package/android/src/test/java/com/mentra/acsmeeting/video/I420PackerTest.kt

@@ -1,126 +0,0 @@
-package com.mentra.acsmeeting.video
-
-import org.assertj.core.api.Assertions.assertThat
-import org.junit.Test
-import java.nio.ByteBuffer
-
-class I420PackerTest {
-  @Test
-  fun percentileMsHandlesEmptyAndOrder() {
-    assertThat(I420Packer.percentileMs(LongArray(0), 0.50)).isEqualTo("na")
-    assertThat(I420Packer.percentileMs(longArrayOf(10_000_000, 20_000_000, 40_000_000), 0.50)).isEqualTo("20.0")
-  }
-
-  @Test
-  fun packedSizeUsesChromaRounding() {
-    assertThat(I420Packer.packedSize(4, 4)).isEqualTo(24)
-    assertThat(I420Packer.packedSize(3, 3)).isEqualTo(3 * 3 + 2 * 2 * 2)
-    assertThat(I420Packer.packedSize(1280, 720)).isEqualTo(1280 * 720 * 3 / 2)
-  }
-
-  @Test
-  fun planeMinBytesIsLastRowPlusFullStrides() {
-    assertThat(I420Packer.planeMinBytes(8, 8, 4)).isEqualTo(32)
-    assertThat(I420Packer.planeMinBytes(16, 8, 4)).isEqualTo(16 * 3 + 8)
-    assertThat(I420Packer.planeMinBytes(4, 8, 4)).isEqualTo(0)
-  }
-
-  @Test
-  fun copyPlaneFromPaddedStrideWritesTightDest() {
-    val width = 4
-    val height = 4
-    val padded = buildPlanes(width, height, strideY = 8, strideUv = 4, seed = 7)
-    val dest = ByteBuffer.allocate(width * height)
-    I420Packer.copyPlane(padded.y, 8, width, height, dest)
-    dest.flip()
-    val out = ByteArray(dest.remaining())
-    dest.get(out)
-    assertThat(out).containsExactly(
-      7, 8, 9, 10,
-      11, 12, 13, 14,
-      15, 16, 17, 18,
-      19, 20, 21, 22,
-    )
-  }
-
-  @Test
-  fun chromaStrideMatchesAcsI420Offer() {
-    assertThat(I420Packer.chromaStride(1280)).isEqualTo(640)
-    assertThat(I420Packer.chromaStride(3)).isEqualTo(2)
-  }
-
-  @Test
-  fun paddedStridesPackIdenticallyToTightStrides() {
-    val width = 4
-    val height = 4
-    val tight = buildPlanes(width, height, strideY = 4, strideUv = 2, seed = 7)
-    val padded = buildPlanes(width, height, strideY = 8, strideUv = 4, seed = 7)
-    val tightOut = ByteBuffer.allocate(I420Packer.packedSize(width, height))
-    val paddedOut = ByteBuffer.allocate(I420Packer.packedSize(width, height))
-    I420Packer.pack(
-      tight.y, 4, tight.u, 2, tight.v, 2, width, height, tightOut,
-    )
-    I420Packer.pack(
-      padded.y, 8, padded.u, 4, padded.v, 4, width, height, paddedOut,
-    )
-    assertThat(bytes(paddedOut)).isEqualTo(bytes(tightOut))
-    assertThat(tightOut.remaining()).isEqualTo(I420Packer.packedSize(width, height))
-  }
-
-  @Test
-  fun oddDimensionsPackExactBytes() {
-    val width = 3
-    val height = 3
-    val planes = buildPlanes(width, height, strideY = 5, strideUv = 3, seed = 11)
-    val dest = ByteBuffer.allocate(I420Packer.packedSize(width, height))
-    I420Packer.pack(planes.y, 5, planes.u, 3, planes.v, 3, width, height, dest)
-    val packed = bytes(dest)
-    assertThat(packed).hasSize(3 * 3 + 2 * 2 * 2)
-    // Visible Y is seed + row*width + col; chroma is 2x2 after (3+1)/2 rounding.
-    assertThat(packed.copyOfRange(0, 9)).containsExactly(
-      11, 12, 13,
-      14, 15, 16,
-      17, 18, 19,
-    )
-    assertThat(packed.copyOfRange(9, 13)).containsExactly(31, 32, 33, 34)
-    assertThat(packed.copyOfRange(13, 17)).containsExactly(41, 42, 43, 44)
-  }
-
-  private fun buildPlanes(
-    width: Int,
-    height: Int,
-    strideY: Int,
-    strideUv: Int,
-    seed: Int,
-  ): Planes {
-    val chromaW = (width + 1) / 2
-    val chromaH = (height + 1) / 2
-    val y = ByteBuffer.allocate(strideY * height)
-    val u = ByteBuffer.allocate(strideUv * chromaH)
-    val v = ByteBuffer.allocate(strideUv * chromaH)
-    for (row in 0 until height) {
-      for (col in 0 until width) {
-        y.put(row * strideY + col, (seed + row * width + col).toByte())
-      }
-    }
-    for (row in 0 until chromaH) {
-      for (col in 0 until chromaW) {
-        u.put(row * strideUv + col, (seed + 20 + row * chromaW + col).toByte())
-        v.put(row * strideUv + col, (seed + 30 + row * chromaW + col).toByte())
-      }
-    }
-    y.rewind()
-    u.rewind()
-    v.rewind()
-    return Planes(y, u, v)
-  }
-
-  private fun bytes(buffer: ByteBuffer): ByteArray {
-    val copy = buffer.duplicate()
-    val out = ByteArray(copy.remaining())
-    copy.get(out)
-    return out
-  }
-
-  private data class Planes(val y: ByteBuffer, val u: ByteBuffer, val v: ByteBuffer)
-}

package/ios/PolicyKit/Sources/AcsAudioPolicy/GlassesMediaSource.swift

@@ -1,68 +0,0 @@
-import Foundation
-
-public enum SourceKind: String, Sendable {
-  case whep
-  case direct
-}
-
-public enum SourceState: String, Sendable {
-  case idle
-  case connecting
-  case live
-  case failed
-}
-
-public struct SourceConfig: Equatable, Sendable {
-  public var url: String
-  public var kind: SourceKind
-
-  public init(url: String, kind: SourceKind = .whep) {
-    self.url = url
-    self.kind = kind
-  }
-}
-
-public protocol GlassesMediaSource: AnyObject {
-  var state: SourceState { get }
-  func start(config: SourceConfig)
-  func restart(config: SourceConfig)
-  func stop()
-  func setPcmDeliveryEnabled(_ enabled: Bool)
-}
-
-public final class GlassesMediaController {
-  private let factory: () -> GlassesMediaSource
-  private var source: GlassesMediaSource?
-
-  public init(factory: @escaping () -> GlassesMediaSource) {
-    self.factory = factory
-  }
-
-  public var state: SourceState { source?.state ?? .idle }
-
-  public func attach(config: SourceConfig) {
-    source?.stop()
-    let next = factory()
-    source = next
-    next.start(config: config)
-  }
-
-  public func restart(config: SourceConfig) {
-    source?.restart(config: config)
-  }
-
-  public func stop() {
-    source?.stop()
-    source = nil
-  }
-
-  public func setPcmDeliveryEnabled(_ enabled: Bool) {
-    source?.setPcmDeliveryEnabled(enabled)
-  }
-}
-
-public enum CapturePolicy {
-  public static func captureGlassesMic(_ source: AudioSourceKind) -> Bool {
-    source == .glasses
-  }
-}

package/ios/WhepVideoSource.swift

@@ -1,356 +0,0 @@
-import AVFoundation
-import CoreVideo
-import Foundation
-import WebRTC
-
-/// Recvonly WHEP subscriber. Decoded frames become CVPixelBuffers for ACS.
-/// Remote PCM is the P4 hard gate (RTCAudioRenderer on LiveKit/WebRTC-SDK 137).
-final class WhepVideoSource: NSObject, GlassesMediaSource {
-  var onFrame: ((CVPixelBuffer) -> Void)?
-  var onPcm: ((Data, Int, Int) -> Void)?
-  /// Fires on every health transition. `.failed` is the one nothing else can see:
-  /// ICE dropped or the WHEP endpoint went away while the ACS call stays connected.
-  var onStateChange: ((SourceState, String) -> Void)?
-  private(set) var state: SourceState = .idle
-
-  private var factory: RTCPeerConnectionFactory?
-  private var pc: RTCPeerConnection?
-  private var currentUrl: String?
-  private var offerPosted = false
-  private var pendingOfferPost: DispatchWorkItem?
-  // Fail closed like Android: the audio policy turns delivery on once the ACS
-  // virtual stream is live. Survives restart() so a WHEP rebuild cannot silently
-  // re-open the uplink against the current mute decision.
-  private var pcmEnabled = false
-  private var audioTracks: [RTCAudioTrack] = []
-  private var attachedVideo: RTCVideoTrack?
-  // Every peer generation gets its own id so a callback from a peer being torn
-  // down (ICE failed racing a rebuild) cannot mark the new one failed.
-  private var generation = 0
-  // `.live` means "a frame reached the renderer", not "the WHEP endpoint
-  // answered" — the answer lands seconds before ACS submits anything, and an
-  // answer that never produces a frame used to read healthy forever behind a
-  // frozen image. Armed per generation so a frame from a peer being torn down
-  // cannot promote its replacement.
-  private var firstFrameGeneration = -1
-  private var firstFramePromoted = false
-  private var firstFrameDeadline: DispatchWorkItem?
-  private var frameCount = 0
-  private var lastFpsLog = Date()
-  private lazy var http: URLSession = {
-    let config = URLSessionConfiguration.ephemeral
-    config.timeoutIntervalForRequest = 20
-    config.timeoutIntervalForResource = 20
-    return URLSession(configuration: config)
-  }()
-
-  func start(config: SourceConfig) {
-    start(whepUrl: config.url)
-  }
-
-  func start(whepUrl: String) {
-    stop()
-    currentUrl = whepUrl
-    offerPosted = false
-    generation += 1
-    let gen = generation
-    transition(.connecting, reason: "start")
-    RTCInitializeSSL()
-    let encoder = RTCDefaultVideoEncoderFactory()
-    let decoder = RTCDefaultVideoDecoderFactory()
-    factory = RTCPeerConnectionFactory(encoderFactory: encoder, decoderFactory: decoder)
-    let config = RTCConfiguration()
-    config.iceServers = [RTCIceServer(urlStrings: ["stun:stun.l.google.com:19302"])]
-    let constraints = RTCMediaConstraints(mandatoryConstraints: nil, optionalConstraints: nil)
-    guard let factory, let peer = factory.peerConnection(with: config, constraints: constraints, delegate: self) else {
-      transition(.failed, reason: "peer_create_failed")
-      return
-    }
-    pc = peer
-    peer.addTransceiver(of: .video, init: RTCRtpTransceiverInit.recvOnly())
-    peer.addTransceiver(of: .audio, init: RTCRtpTransceiverInit.recvOnly())
-    peer.offer(for: constraints) { [weak self] sdp, error in
-      guard let self, self.generation == gen else { return }
-      guard let sdp, error == nil else {
-        self.transition(.failed, reason: "offer_failed")
-        return
-      }
-      peer.setLocalDescription(sdp) { [weak self] _ in
-        // Track the delayed post so a same-instance restart can cancel it and
-        // avoid publishing a stale peer's SDP before ICE finishes.
-        let work = DispatchWorkItem { [weak self] in self?.postOfferIfNeeded() }
-        self?.pendingOfferPost = work
-        DispatchQueue.main.asyncAfter(deadline: .now() + 2, execute: work)
-      }
-    }
-  }
-
-  func restart(config: SourceConfig) {
-    // Only reuse the existing subscriber when the URL is unchanged AND the peer
-    // is still healthy. After ICE failed/disconnected (e.g. a Wi-Fi drop reusing
-    // the same WHEP URL) we must rebuild, or glasses video and mic never recover.
-    // `.connecting` is reusable and now spans answer → first frame; rebuilding
-    // there would only restart a wait that is bounded by the deadline below.
-    if config.url == currentUrl, state != .failed { return }
-    start(config: config)
-  }
-
-  /// Rebuild on the current URL even when the peer looks healthy (phone changed
-  /// networks; ICE may not have noticed the dead candidate pair yet).
-  func forceRestart() {
-    guard let url = currentUrl else { return }
-    NSLog("ACS-SPIKE WHEP forced rebuild state=\(state.rawValue)")
-    start(whepUrl: url)
-  }
-
-  func updateUrl(_ whepUrl: String) {
-    restart(config: SourceConfig(url: whepUrl))
-  }
-
-  func setPcmDeliveryEnabled(_ enabled: Bool) {
-    setPcmEnabled(enabled)
-  }
-
-  func setPcmEnabled(_ enabled: Bool) {
-    pcmEnabled = enabled
-    audioTracks.forEach { $0.isEnabled = enabled }
-    NSLog("ACS-SPIKE WHEP audio track enabled=\(enabled)")
-  }
-
-  func stop() {
-    currentUrl = nil
-    offerPosted = false
-    pendingOfferPost?.cancel()
-    pendingOfferPost = nil
-    // Disarmed before the renderer is removed, so a frame already in flight
-    // cannot promote the source we are tearing down.
-    cancelFirstFrameDeadline()
-    firstFrameGeneration = -1
-    firstFramePromoted = false
-    audioTracks.removeAll()
-    attachedVideo?.remove(self as RTCVideoRenderer)
-    attachedVideo = nil
-    // Invalidate callbacks from the peer being closed below before they can
-    // observe the idle we are about to set.
-    generation += 1
-    transition(.idle, reason: "stop")
-    // Drop our reference first so delegate callbacks fired by close() fail isCurrent().
-    let peer = pc
-    pc = nil
-    peer?.close()
-  }
-
-  private func transition(_ next: SourceState, reason: String) {
-    let previous = state
-    state = next
-    guard previous != next else { return }
-    NSLog("ACS-SPIKE WHEP source \(previous.rawValue) -> \(next.rawValue) (\(reason))")
-    onStateChange?(next, reason)
-  }
-
-  private func attachVideo(_ track: RTCVideoTrack) {
-    // Unified Plan delivers the same remote video via didAdd stream and
-    // didStartReceivingOn; rendering it twice doubles the frames handed to ACS.
-    if let attachedVideo, attachedVideo === track { return }
-    attachedVideo?.remove(self as RTCVideoRenderer)
-    attachedVideo = track
-    track.add(self as RTCVideoRenderer)
-  }
-
-  private func attachAudio(_ track: RTCAudioTrack) {
-    // Unified Plan fires didAdd stream, didStartReceivingOn and add-track for
-    // the same remote audio; only wire the renderer once or Teams hears doubled
-    // PCM.
-    if audioTracks.contains(where: { $0 === track }) {
-      track.isEnabled = pcmEnabled
-      return
-    }
-    audioTracks.append(track)
-    track.isEnabled = pcmEnabled
-    track.add(self as RTCAudioRenderer)
-  }
-
-  private func postOfferIfNeeded() {
-    guard !offerPosted, let url = currentUrl, let peer = pc, let sdp = peer.localDescription?.sdp else { return }
-    offerPosted = true
-    postOffer(url: url, sdp: sdp, peer: peer, gen: generation)
-  }
-
-  private func postOffer(url: String, sdp: String, peer: RTCPeerConnection, gen: Int) {
-    guard let endpoint = URL(string: url) else {
-      transition(.failed, reason: "whep_bad_url")
-      return
-    }
-    var request = URLRequest(url: endpoint)
-    request.httpMethod = "POST"
-    request.setValue("application/sdp", forHTTPHeaderField: "Content-Type")
-    request.httpBody = sdp.data(using: .utf8)
-    http.dataTask(with: request) { [weak self] data, response, error in
-      guard let self, self.generation == gen else { return }
-      let code = (response as? HTTPURLResponse)?.statusCode ?? -1
-      NSLog("ACS-SPIKE P3 WHEP \(code) answer bytes=\(data?.count ?? 0)")
-      guard error == nil, (200 ..< 300).contains(code),
-            let data, let answer = String(data: data, encoding: .utf8) else {
-        NSLog("ACS-SPIKE WHEP answer rejected code=\(code) error=\(error?.localizedDescription ?? "none")")
-        self.transition(.failed, reason: error != nil ? "whep_post_failed" : "whep_http_\(code)")
-        return
-      }
-      // Stay `.connecting`: setRemoteDescription, ICE and the decoder still have
-      // to run before ACS is handed anything.
-      self.armFirstFrame(gen: gen)
-      let desc = RTCSessionDescription(type: .answer, sdp: answer)
-      peer.setRemoteDescription(desc) { _ in }
-    }.resume()
-  }
-
-  /// Arms the first-frame gate for `gen` and bounds the wait. An answered WHEP
-  /// that never delivers is invisible to every other layer — ACS keeps the call
-  /// up on a frozen frame — so it has to expire into `.failed` and let the
-  /// module's rebuild backoff repair it.
-  private func armFirstFrame(gen: Int) {
-    firstFrameGeneration = gen
-    firstFramePromoted = false
-    cancelFirstFrameDeadline()
-    let work = DispatchWorkItem { [weak self] in
-      guard let self else { return }
-      self.firstFrameDeadline = nil
-      guard self.generation == gen, self.firstFrameGeneration == gen, !self.firstFramePromoted else { return }
-      NSLog("ACS-SPIKE WHEP answered but delivered no frame in \(Self.firstFrameTimeout)s")
-      self.transition(.failed, reason: "no_first_frame")
-    }
-    firstFrameDeadline = work
-    DispatchQueue.main.asyncAfter(deadline: .now() + Self.firstFrameTimeout, execute: work)
-  }
-
-  private func cancelFirstFrameDeadline() {
-    firstFrameDeadline?.cancel()
-    firstFrameDeadline = nil
-  }
-
-  /// The one honest `.live`: a decoded frame is on its way to ACS.
-  private func notePromotableFrame() {
-    guard !firstFramePromoted, firstFrameGeneration == generation else { return }
-    firstFramePromoted = true
-    cancelFirstFrameDeadline()
-    transition(.live, reason: "first_frame")
-  }
-
-  private func isCurrent(_ peerConnection: RTCPeerConnection) -> Bool {
-    pc === peerConnection
-  }
-
-  /// Matches Android's FIRST_FRAME_TIMEOUT_MS: a measured answer → first frame
-  /// gap is ~3 s, and a cold subscribe on a bad network is slower.
-  private static let firstFrameTimeout: TimeInterval = 9
-}
-
-extension WhepVideoSource: RTCPeerConnectionDelegate {
-  func peerConnection(_ peerConnection: RTCPeerConnection, didChange stateChanged: RTCSignalingState) {}
-  func peerConnection(_ peerConnection: RTCPeerConnection, didAdd stream: RTCMediaStream) {
-    guard isCurrent(peerConnection) else { return }
-    stream.videoTracks.first.map(attachVideo)
-    stream.audioTracks.first.map(attachAudio)
-  }
-  func peerConnection(_ peerConnection: RTCPeerConnection, didRemove stream: RTCMediaStream) {}
-  func peerConnectionShouldNegotiate(_ peerConnection: RTCPeerConnection) {}
-  func peerConnection(_ peerConnection: RTCPeerConnection, didChange newState: RTCIceConnectionState) {
-    NSLog("ACS-SPIKE ICE \(newState.rawValue)")
-    guard isCurrent(peerConnection) else { return }
-    if newState == .failed || newState == .disconnected {
-      transition(.failed, reason: newState == .failed ? "ice_failed" : "ice_disconnected")
-    } else if newState == .connected || newState == .completed {
-      // ICE can bounce disconnected → connected on its own; only act once the
-      // answer has been applied. A recovered candidate pair is a promise of
-      // frames, not a frame, so go back to `.connecting` and let the next one
-      // promote — immediately if media was already flowing, and if nothing
-      // arrives the rearmed deadline fails us again for the rebuild backoff.
-      if state == .failed, offerPosted {
-        transition(.connecting, reason: "ice_recovered")
-        armFirstFrame(gen: generation)
-      }
-    }
-  }
-  func peerConnection(_ peerConnection: RTCPeerConnection, didChange newState: RTCIceGatheringState) {
-    guard isCurrent(peerConnection) else { return }
-    if newState == .complete { postOfferIfNeeded() }
-  }
-  func peerConnection(_ peerConnection: RTCPeerConnection, didGenerate candidate: RTCIceCandidate) {}
-  func peerConnection(_ peerConnection: RTCPeerConnection, didRemove candidates: [RTCIceCandidate]) {}
-  func peerConnection(_ peerConnection: RTCPeerConnection, didOpen dataChannel: RTCDataChannel) {}
-  func peerConnection(_ peerConnection: RTCPeerConnection, didStartReceivingOn transceiver: RTCRtpTransceiver) {
-    guard isCurrent(peerConnection) else { return }
-    if let video = transceiver.receiver.track as? RTCVideoTrack {
-      attachVideo(video)
-    }
-    if let audio = transceiver.receiver.track as? RTCAudioTrack {
-      attachAudio(audio)
-    }
-  }
-}
-
-extension WhepVideoSource: RTCVideoRenderer {
-  func setSize(_ size: CGSize) {}
-  func renderFrame(_ frame: RTCVideoFrame?) {
-    guard let frame, let buffer = frame.buffer as? RTCCVPixelBuffer else { return }
-    notePromotableFrame()
-    frameCount += 1
-    let now = Date()
-    if now.timeIntervalSince(lastFpsLog) >= 1 {
-      NSLog("ACS-SPIKE P3 video \(Int(frame.width))x\(Int(frame.height)) fps=\(frameCount)")
-      frameCount = 0
-      lastFpsLog = now
-    }
-    onFrame?(buffer.pixelBuffer)
-  }
-}
-
-extension WhepVideoSource: RTCAudioRenderer {
-  func render(pcmBuffer: AVAudioPCMBuffer) {
-    guard pcmEnabled else { return }
-    let channels = Int(pcmBuffer.format.channelCount)
-    let rate = Int(pcmBuffer.format.sampleRate)
-    guard let data = Self.pcm16(from: pcmBuffer) else { return }
-    onPcm?(data, rate, channels)
-  }
-}
-
-private extension WhepVideoSource {
-  static func pcm16(from buffer: AVAudioPCMBuffer) -> Data? {
-    let channels = Int(buffer.format.channelCount)
-    let frames = Int(buffer.frameLength)
-    if frames == 0 || channels == 0 { return nil }
-    var data = Data(count: frames * channels * 2)
-    if let int16 = buffer.int16ChannelData {
-      data.withUnsafeMutableBytes { raw in
-        let dest = raw.bindMemory(to: Int16.self)
-        for frame in 0 ..< frames {
-          for channel in 0 ..< channels {
-            dest[frame * channels + channel] = int16[channel][frame]
-          }
-        }
-      }
-      return data
-    }
-    if let float = buffer.floatChannelData {
-      data.withUnsafeMutableBytes { raw in
-        let dest = raw.bindMemory(to: Int16.self)
-        for frame in 0 ..< frames {
-          for channel in 0 ..< channels {
-            let sample = max(-1, min(1, float[channel][frame]))
-            dest[frame * channels + channel] = Int16(sample * Float(Int16.max))
-          }
-        }
-      }
-      return data
-    }
-    return nil
-  }
-}
-
-private extension RTCRtpTransceiverInit {
-  static func recvOnly() -> RTCRtpTransceiverInit {
-    let value = RTCRtpTransceiverInit()
-    value.direction = .recvOnly
-    return value
-  }
-}