@mentra/acs-meeting 3.2.0-dev.120

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. package/NOTICE +20 -0
  2. package/README.md +412 -0
  3. package/THIRD_PARTY +7 -0
  4. package/android/build.gradle +64 -0
  5. package/android/src/main/AndroidManifest.xml +5 -0
  6. package/android/src/main/java/com/mentra/acsmeeting/AcsMeetingModule.kt +80 -0
  7. package/android/src/main/java/com/mentra/acsmeeting/AcsMeetingSession.kt +846 -0
  8. package/android/src/main/java/com/mentra/acsmeeting/RemoteRoster.kt +123 -0
  9. package/android/src/main/java/com/mentra/acsmeeting/audio/AcsAudioPolicy.kt +90 -0
  10. package/android/src/main/java/com/mentra/acsmeeting/audio/AudioPolicyApplier.kt +176 -0
  11. package/android/src/main/java/com/mentra/acsmeeting/audio/IncomingAudioPump.kt +112 -0
  12. package/android/src/main/java/com/mentra/acsmeeting/audio/IncomingRateProbe.kt +65 -0
  13. package/android/src/main/java/com/mentra/acsmeeting/audio/PcmBridge.kt +108 -0
  14. package/android/src/main/java/com/mentra/acsmeeting/audio/PcmResampler.kt +159 -0
  15. package/android/src/main/java/com/mentra/acsmeeting/audio/PhoneMicCapturer.kt +115 -0
  16. package/android/src/main/java/com/mentra/acsmeeting/audio/UplinkPacer.kt +302 -0
  17. package/android/src/main/java/com/mentra/acsmeeting/audio/UplinkSender.kt +230 -0
  18. package/android/src/main/java/com/mentra/acsmeeting/source/BitmapFont.kt +99 -0
  19. package/android/src/main/java/com/mentra/acsmeeting/source/CloudflareWhepSource.kt +578 -0
  20. package/android/src/main/java/com/mentra/acsmeeting/source/FirstFrameGate.kt +45 -0
  21. package/android/src/main/java/com/mentra/acsmeeting/source/GlassesMediaSource.kt +101 -0
  22. package/android/src/main/java/com/mentra/acsmeeting/source/MediaListeners.kt +57 -0
  23. package/android/src/main/java/com/mentra/acsmeeting/source/SyntheticFrameFactory.kt +205 -0
  24. package/android/src/main/java/com/mentra/acsmeeting/source/SyntheticI420Source.kt +214 -0
  25. package/android/src/main/java/com/mentra/acsmeeting/source/TrackRegistry.kt +31 -0
  26. package/android/src/main/java/com/mentra/acsmeeting/source/VideoSourceArm.kt +41 -0
  27. package/android/src/main/java/com/mentra/acsmeeting/telemetry/ChromaProbe.kt +78 -0
  28. package/android/src/main/java/com/mentra/acsmeeting/telemetry/PipelineStats.kt +281 -0
  29. package/android/src/main/java/com/mentra/acsmeeting/telemetry/PipelineTicker.kt +51 -0
  30. package/android/src/main/java/com/mentra/acsmeeting/telemetry/ProcessCpu.kt +15 -0
  31. package/android/src/main/java/com/mentra/acsmeeting/telemetry/RingPercentile.kt +33 -0
  32. package/android/src/main/java/com/mentra/acsmeeting/video/AcsFrameSender.kt +475 -0
  33. package/android/src/main/java/com/mentra/acsmeeting/video/AcsTimestamp.kt +26 -0
  34. package/android/src/main/java/com/mentra/acsmeeting/video/FrameGeometry.kt +22 -0
  35. package/android/src/main/java/com/mentra/acsmeeting/video/I420FormatSpec.kt +90 -0
  36. package/android/src/main/java/com/mentra/acsmeeting/video/I420Packer.kt +72 -0
  37. package/android/src/main/java/com/mentra/acsmeeting/video/Nv12FormatSpec.kt +36 -0
  38. package/android/src/main/java/com/mentra/acsmeeting/video/Nv12Packer.kt +51 -0
  39. package/android/src/main/java/com/mentra/acsmeeting/video/SendGate.kt +27 -0
  40. package/android/src/main/java/com/mentra/acsmeeting/video/VideoProfile.kt +57 -0
  41. package/android/src/test/java/com/mentra/acsmeeting/RemoteRosterTest.kt +40 -0
  42. package/android/src/test/java/com/mentra/acsmeeting/audio/AcsAudioPolicyTest.kt +204 -0
  43. package/android/src/test/java/com/mentra/acsmeeting/audio/AudioPolicyApplierTest.kt +223 -0
  44. package/android/src/test/java/com/mentra/acsmeeting/audio/CapturePolicyTest.kt +12 -0
  45. package/android/src/test/java/com/mentra/acsmeeting/audio/IncomingAudioPumpTest.kt +97 -0
  46. package/android/src/test/java/com/mentra/acsmeeting/audio/IncomingRateProbeTest.kt +78 -0
  47. package/android/src/test/java/com/mentra/acsmeeting/audio/PcmResamplerTest.kt +125 -0
  48. package/android/src/test/java/com/mentra/acsmeeting/audio/UplinkPacerTest.kt +201 -0
  49. package/android/src/test/java/com/mentra/acsmeeting/audio/UplinkSenderTest.kt +166 -0
  50. package/android/src/test/java/com/mentra/acsmeeting/source/AcsInvestigationTest.kt +26 -0
  51. package/android/src/test/java/com/mentra/acsmeeting/source/BitmapFontTest.kt +22 -0
  52. package/android/src/test/java/com/mentra/acsmeeting/source/FirstFrameGateTest.kt +60 -0
  53. package/android/src/test/java/com/mentra/acsmeeting/source/GlassesMediaSourceTest.kt +133 -0
  54. package/android/src/test/java/com/mentra/acsmeeting/source/I420PlanesTest.kt +67 -0
  55. package/android/src/test/java/com/mentra/acsmeeting/source/SyntheticFrameFactoryTest.kt +193 -0
  56. package/android/src/test/java/com/mentra/acsmeeting/source/SyntheticI420SourceTest.kt +176 -0
  57. package/android/src/test/java/com/mentra/acsmeeting/source/TrackRegistryTest.kt +36 -0
  58. package/android/src/test/java/com/mentra/acsmeeting/telemetry/ChromaProbeTest.kt +61 -0
  59. package/android/src/test/java/com/mentra/acsmeeting/telemetry/PipelineStatsTest.kt +256 -0
  60. package/android/src/test/java/com/mentra/acsmeeting/telemetry/ProcessCpuTest.kt +18 -0
  61. package/android/src/test/java/com/mentra/acsmeeting/telemetry/RingPercentileTest.kt +49 -0
  62. package/android/src/test/java/com/mentra/acsmeeting/video/AcsTimestampTest.kt +43 -0
  63. package/android/src/test/java/com/mentra/acsmeeting/video/FrameGeometryTest.kt +33 -0
  64. package/android/src/test/java/com/mentra/acsmeeting/video/I420FormatSpecTest.kt +77 -0
  65. package/android/src/test/java/com/mentra/acsmeeting/video/I420PackerTest.kt +126 -0
  66. package/android/src/test/java/com/mentra/acsmeeting/video/Nv12FormatSpecTest.kt +36 -0
  67. package/android/src/test/java/com/mentra/acsmeeting/video/Nv12PackerTest.kt +58 -0
  68. package/android/src/test/java/com/mentra/acsmeeting/video/SendGateTest.kt +79 -0
  69. package/android/src/test/java/com/mentra/acsmeeting/video/VideoProfileTest.kt +68 -0
  70. package/build/AcsMeeting.types.d.ts +59 -0
  71. package/build/AcsMeeting.types.d.ts.map +1 -0
  72. package/build/AcsMeeting.types.js +2 -0
  73. package/build/AcsMeeting.types.js.map +1 -0
  74. package/build/AcsMeetingModule.d.ts +15 -0
  75. package/build/AcsMeetingModule.d.ts.map +1 -0
  76. package/build/AcsMeetingModule.js +3 -0
  77. package/build/AcsMeetingModule.js.map +1 -0
  78. package/build/AcsMeetingModule.web.d.ts +16 -0
  79. package/build/AcsMeetingModule.web.d.ts.map +1 -0
  80. package/build/AcsMeetingModule.web.js +31 -0
  81. package/build/AcsMeetingModule.web.js.map +1 -0
  82. package/build/index.d.ts +3 -0
  83. package/build/index.d.ts.map +1 -0
  84. package/build/index.js +3 -0
  85. package/build/index.js.map +1 -0
  86. package/expo-module.config.json +9 -0
  87. package/ios/AcsFrameSender.swift +47 -0
  88. package/ios/AcsMeeting.podspec +27 -0
  89. package/ios/AcsMeetingModule.swift +558 -0
  90. package/ios/PcmBridge.swift +117 -0
  91. package/ios/PhoneMicCapturer.swift +64 -0
  92. package/ios/PolicyKit/Package.swift +17 -0
  93. package/ios/PolicyKit/Sources/AcsAudioPolicy/AcsAudioPolicy.swift +102 -0
  94. package/ios/PolicyKit/Sources/AcsAudioPolicy/AudioPolicyApplier.swift +162 -0
  95. package/ios/PolicyKit/Sources/AcsAudioPolicy/GlassesMediaSource.swift +68 -0
  96. package/ios/PolicyKit/Tests/AcsAudioPolicyTests/AcsAudioPolicyTests.swift +273 -0
  97. package/ios/WhepVideoSource.swift +356 -0
  98. package/package.json +56 -0
  99. package/src/AcsMeeting.types.ts +66 -0
  100. package/src/AcsMeetingModule.ts +16 -0
  101. package/src/AcsMeetingModule.web.ts +33 -0
  102. package/src/index.ts +2 -0
@@ -0,0 +1,475 @@
1
+ package com.mentra.acsmeeting.video
2
+
3
+ import android.util.Log
4
+ import com.azure.android.communication.calling.RawOutgoingVideoStream
5
+ import com.azure.android.communication.calling.RawVideoFrameBuffer
6
+ import com.azure.android.communication.calling.VideoStreamFormat
7
+ import com.azure.android.communication.calling.VideoStreamFormatChangedListener
8
+ import com.azure.android.communication.calling.VideoStreamPixelFormat
9
+ import com.azure.android.communication.calling.VideoStreamResolution
10
+ import com.azure.android.communication.calling.VideoStreamState
11
+ import com.azure.android.communication.calling.VideoStreamStateChangedListener
12
+ import com.azure.android.communication.calling.VirtualOutgoingVideoStream
13
+ import com.mentra.acsmeeting.source.AcsInvestigation
14
+ import com.mentra.acsmeeting.source.I420Planes
15
+ import com.mentra.acsmeeting.source.PixelFormatArm
16
+ import com.mentra.acsmeeting.source.TargetSize
17
+ import com.mentra.acsmeeting.telemetry.ChromaProbe
18
+ import com.mentra.acsmeeting.telemetry.PipelineStats
19
+ import java.nio.ByteBuffer
20
+ import java.util.concurrent.ConcurrentHashMap
21
+ import java.util.concurrent.ConcurrentLinkedQueue
22
+ import java.util.concurrent.Executors
23
+ import java.util.concurrent.RejectedExecutionException
24
+ import java.util.concurrent.TimeUnit
25
+ import java.util.concurrent.TimeoutException
26
+ import java.util.concurrent.atomic.AtomicBoolean
27
+ import java.util.concurrent.atomic.AtomicInteger
28
+ import java.util.concurrent.atomic.AtomicReference
29
+
30
+ /**
31
+ * I420 to ACS: three independent direct planes (Y, U, V). No inter-arrival
32
+ * pacer — [SendGate] is the only backpressure. Size mismatch drops; the sink
33
+ * owns scaling to the negotiated size.
34
+ */
35
+ class AcsFrameSender(
36
+ private val stats: PipelineStats = PipelineStats(),
37
+ ) {
38
+ private val running = AtomicBoolean(false)
39
+ private val stream = AtomicReference<RawOutgoingVideoStream?>(null)
40
+ private val format = AtomicReference<VideoStreamFormat?>(null)
41
+ private val sender = Executors.newSingleThreadExecutor { runnable ->
42
+ Thread(runnable, "acs-i420-send").apply { isDaemon = true }
43
+ }
44
+ private val gate = SendGate()
45
+ private val pool = ConcurrentHashMap<Int, ConcurrentLinkedQueue<ByteBuffer>>()
46
+ private val sendSeq = AtomicInteger(0)
47
+ private val lastTicks = AtomicReference(0L)
48
+ private val held = AtomicInteger(0)
49
+ private val cleanup = Executors.newSingleThreadScheduledExecutor { runnable ->
50
+ Thread(runnable, "acs-zc-release").apply { isDaemon = true }
51
+ }
52
+ // Set on the session thread, read from ACS state/format listener threads.
53
+ @Volatile private var onFormat: ((TargetSize) -> Unit)? = null
54
+ private var attachedStream: VirtualOutgoingVideoStream? = null
55
+ private var stateListener: VideoStreamStateChangedListener? = null
56
+ private var formatListener: VideoStreamFormatChangedListener? = null
57
+
58
+ fun attach(outgoing: VirtualOutgoingVideoStream, onFormat: ((TargetSize) -> Unit)? = null) {
59
+ detach()
60
+ this.onFormat = onFormat
61
+ stream.set(outgoing)
62
+ attachedStream = outgoing
63
+ val onState = VideoStreamStateChangedListener {
64
+ // Ignore late events from a previous call's stream so a stale STOPPED
65
+ // cannot freeze outgoing video for the current meeting.
66
+ if (stream.get() !== outgoing) return@VideoStreamStateChangedListener
67
+ val state = outgoing.state
68
+ Log.i(TAG, "raw video state=$state")
69
+ running.set(state == VideoStreamState.STARTED)
70
+ if (state == VideoStreamState.STARTED) {
71
+ format.set(outgoing.format)
72
+ logFormat(outgoing.format)
73
+ pushTarget(outgoing.format)
74
+ }
75
+ }
76
+ val onFormatChanged = VideoStreamFormatChangedListener { args ->
77
+ if (stream.get() !== outgoing) return@VideoStreamFormatChangedListener
78
+ format.set(args.format)
79
+ logFormat(args.format)
80
+ pushTarget(args.format)
81
+ }
82
+ stateListener = onState
83
+ formatListener = onFormatChanged
84
+ outgoing.addOnStateChangedListener(onState)
85
+ outgoing.addOnFormatChangedListener(onFormatChanged)
86
+ }
87
+
88
+ fun isReady(): Boolean = stream.get() != null && running.get() && format.get() != null
89
+
90
+ fun sendPlanes(planes: I420Planes) {
91
+ val out = stream.get()
92
+ if (out == null || !running.get()) {
93
+ stats.onDropNotStarted()
94
+ return
95
+ }
96
+ val negotiated = format.get()
97
+ if (negotiated == null) {
98
+ stats.onDropNotStarted()
99
+ return
100
+ }
101
+ if (negotiated.width != planes.width || negotiated.height != planes.height) {
102
+ stats.onDropSize()
103
+ return
104
+ }
105
+ if (!planes.planesReadable()) {
106
+ stats.onDropMalformed()
107
+ Log.w(
108
+ TAG,
109
+ "P5 send plane too small ${planes.width}x${planes.height} " +
110
+ "stride=${planes.strideY}/${planes.strideU}/${planes.strideV}",
111
+ )
112
+ return
113
+ }
114
+
115
+ val prepared = prepareSend(planes, negotiated)
116
+ stats.setChroma(prepared.chroma)
117
+
118
+ if (!gate.tryAcquire()) {
119
+ stats.onDropBusy()
120
+ releaseSendBuffers(prepared, planes)
121
+ return
122
+ }
123
+
124
+ val seq = sendSeq.incrementAndGet()
125
+ stats.onQueued()
126
+ if (seq <= TRACE_SENDS) {
127
+ Log.i(
128
+ TAG,
129
+ "P5 send-enter seq=$seq mode=${prepared.mode} buffers=${prepared.buffers.size} " +
130
+ "sizes=${prepared.buffers.joinToString("/") { it.remaining().toString() }} " +
131
+ "direct=${prepared.buffers.all { it.isDirect }} tight=${planes.isTight()} " +
132
+ "sinkThread=${Thread.currentThread().name}",
133
+ )
134
+ }
135
+
136
+ val submitted = try {
137
+ sender.execute {
138
+ var result = "ok"
139
+ // A timed-out future is NOT cancelled: ACS still reads these direct buffers.
140
+ var reclaimable = true
141
+ var frame: RawVideoFrameBuffer? = null
142
+ try {
143
+ val streamTicks = try {
144
+ out.timestampInTicks
145
+ } catch (_: Exception) {
146
+ 0L
147
+ }
148
+ val ticks = AcsTimestamp.resolve(
149
+ streamTicks = streamTicks,
150
+ captureNs = if (planes.timestampNs > 0) planes.timestampNs else System.nanoTime(),
151
+ lastTicks = lastTicks.get(),
152
+ )
153
+ lastTicks.set(ticks)
154
+ frame = RawVideoFrameBuffer()
155
+ frame.buffers = prepared.buffers
156
+ frame.streamFormat = negotiated
157
+ frame.timestampInTicks = ticks
158
+ val sendStart = System.nanoTime()
159
+ out.sendRawVideoFrame(frame).get(SEND_TIMEOUT_MS, TimeUnit.MILLISECONDS)
160
+ stats.send.record(System.nanoTime() - sendStart)
161
+ stats.onSub()
162
+ if (seq <= TRACE_SENDS) {
163
+ Log.i(
164
+ TAG,
165
+ "P5 send-exit seq=$seq result=ok ticks=$ticks streamTicks=$streamTicks " +
166
+ "thread=${Thread.currentThread().name}",
167
+ )
168
+ }
169
+ } catch (error: TimeoutException) {
170
+ result = "timeout"
171
+ reclaimable = false
172
+ stats.onDropFail()
173
+ stats.onAbandoned()
174
+ if (prepared.zeroCopy) {
175
+ stats.onZcTimeout()
176
+ scheduleZeroCopyRelease(planes)
177
+ }
178
+ Log.w(TAG, "P5 send-exit seq=$seq result=timeout getMs>$SEND_TIMEOUT_MS buffers=abandoned")
179
+ } catch (error: Exception) {
180
+ result = "failed"
181
+ stats.onDropFail()
182
+ Log.w(TAG, "P5 send-exit seq=$seq result=failed ${describeAcs(error)}")
183
+ } finally {
184
+ if (reclaimable) {
185
+ try {
186
+ frame?.close()
187
+ } catch (_: Exception) {
188
+ }
189
+ releaseSendBuffers(prepared, planes)
190
+ }
191
+ gate.release()
192
+ if (seq == 1 && result != "ok") {
193
+ Log.w(TAG, "P5 send first frame did not complete result=$result mode=${prepared.mode}")
194
+ }
195
+ }
196
+ }
197
+ true
198
+ } catch (error: RejectedExecutionException) {
199
+ false
200
+ }
201
+ if (!submitted) {
202
+ stats.onDropFail()
203
+ gate.release()
204
+ releaseSendBuffers(prepared, planes)
205
+ }
206
+ }
207
+
208
+ private data class PreparedSend(
209
+ val buffers: List<ByteBuffer>,
210
+ val zeroCopy: Boolean,
211
+ val mode: String,
212
+ val chroma: ChromaProbe.Sample,
213
+ )
214
+
215
+ private fun prepareSend(planes: I420Planes, negotiated: VideoStreamFormat): PreparedSend {
216
+ if (negotiatedPixelIsNv12(negotiated)) {
217
+ return prepareNv12(planes)
218
+ }
219
+ val ySize = planes.width * planes.height
220
+ val uvSize = I420Packer.chromaStride(planes.width) * I420Packer.chromaStride(planes.height)
221
+ val zeroCopy = tryZeroCopy(planes)
222
+ val y: ByteBuffer
223
+ val u: ByteBuffer
224
+ val v: ByteBuffer
225
+ if (zeroCopy) {
226
+ planes.retain!!()
227
+ val nowHeld = held.incrementAndGet()
228
+ stats.onZcUsed(nowHeld)
229
+ y = planes.y.duplicate()
230
+ u = planes.u.duplicate()
231
+ v = planes.v.duplicate()
232
+ } else {
233
+ y = borrow(ySize)
234
+ u = borrow(uvSize)
235
+ v = borrow(uvSize)
236
+ val chromaW = I420Packer.chromaStride(planes.width)
237
+ val chromaH = I420Packer.chromaStride(planes.height)
238
+ val copyStart = System.nanoTime()
239
+ I420Packer.copyPlane(planes.y, planes.strideY, planes.width, planes.height, y)
240
+ I420Packer.copyPlane(planes.u, planes.strideU, chromaW, chromaH, u)
241
+ I420Packer.copyPlane(planes.v, planes.strideV, chromaW, chromaH, v)
242
+ y.flip()
243
+ u.flip()
244
+ v.flip()
245
+ stats.copy.record(System.nanoTime() - copyStart)
246
+ }
247
+ return PreparedSend(
248
+ buffers = listOf(y, u, v),
249
+ zeroCopy = zeroCopy,
250
+ mode = if (zeroCopy) "zerocopy" else "planes",
251
+ chroma = ChromaProbe.samplePlanes(y, u, v),
252
+ )
253
+ }
254
+
255
+ private fun prepareNv12(planes: I420Planes): PreparedSend {
256
+ val ySize = planes.width * planes.height
257
+ val uvSize = Nv12Packer.uvSize(planes.width, planes.height)
258
+ val y = borrow(ySize)
259
+ val uv = borrow(uvSize)
260
+ val chromaW = I420Packer.chromaStride(planes.width)
261
+ val chromaH = Nv12Packer.chromaHeight(planes.height)
262
+ val copyStart = System.nanoTime()
263
+ Nv12Packer.copyY(planes.y, planes.strideY, planes.width, planes.height, y)
264
+ Nv12Packer.interleaveUv(
265
+ planes.u, planes.strideU, planes.v, planes.strideV, chromaW, chromaH, uv,
266
+ )
267
+ y.flip()
268
+ uv.flip()
269
+ stats.copy.record(System.nanoTime() - copyStart)
270
+ return PreparedSend(
271
+ buffers = listOf(y, uv),
272
+ zeroCopy = false,
273
+ mode = "nv12",
274
+ chroma = ChromaProbe.sampleNv12(y, uv),
275
+ )
276
+ }
277
+
278
+ private fun negotiatedPixelIsNv12(negotiated: VideoStreamFormat): Boolean {
279
+ return try {
280
+ negotiated.pixelFormat == VideoStreamPixelFormat.NV12
281
+ } catch (_: Exception) {
282
+ AcsInvestigation.pixelFormat == PixelFormatArm.NV12
283
+ }
284
+ }
285
+
286
+ private fun tryZeroCopy(planes: I420Planes): Boolean {
287
+ if (!AcsInvestigation.zeroCopy) return false
288
+ if (planes.retain == null || planes.release == null) {
289
+ stats.onZcFell()
290
+ return false
291
+ }
292
+ if (!planes.isDirect()) {
293
+ stats.onZcFell()
294
+ return false
295
+ }
296
+ if (!planes.isTight()) {
297
+ stats.onZcPadded()
298
+ stats.onZcFell()
299
+ return false
300
+ }
301
+ if (held.get() >= MAX_HELD) {
302
+ stats.onZcFell()
303
+ return false
304
+ }
305
+ return true
306
+ }
307
+
308
+ private fun releaseSendBuffers(prepared: PreparedSend, planes: I420Planes) {
309
+ if (prepared.zeroCopy) {
310
+ try {
311
+ planes.release?.invoke()
312
+ } catch (_: Exception) {
313
+ }
314
+ held.decrementAndGet()
315
+ } else {
316
+ for (buffer in prepared.buffers) recycle(buffer)
317
+ }
318
+ }
319
+
320
+ private fun scheduleZeroCopyRelease(planes: I420Planes) {
321
+ cleanup.schedule({
322
+ try {
323
+ planes.release?.invoke()
324
+ } catch (_: Exception) {
325
+ } finally {
326
+ held.decrementAndGet()
327
+ }
328
+ }, ZC_GRACE_MS, TimeUnit.MILLISECONDS)
329
+ }
330
+
331
+ fun detach() {
332
+ running.set(false)
333
+ val previous = attachedStream
334
+ if (previous != null) {
335
+ stateListener?.let { previous.removeOnStateChangedListener(it) }
336
+ formatListener?.let { previous.removeOnFormatChangedListener(it) }
337
+ }
338
+ attachedStream = null
339
+ stateListener = null
340
+ formatListener = null
341
+ stream.set(null)
342
+ format.set(null)
343
+ sendSeq.set(0)
344
+ lastTicks.set(0L)
345
+ pool.clear()
346
+ onFormat = null
347
+ }
348
+
349
+ private fun pushTarget(fmt: VideoStreamFormat?) {
350
+ if (fmt == null || fmt.width <= 0 || fmt.height <= 0) return
351
+ onFormat?.invoke(TargetSize(fmt.width, fmt.height))
352
+ }
353
+
354
+ // Keyed by exact capacity: luma and chroma differ 4x, and a single queue made
355
+ // a luma borrow evict a chroma buffer it could not use.
356
+ private fun borrow(capacity: Int): ByteBuffer {
357
+ val existing = pool[capacity]?.poll()
358
+ if (existing != null) {
359
+ existing.clear()
360
+ return existing
361
+ }
362
+ stats.onPlaneAlloc()
363
+ return ByteBuffer.allocateDirect(capacity)
364
+ }
365
+
366
+ private fun recycle(buffer: ByteBuffer) {
367
+ val queue = pool.getOrPut(buffer.capacity()) { ConcurrentLinkedQueue() }
368
+ if (queue.size >= POOL_PER_SIZE) return
369
+ buffer.clear()
370
+ queue.offer(buffer)
371
+ }
372
+
373
+ private fun logFormat(fmt: VideoStreamFormat?) {
374
+ if (fmt == null) return
375
+ Log.i(
376
+ TAG,
377
+ "P5 negotiated format pixel=${fmt.pixelFormat} ${fmt.width}x${fmt.height} fps=${fmt.framesPerSecond} " +
378
+ "stride=${fmt.stride1}/${fmt.stride2}/${fmt.stride3}",
379
+ )
380
+ }
381
+
382
+ companion object {
383
+ private const val TAG = "ACS-SPIKE"
384
+ private const val SEND_TIMEOUT_MS = 200L
385
+ private const val TRACE_SENDS = 8
386
+ private const val POOL_PER_SIZE = 4
387
+ private const val MAX_HELD = 2
388
+ private const val ZC_GRACE_MS = 1_000L
389
+
390
+ fun outgoingFormat(profile: VideoProfile = VideoProfile.DEFAULT): VideoStreamFormat =
391
+ when (AcsInvestigation.pixelFormat) {
392
+ PixelFormatArm.NV12 -> nv12Format(profile)
393
+ PixelFormatArm.I420 -> i420Format(profile)
394
+ }
395
+
396
+ fun i420Format(profile: VideoProfile = VideoProfile.DEFAULT): VideoStreamFormat =
397
+ i420Format(profile.width, profile.height, profile.fps.toFloat())
398
+
399
+ fun nv12Format(profile: VideoProfile = VideoProfile.DEFAULT): VideoStreamFormat =
400
+ nv12Format(profile.width, profile.height, profile.fps.toFloat())
401
+
402
+ fun i420Format(width: Int, height: Int, fps: Float): VideoStreamFormat {
403
+ val spec = I420FormatSpec.of(width, height, fps)
404
+ val format = VideoStreamFormat()
405
+ format.pixelFormat = VideoStreamPixelFormat.I420
406
+ applyNamedSize(format, spec.width, spec.height)
407
+ format.framesPerSecond = spec.fps
408
+ format.stride1 = spec.strideY
409
+ format.stride2 = spec.strideU
410
+ format.stride3 = spec.strideV
411
+ return format
412
+ }
413
+
414
+ fun nv12Format(width: Int, height: Int, fps: Float): VideoStreamFormat {
415
+ val spec = Nv12FormatSpec.of(width, height, fps)
416
+ val format = VideoStreamFormat()
417
+ format.pixelFormat = VideoStreamPixelFormat.NV12
418
+ applyNamedSize(format, spec.width, spec.height)
419
+ format.framesPerSecond = spec.fps
420
+ format.stride1 = spec.strideY
421
+ format.stride2 = spec.strideUv
422
+ return format
423
+ }
424
+
425
+ private fun applyNamedSize(format: VideoStreamFormat, width: Int, height: Int) {
426
+ when (I420FormatSpec.of(width, height).namedResolution()) {
427
+ AcsNamedResolution.P1080 -> format.resolution = VideoStreamResolution.P1080
428
+ AcsNamedResolution.P720 -> format.resolution = VideoStreamResolution.P720
429
+ AcsNamedResolution.P540 -> format.resolution = VideoStreamResolution.P540
430
+ AcsNamedResolution.P480 -> format.resolution = VideoStreamResolution.P480
431
+ AcsNamedResolution.P360 -> format.resolution = VideoStreamResolution.P360
432
+ AcsNamedResolution.P270 -> format.resolution = VideoStreamResolution.P270
433
+ AcsNamedResolution.P240 -> format.resolution = VideoStreamResolution.P240
434
+ AcsNamedResolution.P180 -> format.resolution = VideoStreamResolution.P180
435
+ AcsNamedResolution.VGA -> format.resolution = VideoStreamResolution.VGA
436
+ AcsNamedResolution.QVGA -> format.resolution = VideoStreamResolution.QVGA
437
+ null -> {
438
+ Log.w(TAG, "ACS format ${width}x${height} is not a VideoStreamResolution; advertising raw size")
439
+ format.width = width
440
+ format.height = height
441
+ }
442
+ }
443
+ }
444
+
445
+ fun describeAcs(error: Throwable): String {
446
+ val parts = ArrayList<String>(4)
447
+ var current: Throwable? = error
448
+ var depth = 0
449
+ while (current != null && depth < 5) {
450
+ val code = errorCodeOf(current)
451
+ val message = current.message?.trim().orEmpty()
452
+ parts.add(
453
+ buildString {
454
+ append(current!!.javaClass.simpleName)
455
+ if (message.isNotEmpty()) append(':').append(message)
456
+ if (code != null) append(" code=").append(code)
457
+ },
458
+ )
459
+ current = current.cause
460
+ depth += 1
461
+ }
462
+ return parts.joinToString(" | ")
463
+ }
464
+
465
+ private fun errorCodeOf(error: Throwable): String? {
466
+ return try {
467
+ val method = error.javaClass.methods.firstOrNull { it.name == "getErrorCode" && it.parameterCount == 0 }
468
+ ?: return null
469
+ method.invoke(error)?.toString()
470
+ } catch (_: Exception) {
471
+ null
472
+ }
473
+ }
474
+ }
475
+ }
@@ -0,0 +1,26 @@
1
+ package com.mentra.acsmeeting.video
2
+
3
+ /**
4
+ * ACS [RawVideoFrame.setTimestampInTicks] uses Windows / .NET ticks:
5
+ * 100-nanosecond units. A frame with timestamp 0 is "no time", and a run of
6
+ * those makes the Teams jitter buffer hold the last picture until it decides
7
+ * the stream has moved — which looks like a random freeze.
8
+ *
9
+ * Prefer the stream clock when ACS is advancing it (Microsoft's sample does
10
+ * exactly that). Otherwise convert a capture timestamp. Always emit a tick
11
+ * strictly after [lastTicks] so two frames never share a presentation time.
12
+ */
13
+ object AcsTimestamp {
14
+ const val NS_PER_TICK = 100L
15
+
16
+ fun resolve(streamTicks: Long, captureNs: Long, lastTicks: Long): Long {
17
+ val fromStream = streamTicks > 0
18
+ val fromCapture = captureNs > 0
19
+ val candidate = when {
20
+ fromStream && streamTicks > lastTicks -> streamTicks
21
+ fromCapture -> captureNs / NS_PER_TICK
22
+ else -> lastTicks + 1
23
+ }
24
+ return if (candidate <= lastTicks) lastTicks + 1 else candidate
25
+ }
26
+ }
@@ -0,0 +1,22 @@
1
+ package com.mentra.acsmeeting.video
2
+
3
+ /**
4
+ * Pack I420 in the buffer's coordinate system, never [VideoFrame.rotatedWidth].
5
+ * cropAndScale and toI420 planes are un-rotated; mixing them with display size
6
+ * overruns the plane at rotation 90/270.
7
+ */
8
+ object FrameGeometry {
9
+ data class PackSize(
10
+ val width: Int,
11
+ val height: Int,
12
+ val rotationNonZero: Boolean,
13
+ )
14
+
15
+ fun packSize(bufferWidth: Int, bufferHeight: Int, rotationDegrees: Int): PackSize {
16
+ return PackSize(
17
+ width = bufferWidth,
18
+ height = bufferHeight,
19
+ rotationNonZero = rotationDegrees % 360 != 0,
20
+ )
21
+ }
22
+ }
@@ -0,0 +1,90 @@
1
+ package com.mentra.acsmeeting.video
2
+
3
+ /**
4
+ * ACS Android [com.azure.android.communication.calling.VideoStreamResolution]
5
+ * names. The virtual outgoing stream wants these exact sizes. Near-matches
6
+ * fail even when WHEP preview works: P480 is 858×480, not 854×480. Portrait
7
+ * 540×960 is not a named size; ACS P540 is 960×540.
8
+ */
9
+ enum class AcsNamedResolution {
10
+ P1080,
11
+ P720,
12
+ P540,
13
+ P480,
14
+ P360,
15
+ P270,
16
+ P240,
17
+ P180,
18
+ VGA,
19
+ QVGA,
20
+ }
21
+
22
+ /**
23
+ * The I420 format we advertise to ACS, as plain values.
24
+ *
25
+ * Split out from [AcsFrameSender.i420Format] because VideoStreamFormat is a
26
+ * native-backed ACS class that cannot initialize under a JVM unit test. The
27
+ * arithmetic is the part worth asserting; applying it to the SDK object is not.
28
+ *
29
+ * Strides must agree with [I420Packer], or ACS reads the chroma planes at the
30
+ * wrong offsets and the picture greens out.
31
+ */
32
+ data class I420FormatSpec(
33
+ val width: Int,
34
+ val height: Int,
35
+ val fps: Float,
36
+ val strideY: Int,
37
+ val strideU: Int,
38
+ val strideV: Int,
39
+ ) {
40
+ companion object {
41
+ const val MIN_WIDTH = 240
42
+ const val MAX_WIDTH = 1920
43
+ const val MIN_HEIGHT = 180
44
+ const val MAX_HEIGHT = 1080
45
+ const val MIN_FPS = 1f
46
+ const val MAX_FPS = 30f
47
+
48
+ /** VirtualOutgoingVideoStream join allowlist. P540 is 960×540, never 540×960. */
49
+ private val ALLOWED_SIZES = setOf(1280 to 720, 960 to 540)
50
+
51
+ fun of(width: Int = 1280, height: Int = 720, fps: Float = 20f): I420FormatSpec {
52
+ val chroma = I420Packer.chromaStride(width)
53
+ return I420FormatSpec(
54
+ width = width,
55
+ height = height,
56
+ fps = fps,
57
+ strideY = width,
58
+ strideU = chroma,
59
+ strideV = chroma,
60
+ )
61
+ }
62
+
63
+ fun parseOrNull(width: Int, height: Int, fps: Float): I420FormatSpec? {
64
+ if (width to height !in ALLOWED_SIZES) return null
65
+ val spec = of(width, height, fps)
66
+ return if (spec.withinAcsBounds()) spec else null
67
+ }
68
+ }
69
+
70
+ fun withinAcsBounds(): Boolean =
71
+ width in MIN_WIDTH..MAX_WIDTH && height in MIN_HEIGHT..MAX_HEIGHT && fps in MIN_FPS..MAX_FPS
72
+
73
+ /** Named ACS resolution for this geometry, or null if we must advertise raw size. */
74
+ fun namedResolution(): AcsNamedResolution? = when (width to height) {
75
+ 1920 to 1080 -> AcsNamedResolution.P1080
76
+ 1280 to 720 -> AcsNamedResolution.P720
77
+ 960 to 540 -> AcsNamedResolution.P540
78
+ 858 to 480 -> AcsNamedResolution.P480
79
+ 640 to 360 -> AcsNamedResolution.P360
80
+ 480 to 270 -> AcsNamedResolution.P270
81
+ 352 to 240 -> AcsNamedResolution.P240
82
+ 320 to 180 -> AcsNamedResolution.P180
83
+ 640 to 480 -> AcsNamedResolution.VGA
84
+ 320 to 240 -> AcsNamedResolution.QVGA
85
+ else -> null
86
+ }
87
+
88
+ /** Bytes ACS will read given these strides; must equal what [I420Packer] writes. */
89
+ fun packedSize(): Int = strideY * height + 2 * (strideU * I420Packer.chromaStride(height))
90
+ }
@@ -0,0 +1,72 @@
1
+ package com.mentra.acsmeeting.video
2
+
3
+ import java.nio.ByteBuffer
4
+ import kotlin.math.roundToInt
5
+
6
+ /** Tight I420 pack + stride math. No Android / ACS / WebRTC types so JVM tests stay cheap. */
7
+ object I420Packer {
8
+ fun packedSize(width: Int, height: Int): Int {
9
+ val chromaW = (width + 1) / 2
10
+ val chromaH = (height + 1) / 2
11
+ return width * height + 2 * chromaW * chromaH
12
+ }
13
+
14
+ fun chromaStride(width: Int): Int = (width + 1) / 2
15
+
16
+ /**
17
+ * Bytes a reader must be able to see for a plane with [stride], starting at
18
+ * the buffer's current position. Last row is only [width] bytes, not a full
19
+ * stride, so this is `stride*(height-1)+width`.
20
+ */
21
+ fun planeMinBytes(stride: Int, width: Int, height: Int): Int {
22
+ if (width <= 0 || height <= 0 || stride < width) return 0
23
+ return stride * (height - 1) + width
24
+ }
25
+
26
+ /**
27
+ * Packs padded WebRTC planes into tight I420 (strideY=width, strideU=strideV=(width+1)/2).
28
+ */
29
+ fun pack(
30
+ dataY: ByteBuffer,
31
+ strideY: Int,
32
+ dataU: ByteBuffer,
33
+ strideU: Int,
34
+ dataV: ByteBuffer,
35
+ strideV: Int,
36
+ width: Int,
37
+ height: Int,
38
+ dest: ByteBuffer,
39
+ ) {
40
+ dest.clear()
41
+ val chromaW = chromaStride(width)
42
+ val chromaH = chromaStride(height)
43
+ copyPlane(dataY, strideY, width, height, dest)
44
+ copyPlane(dataU, strideU, chromaW, chromaH, dest)
45
+ copyPlane(dataV, strideV, chromaW, chromaH, dest)
46
+ dest.flip()
47
+ }
48
+
49
+ fun copyPlane(src: ByteBuffer, stride: Int, width: Int, height: Int, dest: ByteBuffer) {
50
+ val plane = src.duplicate()
51
+ val origin = plane.position()
52
+ if (stride == width && plane.remaining() >= width * height) {
53
+ plane.limit(origin + width * height)
54
+ dest.put(plane)
55
+ return
56
+ }
57
+ for (row in 0 until height) {
58
+ val start = origin + row * stride
59
+ plane.limit(start + width)
60
+ plane.position(start)
61
+ dest.put(plane)
62
+ }
63
+ }
64
+
65
+ fun percentileMs(samplesNs: LongArray, quantile: Double): String {
66
+ if (samplesNs.isEmpty()) return "na"
67
+ val sorted = samplesNs.copyOf()
68
+ sorted.sort()
69
+ val index = ((sorted.size - 1) * quantile).roundToInt().coerceIn(0, sorted.lastIndex)
70
+ return ((sorted[index] / 1_000_000.0) * 10).roundToInt().div(10.0).toString()
71
+ }
72
+ }