@mentra/acs-meeting 3.2.0-dev.120

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. package/NOTICE +20 -0
  2. package/README.md +412 -0
  3. package/THIRD_PARTY +7 -0
  4. package/android/build.gradle +64 -0
  5. package/android/src/main/AndroidManifest.xml +5 -0
  6. package/android/src/main/java/com/mentra/acsmeeting/AcsMeetingModule.kt +80 -0
  7. package/android/src/main/java/com/mentra/acsmeeting/AcsMeetingSession.kt +846 -0
  8. package/android/src/main/java/com/mentra/acsmeeting/RemoteRoster.kt +123 -0
  9. package/android/src/main/java/com/mentra/acsmeeting/audio/AcsAudioPolicy.kt +90 -0
  10. package/android/src/main/java/com/mentra/acsmeeting/audio/AudioPolicyApplier.kt +176 -0
  11. package/android/src/main/java/com/mentra/acsmeeting/audio/IncomingAudioPump.kt +112 -0
  12. package/android/src/main/java/com/mentra/acsmeeting/audio/IncomingRateProbe.kt +65 -0
  13. package/android/src/main/java/com/mentra/acsmeeting/audio/PcmBridge.kt +108 -0
  14. package/android/src/main/java/com/mentra/acsmeeting/audio/PcmResampler.kt +159 -0
  15. package/android/src/main/java/com/mentra/acsmeeting/audio/PhoneMicCapturer.kt +115 -0
  16. package/android/src/main/java/com/mentra/acsmeeting/audio/UplinkPacer.kt +302 -0
  17. package/android/src/main/java/com/mentra/acsmeeting/audio/UplinkSender.kt +230 -0
  18. package/android/src/main/java/com/mentra/acsmeeting/source/BitmapFont.kt +99 -0
  19. package/android/src/main/java/com/mentra/acsmeeting/source/CloudflareWhepSource.kt +578 -0
  20. package/android/src/main/java/com/mentra/acsmeeting/source/FirstFrameGate.kt +45 -0
  21. package/android/src/main/java/com/mentra/acsmeeting/source/GlassesMediaSource.kt +101 -0
  22. package/android/src/main/java/com/mentra/acsmeeting/source/MediaListeners.kt +57 -0
  23. package/android/src/main/java/com/mentra/acsmeeting/source/SyntheticFrameFactory.kt +205 -0
  24. package/android/src/main/java/com/mentra/acsmeeting/source/SyntheticI420Source.kt +214 -0
  25. package/android/src/main/java/com/mentra/acsmeeting/source/TrackRegistry.kt +31 -0
  26. package/android/src/main/java/com/mentra/acsmeeting/source/VideoSourceArm.kt +41 -0
  27. package/android/src/main/java/com/mentra/acsmeeting/telemetry/ChromaProbe.kt +78 -0
  28. package/android/src/main/java/com/mentra/acsmeeting/telemetry/PipelineStats.kt +281 -0
  29. package/android/src/main/java/com/mentra/acsmeeting/telemetry/PipelineTicker.kt +51 -0
  30. package/android/src/main/java/com/mentra/acsmeeting/telemetry/ProcessCpu.kt +15 -0
  31. package/android/src/main/java/com/mentra/acsmeeting/telemetry/RingPercentile.kt +33 -0
  32. package/android/src/main/java/com/mentra/acsmeeting/video/AcsFrameSender.kt +475 -0
  33. package/android/src/main/java/com/mentra/acsmeeting/video/AcsTimestamp.kt +26 -0
  34. package/android/src/main/java/com/mentra/acsmeeting/video/FrameGeometry.kt +22 -0
  35. package/android/src/main/java/com/mentra/acsmeeting/video/I420FormatSpec.kt +90 -0
  36. package/android/src/main/java/com/mentra/acsmeeting/video/I420Packer.kt +72 -0
  37. package/android/src/main/java/com/mentra/acsmeeting/video/Nv12FormatSpec.kt +36 -0
  38. package/android/src/main/java/com/mentra/acsmeeting/video/Nv12Packer.kt +51 -0
  39. package/android/src/main/java/com/mentra/acsmeeting/video/SendGate.kt +27 -0
  40. package/android/src/main/java/com/mentra/acsmeeting/video/VideoProfile.kt +57 -0
  41. package/android/src/test/java/com/mentra/acsmeeting/RemoteRosterTest.kt +40 -0
  42. package/android/src/test/java/com/mentra/acsmeeting/audio/AcsAudioPolicyTest.kt +204 -0
  43. package/android/src/test/java/com/mentra/acsmeeting/audio/AudioPolicyApplierTest.kt +223 -0
  44. package/android/src/test/java/com/mentra/acsmeeting/audio/CapturePolicyTest.kt +12 -0
  45. package/android/src/test/java/com/mentra/acsmeeting/audio/IncomingAudioPumpTest.kt +97 -0
  46. package/android/src/test/java/com/mentra/acsmeeting/audio/IncomingRateProbeTest.kt +78 -0
  47. package/android/src/test/java/com/mentra/acsmeeting/audio/PcmResamplerTest.kt +125 -0
  48. package/android/src/test/java/com/mentra/acsmeeting/audio/UplinkPacerTest.kt +201 -0
  49. package/android/src/test/java/com/mentra/acsmeeting/audio/UplinkSenderTest.kt +166 -0
  50. package/android/src/test/java/com/mentra/acsmeeting/source/AcsInvestigationTest.kt +26 -0
  51. package/android/src/test/java/com/mentra/acsmeeting/source/BitmapFontTest.kt +22 -0
  52. package/android/src/test/java/com/mentra/acsmeeting/source/FirstFrameGateTest.kt +60 -0
  53. package/android/src/test/java/com/mentra/acsmeeting/source/GlassesMediaSourceTest.kt +133 -0
  54. package/android/src/test/java/com/mentra/acsmeeting/source/I420PlanesTest.kt +67 -0
  55. package/android/src/test/java/com/mentra/acsmeeting/source/SyntheticFrameFactoryTest.kt +193 -0
  56. package/android/src/test/java/com/mentra/acsmeeting/source/SyntheticI420SourceTest.kt +176 -0
  57. package/android/src/test/java/com/mentra/acsmeeting/source/TrackRegistryTest.kt +36 -0
  58. package/android/src/test/java/com/mentra/acsmeeting/telemetry/ChromaProbeTest.kt +61 -0
  59. package/android/src/test/java/com/mentra/acsmeeting/telemetry/PipelineStatsTest.kt +256 -0
  60. package/android/src/test/java/com/mentra/acsmeeting/telemetry/ProcessCpuTest.kt +18 -0
  61. package/android/src/test/java/com/mentra/acsmeeting/telemetry/RingPercentileTest.kt +49 -0
  62. package/android/src/test/java/com/mentra/acsmeeting/video/AcsTimestampTest.kt +43 -0
  63. package/android/src/test/java/com/mentra/acsmeeting/video/FrameGeometryTest.kt +33 -0
  64. package/android/src/test/java/com/mentra/acsmeeting/video/I420FormatSpecTest.kt +77 -0
  65. package/android/src/test/java/com/mentra/acsmeeting/video/I420PackerTest.kt +126 -0
  66. package/android/src/test/java/com/mentra/acsmeeting/video/Nv12FormatSpecTest.kt +36 -0
  67. package/android/src/test/java/com/mentra/acsmeeting/video/Nv12PackerTest.kt +58 -0
  68. package/android/src/test/java/com/mentra/acsmeeting/video/SendGateTest.kt +79 -0
  69. package/android/src/test/java/com/mentra/acsmeeting/video/VideoProfileTest.kt +68 -0
  70. package/build/AcsMeeting.types.d.ts +59 -0
  71. package/build/AcsMeeting.types.d.ts.map +1 -0
  72. package/build/AcsMeeting.types.js +2 -0
  73. package/build/AcsMeeting.types.js.map +1 -0
  74. package/build/AcsMeetingModule.d.ts +15 -0
  75. package/build/AcsMeetingModule.d.ts.map +1 -0
  76. package/build/AcsMeetingModule.js +3 -0
  77. package/build/AcsMeetingModule.js.map +1 -0
  78. package/build/AcsMeetingModule.web.d.ts +16 -0
  79. package/build/AcsMeetingModule.web.d.ts.map +1 -0
  80. package/build/AcsMeetingModule.web.js +31 -0
  81. package/build/AcsMeetingModule.web.js.map +1 -0
  82. package/build/index.d.ts +3 -0
  83. package/build/index.d.ts.map +1 -0
  84. package/build/index.js +3 -0
  85. package/build/index.js.map +1 -0
  86. package/expo-module.config.json +9 -0
  87. package/ios/AcsFrameSender.swift +47 -0
  88. package/ios/AcsMeeting.podspec +27 -0
  89. package/ios/AcsMeetingModule.swift +558 -0
  90. package/ios/PcmBridge.swift +117 -0
  91. package/ios/PhoneMicCapturer.swift +64 -0
  92. package/ios/PolicyKit/Package.swift +17 -0
  93. package/ios/PolicyKit/Sources/AcsAudioPolicy/AcsAudioPolicy.swift +102 -0
  94. package/ios/PolicyKit/Sources/AcsAudioPolicy/AudioPolicyApplier.swift +162 -0
  95. package/ios/PolicyKit/Sources/AcsAudioPolicy/GlassesMediaSource.swift +68 -0
  96. package/ios/PolicyKit/Tests/AcsAudioPolicyTests/AcsAudioPolicyTests.swift +273 -0
  97. package/ios/WhepVideoSource.swift +356 -0
  98. package/package.json +56 -0
  99. package/src/AcsMeeting.types.ts +66 -0
  100. package/src/AcsMeetingModule.ts +16 -0
  101. package/src/AcsMeetingModule.web.ts +33 -0
  102. package/src/index.ts +2 -0
@@ -0,0 +1,101 @@
1
+ package com.mentra.acsmeeting.source
2
+
3
+ enum class SourceKind { WHEP, DIRECT }
4
+
5
+ enum class SourceState { IDLE, CONNECTING, LIVE, FAILED }
6
+
7
+ data class SourceConfig(
8
+ val url: String,
9
+ val kind: SourceKind = SourceKind.WHEP,
10
+ )
11
+
12
+ /** ACS-negotiated output size. One object so a frame cannot observe a torn width/height. */
13
+ data class TargetSize(val width: Int, val height: Int)
14
+
15
+ /**
16
+ * Whether a subscriber already on [currentUrl] can serve [next] as-is instead of
17
+ * being rebuilt.
18
+ *
19
+ * Reuse is the whole point: a rebuild mints a new peer and restarts the wait for
20
+ * the first frame, which is the outage we are trying to shorten. `CONNECTING`
21
+ * counts as reusable, and since `LIVE` means "a frame reached the sink" that now
22
+ * spans the answer → first frame window too. It cannot strand a caller, because
23
+ * every `CONNECTING` is bounded: the offer post fails, or the answer's
24
+ * first-frame deadline expires into `FAILED` and this returns false.
25
+ */
26
+ fun canReuseSource(currentUrl: String?, state: SourceState, next: SourceConfig): Boolean =
27
+ currentUrl == next.url && next.kind == SourceKind.WHEP && state != SourceState.FAILED
28
+
29
+ /**
30
+ * Observes source health transitions. `FAILED` is the one that matters: ICE
31
+ * dropped, the WHEP endpoint went away, or an answered subscription never
32
+ * delivered a frame — and nothing downstream will notice on its own, because
33
+ * ACS keeps the call up with a frozen last frame.
34
+ */
35
+ fun interface SourceStateListener {
36
+ fun onSourceState(state: SourceState, reason: String?)
37
+ }
38
+
39
+ /**
40
+ * Transports whatever the glasses already captured. Capture policy lives
41
+ * outside this interface so a Cloudflare hop can be replaced without touching ACS.
42
+ */
43
+ interface GlassesMediaSource {
44
+ fun start(config: SourceConfig)
45
+ fun restart(config: SourceConfig)
46
+ /** Rebuild the transport for the current config even if it looks healthy. */
47
+ fun forceRestart() {}
48
+ fun stop()
49
+ val state: SourceState
50
+ fun setPcmDeliveryEnabled(enabled: Boolean)
51
+ fun setTargetSize(size: TargetSize?) {}
52
+ fun setStateListener(listener: SourceStateListener?) {}
53
+ }
54
+
55
+ fun interface GlassesMediaSourceFactory {
56
+ fun create(video: VideoFrameListener, pcm: PcmListener): GlassesMediaSource
57
+ }
58
+
59
+ class GlassesMediaController(
60
+ private val factory: GlassesMediaSourceFactory,
61
+ ) {
62
+ private var source: GlassesMediaSource? = null
63
+ private var stateListener: SourceStateListener? = null
64
+
65
+ val state: SourceState
66
+ get() = source?.state ?: SourceState.IDLE
67
+
68
+ fun attach(video: VideoFrameListener, pcm: PcmListener, config: SourceConfig) {
69
+ source?.stop()
70
+ source = factory.create(video, pcm).also {
71
+ it.setStateListener(stateListener)
72
+ it.start(config)
73
+ }
74
+ }
75
+
76
+ fun restart(config: SourceConfig) {
77
+ source?.restart(config)
78
+ }
79
+
80
+ fun forceRestart() {
81
+ source?.forceRestart()
82
+ }
83
+
84
+ fun setStateListener(listener: SourceStateListener?) {
85
+ stateListener = listener
86
+ source?.setStateListener(listener)
87
+ }
88
+
89
+ fun stop() {
90
+ source?.stop()
91
+ source = null
92
+ }
93
+
94
+ fun setPcmDeliveryEnabled(enabled: Boolean) {
95
+ source?.setPcmDeliveryEnabled(enabled)
96
+ }
97
+
98
+ fun setTargetSize(size: TargetSize?) {
99
+ source?.setTargetSize(size)
100
+ }
101
+ }
@@ -0,0 +1,57 @@
1
+ package com.mentra.acsmeeting.source
2
+
3
+ import com.mentra.acsmeeting.video.I420Packer
4
+ import java.nio.ByteBuffer
5
+
6
+ /** Replaceable later by LocalGlassesVideoSource (SoftAP / local WebRTC). */
7
+ interface VideoSource {
8
+ fun start(whepUrl: String)
9
+ fun updateUrl(whepUrl: String)
10
+ fun stop()
11
+ }
12
+
13
+ /**
14
+ * Planes are valid only for the duration of the callback unless [retain] is
15
+ * non-null and invoked before the callback returns. [release] must then be
16
+ * called exactly once after ACS is done with the buffers.
17
+ */
18
+ class I420Planes(
19
+ val y: ByteBuffer,
20
+ val strideY: Int,
21
+ val u: ByteBuffer,
22
+ val strideU: Int,
23
+ val v: ByteBuffer,
24
+ val strideV: Int,
25
+ val width: Int,
26
+ val height: Int,
27
+ val timestampNs: Long,
28
+ val retain: (() -> Unit)? = null,
29
+ val release: (() -> Unit)? = null,
30
+ ) {
31
+ fun isTight(): Boolean {
32
+ val chroma = I420Packer.chromaStride(width)
33
+ return strideY == width && strideU == chroma && strideV == chroma
34
+ }
35
+
36
+ fun isDirect(): Boolean = y.isDirect && u.isDirect && v.isDirect
37
+
38
+ fun planesReadable(): Boolean {
39
+ val chromaW = I420Packer.chromaStride(width)
40
+ val chromaH = I420Packer.chromaStride(height)
41
+ return remaining(y) >= I420Packer.planeMinBytes(strideY, width, height) &&
42
+ remaining(u) >= I420Packer.planeMinBytes(strideU, chromaW, chromaH) &&
43
+ remaining(v) >= I420Packer.planeMinBytes(strideV, chromaW, chromaH)
44
+ }
45
+
46
+ companion object {
47
+ fun remaining(buffer: ByteBuffer): Int = buffer.duplicate().remaining()
48
+ }
49
+ }
50
+
51
+ fun interface VideoFrameListener {
52
+ fun onVideoFrame(planes: I420Planes)
53
+ }
54
+
55
+ fun interface PcmListener {
56
+ fun onPcm(pcm16Le: ByteArray, sampleRate: Int, channels: Int)
57
+ }
@@ -0,0 +1,205 @@
1
+ package com.mentra.acsmeeting.source
2
+
3
+ import com.mentra.acsmeeting.video.I420Packer
4
+ import java.nio.ByteBuffer
5
+ import kotlin.math.max
6
+
7
+ /**
8
+ * Packed I420 generator. No Android / ACS / WebRTC types so JVM tests stay cheap.
9
+ *
10
+ * CHEAP paints a flat field, a moving bar, and a Recall-style stamp (6-digit
11
+ * frame id + 1s FPS). Low motion: enough change that ACS cannot skip, still
12
+ * trivially compressible.
13
+ *
14
+ * MOTION pans a tiled scene across the frame. That is the glasses-turn proxy:
15
+ * temporal prediction dies (almost every pixel changes) but spatial structure
16
+ * stays, so an I-frame still compresses. Use this to ask whether ACS holds
17
+ * 15 fps under high motion.
18
+ *
19
+ * NOISE rewrites the whole Y plane from an LCG. That is not camera motion.
20
+ * Neighbors are uncorrelated, so even I-frames stay huge. Measured on ACS at
21
+ * 720p it pinned ~1.9 Mbps and collapsed the wire rate to 1.4 fps while we
22
+ * submitted 15. Use it to stress the encoder, never to model glasses movement.
23
+ *
24
+ * Chroma stays at 128 in every mode.
25
+ */
26
+ class SyntheticFrameFactory(
27
+ private val entropy: SyntheticEntropy = SyntheticEntropy.CHEAP,
28
+ ) {
29
+ private var lastDest: ByteBuffer? = null
30
+ private var lastWidth = 0
31
+ private var lastHeight = 0
32
+ private var background: ByteArray? = null
33
+ private var scene: ByteArray? = null
34
+ private var sceneWidth = 0
35
+ private var sceneHeight = 0
36
+
37
+ fun write(dest: ByteBuffer, width: Int, height: Int, frameIndex: Int, emitFps: Double? = null) {
38
+ require(width > 0 && height > 0) { "size must be positive" }
39
+ val needed = I420Packer.packedSize(width, height)
40
+ require(dest.capacity() >= needed) { "dest capacity ${dest.capacity()} < $needed" }
41
+ when (entropy) {
42
+ SyntheticEntropy.NOISE -> writeNoise(dest, width, height, frameIndex)
43
+ SyntheticEntropy.MOTION -> writeMotion(dest, width, height, frameIndex)
44
+ SyntheticEntropy.CHEAP -> writeCheap(dest, width, height, frameIndex)
45
+ }
46
+ writeStamp(dest, width, height, frameIndex, emitFps)
47
+ dest.position(0)
48
+ dest.limit(needed)
49
+ }
50
+
51
+ private fun writeCheap(dest: ByteBuffer, width: Int, height: Int, frameIndex: Int) {
52
+ // Bulk-restore the field instead of a per-pixel loop: this runs on the emit
53
+ // thread every frame, and 921600 single-byte puts is exactly the Kotlin
54
+ // per-pixel cost the rest of this module exists to avoid.
55
+ val ySize = width * height
56
+ val template = background.takeIf { it != null && it.size == ySize }
57
+ ?: ByteArray(ySize) { BG_Y }.also { background = it }
58
+ dest.clear()
59
+ dest.put(template, 0, ySize)
60
+ if (dest !== lastDest || width != lastWidth || height != lastHeight) {
61
+ fillChroma(dest, width, height)
62
+ lastDest = dest
63
+ lastWidth = width
64
+ lastHeight = height
65
+ }
66
+ val barWidth = max(8, width / 80)
67
+ val travel = max(1, width - barWidth)
68
+ val barX = ((frameIndex.toLong() * barWidth) % travel).toInt()
69
+ fillYRect(dest, width, height, barX, 0, barWidth, height, BAR_Y)
70
+ }
71
+
72
+ /**
73
+ * Copy a pre-painted tiled scene with a wrap-safe offset. One bulk put per
74
+ * row, not one put per pixel — the scene is the glasses-turn stand-in, and
75
+ * generating it in Kotlin every frame would be the cost we are trying to
76
+ * isolate out of this measurement.
77
+ */
78
+ private fun writeMotion(dest: ByteBuffer, width: Int, height: Int, frameIndex: Int) {
79
+ val field = ensureScene(width, height)
80
+ val ox = ((frameIndex.toLong() * PAN_X) % TILE).toInt()
81
+ val oy = ((frameIndex.toLong() * PAN_Y) % TILE).toInt()
82
+ for (row in 0 until height) {
83
+ dest.position(row * width)
84
+ dest.put(field, (row + oy) * sceneWidth + ox, width)
85
+ }
86
+ if (dest !== lastDest || width != lastWidth || height != lastHeight) {
87
+ fillChroma(dest, width, height)
88
+ lastDest = dest
89
+ lastWidth = width
90
+ lastHeight = height
91
+ }
92
+ }
93
+
94
+ private fun ensureScene(width: Int, height: Int): ByteArray {
95
+ val sw = width + TILE
96
+ val sh = height + TILE
97
+ val existing = scene
98
+ if (existing != null && sceneWidth == sw && sceneHeight == sh) return existing
99
+ val field = ByteArray(sw * sh)
100
+ for (y in 0 until sh) {
101
+ val rowOff = y * sw
102
+ val tileY = y / TILE
103
+ val gy = y % TILE
104
+ for (x in 0 until sw) {
105
+ val tile = (x / TILE + tileY) and 1
106
+ val gx = x % TILE
107
+ val base = if (tile == 0) DARK_TILE else LIGHT_TILE
108
+ field[rowOff + x] = (base + gx - TILE / 2 + gy / 4).toByte()
109
+ }
110
+ }
111
+ scene = field
112
+ sceneWidth = sw
113
+ sceneHeight = sh
114
+ return field
115
+ }
116
+
117
+ private fun writeNoise(dest: ByteBuffer, width: Int, height: Int, frameIndex: Int) {
118
+ writeNoiseY(dest, width, height, frameIndex)
119
+ fillChroma(dest, width, height)
120
+ lastDest = dest
121
+ lastWidth = width
122
+ lastHeight = height
123
+ }
124
+
125
+ private fun writeNoiseY(dest: ByteBuffer, width: Int, height: Int, frameIndex: Int) {
126
+ val ySize = width * height
127
+ var seed = (frameIndex + 1) * LCG_SEED
128
+ for (i in 0 until ySize) {
129
+ seed = seed * LCG_MUL + LCG_ADD
130
+ dest.put(i, ((seed ushr 24) and 0xFF).toByte())
131
+ }
132
+ }
133
+
134
+ /**
135
+ * Recall burns `F 000123` plus live rates into the captured page. Same idea here:
136
+ * a 6-digit unique-frame id and the last 1s emit rate, readable on the Teams tile.
137
+ */
138
+ private fun writeStamp(
139
+ dest: ByteBuffer,
140
+ width: Int,
141
+ height: Int,
142
+ frameIndex: Int,
143
+ emitFps: Double?,
144
+ ) {
145
+ val scale = (height / 60).coerceIn(8, 16)
146
+ val line1 = "F ${frameIndex.coerceAtLeast(0).toString().padStart(6, '0')}"
147
+ val fpsLabel = emitFps?.let { ((it * 10).toInt() / 10.0).toString() } ?: "--"
148
+ val line2 = "FPS $fpsLabel"
149
+ val pad = scale * 2
150
+ val lineH = BitmapFont.textHeight(scale)
151
+ val gap = scale
152
+ val boxW = max(BitmapFont.textWidth(line1.length, scale), BitmapFont.textWidth(line2.length, scale)) + pad * 2
153
+ val boxH = lineH * 2 + gap + pad * 2
154
+ val boxX = (width / 16).coerceAtLeast(16)
155
+ val boxY = (height / 2 - boxH / 2).coerceAtLeast(0)
156
+ fillYRect(dest, width, height, boxX, boxY, boxW, boxH, OFF_Y)
157
+ BitmapFont.draw(dest, width, height, line1, boxX + pad, boxY + pad, scale, BAR_Y)
158
+ BitmapFont.draw(dest, width, height, line2, boxX + pad, boxY + pad + lineH + gap, scale, BAR_Y)
159
+ }
160
+
161
+ private fun fillChroma(dest: ByteBuffer, width: Int, height: Int) {
162
+ val ySize = width * height
163
+ val uvSize = I420Packer.chromaStride(width) * I420Packer.chromaStride(height)
164
+ val end = ySize + 2 * uvSize
165
+ for (i in ySize until end) dest.put(i, CHROMA)
166
+ }
167
+
168
+ private fun fillYRect(
169
+ dest: ByteBuffer,
170
+ width: Int,
171
+ height: Int,
172
+ x0: Int,
173
+ y0: Int,
174
+ rectW: Int,
175
+ rectH: Int,
176
+ value: Byte,
177
+ ) {
178
+ val xStart = x0.coerceIn(0, width)
179
+ val yStart = y0.coerceIn(0, height)
180
+ val xEnd = (x0 + rectW).coerceIn(0, width)
181
+ val yEnd = (y0 + rectH).coerceIn(0, height)
182
+ if (xStart >= xEnd || yStart >= yEnd) return
183
+ for (row in yStart until yEnd) {
184
+ val rowOff = row * width
185
+ for (col in xStart until xEnd) {
186
+ dest.put(rowOff + col, value)
187
+ }
188
+ }
189
+ }
190
+
191
+ companion object {
192
+ const val BG_Y: Byte = 40
193
+ const val OFF_Y: Byte = 16
194
+ const val BAR_Y: Byte = 235.toByte()
195
+ const val CHROMA: Byte = 128.toByte()
196
+ const val TILE = 32
197
+ const val PAN_X = 8
198
+ const val PAN_Y = 4
199
+ private const val DARK_TILE = 70
200
+ private const val LIGHT_TILE = 170
201
+ private const val LCG_SEED = -1640531527
202
+ private const val LCG_MUL = 1664525
203
+ private const val LCG_ADD = 1013904223
204
+ }
205
+ }
@@ -0,0 +1,214 @@
1
+ package com.mentra.acsmeeting.source
2
+
3
+ import android.util.Log
4
+ import com.mentra.acsmeeting.telemetry.PipelineStats
5
+ import com.mentra.acsmeeting.video.I420Packer
6
+ import java.nio.ByteBuffer
7
+ import java.util.concurrent.Executors
8
+ import java.util.concurrent.ScheduledExecutorService
9
+ import java.util.concurrent.TimeUnit
10
+ import java.util.concurrent.atomic.AtomicBoolean
11
+ import java.util.concurrent.atomic.AtomicInteger
12
+ import java.util.concurrent.atomic.AtomicLong
13
+ import java.util.concurrent.atomic.AtomicReference
14
+
15
+ /**
16
+ * Fixed-rate packed-I420 source. Bypasses glasses, Cloudflare, WHEP and decode.
17
+ * Ticks that fail [isReady] or lack a target size are withheld and uncounted.
18
+ */
19
+ class SyntheticI420Source(
20
+ private val video: VideoFrameListener,
21
+ private val stats: PipelineStats,
22
+ private val isReady: () -> Boolean,
23
+ private val config: SyntheticConfig = SyntheticConfig(),
24
+ ) : GlassesMediaSource {
25
+ private val factory = SyntheticFrameFactory(config.entropy)
26
+ private val targetSize = AtomicReference<TargetSize?>(null)
27
+ private val pcmEnabled = AtomicBoolean(false)
28
+ private val frameIndex = AtomicInteger(0)
29
+ private val withheld = AtomicInteger(0)
30
+ private val withheldThisGap = AtomicInteger(0)
31
+ private val emitted = AtomicInteger(0)
32
+ private val emitting = AtomicBoolean(false)
33
+ private val firstLogged = AtomicBoolean(false)
34
+ private val fpsWindowStartNs = AtomicLong(0)
35
+ private val fpsWindowCount = AtomicInteger(0)
36
+ @Volatile private var stampFps: Double? = null
37
+ @Volatile override var state: SourceState = SourceState.IDLE
38
+ private set
39
+ @Volatile private var dest: ByteBuffer? = null
40
+ @Volatile private var destWidth = 0
41
+ @Volatile private var destHeight = 0
42
+ @Volatile private var destY: ByteBuffer? = null
43
+ @Volatile private var destU: ByteBuffer? = null
44
+ @Volatile private var destV: ByteBuffer? = null
45
+ private var scheduler: ScheduledExecutorService? = null
46
+
47
+ /** Test-visible gate transitions. Production also writes them to ACS-SPIKE. */
48
+ internal val events = mutableListOf<String>()
49
+
50
+ /** Puts the source in LIVE without starting the scheduler. For emitOnce tests. */
51
+ internal fun becomeLive() {
52
+ state = SourceState.LIVE
53
+ }
54
+
55
+ override fun start(config: SourceConfig) {
56
+ stop()
57
+ becomeLive()
58
+ val fps = this.config.fps.coerceAtLeast(1)
59
+ val periodUs = 1_000_000L / fps
60
+ val exec = Executors.newSingleThreadScheduledExecutor { runnable ->
61
+ Thread(runnable, "acs-synthetic-i420").apply { isDaemon = true }
62
+ }
63
+ scheduler = exec
64
+ exec.scheduleAtFixedRate({ emitOnce() }, 0, periodUs, TimeUnit.MICROSECONDS)
65
+ }
66
+
67
+ override fun restart(config: SourceConfig) {
68
+ stop()
69
+ start(config)
70
+ }
71
+
72
+ override fun stop() {
73
+ scheduler?.shutdownNow()
74
+ scheduler = null
75
+ frameIndex.set(0)
76
+ withheld.set(0)
77
+ withheldThisGap.set(0)
78
+ emitted.set(0)
79
+ emitting.set(false)
80
+ firstLogged.set(false)
81
+ fpsWindowStartNs.set(0)
82
+ fpsWindowCount.set(0)
83
+ stampFps = null
84
+ dest = null
85
+ destWidth = 0
86
+ destHeight = 0
87
+ destY = null
88
+ destU = null
89
+ destV = null
90
+ state = SourceState.IDLE
91
+ }
92
+
93
+ override fun setPcmDeliveryEnabled(enabled: Boolean) {
94
+ pcmEnabled.set(enabled)
95
+ }
96
+
97
+ override fun setTargetSize(size: TargetSize?) {
98
+ targetSize.set(size)
99
+ }
100
+
101
+ internal fun pcmDeliveryEnabled(): Boolean = pcmEnabled.get()
102
+
103
+ /**
104
+ * Generate-and-emit one frame if the sender is ready and a target size exists.
105
+ * Returns true only when a frame was handed to [video].
106
+ */
107
+ internal fun emitOnce(): Boolean {
108
+ if (state != SourceState.LIVE) return false
109
+ val size = targetSize.get()
110
+ val ready = isReady() && size != null
111
+ if (!ready) {
112
+ withheld.incrementAndGet()
113
+ withheldThisGap.incrementAndGet()
114
+ if (emitting.compareAndSet(true, false)) {
115
+ logEvent("P6 synthetic paused reason=not-ready afterFrames=${emitted.get()}")
116
+ }
117
+ return false
118
+ }
119
+ val gap = withheldThisGap.getAndSet(0)
120
+ if (!emitting.get() && emitted.get() > 0) {
121
+ logEvent("P6 synthetic resumed withheld=$gap")
122
+ }
123
+ emitting.set(true)
124
+
125
+ val w = size!!.width
126
+ val h = size.height
127
+ val buffer = ensureDest(w, h)
128
+ val index = frameIndex.getAndIncrement()
129
+ stampFps = sampleStampFps()
130
+ factory.write(buffer, w, h, index, stampFps)
131
+ stats.onSink()
132
+ stats.recordGap()
133
+ stats.setSize(w, h)
134
+ val chroma = I420Packer.chromaStride(w)
135
+ video.onVideoFrame(
136
+ I420Planes(
137
+ y = destY!!,
138
+ strideY = w,
139
+ u = destU!!,
140
+ strideU = chroma,
141
+ v = destV!!,
142
+ strideV = chroma,
143
+ width = w,
144
+ height = h,
145
+ timestampNs = System.nanoTime(),
146
+ ),
147
+ )
148
+ val count = emitted.incrementAndGet()
149
+ if (firstLogged.compareAndSet(false, true)) {
150
+ logEvent(
151
+ "P6 synthetic first-frame withheld=${withheld.get()} size=${w}x${h} " +
152
+ "fps=${config.fps} entropy=${config.entropy}",
153
+ )
154
+ }
155
+ return count > 0
156
+ }
157
+
158
+ /** 1-second emit rate, same window Recall uses for the P column on the stamp. */
159
+ private fun sampleStampFps(): Double? {
160
+ val now = System.nanoTime()
161
+ val started = fpsWindowStartNs.get()
162
+ if (started == 0L) {
163
+ fpsWindowStartNs.set(now)
164
+ fpsWindowCount.set(1)
165
+ return stampFps
166
+ }
167
+ fpsWindowCount.incrementAndGet()
168
+ val elapsedNs = now - started
169
+ if (elapsedNs >= 1_000_000_000L) {
170
+ val rate = fpsWindowCount.get() * 1_000_000_000.0 / elapsedNs
171
+ fpsWindowCount.set(0)
172
+ fpsWindowStartNs.set(now)
173
+ return rate
174
+ }
175
+ return stampFps
176
+ }
177
+
178
+ private fun ensureDest(width: Int, height: Int): ByteBuffer {
179
+ val needed = I420Packer.packedSize(width, height)
180
+ val existing = dest
181
+ if (existing != null && destWidth == width && destHeight == height && existing.capacity() >= needed) {
182
+ existing.clear()
183
+ return existing
184
+ }
185
+ val next = ByteBuffer.allocateDirect(needed)
186
+ dest = next
187
+ destWidth = width
188
+ destHeight = height
189
+ slicePlanes(next, width, height)
190
+ return next
191
+ }
192
+
193
+ private fun slicePlanes(packed: ByteBuffer, width: Int, height: Int) {
194
+ val ySize = width * height
195
+ val uvSize = I420Packer.chromaStride(width) * I420Packer.chromaStride(height)
196
+ val view = packed.duplicate()
197
+ view.clear()
198
+ destY = view.duplicate().apply { position(0); limit(ySize) }.slice()
199
+ destU = view.duplicate().apply { position(ySize); limit(ySize + uvSize) }.slice()
200
+ destV = view.duplicate().apply { position(ySize + uvSize); limit(ySize + 2 * uvSize) }.slice()
201
+ }
202
+
203
+ private fun logEvent(line: String) {
204
+ synchronized(events) { events.add(line) }
205
+ try {
206
+ Log.i(TAG, line)
207
+ } catch (_: Throwable) {
208
+ }
209
+ }
210
+
211
+ companion object {
212
+ private const val TAG = "ACS-SPIKE"
213
+ }
214
+ }
@@ -0,0 +1,31 @@
1
+ package com.mentra.acsmeeting.source
2
+
3
+ /**
4
+ * Dedupe WebRTC track attachments by native track id.
5
+ * Three observer callbacks can deliver distinct Java wrappers of one track;
6
+ * object identity cannot be used. [reset] is load-bearing: [CloudflareWhepSource.start]
7
+ * calls stop first, and a stale claim set makes reconnect never attach a sink.
8
+ */
9
+ class TrackRegistry {
10
+ private val ids = mutableSetOf<String>()
11
+ private var skipped = 0
12
+
13
+ @Synchronized
14
+ fun claim(id: String): Boolean {
15
+ if (ids.add(id)) return true
16
+ skipped += 1
17
+ return false
18
+ }
19
+
20
+ @Synchronized
21
+ fun reset() {
22
+ ids.clear()
23
+ skipped = 0
24
+ }
25
+
26
+ @Synchronized
27
+ fun size(): Int = ids.size
28
+
29
+ @Synchronized
30
+ fun skipped(): Int = skipped
31
+ }
@@ -0,0 +1,41 @@
1
+ package com.mentra.acsmeeting.source
2
+
3
+ enum class VideoSourceArm { WHEP, SYNTHETIC }
4
+
5
+ enum class SyntheticEntropy { CHEAP, MOTION, NOISE }
6
+
7
+ enum class DecoderMode { TEXTURE, BYTE_BUFFER }
8
+
9
+ enum class PixelFormatArm { I420, NV12 }
10
+
11
+ /**
12
+ * Investigation arm. SYNTHETIC bypasses glasses, Cloudflare, WHEP and decode.
13
+ * Ships as WHEP. Flip locally to run the experiment; do not commit SYNTHETIC.
14
+ *
15
+ * [decoderMode] TEXTURE is the production default (shared EGL, MediaCodec to
16
+ * Surface). BYTE_BUFFER is the A/B: no shared context, CPU planes, no
17
+ * glReadPixels. 720p BYTE_BUFFER on SM_S948U decoded on hardware with
18
+ * i420P95=0, but wire/codec never attached and busy drops climbed. Do not
19
+ * commit BYTE_BUFFER until a same-phone 540p A/B clears the campaign gates.
20
+ *
21
+ * [zeroCopy] hands WebRTC I420 planes straight to ACS when they are tight and
22
+ * retainable. Ships off. Do not commit true.
23
+ *
24
+ * [pixelFormat] I420 is the production default. NV12 is the encoder-flip A/B:
25
+ * advertise and send biplanar NV12 in case ACS picks a hardware H.264
26
+ * encoder. Revert unless `codecName` leaves `h264 sw`. Do not commit NV12
27
+ * on a failed flip.
28
+ */
29
+ object AcsInvestigation {
30
+ val videoArm = VideoSourceArm.WHEP
31
+ val syntheticFps = 15
32
+ val syntheticEntropy = SyntheticEntropy.MOTION
33
+ val decoderMode = DecoderMode.TEXTURE
34
+ val zeroCopy = false
35
+ val pixelFormat = PixelFormatArm.I420
36
+ }
37
+
38
+ data class SyntheticConfig(
39
+ val fps: Int = AcsInvestigation.syntheticFps,
40
+ val entropy: SyntheticEntropy = AcsInvestigation.syntheticEntropy,
41
+ )