@mentra/acs-meeting 3.2.0-dev.120
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +20 -0
- package/README.md +412 -0
- package/THIRD_PARTY +7 -0
- package/android/build.gradle +64 -0
- package/android/src/main/AndroidManifest.xml +5 -0
- package/android/src/main/java/com/mentra/acsmeeting/AcsMeetingModule.kt +80 -0
- package/android/src/main/java/com/mentra/acsmeeting/AcsMeetingSession.kt +846 -0
- package/android/src/main/java/com/mentra/acsmeeting/RemoteRoster.kt +123 -0
- package/android/src/main/java/com/mentra/acsmeeting/audio/AcsAudioPolicy.kt +90 -0
- package/android/src/main/java/com/mentra/acsmeeting/audio/AudioPolicyApplier.kt +176 -0
- package/android/src/main/java/com/mentra/acsmeeting/audio/IncomingAudioPump.kt +112 -0
- package/android/src/main/java/com/mentra/acsmeeting/audio/IncomingRateProbe.kt +65 -0
- package/android/src/main/java/com/mentra/acsmeeting/audio/PcmBridge.kt +108 -0
- package/android/src/main/java/com/mentra/acsmeeting/audio/PcmResampler.kt +159 -0
- package/android/src/main/java/com/mentra/acsmeeting/audio/PhoneMicCapturer.kt +115 -0
- package/android/src/main/java/com/mentra/acsmeeting/audio/UplinkPacer.kt +302 -0
- package/android/src/main/java/com/mentra/acsmeeting/audio/UplinkSender.kt +230 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/BitmapFont.kt +99 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/CloudflareWhepSource.kt +578 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/FirstFrameGate.kt +45 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/GlassesMediaSource.kt +101 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/MediaListeners.kt +57 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/SyntheticFrameFactory.kt +205 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/SyntheticI420Source.kt +214 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/TrackRegistry.kt +31 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/VideoSourceArm.kt +41 -0
- package/android/src/main/java/com/mentra/acsmeeting/telemetry/ChromaProbe.kt +78 -0
- package/android/src/main/java/com/mentra/acsmeeting/telemetry/PipelineStats.kt +281 -0
- package/android/src/main/java/com/mentra/acsmeeting/telemetry/PipelineTicker.kt +51 -0
- package/android/src/main/java/com/mentra/acsmeeting/telemetry/ProcessCpu.kt +15 -0
- package/android/src/main/java/com/mentra/acsmeeting/telemetry/RingPercentile.kt +33 -0
- package/android/src/main/java/com/mentra/acsmeeting/video/AcsFrameSender.kt +475 -0
- package/android/src/main/java/com/mentra/acsmeeting/video/AcsTimestamp.kt +26 -0
- package/android/src/main/java/com/mentra/acsmeeting/video/FrameGeometry.kt +22 -0
- package/android/src/main/java/com/mentra/acsmeeting/video/I420FormatSpec.kt +90 -0
- package/android/src/main/java/com/mentra/acsmeeting/video/I420Packer.kt +72 -0
- package/android/src/main/java/com/mentra/acsmeeting/video/Nv12FormatSpec.kt +36 -0
- package/android/src/main/java/com/mentra/acsmeeting/video/Nv12Packer.kt +51 -0
- package/android/src/main/java/com/mentra/acsmeeting/video/SendGate.kt +27 -0
- package/android/src/main/java/com/mentra/acsmeeting/video/VideoProfile.kt +57 -0
- package/android/src/test/java/com/mentra/acsmeeting/RemoteRosterTest.kt +40 -0
- package/android/src/test/java/com/mentra/acsmeeting/audio/AcsAudioPolicyTest.kt +204 -0
- package/android/src/test/java/com/mentra/acsmeeting/audio/AudioPolicyApplierTest.kt +223 -0
- package/android/src/test/java/com/mentra/acsmeeting/audio/CapturePolicyTest.kt +12 -0
- package/android/src/test/java/com/mentra/acsmeeting/audio/IncomingAudioPumpTest.kt +97 -0
- package/android/src/test/java/com/mentra/acsmeeting/audio/IncomingRateProbeTest.kt +78 -0
- package/android/src/test/java/com/mentra/acsmeeting/audio/PcmResamplerTest.kt +125 -0
- package/android/src/test/java/com/mentra/acsmeeting/audio/UplinkPacerTest.kt +201 -0
- package/android/src/test/java/com/mentra/acsmeeting/audio/UplinkSenderTest.kt +166 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/AcsInvestigationTest.kt +26 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/BitmapFontTest.kt +22 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/FirstFrameGateTest.kt +60 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/GlassesMediaSourceTest.kt +133 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/I420PlanesTest.kt +67 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/SyntheticFrameFactoryTest.kt +193 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/SyntheticI420SourceTest.kt +176 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/TrackRegistryTest.kt +36 -0
- package/android/src/test/java/com/mentra/acsmeeting/telemetry/ChromaProbeTest.kt +61 -0
- package/android/src/test/java/com/mentra/acsmeeting/telemetry/PipelineStatsTest.kt +256 -0
- package/android/src/test/java/com/mentra/acsmeeting/telemetry/ProcessCpuTest.kt +18 -0
- package/android/src/test/java/com/mentra/acsmeeting/telemetry/RingPercentileTest.kt +49 -0
- package/android/src/test/java/com/mentra/acsmeeting/video/AcsTimestampTest.kt +43 -0
- package/android/src/test/java/com/mentra/acsmeeting/video/FrameGeometryTest.kt +33 -0
- package/android/src/test/java/com/mentra/acsmeeting/video/I420FormatSpecTest.kt +77 -0
- package/android/src/test/java/com/mentra/acsmeeting/video/I420PackerTest.kt +126 -0
- package/android/src/test/java/com/mentra/acsmeeting/video/Nv12FormatSpecTest.kt +36 -0
- package/android/src/test/java/com/mentra/acsmeeting/video/Nv12PackerTest.kt +58 -0
- package/android/src/test/java/com/mentra/acsmeeting/video/SendGateTest.kt +79 -0
- package/android/src/test/java/com/mentra/acsmeeting/video/VideoProfileTest.kt +68 -0
- package/build/AcsMeeting.types.d.ts +59 -0
- package/build/AcsMeeting.types.d.ts.map +1 -0
- package/build/AcsMeeting.types.js +2 -0
- package/build/AcsMeeting.types.js.map +1 -0
- package/build/AcsMeetingModule.d.ts +15 -0
- package/build/AcsMeetingModule.d.ts.map +1 -0
- package/build/AcsMeetingModule.js +3 -0
- package/build/AcsMeetingModule.js.map +1 -0
- package/build/AcsMeetingModule.web.d.ts +16 -0
- package/build/AcsMeetingModule.web.d.ts.map +1 -0
- package/build/AcsMeetingModule.web.js +31 -0
- package/build/AcsMeetingModule.web.js.map +1 -0
- package/build/index.d.ts +3 -0
- package/build/index.d.ts.map +1 -0
- package/build/index.js +3 -0
- package/build/index.js.map +1 -0
- package/expo-module.config.json +9 -0
- package/ios/AcsFrameSender.swift +47 -0
- package/ios/AcsMeeting.podspec +27 -0
- package/ios/AcsMeetingModule.swift +558 -0
- package/ios/PcmBridge.swift +117 -0
- package/ios/PhoneMicCapturer.swift +64 -0
- package/ios/PolicyKit/Package.swift +17 -0
- package/ios/PolicyKit/Sources/AcsAudioPolicy/AcsAudioPolicy.swift +102 -0
- package/ios/PolicyKit/Sources/AcsAudioPolicy/AudioPolicyApplier.swift +162 -0
- package/ios/PolicyKit/Sources/AcsAudioPolicy/GlassesMediaSource.swift +68 -0
- package/ios/PolicyKit/Tests/AcsAudioPolicyTests/AcsAudioPolicyTests.swift +273 -0
- package/ios/WhepVideoSource.swift +356 -0
- package/package.json +56 -0
- package/src/AcsMeeting.types.ts +66 -0
- package/src/AcsMeetingModule.ts +16 -0
- package/src/AcsMeetingModule.web.ts +33 -0
- package/src/index.ts +2 -0
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
package com.mentra.acsmeeting.source
|
|
2
|
+
|
|
3
|
+
enum class SourceKind { WHEP, DIRECT }
|
|
4
|
+
|
|
5
|
+
enum class SourceState { IDLE, CONNECTING, LIVE, FAILED }
|
|
6
|
+
|
|
7
|
+
data class SourceConfig(
|
|
8
|
+
val url: String,
|
|
9
|
+
val kind: SourceKind = SourceKind.WHEP,
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
/** ACS-negotiated output size. One object so a frame cannot observe a torn width/height. */
|
|
13
|
+
data class TargetSize(val width: Int, val height: Int)
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Whether a subscriber already on [currentUrl] can serve [next] as-is instead of
|
|
17
|
+
* being rebuilt.
|
|
18
|
+
*
|
|
19
|
+
* Reuse is the whole point: a rebuild mints a new peer and restarts the wait for
|
|
20
|
+
* the first frame, which is the outage we are trying to shorten. `CONNECTING`
|
|
21
|
+
* counts as reusable, and since `LIVE` means "a frame reached the sink" that now
|
|
22
|
+
* spans the answer → first frame window too. It cannot strand a caller, because
|
|
23
|
+
* every `CONNECTING` is bounded: the offer post fails, or the answer's
|
|
24
|
+
* first-frame deadline expires into `FAILED` and this returns false.
|
|
25
|
+
*/
|
|
26
|
+
fun canReuseSource(currentUrl: String?, state: SourceState, next: SourceConfig): Boolean =
|
|
27
|
+
currentUrl == next.url && next.kind == SourceKind.WHEP && state != SourceState.FAILED
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Observes source health transitions. `FAILED` is the one that matters: ICE
|
|
31
|
+
* dropped, the WHEP endpoint went away, or an answered subscription never
|
|
32
|
+
* delivered a frame — and nothing downstream will notice on its own, because
|
|
33
|
+
* ACS keeps the call up with a frozen last frame.
|
|
34
|
+
*/
|
|
35
|
+
fun interface SourceStateListener {
|
|
36
|
+
fun onSourceState(state: SourceState, reason: String?)
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Transports whatever the glasses already captured. Capture policy lives
|
|
41
|
+
* outside this interface so a Cloudflare hop can be replaced without touching ACS.
|
|
42
|
+
*/
|
|
43
|
+
interface GlassesMediaSource {
|
|
44
|
+
fun start(config: SourceConfig)
|
|
45
|
+
fun restart(config: SourceConfig)
|
|
46
|
+
/** Rebuild the transport for the current config even if it looks healthy. */
|
|
47
|
+
fun forceRestart() {}
|
|
48
|
+
fun stop()
|
|
49
|
+
val state: SourceState
|
|
50
|
+
fun setPcmDeliveryEnabled(enabled: Boolean)
|
|
51
|
+
fun setTargetSize(size: TargetSize?) {}
|
|
52
|
+
fun setStateListener(listener: SourceStateListener?) {}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
fun interface GlassesMediaSourceFactory {
|
|
56
|
+
fun create(video: VideoFrameListener, pcm: PcmListener): GlassesMediaSource
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
class GlassesMediaController(
|
|
60
|
+
private val factory: GlassesMediaSourceFactory,
|
|
61
|
+
) {
|
|
62
|
+
private var source: GlassesMediaSource? = null
|
|
63
|
+
private var stateListener: SourceStateListener? = null
|
|
64
|
+
|
|
65
|
+
val state: SourceState
|
|
66
|
+
get() = source?.state ?: SourceState.IDLE
|
|
67
|
+
|
|
68
|
+
fun attach(video: VideoFrameListener, pcm: PcmListener, config: SourceConfig) {
|
|
69
|
+
source?.stop()
|
|
70
|
+
source = factory.create(video, pcm).also {
|
|
71
|
+
it.setStateListener(stateListener)
|
|
72
|
+
it.start(config)
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
fun restart(config: SourceConfig) {
|
|
77
|
+
source?.restart(config)
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
fun forceRestart() {
|
|
81
|
+
source?.forceRestart()
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
fun setStateListener(listener: SourceStateListener?) {
|
|
85
|
+
stateListener = listener
|
|
86
|
+
source?.setStateListener(listener)
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
fun stop() {
|
|
90
|
+
source?.stop()
|
|
91
|
+
source = null
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
fun setPcmDeliveryEnabled(enabled: Boolean) {
|
|
95
|
+
source?.setPcmDeliveryEnabled(enabled)
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
fun setTargetSize(size: TargetSize?) {
|
|
99
|
+
source?.setTargetSize(size)
|
|
100
|
+
}
|
|
101
|
+
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
package com.mentra.acsmeeting.source
|
|
2
|
+
|
|
3
|
+
import com.mentra.acsmeeting.video.I420Packer
|
|
4
|
+
import java.nio.ByteBuffer
|
|
5
|
+
|
|
6
|
+
/** Replaceable later by LocalGlassesVideoSource (SoftAP / local WebRTC). */
|
|
7
|
+
interface VideoSource {
|
|
8
|
+
fun start(whepUrl: String)
|
|
9
|
+
fun updateUrl(whepUrl: String)
|
|
10
|
+
fun stop()
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Planes are valid only for the duration of the callback unless [retain] is
|
|
15
|
+
* non-null and invoked before the callback returns. [release] must then be
|
|
16
|
+
* called exactly once after ACS is done with the buffers.
|
|
17
|
+
*/
|
|
18
|
+
class I420Planes(
|
|
19
|
+
val y: ByteBuffer,
|
|
20
|
+
val strideY: Int,
|
|
21
|
+
val u: ByteBuffer,
|
|
22
|
+
val strideU: Int,
|
|
23
|
+
val v: ByteBuffer,
|
|
24
|
+
val strideV: Int,
|
|
25
|
+
val width: Int,
|
|
26
|
+
val height: Int,
|
|
27
|
+
val timestampNs: Long,
|
|
28
|
+
val retain: (() -> Unit)? = null,
|
|
29
|
+
val release: (() -> Unit)? = null,
|
|
30
|
+
) {
|
|
31
|
+
fun isTight(): Boolean {
|
|
32
|
+
val chroma = I420Packer.chromaStride(width)
|
|
33
|
+
return strideY == width && strideU == chroma && strideV == chroma
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
fun isDirect(): Boolean = y.isDirect && u.isDirect && v.isDirect
|
|
37
|
+
|
|
38
|
+
fun planesReadable(): Boolean {
|
|
39
|
+
val chromaW = I420Packer.chromaStride(width)
|
|
40
|
+
val chromaH = I420Packer.chromaStride(height)
|
|
41
|
+
return remaining(y) >= I420Packer.planeMinBytes(strideY, width, height) &&
|
|
42
|
+
remaining(u) >= I420Packer.planeMinBytes(strideU, chromaW, chromaH) &&
|
|
43
|
+
remaining(v) >= I420Packer.planeMinBytes(strideV, chromaW, chromaH)
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
companion object {
|
|
47
|
+
fun remaining(buffer: ByteBuffer): Int = buffer.duplicate().remaining()
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
fun interface VideoFrameListener {
|
|
52
|
+
fun onVideoFrame(planes: I420Planes)
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
fun interface PcmListener {
|
|
56
|
+
fun onPcm(pcm16Le: ByteArray, sampleRate: Int, channels: Int)
|
|
57
|
+
}
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
package com.mentra.acsmeeting.source
|
|
2
|
+
|
|
3
|
+
import com.mentra.acsmeeting.video.I420Packer
|
|
4
|
+
import java.nio.ByteBuffer
|
|
5
|
+
import kotlin.math.max
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Packed I420 generator. No Android / ACS / WebRTC types so JVM tests stay cheap.
|
|
9
|
+
*
|
|
10
|
+
* CHEAP paints a flat field, a moving bar, and a Recall-style stamp (6-digit
|
|
11
|
+
* frame id + 1s FPS). Low motion: enough change that ACS cannot skip, still
|
|
12
|
+
* trivially compressible.
|
|
13
|
+
*
|
|
14
|
+
* MOTION pans a tiled scene across the frame. That is the glasses-turn proxy:
|
|
15
|
+
* temporal prediction dies (almost every pixel changes) but spatial structure
|
|
16
|
+
* stays, so an I-frame still compresses. Use this to ask whether ACS holds
|
|
17
|
+
* 15 fps under high motion.
|
|
18
|
+
*
|
|
19
|
+
* NOISE rewrites the whole Y plane from an LCG. That is not camera motion.
|
|
20
|
+
* Neighbors are uncorrelated, so even I-frames stay huge. Measured on ACS at
|
|
21
|
+
* 720p it pinned ~1.9 Mbps and collapsed the wire rate to 1.4 fps while we
|
|
22
|
+
* submitted 15. Use it to stress the encoder, never to model glasses movement.
|
|
23
|
+
*
|
|
24
|
+
* Chroma stays at 128 in every mode.
|
|
25
|
+
*/
|
|
26
|
+
class SyntheticFrameFactory(
|
|
27
|
+
private val entropy: SyntheticEntropy = SyntheticEntropy.CHEAP,
|
|
28
|
+
) {
|
|
29
|
+
private var lastDest: ByteBuffer? = null
|
|
30
|
+
private var lastWidth = 0
|
|
31
|
+
private var lastHeight = 0
|
|
32
|
+
private var background: ByteArray? = null
|
|
33
|
+
private var scene: ByteArray? = null
|
|
34
|
+
private var sceneWidth = 0
|
|
35
|
+
private var sceneHeight = 0
|
|
36
|
+
|
|
37
|
+
fun write(dest: ByteBuffer, width: Int, height: Int, frameIndex: Int, emitFps: Double? = null) {
|
|
38
|
+
require(width > 0 && height > 0) { "size must be positive" }
|
|
39
|
+
val needed = I420Packer.packedSize(width, height)
|
|
40
|
+
require(dest.capacity() >= needed) { "dest capacity ${dest.capacity()} < $needed" }
|
|
41
|
+
when (entropy) {
|
|
42
|
+
SyntheticEntropy.NOISE -> writeNoise(dest, width, height, frameIndex)
|
|
43
|
+
SyntheticEntropy.MOTION -> writeMotion(dest, width, height, frameIndex)
|
|
44
|
+
SyntheticEntropy.CHEAP -> writeCheap(dest, width, height, frameIndex)
|
|
45
|
+
}
|
|
46
|
+
writeStamp(dest, width, height, frameIndex, emitFps)
|
|
47
|
+
dest.position(0)
|
|
48
|
+
dest.limit(needed)
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
private fun writeCheap(dest: ByteBuffer, width: Int, height: Int, frameIndex: Int) {
|
|
52
|
+
// Bulk-restore the field instead of a per-pixel loop: this runs on the emit
|
|
53
|
+
// thread every frame, and 921600 single-byte puts is exactly the Kotlin
|
|
54
|
+
// per-pixel cost the rest of this module exists to avoid.
|
|
55
|
+
val ySize = width * height
|
|
56
|
+
val template = background.takeIf { it != null && it.size == ySize }
|
|
57
|
+
?: ByteArray(ySize) { BG_Y }.also { background = it }
|
|
58
|
+
dest.clear()
|
|
59
|
+
dest.put(template, 0, ySize)
|
|
60
|
+
if (dest !== lastDest || width != lastWidth || height != lastHeight) {
|
|
61
|
+
fillChroma(dest, width, height)
|
|
62
|
+
lastDest = dest
|
|
63
|
+
lastWidth = width
|
|
64
|
+
lastHeight = height
|
|
65
|
+
}
|
|
66
|
+
val barWidth = max(8, width / 80)
|
|
67
|
+
val travel = max(1, width - barWidth)
|
|
68
|
+
val barX = ((frameIndex.toLong() * barWidth) % travel).toInt()
|
|
69
|
+
fillYRect(dest, width, height, barX, 0, barWidth, height, BAR_Y)
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Copy a pre-painted tiled scene with a wrap-safe offset. One bulk put per
|
|
74
|
+
* row, not one put per pixel — the scene is the glasses-turn stand-in, and
|
|
75
|
+
* generating it in Kotlin every frame would be the cost we are trying to
|
|
76
|
+
* isolate out of this measurement.
|
|
77
|
+
*/
|
|
78
|
+
private fun writeMotion(dest: ByteBuffer, width: Int, height: Int, frameIndex: Int) {
|
|
79
|
+
val field = ensureScene(width, height)
|
|
80
|
+
val ox = ((frameIndex.toLong() * PAN_X) % TILE).toInt()
|
|
81
|
+
val oy = ((frameIndex.toLong() * PAN_Y) % TILE).toInt()
|
|
82
|
+
for (row in 0 until height) {
|
|
83
|
+
dest.position(row * width)
|
|
84
|
+
dest.put(field, (row + oy) * sceneWidth + ox, width)
|
|
85
|
+
}
|
|
86
|
+
if (dest !== lastDest || width != lastWidth || height != lastHeight) {
|
|
87
|
+
fillChroma(dest, width, height)
|
|
88
|
+
lastDest = dest
|
|
89
|
+
lastWidth = width
|
|
90
|
+
lastHeight = height
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
private fun ensureScene(width: Int, height: Int): ByteArray {
|
|
95
|
+
val sw = width + TILE
|
|
96
|
+
val sh = height + TILE
|
|
97
|
+
val existing = scene
|
|
98
|
+
if (existing != null && sceneWidth == sw && sceneHeight == sh) return existing
|
|
99
|
+
val field = ByteArray(sw * sh)
|
|
100
|
+
for (y in 0 until sh) {
|
|
101
|
+
val rowOff = y * sw
|
|
102
|
+
val tileY = y / TILE
|
|
103
|
+
val gy = y % TILE
|
|
104
|
+
for (x in 0 until sw) {
|
|
105
|
+
val tile = (x / TILE + tileY) and 1
|
|
106
|
+
val gx = x % TILE
|
|
107
|
+
val base = if (tile == 0) DARK_TILE else LIGHT_TILE
|
|
108
|
+
field[rowOff + x] = (base + gx - TILE / 2 + gy / 4).toByte()
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
scene = field
|
|
112
|
+
sceneWidth = sw
|
|
113
|
+
sceneHeight = sh
|
|
114
|
+
return field
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
private fun writeNoise(dest: ByteBuffer, width: Int, height: Int, frameIndex: Int) {
|
|
118
|
+
writeNoiseY(dest, width, height, frameIndex)
|
|
119
|
+
fillChroma(dest, width, height)
|
|
120
|
+
lastDest = dest
|
|
121
|
+
lastWidth = width
|
|
122
|
+
lastHeight = height
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
private fun writeNoiseY(dest: ByteBuffer, width: Int, height: Int, frameIndex: Int) {
|
|
126
|
+
val ySize = width * height
|
|
127
|
+
var seed = (frameIndex + 1) * LCG_SEED
|
|
128
|
+
for (i in 0 until ySize) {
|
|
129
|
+
seed = seed * LCG_MUL + LCG_ADD
|
|
130
|
+
dest.put(i, ((seed ushr 24) and 0xFF).toByte())
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Recall burns `F 000123` plus live rates into the captured page. Same idea here:
|
|
136
|
+
* a 6-digit unique-frame id and the last 1s emit rate, readable on the Teams tile.
|
|
137
|
+
*/
|
|
138
|
+
private fun writeStamp(
|
|
139
|
+
dest: ByteBuffer,
|
|
140
|
+
width: Int,
|
|
141
|
+
height: Int,
|
|
142
|
+
frameIndex: Int,
|
|
143
|
+
emitFps: Double?,
|
|
144
|
+
) {
|
|
145
|
+
val scale = (height / 60).coerceIn(8, 16)
|
|
146
|
+
val line1 = "F ${frameIndex.coerceAtLeast(0).toString().padStart(6, '0')}"
|
|
147
|
+
val fpsLabel = emitFps?.let { ((it * 10).toInt() / 10.0).toString() } ?: "--"
|
|
148
|
+
val line2 = "FPS $fpsLabel"
|
|
149
|
+
val pad = scale * 2
|
|
150
|
+
val lineH = BitmapFont.textHeight(scale)
|
|
151
|
+
val gap = scale
|
|
152
|
+
val boxW = max(BitmapFont.textWidth(line1.length, scale), BitmapFont.textWidth(line2.length, scale)) + pad * 2
|
|
153
|
+
val boxH = lineH * 2 + gap + pad * 2
|
|
154
|
+
val boxX = (width / 16).coerceAtLeast(16)
|
|
155
|
+
val boxY = (height / 2 - boxH / 2).coerceAtLeast(0)
|
|
156
|
+
fillYRect(dest, width, height, boxX, boxY, boxW, boxH, OFF_Y)
|
|
157
|
+
BitmapFont.draw(dest, width, height, line1, boxX + pad, boxY + pad, scale, BAR_Y)
|
|
158
|
+
BitmapFont.draw(dest, width, height, line2, boxX + pad, boxY + pad + lineH + gap, scale, BAR_Y)
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
private fun fillChroma(dest: ByteBuffer, width: Int, height: Int) {
|
|
162
|
+
val ySize = width * height
|
|
163
|
+
val uvSize = I420Packer.chromaStride(width) * I420Packer.chromaStride(height)
|
|
164
|
+
val end = ySize + 2 * uvSize
|
|
165
|
+
for (i in ySize until end) dest.put(i, CHROMA)
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
private fun fillYRect(
|
|
169
|
+
dest: ByteBuffer,
|
|
170
|
+
width: Int,
|
|
171
|
+
height: Int,
|
|
172
|
+
x0: Int,
|
|
173
|
+
y0: Int,
|
|
174
|
+
rectW: Int,
|
|
175
|
+
rectH: Int,
|
|
176
|
+
value: Byte,
|
|
177
|
+
) {
|
|
178
|
+
val xStart = x0.coerceIn(0, width)
|
|
179
|
+
val yStart = y0.coerceIn(0, height)
|
|
180
|
+
val xEnd = (x0 + rectW).coerceIn(0, width)
|
|
181
|
+
val yEnd = (y0 + rectH).coerceIn(0, height)
|
|
182
|
+
if (xStart >= xEnd || yStart >= yEnd) return
|
|
183
|
+
for (row in yStart until yEnd) {
|
|
184
|
+
val rowOff = row * width
|
|
185
|
+
for (col in xStart until xEnd) {
|
|
186
|
+
dest.put(rowOff + col, value)
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
companion object {
|
|
192
|
+
const val BG_Y: Byte = 40
|
|
193
|
+
const val OFF_Y: Byte = 16
|
|
194
|
+
const val BAR_Y: Byte = 235.toByte()
|
|
195
|
+
const val CHROMA: Byte = 128.toByte()
|
|
196
|
+
const val TILE = 32
|
|
197
|
+
const val PAN_X = 8
|
|
198
|
+
const val PAN_Y = 4
|
|
199
|
+
private const val DARK_TILE = 70
|
|
200
|
+
private const val LIGHT_TILE = 170
|
|
201
|
+
private const val LCG_SEED = -1640531527
|
|
202
|
+
private const val LCG_MUL = 1664525
|
|
203
|
+
private const val LCG_ADD = 1013904223
|
|
204
|
+
}
|
|
205
|
+
}
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
package com.mentra.acsmeeting.source
|
|
2
|
+
|
|
3
|
+
import android.util.Log
|
|
4
|
+
import com.mentra.acsmeeting.telemetry.PipelineStats
|
|
5
|
+
import com.mentra.acsmeeting.video.I420Packer
|
|
6
|
+
import java.nio.ByteBuffer
|
|
7
|
+
import java.util.concurrent.Executors
|
|
8
|
+
import java.util.concurrent.ScheduledExecutorService
|
|
9
|
+
import java.util.concurrent.TimeUnit
|
|
10
|
+
import java.util.concurrent.atomic.AtomicBoolean
|
|
11
|
+
import java.util.concurrent.atomic.AtomicInteger
|
|
12
|
+
import java.util.concurrent.atomic.AtomicLong
|
|
13
|
+
import java.util.concurrent.atomic.AtomicReference
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Fixed-rate packed-I420 source. Bypasses glasses, Cloudflare, WHEP and decode.
|
|
17
|
+
* Ticks that fail [isReady] or lack a target size are withheld and uncounted.
|
|
18
|
+
*/
|
|
19
|
+
class SyntheticI420Source(
|
|
20
|
+
private val video: VideoFrameListener,
|
|
21
|
+
private val stats: PipelineStats,
|
|
22
|
+
private val isReady: () -> Boolean,
|
|
23
|
+
private val config: SyntheticConfig = SyntheticConfig(),
|
|
24
|
+
) : GlassesMediaSource {
|
|
25
|
+
private val factory = SyntheticFrameFactory(config.entropy)
|
|
26
|
+
private val targetSize = AtomicReference<TargetSize?>(null)
|
|
27
|
+
private val pcmEnabled = AtomicBoolean(false)
|
|
28
|
+
private val frameIndex = AtomicInteger(0)
|
|
29
|
+
private val withheld = AtomicInteger(0)
|
|
30
|
+
private val withheldThisGap = AtomicInteger(0)
|
|
31
|
+
private val emitted = AtomicInteger(0)
|
|
32
|
+
private val emitting = AtomicBoolean(false)
|
|
33
|
+
private val firstLogged = AtomicBoolean(false)
|
|
34
|
+
private val fpsWindowStartNs = AtomicLong(0)
|
|
35
|
+
private val fpsWindowCount = AtomicInteger(0)
|
|
36
|
+
@Volatile private var stampFps: Double? = null
|
|
37
|
+
@Volatile override var state: SourceState = SourceState.IDLE
|
|
38
|
+
private set
|
|
39
|
+
@Volatile private var dest: ByteBuffer? = null
|
|
40
|
+
@Volatile private var destWidth = 0
|
|
41
|
+
@Volatile private var destHeight = 0
|
|
42
|
+
@Volatile private var destY: ByteBuffer? = null
|
|
43
|
+
@Volatile private var destU: ByteBuffer? = null
|
|
44
|
+
@Volatile private var destV: ByteBuffer? = null
|
|
45
|
+
private var scheduler: ScheduledExecutorService? = null
|
|
46
|
+
|
|
47
|
+
/** Test-visible gate transitions. Production also writes them to ACS-SPIKE. */
|
|
48
|
+
internal val events = mutableListOf<String>()
|
|
49
|
+
|
|
50
|
+
/** Puts the source in LIVE without starting the scheduler. For emitOnce tests. */
|
|
51
|
+
internal fun becomeLive() {
|
|
52
|
+
state = SourceState.LIVE
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
override fun start(config: SourceConfig) {
|
|
56
|
+
stop()
|
|
57
|
+
becomeLive()
|
|
58
|
+
val fps = this.config.fps.coerceAtLeast(1)
|
|
59
|
+
val periodUs = 1_000_000L / fps
|
|
60
|
+
val exec = Executors.newSingleThreadScheduledExecutor { runnable ->
|
|
61
|
+
Thread(runnable, "acs-synthetic-i420").apply { isDaemon = true }
|
|
62
|
+
}
|
|
63
|
+
scheduler = exec
|
|
64
|
+
exec.scheduleAtFixedRate({ emitOnce() }, 0, periodUs, TimeUnit.MICROSECONDS)
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
override fun restart(config: SourceConfig) {
|
|
68
|
+
stop()
|
|
69
|
+
start(config)
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
override fun stop() {
|
|
73
|
+
scheduler?.shutdownNow()
|
|
74
|
+
scheduler = null
|
|
75
|
+
frameIndex.set(0)
|
|
76
|
+
withheld.set(0)
|
|
77
|
+
withheldThisGap.set(0)
|
|
78
|
+
emitted.set(0)
|
|
79
|
+
emitting.set(false)
|
|
80
|
+
firstLogged.set(false)
|
|
81
|
+
fpsWindowStartNs.set(0)
|
|
82
|
+
fpsWindowCount.set(0)
|
|
83
|
+
stampFps = null
|
|
84
|
+
dest = null
|
|
85
|
+
destWidth = 0
|
|
86
|
+
destHeight = 0
|
|
87
|
+
destY = null
|
|
88
|
+
destU = null
|
|
89
|
+
destV = null
|
|
90
|
+
state = SourceState.IDLE
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
override fun setPcmDeliveryEnabled(enabled: Boolean) {
|
|
94
|
+
pcmEnabled.set(enabled)
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
override fun setTargetSize(size: TargetSize?) {
|
|
98
|
+
targetSize.set(size)
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
internal fun pcmDeliveryEnabled(): Boolean = pcmEnabled.get()
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Generate-and-emit one frame if the sender is ready and a target size exists.
|
|
105
|
+
* Returns true only when a frame was handed to [video].
|
|
106
|
+
*/
|
|
107
|
+
internal fun emitOnce(): Boolean {
|
|
108
|
+
if (state != SourceState.LIVE) return false
|
|
109
|
+
val size = targetSize.get()
|
|
110
|
+
val ready = isReady() && size != null
|
|
111
|
+
if (!ready) {
|
|
112
|
+
withheld.incrementAndGet()
|
|
113
|
+
withheldThisGap.incrementAndGet()
|
|
114
|
+
if (emitting.compareAndSet(true, false)) {
|
|
115
|
+
logEvent("P6 synthetic paused reason=not-ready afterFrames=${emitted.get()}")
|
|
116
|
+
}
|
|
117
|
+
return false
|
|
118
|
+
}
|
|
119
|
+
val gap = withheldThisGap.getAndSet(0)
|
|
120
|
+
if (!emitting.get() && emitted.get() > 0) {
|
|
121
|
+
logEvent("P6 synthetic resumed withheld=$gap")
|
|
122
|
+
}
|
|
123
|
+
emitting.set(true)
|
|
124
|
+
|
|
125
|
+
val w = size!!.width
|
|
126
|
+
val h = size.height
|
|
127
|
+
val buffer = ensureDest(w, h)
|
|
128
|
+
val index = frameIndex.getAndIncrement()
|
|
129
|
+
stampFps = sampleStampFps()
|
|
130
|
+
factory.write(buffer, w, h, index, stampFps)
|
|
131
|
+
stats.onSink()
|
|
132
|
+
stats.recordGap()
|
|
133
|
+
stats.setSize(w, h)
|
|
134
|
+
val chroma = I420Packer.chromaStride(w)
|
|
135
|
+
video.onVideoFrame(
|
|
136
|
+
I420Planes(
|
|
137
|
+
y = destY!!,
|
|
138
|
+
strideY = w,
|
|
139
|
+
u = destU!!,
|
|
140
|
+
strideU = chroma,
|
|
141
|
+
v = destV!!,
|
|
142
|
+
strideV = chroma,
|
|
143
|
+
width = w,
|
|
144
|
+
height = h,
|
|
145
|
+
timestampNs = System.nanoTime(),
|
|
146
|
+
),
|
|
147
|
+
)
|
|
148
|
+
val count = emitted.incrementAndGet()
|
|
149
|
+
if (firstLogged.compareAndSet(false, true)) {
|
|
150
|
+
logEvent(
|
|
151
|
+
"P6 synthetic first-frame withheld=${withheld.get()} size=${w}x${h} " +
|
|
152
|
+
"fps=${config.fps} entropy=${config.entropy}",
|
|
153
|
+
)
|
|
154
|
+
}
|
|
155
|
+
return count > 0
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/** 1-second emit rate, same window Recall uses for the P column on the stamp. */
|
|
159
|
+
private fun sampleStampFps(): Double? {
|
|
160
|
+
val now = System.nanoTime()
|
|
161
|
+
val started = fpsWindowStartNs.get()
|
|
162
|
+
if (started == 0L) {
|
|
163
|
+
fpsWindowStartNs.set(now)
|
|
164
|
+
fpsWindowCount.set(1)
|
|
165
|
+
return stampFps
|
|
166
|
+
}
|
|
167
|
+
fpsWindowCount.incrementAndGet()
|
|
168
|
+
val elapsedNs = now - started
|
|
169
|
+
if (elapsedNs >= 1_000_000_000L) {
|
|
170
|
+
val rate = fpsWindowCount.get() * 1_000_000_000.0 / elapsedNs
|
|
171
|
+
fpsWindowCount.set(0)
|
|
172
|
+
fpsWindowStartNs.set(now)
|
|
173
|
+
return rate
|
|
174
|
+
}
|
|
175
|
+
return stampFps
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
private fun ensureDest(width: Int, height: Int): ByteBuffer {
|
|
179
|
+
val needed = I420Packer.packedSize(width, height)
|
|
180
|
+
val existing = dest
|
|
181
|
+
if (existing != null && destWidth == width && destHeight == height && existing.capacity() >= needed) {
|
|
182
|
+
existing.clear()
|
|
183
|
+
return existing
|
|
184
|
+
}
|
|
185
|
+
val next = ByteBuffer.allocateDirect(needed)
|
|
186
|
+
dest = next
|
|
187
|
+
destWidth = width
|
|
188
|
+
destHeight = height
|
|
189
|
+
slicePlanes(next, width, height)
|
|
190
|
+
return next
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
private fun slicePlanes(packed: ByteBuffer, width: Int, height: Int) {
|
|
194
|
+
val ySize = width * height
|
|
195
|
+
val uvSize = I420Packer.chromaStride(width) * I420Packer.chromaStride(height)
|
|
196
|
+
val view = packed.duplicate()
|
|
197
|
+
view.clear()
|
|
198
|
+
destY = view.duplicate().apply { position(0); limit(ySize) }.slice()
|
|
199
|
+
destU = view.duplicate().apply { position(ySize); limit(ySize + uvSize) }.slice()
|
|
200
|
+
destV = view.duplicate().apply { position(ySize + uvSize); limit(ySize + 2 * uvSize) }.slice()
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
private fun logEvent(line: String) {
|
|
204
|
+
synchronized(events) { events.add(line) }
|
|
205
|
+
try {
|
|
206
|
+
Log.i(TAG, line)
|
|
207
|
+
} catch (_: Throwable) {
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
companion object {
|
|
212
|
+
private const val TAG = "ACS-SPIKE"
|
|
213
|
+
}
|
|
214
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
package com.mentra.acsmeeting.source
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Dedupe WebRTC track attachments by native track id.
|
|
5
|
+
* Three observer callbacks can deliver distinct Java wrappers of one track;
|
|
6
|
+
* object identity cannot be used. [reset] is load-bearing: [CloudflareWhepSource.start]
|
|
7
|
+
* calls stop first, and a stale claim set makes reconnect never attach a sink.
|
|
8
|
+
*/
|
|
9
|
+
class TrackRegistry {
|
|
10
|
+
private val ids = mutableSetOf<String>()
|
|
11
|
+
private var skipped = 0
|
|
12
|
+
|
|
13
|
+
@Synchronized
|
|
14
|
+
fun claim(id: String): Boolean {
|
|
15
|
+
if (ids.add(id)) return true
|
|
16
|
+
skipped += 1
|
|
17
|
+
return false
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
@Synchronized
|
|
21
|
+
fun reset() {
|
|
22
|
+
ids.clear()
|
|
23
|
+
skipped = 0
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
@Synchronized
|
|
27
|
+
fun size(): Int = ids.size
|
|
28
|
+
|
|
29
|
+
@Synchronized
|
|
30
|
+
fun skipped(): Int = skipped
|
|
31
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
package com.mentra.acsmeeting.source
|
|
2
|
+
|
|
3
|
+
enum class VideoSourceArm { WHEP, SYNTHETIC }
|
|
4
|
+
|
|
5
|
+
enum class SyntheticEntropy { CHEAP, MOTION, NOISE }
|
|
6
|
+
|
|
7
|
+
enum class DecoderMode { TEXTURE, BYTE_BUFFER }
|
|
8
|
+
|
|
9
|
+
enum class PixelFormatArm { I420, NV12 }
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Investigation arm. SYNTHETIC bypasses glasses, Cloudflare, WHEP and decode.
|
|
13
|
+
* Ships as WHEP. Flip locally to run the experiment; do not commit SYNTHETIC.
|
|
14
|
+
*
|
|
15
|
+
* [decoderMode] TEXTURE is the production default (shared EGL, MediaCodec to
|
|
16
|
+
* Surface). BYTE_BUFFER is the A/B: no shared context, CPU planes, no
|
|
17
|
+
* glReadPixels. 720p BYTE_BUFFER on SM_S948U decoded on hardware with
|
|
18
|
+
* i420P95=0, but wire/codec never attached and busy drops climbed. Do not
|
|
19
|
+
* commit BYTE_BUFFER until a same-phone 540p A/B clears the campaign gates.
|
|
20
|
+
*
|
|
21
|
+
* [zeroCopy] hands WebRTC I420 planes straight to ACS when they are tight and
|
|
22
|
+
* retainable. Ships off. Do not commit true.
|
|
23
|
+
*
|
|
24
|
+
* [pixelFormat] I420 is the production default. NV12 is the encoder-flip A/B:
|
|
25
|
+
* advertise and send biplanar NV12 in case ACS picks a hardware H.264
|
|
26
|
+
* encoder. Revert unless `codecName` leaves `h264 sw`. Do not commit NV12
|
|
27
|
+
* on a failed flip.
|
|
28
|
+
*/
|
|
29
|
+
object AcsInvestigation {
|
|
30
|
+
val videoArm = VideoSourceArm.WHEP
|
|
31
|
+
val syntheticFps = 15
|
|
32
|
+
val syntheticEntropy = SyntheticEntropy.MOTION
|
|
33
|
+
val decoderMode = DecoderMode.TEXTURE
|
|
34
|
+
val zeroCopy = false
|
|
35
|
+
val pixelFormat = PixelFormatArm.I420
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
data class SyntheticConfig(
|
|
39
|
+
val fps: Int = AcsInvestigation.syntheticFps,
|
|
40
|
+
val entropy: SyntheticEntropy = AcsInvestigation.syntheticEntropy,
|
|
41
|
+
)
|