@mentra/acs-meeting 3.2.0-dev.120

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. package/NOTICE +20 -0
  2. package/README.md +412 -0
  3. package/THIRD_PARTY +7 -0
  4. package/android/build.gradle +64 -0
  5. package/android/src/main/AndroidManifest.xml +5 -0
  6. package/android/src/main/java/com/mentra/acsmeeting/AcsMeetingModule.kt +80 -0
  7. package/android/src/main/java/com/mentra/acsmeeting/AcsMeetingSession.kt +846 -0
  8. package/android/src/main/java/com/mentra/acsmeeting/RemoteRoster.kt +123 -0
  9. package/android/src/main/java/com/mentra/acsmeeting/audio/AcsAudioPolicy.kt +90 -0
  10. package/android/src/main/java/com/mentra/acsmeeting/audio/AudioPolicyApplier.kt +176 -0
  11. package/android/src/main/java/com/mentra/acsmeeting/audio/IncomingAudioPump.kt +112 -0
  12. package/android/src/main/java/com/mentra/acsmeeting/audio/IncomingRateProbe.kt +65 -0
  13. package/android/src/main/java/com/mentra/acsmeeting/audio/PcmBridge.kt +108 -0
  14. package/android/src/main/java/com/mentra/acsmeeting/audio/PcmResampler.kt +159 -0
  15. package/android/src/main/java/com/mentra/acsmeeting/audio/PhoneMicCapturer.kt +115 -0
  16. package/android/src/main/java/com/mentra/acsmeeting/audio/UplinkPacer.kt +302 -0
  17. package/android/src/main/java/com/mentra/acsmeeting/audio/UplinkSender.kt +230 -0
  18. package/android/src/main/java/com/mentra/acsmeeting/source/BitmapFont.kt +99 -0
  19. package/android/src/main/java/com/mentra/acsmeeting/source/CloudflareWhepSource.kt +578 -0
  20. package/android/src/main/java/com/mentra/acsmeeting/source/FirstFrameGate.kt +45 -0
  21. package/android/src/main/java/com/mentra/acsmeeting/source/GlassesMediaSource.kt +101 -0
  22. package/android/src/main/java/com/mentra/acsmeeting/source/MediaListeners.kt +57 -0
  23. package/android/src/main/java/com/mentra/acsmeeting/source/SyntheticFrameFactory.kt +205 -0
  24. package/android/src/main/java/com/mentra/acsmeeting/source/SyntheticI420Source.kt +214 -0
  25. package/android/src/main/java/com/mentra/acsmeeting/source/TrackRegistry.kt +31 -0
  26. package/android/src/main/java/com/mentra/acsmeeting/source/VideoSourceArm.kt +41 -0
  27. package/android/src/main/java/com/mentra/acsmeeting/telemetry/ChromaProbe.kt +78 -0
  28. package/android/src/main/java/com/mentra/acsmeeting/telemetry/PipelineStats.kt +281 -0
  29. package/android/src/main/java/com/mentra/acsmeeting/telemetry/PipelineTicker.kt +51 -0
  30. package/android/src/main/java/com/mentra/acsmeeting/telemetry/ProcessCpu.kt +15 -0
  31. package/android/src/main/java/com/mentra/acsmeeting/telemetry/RingPercentile.kt +33 -0
  32. package/android/src/main/java/com/mentra/acsmeeting/video/AcsFrameSender.kt +475 -0
  33. package/android/src/main/java/com/mentra/acsmeeting/video/AcsTimestamp.kt +26 -0
  34. package/android/src/main/java/com/mentra/acsmeeting/video/FrameGeometry.kt +22 -0
  35. package/android/src/main/java/com/mentra/acsmeeting/video/I420FormatSpec.kt +90 -0
  36. package/android/src/main/java/com/mentra/acsmeeting/video/I420Packer.kt +72 -0
  37. package/android/src/main/java/com/mentra/acsmeeting/video/Nv12FormatSpec.kt +36 -0
  38. package/android/src/main/java/com/mentra/acsmeeting/video/Nv12Packer.kt +51 -0
  39. package/android/src/main/java/com/mentra/acsmeeting/video/SendGate.kt +27 -0
  40. package/android/src/main/java/com/mentra/acsmeeting/video/VideoProfile.kt +57 -0
  41. package/android/src/test/java/com/mentra/acsmeeting/RemoteRosterTest.kt +40 -0
  42. package/android/src/test/java/com/mentra/acsmeeting/audio/AcsAudioPolicyTest.kt +204 -0
  43. package/android/src/test/java/com/mentra/acsmeeting/audio/AudioPolicyApplierTest.kt +223 -0
  44. package/android/src/test/java/com/mentra/acsmeeting/audio/CapturePolicyTest.kt +12 -0
  45. package/android/src/test/java/com/mentra/acsmeeting/audio/IncomingAudioPumpTest.kt +97 -0
  46. package/android/src/test/java/com/mentra/acsmeeting/audio/IncomingRateProbeTest.kt +78 -0
  47. package/android/src/test/java/com/mentra/acsmeeting/audio/PcmResamplerTest.kt +125 -0
  48. package/android/src/test/java/com/mentra/acsmeeting/audio/UplinkPacerTest.kt +201 -0
  49. package/android/src/test/java/com/mentra/acsmeeting/audio/UplinkSenderTest.kt +166 -0
  50. package/android/src/test/java/com/mentra/acsmeeting/source/AcsInvestigationTest.kt +26 -0
  51. package/android/src/test/java/com/mentra/acsmeeting/source/BitmapFontTest.kt +22 -0
  52. package/android/src/test/java/com/mentra/acsmeeting/source/FirstFrameGateTest.kt +60 -0
  53. package/android/src/test/java/com/mentra/acsmeeting/source/GlassesMediaSourceTest.kt +133 -0
  54. package/android/src/test/java/com/mentra/acsmeeting/source/I420PlanesTest.kt +67 -0
  55. package/android/src/test/java/com/mentra/acsmeeting/source/SyntheticFrameFactoryTest.kt +193 -0
  56. package/android/src/test/java/com/mentra/acsmeeting/source/SyntheticI420SourceTest.kt +176 -0
  57. package/android/src/test/java/com/mentra/acsmeeting/source/TrackRegistryTest.kt +36 -0
  58. package/android/src/test/java/com/mentra/acsmeeting/telemetry/ChromaProbeTest.kt +61 -0
  59. package/android/src/test/java/com/mentra/acsmeeting/telemetry/PipelineStatsTest.kt +256 -0
  60. package/android/src/test/java/com/mentra/acsmeeting/telemetry/ProcessCpuTest.kt +18 -0
  61. package/android/src/test/java/com/mentra/acsmeeting/telemetry/RingPercentileTest.kt +49 -0
  62. package/android/src/test/java/com/mentra/acsmeeting/video/AcsTimestampTest.kt +43 -0
  63. package/android/src/test/java/com/mentra/acsmeeting/video/FrameGeometryTest.kt +33 -0
  64. package/android/src/test/java/com/mentra/acsmeeting/video/I420FormatSpecTest.kt +77 -0
  65. package/android/src/test/java/com/mentra/acsmeeting/video/I420PackerTest.kt +126 -0
  66. package/android/src/test/java/com/mentra/acsmeeting/video/Nv12FormatSpecTest.kt +36 -0
  67. package/android/src/test/java/com/mentra/acsmeeting/video/Nv12PackerTest.kt +58 -0
  68. package/android/src/test/java/com/mentra/acsmeeting/video/SendGateTest.kt +79 -0
  69. package/android/src/test/java/com/mentra/acsmeeting/video/VideoProfileTest.kt +68 -0
  70. package/build/AcsMeeting.types.d.ts +59 -0
  71. package/build/AcsMeeting.types.d.ts.map +1 -0
  72. package/build/AcsMeeting.types.js +2 -0
  73. package/build/AcsMeeting.types.js.map +1 -0
  74. package/build/AcsMeetingModule.d.ts +15 -0
  75. package/build/AcsMeetingModule.d.ts.map +1 -0
  76. package/build/AcsMeetingModule.js +3 -0
  77. package/build/AcsMeetingModule.js.map +1 -0
  78. package/build/AcsMeetingModule.web.d.ts +16 -0
  79. package/build/AcsMeetingModule.web.d.ts.map +1 -0
  80. package/build/AcsMeetingModule.web.js +31 -0
  81. package/build/AcsMeetingModule.web.js.map +1 -0
  82. package/build/index.d.ts +3 -0
  83. package/build/index.d.ts.map +1 -0
  84. package/build/index.js +3 -0
  85. package/build/index.js.map +1 -0
  86. package/expo-module.config.json +9 -0
  87. package/ios/AcsFrameSender.swift +47 -0
  88. package/ios/AcsMeeting.podspec +27 -0
  89. package/ios/AcsMeetingModule.swift +558 -0
  90. package/ios/PcmBridge.swift +117 -0
  91. package/ios/PhoneMicCapturer.swift +64 -0
  92. package/ios/PolicyKit/Package.swift +17 -0
  93. package/ios/PolicyKit/Sources/AcsAudioPolicy/AcsAudioPolicy.swift +102 -0
  94. package/ios/PolicyKit/Sources/AcsAudioPolicy/AudioPolicyApplier.swift +162 -0
  95. package/ios/PolicyKit/Sources/AcsAudioPolicy/GlassesMediaSource.swift +68 -0
  96. package/ios/PolicyKit/Tests/AcsAudioPolicyTests/AcsAudioPolicyTests.swift +273 -0
  97. package/ios/WhepVideoSource.swift +356 -0
  98. package/package.json +56 -0
  99. package/src/AcsMeeting.types.ts +66 -0
  100. package/src/AcsMeetingModule.ts +16 -0
  101. package/src/AcsMeetingModule.web.ts +33 -0
  102. package/src/index.ts +2 -0
package/NOTICE ADDED
@@ -0,0 +1,20 @@
1
+ Apache License 2.0 third-party notice
2
+ =====================================
3
+
4
+ This module contains Kotlin adapted from RealWear Collaborate (Apache License 2.0).
5
+
6
+ Upstream: https://github.com/realwear/collaborate
7
+ Commit: 2a143fd18c952c91e1cbae525d8126a858c18edf (HEAD at import)
8
+ Source file: android/apps/rwt/app/src/main/java/com/realwear/acs/util/thermal/VideoFrameSender.kt
9
+ Adapted into: android/src/main/java/com/mentra/acsmeeting/AcsFrameSender.kt
10
+
11
+ Patterns used (not copied verbatim):
12
+ - direct ByteBuffer pooling and reuse
13
+ - format negotiation from RawOutgoingVideoStream constraints
14
+ - frame pacing to the negotiated FPS
15
+ - stream-state handling (start/stop/format-changed) and error paths
16
+
17
+ iOS sources in this module are original Mentra code on the official ACS iOS
18
+ raw-video API (CVPixelBuffer). They do not contain RealWear code.
19
+
20
+ The Apache 2.0 license text is in THIRD_PARTY.
package/README.md ADDED
@@ -0,0 +1,412 @@
1
+ # @mentra/acs-meeting
2
+
3
+ Phone-native Azure Communication Services client that puts a MentraOS wearer into a
4
+ Microsoft Teams meeting as a guest. The glasses provide the camera and the microphone;
5
+ the phone does all the WebRTC and ACS work.
6
+
7
+ The phone is a **relay, not a capture device**. It subscribes to whatever the glasses
8
+ already published to Cloudflare, decodes it, and re-publishes it into ACS. No frame
9
+ crosses the JavaScript bridge. Production never generates pixels on the phone.
10
+
11
+ An investigation-only synthetic arm can generate packed I420 locally at a fixed 15 Hz
12
+ (`AcsInvestigation.videoArm`). It ships as `WHEP`. Flip it locally, rebuild native, and
13
+ do not commit `SYNTHETIC`.
14
+
15
+ `AcsInvestigation.decoderMode` ships as `TEXTURE` (shared EGL, MediaCodec to Surface).
16
+ `BYTE_BUFFER` skips the GL readback. A 720p hop on `SM_S948U` decoded in hardware
17
+ with `i420P95=0`, but did not clear the campaign gates; keep the flag on TEXTURE.
18
+ `AcsInvestigation.zeroCopy` ships off; when on,
19
+ tight retainable WebRTC planes go straight to ACS, with an automatic copy fallback.
20
+ `AcsInvestigation.pixelFormat` ships as `I420`. `NV12` is the encoder-flip A/B:
21
+ advertise and send biplanar NV12 and read `codec=` on the ladder. Revert unless
22
+ `codecName` leaves `h264 sw`.
23
+
24
+ Mentra Call's persisted default is `720p15` (`VideoProfile.DEFAULT = HD`,
25
+ 1280×720@15 / 2.5 Mbps). `540p15` (960×540@15 / 1.5 Mbps) is a user-selectable
26
+ preset on the Home settings picker. It is not migrated onto existing installs
27
+ and is not the native default. Miniapp joins pass width, height, fps, and
28
+ `maxBitrateBps` through to native, so a selected `540p15` reaches both glasses
29
+ WHIP and ACS without changing `VideoProfile.DEFAULT`.
30
+
31
+ ## The pipeline in one picture
32
+
33
+ ```
34
+ glasses ──WHIP──▶ Cloudflare ──WHEP──▶ │ phone (this module) │ ──▶ ACS ──▶ Teams
35
+ │ │
36
+ video: H.264 ─▶ WebRTC decoder ─▶ I420 ─▶ cropAndScale ─▶ I420 copy or NV12 interleave ─▶ sendRawVideoFrame
37
+ audio: Opus ─▶ WebRTC decoder ─▶ PCM16 48k ─▶ sendRawAudioBuffer
38
+
39
+ return audio: ACS RawIncomingAudioStream ─▶ base64 ─▶ Expo event ─▶ AudioPlaybackService ─▶ A2DP
40
+ ```
41
+
42
+ Two things about this are load-bearing and easy to get wrong:
43
+
44
+ **ACS encodes for us.** `RawOutgoingVideoStream` accepts raw pixels only — `I420`, `NV12`,
45
+ `RGBA`, and a few others — never an H.264 bitstream. We cannot forward the Cloudflare
46
+ H.264 as-is; it must be decoded and handed over as pixels. ACS then re-encodes to H.264
47
+ internally. The decode is unavoidable; the *conversion* is not, which is why we feed I420
48
+ (what the WebRTC decoder already produces) rather than converting to RGBA.
49
+
50
+ **Only the video path is hot.** At 720p15 the video path moves ~1.4 MB per frame, 15 times
51
+ a second. Everything in `video/` and `source/` is written to avoid per-pixel work in
52
+ Kotlin: scaling is libyuv via `cropAndScale`, packing is a bulk `ByteBuffer` copy, and
53
+ plane buffers are pooled rather than allocated.
54
+
55
+ ## Layout
56
+
57
+ ```
58
+ android/src/main/java/com/mentra/acsmeeting/
59
+ ├── AcsMeetingModule.kt Expo bridge: join/leave/setMuted/getState, onState + onIncomingPcm events
60
+ ├── AcsMeetingSession.kt Orchestrator. Owns the ACS Call and wires the four packages together
61
+
62
+ ├── source/ Upstream — getting glasses media into the process
63
+ │ ├── MediaListeners.kt I420Planes, VideoFrameListener, PcmListener
64
+ │ ├── GlassesMediaSource.kt The transport interface + controller. WHEP is one implementation
65
+ │ ├── CloudflareWhepSource.kt The recvonly WHEP subscriber (PeerConnection, sinks, scaling)
66
+ │ ├── SyntheticFrameFactory.kt Packed I420 generator (CHEAP / MOTION pan / NOISE)
67
+ │ ├── SyntheticI420Source.kt Fixed-rate GlassesMediaSource wrapping the factory
68
+ │ ├── VideoSourceArm.kt Investigation switch. Ships as WHEP / TEXTURE / I420 / zeroCopy=off
69
+ │ └── TrackRegistry.kt Deduplicates track attachment
70
+
71
+ ├── video/ Downstream — pixels into ACS
72
+ │ ├── AcsFrameSender.kt Owns the RawOutgoingVideoStream and the send executor
73
+ │ ├── I420FormatSpec.kt The I420 format we advertise, as plain values
74
+ │ ├── Nv12FormatSpec.kt The NV12 format we advertise, as plain values
75
+ │ ├── AcsTimestamp.kt 100-ns ticks for RawVideoFrame. Zero means freeze.
76
+ │ ├── I420Packer.kt Stride-aware planar copy; planeMinBytes for malformed guards
77
+ │ ├── Nv12Packer.kt I420 → interleaved UV; used only on the NV12 arm
78
+ │ ├── FrameGeometry.kt Buffer-vs-display coordinates under rotation
79
+ │ └── SendGate.kt Single-in-flight backpressure
80
+
81
+ ├── audio/ Two microphones, one call
82
+ │ ├── AcsAudioPolicy.kt Pure decision functions: which stream, who gets muted
83
+ │ ├── AudioPolicyApplier.kt Applies a decision to the live call, with retries
84
+ │ └── PcmBridge.kt Downmix/rebuffer to ACS's 48 kHz mono 20 ms frames
85
+
86
+ └── telemetry/ The measurement ladder
87
+ ├── PipelineStats.kt Every counter, and the 1 Hz "P6 ladder" line
88
+ ├── PipelineTicker.kt Emits that line on a timer
89
+ ├── RingPercentile.kt p50/p95 over a fixed ring of samples
90
+ └── ChromaProbe.kt Plane averages, to catch a mis-packed frame
91
+ ```
92
+
93
+ Tests mirror this exactly under `android/src/test/java/com/mentra/acsmeeting/`.
94
+
95
+ The dependency direction is `AcsMeetingSession → {source, video, audio, telemetry}`, with
96
+ `source` and `video` both depending on `telemetry` to report, and `telemetry/ChromaProbe`
97
+ reaching into `video/I420Packer` for stride math. Nothing in `audio/` touches video.
98
+
99
+ ## Video path
100
+
101
+ `CloudflareWhepSource` POSTs an SDP offer to the WHEP endpoint, gets an answer, and adds a
102
+ `VideoSink` to the remote track. Per frame, on WebRTC's decode thread:
103
+
104
+ 1. If ACS negotiated a different size than the decoder is producing, scale with
105
+ `buffer.cropAndScale(...)` — libyuv, not Kotlin.
106
+ 2. `toI420()`, then hand the three planes to `AcsFrameSender.sendPlanes` as `I420Planes`.
107
+ 3. The sender copies each plane once into a pooled ACS buffer (`copyP95`), or — when
108
+ `zeroCopy` is on and the planes are tight, direct, and retainable — retains the
109
+ WebRTC buffer and submits those planes without a copy. The NV12 investigation arm
110
+ always converts: Y is copied, U/V are interleaved, and ACS gets two buffers.
111
+
112
+ ACS's I420 contract still requires three separate plane buffers. NV12 is two
113
+ (Y + interleaved UV). The old pack-then-split path is gone. A timed-out zero-copy
114
+ send schedules `release()` after a 1 s grace window so the decoder pool is not starved.
115
+
116
+ Three constraints worth knowing before changing this code:
117
+
118
+ - **No inter-frame pacer.** `SendGate` allows exactly one send in flight and nothing else.
119
+ Any minimum-interarrival gate deletes real frames whenever the decoder delivers in
120
+ bursts, which it does.
121
+ - **Buffer coordinates, never display coordinates.** `cropAndScale` and `toI420` operate on
122
+ the un-rotated buffer. Mixing in `VideoFrame.rotatedWidth/rotatedHeight` overruns the
123
+ plane whenever rotation is 90° or 270°. `FrameGeometry` exists to keep this honest, and
124
+ the ladder prints `rot=` so a non-zero rotation is visible rather than silently corrupting.
125
+ - **A timed-out send does not release its buffers.** `Future.get(timeout)` does not cancel
126
+ the work, so ACS may still be reading those planes. They are deliberately leaked rather
127
+ than recycled, and counted as `abandoned` in the ladder.
128
+
129
+ ### Source health and recovery
130
+
131
+ The ACS call and the WHEP subscription fail independently: ICE can drop (phone switched
132
+ Wi-Fi↔cellular) or the WHEP endpoint can 404 (glasses stopped publishing) while ACS stays
133
+ `connected` and Teams holds the last frame. `CloudflareWhepSource` reports every
134
+ `SourceState` transition to the session, which:
135
+
136
+ - carries it on every snapshot as `mediaSource: idle | connecting | live | failed`, so the
137
+ host and miniapp can tell "call up, glasses feed dead" from a healthy call;
138
+ - on `failed`, rebuilds the subscription itself with exponential backoff (1 s → 10 s cap) for
139
+ as long as the call is alive. A host `updateVideoSource` with a new URL cancels the retry.
140
+
141
+ **`LIVE` means a frame reached the sink, not that the WHEP endpoint answered.** The answer
142
+ lands before `setRemoteDescription`, before ICE CHECKING and before the first decode — on an
143
+ S22 reconnect, 3.2 s before the first frame (`whep_answer` 15:56:47.16, `sub=1.0` 15:56:50.36)
144
+ and 6.2 s before a real rate to Teams. Two things followed from promoting on the answer: a
145
+ miniapp reading `mediaSource: live` was still ahead of Teams, and a subscription that answered
146
+ but never delivered read `LIVE` forever behind a frozen frame with nothing able to see it.
147
+
148
+ So the answer arms `FirstFrameGate` and stays `CONNECTING`. The first video frame for that
149
+ peer generation promotes to `LIVE` (`first_frame`); if none arrives within
150
+ `FIRST_FRAME_TIMEOUT_MS` (9 s) the source transitions `FAILED` with reason `no_first_frame`
151
+ and the backoff above rebuilds it. An ICE bounce back to CONNECTED returns the source to
152
+ `CONNECTING` and rearms rather than claiming `LIVE`: a recovered candidate pair is a promise
153
+ of frames, and if media was already flowing the next frame promotes within milliseconds.
154
+
155
+ `canReuseSource` is why a same-URL `restart` during `CONNECTING` is still a no-op: rebuilding
156
+ a subscriber seconds from its first frame only restarts the wait, and `CONNECTING` is bounded
157
+ by the POST timeout and the first-frame deadline, so nothing can strand there.
158
+
159
+ `restartVideoSource()` forces a rebuild on the current URL even when the peer still looks
160
+ healthy; the host calls it from a NetInfo listener when the phone's network identity changes.
161
+
162
+ iOS `WhepVideoSource` mirrors all of this. `sub=` on the P6 ladder is the ground truth for
163
+ "Teams has video"; `first_frame` is the transition that should sit next to it in logcat.
164
+
165
+ ## Audio path
166
+
167
+ Two possible microphones — the glasses or the handset — and the wrong answer means either
168
+ silence or the wearer being recorded when they think they are muted. The decision is
169
+ therefore a pure function in `AcsAudioPolicy`, unit-tested in isolation, and separate from
170
+ the code that applies it.
171
+
172
+ - **`audioSource: "glasses"`** arms a `RawOutgoingAudioStream` — which ACS reports as
173
+ `VIRTUAL_OUTGOING` — and feeds it WHEP PCM through `PcmBridge`, which downmixes to mono
174
+ 48 kHz (passthrough when WHEP already decoded at 48 kHz) and re-chunks into the 20 ms
175
+ frames ACS expects. The handset mic is never opened. Incoming raw audio stays 16 kHz.
176
+ - **`audioSource: "phone"`** uses ACS's own `LocalOutgoingAudioStream` (`LOCAL_OUTGOING`).
177
+
178
+ `ActiveStreamKind` mirrors that pair, and deliberately reads `NONE` for a stream that is
179
+ attached but not `STARTED` — "attached" is not "live", and treating it as live is how a
180
+ muted wearer ends up audible.
181
+
182
+ `AudioPolicyApplier` reports an `AudioSafety` of `safe`, `degraded`, or `unsafe`. `unsafe`
183
+ means both mute and stop failed and a live mic may be reaching the meeting; it is surfaced
184
+ all the way to JS in `AcsMeetingState.audioSafety`.
185
+
186
+ Return audio does **not** use ACS playback. Incoming PCM is base64'd to JS and played by
187
+ `AudioPlaybackService` / `PcmStreamPlayer`, so it reaches the glasses over A2DP and
188
+ participates in MCU duplex via `setOwnAppAudioPlaying`.
189
+
190
+ ## Threads
191
+
192
+ | Thread | What runs on it |
193
+ |---|---|
194
+ | `AcsMeetingSession` single-thread executor | join, leave, and every audio-policy application |
195
+ | `acs-synthetic-i420` | investigation arm only: generate + emit at fixed fps |
196
+ | WebRTC decode thread | the video sink: scale, pack, hand off |
197
+ | `acs-i420-send` | `sendRawVideoFrame` and nothing else |
198
+ | main looper | the 1 Hz ticker and the WebRTC `getStats` poll |
199
+
200
+ The rule that matters: **nothing blocking may run on the WebRTC decode thread.** A
201
+ `PeerConnection.getStats()` call there once stalled video after roughly one second. The
202
+ stats poll is now timer-driven on the main looper, which also means it keeps sampling
203
+ during a freeze instead of going quiet exactly when the data is interesting.
204
+
205
+ ## Reading the telemetry
206
+
207
+ ```
208
+ adb logcat -s ACS-SPIKE | grep "P6 ladder"
209
+ ```
210
+
211
+ One line per second, tracing a frame through every hop:
212
+
213
+ ```
214
+ P6 ladder arm=whep 1280x720 recv=14.8 dec=8.1 sink=8.0 dup=4 sub=8.0 wire=1280x720@8.0 kbps=1400 codec=h264_sw rot=0
215
+ drop{size=0 busy=0 notStarted=0 fail=0 nullI420=0 abandoned=0}
216
+ recv{drop=6 lost=0 nack=0 pli=0 freeze=2 freezeSec=3.4 jit=4.0 decMs=3.2 jbMs=210.0 decImpl=OMX.qcom.video.decoder.avc}
217
+ path{mode=texture copy=planes pix=i420} buf{tex=12 i420=0 other=0}
218
+ stride{y=1280 u=768 v=768 tight=0 padded=12} zc{on=0 used=0 fell=0 padded=0 heldMax=0 timeout=0}
219
+ ms{gapP50=68.3 gapP95=523.4 i420P95=12.0 packP95=na scaleP95=na sinkCbP95=20.0 splitP95=na copyP95=4.1 sendP95=2.7}
220
+ alloc{dest=0 plane=3} chroma{y=162 u=132 v=132} cum{sink=1000 sub=1000 drop=0 inFlight=0}
221
+ cpu{proc=112.4 cores=8}
222
+ ```
223
+
224
+ The rates form a ladder, and the first place two adjacent numbers diverge is the bottleneck:
225
+
226
+ | Field | Hop |
227
+ |---|---|
228
+ | `recv` | frames WebRTC pulled off the wire (`framesReceived`) |
229
+ | `dec` | frames the decoder produced (`framesDecoded`) |
230
+ | `sink` | frames our `VideoSink` saw |
231
+ | `sub` | frames submitted to ACS |
232
+ | `wire` | frames ACS actually encoded (`Features.MEDIA_STATISTICS`) |
233
+
234
+ So `recv` well under 15 blames the network or Cloudflare; a healthy `recv` with a low `dec`
235
+ blames the decoder or jitter buffer; `sink` below `dec` means frames are being dropped
236
+ before us; and `sub` below `sink` means we are dropping them, with `drop{}` naming which
237
+ reason.
238
+
239
+ The rest:
240
+
241
+ - **`cum{}`** is monotonic and is what pass/fail uses. `inFlight` is *derived* as
242
+ `sink - sub - drop`, which makes conservation an identity — a tracked counter cannot be
243
+ read atomically alongside the others and produced false `CONSERVE_FAIL` alarms on a
244
+ perfectly healthy pipeline.
245
+ - **`ms{}`** is our own cost. At 15 fps the budget is 66 ms. `i420P95` is `toI420()`
246
+ (GL readback when `path.mode=texture`; libyuv when `bytebuf`). `copyP95` is the single
247
+ plane copy into ACS buffers. `packP95`/`splitP95` stay in the line for old captures
248
+ and print `na` on the planes path. `sinkCbP95` is the whole decode-thread callback.
249
+ - **`path{}` / `buf{}` / `stride{}` / `zc{}`** say which arm produced the line. `buf.tex`
250
+ climbing with `mode=texture` is the Surface decoder. `buf.i420` climbing with
251
+ `mode=bytebuf` is the A/B succeeding. `pix=nv12` plus `copy=nv12` is the encoder-flip
252
+ arm. `codec=` is ACS `OutgoingVideoStatistics.codecName` — the NV12 experiment
253
+ succeeds only if this leaves `h264_sw`. `stride.padded` dominating means zero-copy will
254
+ fall back (tight-plane only). `decImpl` turning into a software name is an abort for
255
+ the byte-buffer arm.
256
+ - **`cpu{proc}`** is process CPU% the same way `top` reports it (can exceed 100 on
257
+ multi-core). Pair it with the `ms{}` stages: high `i420`/`sinkCb` + high `proc` supports
258
+ convert-on-decode-thread; high `sendP95` with low `i420` blames ACS encode/submit.
259
+ Preview is JS/UI and is not in this line — toggle it and watch `cpu{proc}`.
260
+ - **`alloc{}`** counts dest/plane direct-buffer allocations. After the first frames these
261
+ should stay flat if pooling holds. Climbing values mean we are still allocating per frame.
262
+
263
+ A/B capture (preview off, ~90 s of steady motion, same room):
264
+
265
+ ```bash
266
+ adb logcat -c
267
+ # join the Teams call, hold steady motion
268
+ adb logcat -d -s ACS-SPIKE > /tmp/acs-<arm>.txt
269
+ bun scripts/acs-ladder.ts --compare /tmp/acs-baseline.txt /tmp/acs-<arm>.txt --skip-ms 20000
270
+ ```
271
+
272
+ Compare uses medians, not the last tick. A 15% `recv` gap or a 2x `lost` gap prints a
273
+ confound warning: the network differed and the CPU delta is not attributable. Capture
274
+ order is baseline (texture + planes, `zeroCopy=false`, `pixelFormat=I420`) first,
275
+ then one flag at a time. The NV12 encoder-flip is `pixelFormat=NV12`; keep it
276
+ only if `codec=` leaves `h264_sw`.
277
+
278
+ ### 540p15 + BYTE_BUFFER campaign (operator steps)
279
+
280
+ `720p15` stays the Home and native default. `540p15` is selectable. Arms 1–2
281
+ were captured on `SM_S906U` (Snapdragon, not the A54) with preview off,
282
+ `pixelFormat=I420`, `zeroCopy=false`. Loss on the 720p window was >2× the 540p
283
+ window, so the CPU delta is **not** attributable. Wire fps is.
284
+
285
+ Hold these constants for arm 3: same phone, same network, same Teams meeting,
286
+ same ~90 s walking/head-turn motion after a 20 s warmup, preview off, picker
287
+ still on `540p · 15 fps`.
288
+
289
+ Confirm the join logs before saving each dump:
290
+
291
+ - glasses WHIP: `whip start (1280x720@15)` or `whip start (960x540@15)`
292
+ - miniapp ACS handoff: `requestedAcs=1280×720 @15` or `requestedAcs=960×540 @15`
293
+ - native: `P5 negotiated ... 1280x720 fps=15` or `P5 negotiated ... 960x540 fps=15`
294
+ - ladder: `wire=1280x720@...` or `wire=960x540@...`
295
+
296
+ **Arm 1 — 720p15 / TEXTURE (baseline).** Leave Glasses video on `720p · 15 fps`.
297
+ Confirm `AcsInvestigation.decoderMode` is still `TEXTURE`. Clear logcat, join,
298
+ hold motion, dump:
299
+
300
+ ```bash
301
+ adb logcat -c
302
+ # join Teams on the A54, 90 s of steady motion, preview off
303
+ adb logcat -d -s ACS-SPIKE > /tmp/acs-720p15-texture.txt
304
+ ```
305
+
306
+ Leave the call.
307
+
308
+ **Arm 2 — 540p15 / TEXTURE (resolution savings).** On Home → Glasses video, select
309
+ `540p · 15 fps`. The summary under the picker should read
310
+ `960×540 @ 15 · 1.5 Mbps`. Decoder stays `TEXTURE`. Rejoin the same meeting,
311
+ same motion, dump `/tmp/acs-540p15-texture.txt`. Leave.
312
+
313
+ Compare resolution-only savings (still no decoder flip):
314
+
315
+ ```bash
316
+ bun scripts/acs-ladder.ts --compare /tmp/acs-720p15-texture.txt /tmp/acs-540p15-texture.txt --skip-ms 20000
317
+ ```
318
+
319
+ Recorded on `SM_S906U` 2026-09-01 (medians, 20 s warmup on 540p; 720p is the
320
+ long 15:52 window). `recv` matched (14.9 vs 14.8). `lost` 39 → 7 (confound).
321
+
322
+ | field | 720p15 / TEXTURE | 540p15 / TEXTURE |
323
+ |---|---:|---:|
324
+ | recv | 14.9 | 14.8 |
325
+ | dec / sink / sub | 11 / 11 / 11 | 12.9 / 12 / 12 |
326
+ | wire fps | 8.6 | 13.2 |
327
+ | kbps | 1332 | 1357 |
328
+ | i420P95 / sinkCbP95 / copyP95 | 10.9 / 16.8 / 5.4 | 10.1 / 14.7 / 3.9 |
329
+ | cpu{proc} | 109 | 95 (confounded) |
330
+ | decImpl | `c2.qti.avc.decoder` | `c2.qti.avc.decoder` |
331
+ | codec | `h264_sw` | `h264_sw` |
332
+
333
+ Wire is the win: same ~1.4 Mbps, ~9 → ~13 fps. Encoder stayed software. Keep
334
+ 540p15 selectable; do not make it the default.
335
+
336
+ **Arm 3 — 540p15 / BYTE_BUFFER (decoder savings).** Keep the picker on 540p15.
337
+ Flip `AcsInvestigation.decoderMode` to `BYTE_BUFFER` locally. Rebuild native,
338
+ rejoin, dump `/tmp/acs-540p15-bytebuffer.txt`. Leave.
339
+
340
+ ```bash
341
+ bun scripts/acs-ladder.ts --compare /tmp/acs-540p15-texture.txt /tmp/acs-540p15-bytebuffer.txt --skip-ms 20000
342
+ ```
343
+
344
+ Recorded on `SM_S948U` 2026-09-01 at **720p15** (wrong profile vs the plan;
345
+ different SoC vs arms 1–2). Mechanical decode passed; campaign gates did not.
346
+
347
+ | field | 720p15 / BYTE_BUFFER (`SM_S948U`) |
348
+ |---|---:|
349
+ | path / buf | `bytebuf`, `tex=0`, `i420` climbing |
350
+ | recv / dec / sink | 15 / 15 / 15 |
351
+ | i420P95 / sinkCbP95 / copyP95 | 0.0 / 1.0 / 0.4 |
352
+ | cpu{proc} | ~44 (not comparable to SM_S906U ~109) |
353
+ | decImpl | `c2.qti.avc.decoder` |
354
+ | wire / codec | `na` / `na` |
355
+ | drop.busy | climbing (~18–34) |
356
+
357
+ Do not promote BYTE_BUFFER: no wire fps, no `codec=`, busy backpressure, and
358
+ no same-phone 540p TEXTURE compare. `decoderMode` stays `TEXTURE`.
359
+
360
+ Promote BYTE_BUFFER as the guarded default only when every gate passes:
361
+
362
+ - `path{mode=bytebuf}`, `buf{i420}` rises, and `buf{tex}=0`
363
+ - `recv` differs by at most 15% and packet loss by at most 2× (else confounded)
364
+ - `decImpl` stays hardware (`OMX.qcom`, `c2.qti`, vendor). Abort on
365
+ `c2.android`, `c2.google`, `OMX.google`, or a blank/stalled decoder
366
+ - median process CPU improves by at least 8 percentage points;
367
+ `i420P95`/`sinkCbP95` improve; `wire` does not regress by more than 5%
368
+ - no malformed frames, busy/fail drops, chroma corruption, or remote freezes
369
+
370
+ If it fails, revert only `decoderMode` to `TEXTURE`. 540p15 stays a selectable
371
+ Home preset. Switching Glasses video back to `720p · 15 fps` is the profile
372
+ rollback; it takes effect on the next join.
373
+
374
+ - **`chroma{}`** should sit near `u≈v≈128` on neutral content. Values pinned at 0 or 255
375
+ mean the planes are mis-packed, which is the signature of a stride or geometry bug.
376
+ - **`dup`** counts refused duplicate track attachments. WebRTC delivers the same track
377
+ through three different observer callbacks as three distinct Java wrappers, so object
378
+ identity cannot dedupe them and `TrackRegistry` keys on `track.id()`. Steady `dup=4`
379
+ (two video, two audio) is correct; climbing `dup` means sinks are being re-added.
380
+
381
+ `scripts/acs-ladder.ts` in the Mentra-Call repo parses these lines and prints pass/fail
382
+ over a trailing 10-second window, including the `recv`-vs-`dec` attribution.
383
+
384
+ ## Tests
385
+
386
+ ```bash
387
+ cd mobile/android && ./gradlew :mentra-acs-meeting:testDebugUnitTest
388
+ ```
389
+
390
+ These are plain JVM tests with no Robolectric. That constrains what can be tested: ACS SDK
391
+ types like `VideoStreamFormat` are native-backed and throw `ExceptionInInitializerError`
392
+ outside an Android runtime. The pattern throughout is to keep the logic worth asserting in
393
+ a pure class and let the thin SDK-facing wrapper go untested — `I420FormatSpec` holds the
394
+ stride arithmetic while `AcsFrameSender.i420Format` just copies it onto the SDK object.
395
+
396
+ Anything concurrent has a test that actually races it: `SendGateTest` contends the gate,
397
+ `RingPercentileTest` writes from multiple threads, and `PipelineStatsTest` reproduces the
398
+ counter race that caused the false conservation failures.
399
+
400
+ ## Known gaps
401
+
402
+ - **Glasses encoder stats are never reported.** The `encoder-stats` event arrives with
403
+ `reported: false`, so the top of the ladder (`src`) stays dark and we cannot yet tell
404
+ whether the glasses are actually producing 15 fps.
405
+ - **Outgoing glasses PCM reads as silence** (`P4 pcm meanAbs=0`) and needs its own
406
+ investigation.
407
+ - **iOS is foreground-only.** The Swift side under `ios/` does not have the telemetry
408
+ ladder, and Android's audio-routing answers do not transfer — re-verify separately.
409
+ iOS also has no `RemoteRoster` yet (`participants` is never emitted) and does not apply
410
+ `maxBitrateBps` to the outgoing stream.
411
+
412
+ The original spike runbook is preserved at [spike/README.md](spike/README.md).
package/THIRD_PARTY ADDED
@@ -0,0 +1,7 @@
1
+ Apache License
2
+ Version 2.0, January 2004
3
+ http://www.apache.org/licenses/
4
+
5
+ This product includes software developed by RealWear, Inc.
6
+ (https://github.com/realwear/collaborate), licensed under the Apache License,
7
+ Version 2.0. A copy of the license is available at the URL above.
@@ -0,0 +1,64 @@
1
+ apply plugin: 'com.android.library'
2
+
3
+ group = 'com.mentra.acsmeeting'
4
+ version = '0.1.0'
5
+
6
+ def expoModulesCorePlugin = new File(project(":expo-modules-core").projectDir.absolutePath, "ExpoModulesCorePlugin.gradle")
7
+ apply from: expoModulesCorePlugin
8
+ applyKotlinExpoModulesCorePlugin()
9
+ useCoreDependencies()
10
+ useExpoPublishing()
11
+
12
+ def useManagedAndroidSdkVersions = false
13
+ if (useManagedAndroidSdkVersions) {
14
+ useDefaultAndroidSdkVersions()
15
+ } else {
16
+ buildscript {
17
+ ext.safeExtGet = { prop, fallback ->
18
+ rootProject.ext.has(prop) ? rootProject.ext.get(prop) : fallback
19
+ }
20
+ }
21
+ project.android {
22
+ compileSdkVersion safeExtGet("compileSdkVersion", 36)
23
+ defaultConfig {
24
+ minSdkVersion safeExtGet("minSdkVersion", 24)
25
+ targetSdkVersion safeExtGet("targetSdkVersion", 36)
26
+ }
27
+ }
28
+ }
29
+
30
+ android {
31
+ namespace "com.mentra.acsmeeting"
32
+ defaultConfig {
33
+ versionCode 1
34
+ versionName "0.1.0"
35
+ }
36
+ compileOptions {
37
+ coreLibraryDesugaringEnabled true
38
+ }
39
+ buildTypes {
40
+ internal {
41
+ initWith release
42
+ matchingFallbacks = ['release']
43
+ }
44
+ }
45
+ testOptions {
46
+ unitTests {
47
+ includeAndroidResources = true
48
+ }
49
+ }
50
+ }
51
+
52
+ dependencies {
53
+ coreLibraryDesugaring 'com.android.tools:desugar_jdk_libs:2.1.4'
54
+ implementation 'com.squareup.okhttp3:okhttp:4.12.0'
55
+ implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-android:1.8.1'
56
+ // ACS Calling SDK with raw-media APIs (I420 VirtualOutgoingVideoStream).
57
+ implementation 'com.azure.android:azure-communication-calling:2.16.0'
58
+ implementation 'com.azure.android:azure-communication-common:1.3.0'
59
+ // Same libwebrtc AAR as @livekit/react-native-webrtc so Gradle dedupes instead
60
+ // of shipping two WebRTC stacks. P4 confirms remote AudioTrackSink.
61
+ implementation 'io.github.webrtc-sdk:android:137.7151.05'
62
+ testImplementation 'junit:junit:4.13.2'
63
+ testImplementation 'org.assertj:assertj-core:3.24.2'
64
+ }
@@ -0,0 +1,5 @@
1
+ <manifest xmlns:android="http://schemas.android.com/apk/res/android">
2
+ <uses-permission android:name="android.permission.INTERNET" />
3
+ <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
4
+ <uses-permission android:name="android.permission.RECORD_AUDIO" />
5
+ </manifest>
@@ -0,0 +1,80 @@
1
+ package com.mentra.acsmeeting
2
+
3
+ import com.mentra.acsmeeting.video.VideoProfile
4
+ import expo.modules.kotlin.modules.Module
5
+ import expo.modules.kotlin.modules.ModuleDefinition
6
+
7
+ class AcsMeetingModule : Module() {
8
+ private var session: AcsMeetingSession? = null
9
+
10
+ override fun definition() = ModuleDefinition {
11
+ Name("MentraAcsMeeting")
12
+ Events("onState", "onIncomingPcm")
13
+
14
+ AsyncFunction("join") { options: Map<String, Any?> ->
15
+ val token = options["token"] as? String ?: throw IllegalArgumentException("token is required")
16
+ val meetingUrl = options["meetingUrl"] as? String ?: throw IllegalArgumentException("meetingUrl is required")
17
+ val whepUrl = options["whepUrl"] as? String ?: throw IllegalArgumentException("whepUrl is required")
18
+ val displayName = options["displayName"] as? String
19
+ val dumpWav = options["dumpPcmWav"] as? Boolean ?: false
20
+ val audioSource = options["audioSource"] as? String ?: "glasses"
21
+ val video = parseVideo(options["video"])
22
+ val context = appContext.reactContext ?: throw IllegalStateException("no react context")
23
+ val meeting = session ?: AcsMeetingSession(
24
+ context.applicationContext,
25
+ onState = { sendEvent("onState", it) },
26
+ onIncomingPcm = { base64, rate, channels ->
27
+ sendEvent(
28
+ "onIncomingPcm",
29
+ mapOf("base64" to base64, "sampleRate" to rate, "channels" to channels),
30
+ )
31
+ },
32
+ ).also { session = it }
33
+ meeting.join(token, meetingUrl, whepUrl, displayName, dumpWav, audioSource, video)
34
+ meeting.getState()
35
+ }
36
+
37
+ AsyncFunction("leave") {
38
+ session?.leave()
39
+ }
40
+
41
+ AsyncFunction("setMuted") { muted: Boolean ->
42
+ session?.setMuted(muted) ?: mapOf("state" to "idle", "muted" to muted)
43
+ }
44
+
45
+ AsyncFunction("setAudioSource") { source: String ->
46
+ session?.setAudioSource(source) ?: mapOf("state" to "idle", "muted" to false, "audioSource" to source)
47
+ }
48
+
49
+ AsyncFunction("updateVideoSource") { whepUrl: String ->
50
+ session?.updateVideoSource(whepUrl)
51
+ }
52
+
53
+ AsyncFunction("restartVideoSource") {
54
+ session?.restartVideoSource()
55
+ }
56
+
57
+ AsyncFunction("getState") {
58
+ session?.getState() ?: mapOf("state" to "idle", "muted" to false)
59
+ }
60
+
61
+ OnDestroy {
62
+ session?.leave()
63
+ session = null
64
+ }
65
+ }
66
+
67
+ private fun parseVideo(raw: Any?): VideoProfile {
68
+ if (raw == null) return VideoProfile.DEFAULT
69
+ val map = raw as? Map<*, *> ?: throw IllegalArgumentException("video must be an object")
70
+ val width = (map["width"] as? Number)?.toInt()
71
+ val height = (map["height"] as? Number)?.toInt()
72
+ val fps = (map["fps"] as? Number)?.toInt()
73
+ val bitrate = (map["maxBitrateBps"] as? Number)?.toInt()
74
+ if (width == null || height == null || fps == null || bitrate == null) {
75
+ throw IllegalArgumentException("video requires width, height, fps, and maxBitrateBps")
76
+ }
77
+ return VideoProfile.parse(width, height, fps, bitrate)
78
+ ?: throw IllegalArgumentException("unsupported ACS video ${width}x${height}@${fps}")
79
+ }
80
+ }