@mentra/acs-meeting 3.2.0-dev.120
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +20 -0
- package/README.md +412 -0
- package/THIRD_PARTY +7 -0
- package/android/build.gradle +64 -0
- package/android/src/main/AndroidManifest.xml +5 -0
- package/android/src/main/java/com/mentra/acsmeeting/AcsMeetingModule.kt +80 -0
- package/android/src/main/java/com/mentra/acsmeeting/AcsMeetingSession.kt +846 -0
- package/android/src/main/java/com/mentra/acsmeeting/RemoteRoster.kt +123 -0
- package/android/src/main/java/com/mentra/acsmeeting/audio/AcsAudioPolicy.kt +90 -0
- package/android/src/main/java/com/mentra/acsmeeting/audio/AudioPolicyApplier.kt +176 -0
- package/android/src/main/java/com/mentra/acsmeeting/audio/IncomingAudioPump.kt +112 -0
- package/android/src/main/java/com/mentra/acsmeeting/audio/IncomingRateProbe.kt +65 -0
- package/android/src/main/java/com/mentra/acsmeeting/audio/PcmBridge.kt +108 -0
- package/android/src/main/java/com/mentra/acsmeeting/audio/PcmResampler.kt +159 -0
- package/android/src/main/java/com/mentra/acsmeeting/audio/PhoneMicCapturer.kt +115 -0
- package/android/src/main/java/com/mentra/acsmeeting/audio/UplinkPacer.kt +302 -0
- package/android/src/main/java/com/mentra/acsmeeting/audio/UplinkSender.kt +230 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/BitmapFont.kt +99 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/CloudflareWhepSource.kt +578 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/FirstFrameGate.kt +45 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/GlassesMediaSource.kt +101 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/MediaListeners.kt +57 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/SyntheticFrameFactory.kt +205 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/SyntheticI420Source.kt +214 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/TrackRegistry.kt +31 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/VideoSourceArm.kt +41 -0
- package/android/src/main/java/com/mentra/acsmeeting/telemetry/ChromaProbe.kt +78 -0
- package/android/src/main/java/com/mentra/acsmeeting/telemetry/PipelineStats.kt +281 -0
- package/android/src/main/java/com/mentra/acsmeeting/telemetry/PipelineTicker.kt +51 -0
- package/android/src/main/java/com/mentra/acsmeeting/telemetry/ProcessCpu.kt +15 -0
- package/android/src/main/java/com/mentra/acsmeeting/telemetry/RingPercentile.kt +33 -0
- package/android/src/main/java/com/mentra/acsmeeting/video/AcsFrameSender.kt +475 -0
- package/android/src/main/java/com/mentra/acsmeeting/video/AcsTimestamp.kt +26 -0
- package/android/src/main/java/com/mentra/acsmeeting/video/FrameGeometry.kt +22 -0
- package/android/src/main/java/com/mentra/acsmeeting/video/I420FormatSpec.kt +90 -0
- package/android/src/main/java/com/mentra/acsmeeting/video/I420Packer.kt +72 -0
- package/android/src/main/java/com/mentra/acsmeeting/video/Nv12FormatSpec.kt +36 -0
- package/android/src/main/java/com/mentra/acsmeeting/video/Nv12Packer.kt +51 -0
- package/android/src/main/java/com/mentra/acsmeeting/video/SendGate.kt +27 -0
- package/android/src/main/java/com/mentra/acsmeeting/video/VideoProfile.kt +57 -0
- package/android/src/test/java/com/mentra/acsmeeting/RemoteRosterTest.kt +40 -0
- package/android/src/test/java/com/mentra/acsmeeting/audio/AcsAudioPolicyTest.kt +204 -0
- package/android/src/test/java/com/mentra/acsmeeting/audio/AudioPolicyApplierTest.kt +223 -0
- package/android/src/test/java/com/mentra/acsmeeting/audio/CapturePolicyTest.kt +12 -0
- package/android/src/test/java/com/mentra/acsmeeting/audio/IncomingAudioPumpTest.kt +97 -0
- package/android/src/test/java/com/mentra/acsmeeting/audio/IncomingRateProbeTest.kt +78 -0
- package/android/src/test/java/com/mentra/acsmeeting/audio/PcmResamplerTest.kt +125 -0
- package/android/src/test/java/com/mentra/acsmeeting/audio/UplinkPacerTest.kt +201 -0
- package/android/src/test/java/com/mentra/acsmeeting/audio/UplinkSenderTest.kt +166 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/AcsInvestigationTest.kt +26 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/BitmapFontTest.kt +22 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/FirstFrameGateTest.kt +60 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/GlassesMediaSourceTest.kt +133 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/I420PlanesTest.kt +67 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/SyntheticFrameFactoryTest.kt +193 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/SyntheticI420SourceTest.kt +176 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/TrackRegistryTest.kt +36 -0
- package/android/src/test/java/com/mentra/acsmeeting/telemetry/ChromaProbeTest.kt +61 -0
- package/android/src/test/java/com/mentra/acsmeeting/telemetry/PipelineStatsTest.kt +256 -0
- package/android/src/test/java/com/mentra/acsmeeting/telemetry/ProcessCpuTest.kt +18 -0
- package/android/src/test/java/com/mentra/acsmeeting/telemetry/RingPercentileTest.kt +49 -0
- package/android/src/test/java/com/mentra/acsmeeting/video/AcsTimestampTest.kt +43 -0
- package/android/src/test/java/com/mentra/acsmeeting/video/FrameGeometryTest.kt +33 -0
- package/android/src/test/java/com/mentra/acsmeeting/video/I420FormatSpecTest.kt +77 -0
- package/android/src/test/java/com/mentra/acsmeeting/video/I420PackerTest.kt +126 -0
- package/android/src/test/java/com/mentra/acsmeeting/video/Nv12FormatSpecTest.kt +36 -0
- package/android/src/test/java/com/mentra/acsmeeting/video/Nv12PackerTest.kt +58 -0
- package/android/src/test/java/com/mentra/acsmeeting/video/SendGateTest.kt +79 -0
- package/android/src/test/java/com/mentra/acsmeeting/video/VideoProfileTest.kt +68 -0
- package/build/AcsMeeting.types.d.ts +59 -0
- package/build/AcsMeeting.types.d.ts.map +1 -0
- package/build/AcsMeeting.types.js +2 -0
- package/build/AcsMeeting.types.js.map +1 -0
- package/build/AcsMeetingModule.d.ts +15 -0
- package/build/AcsMeetingModule.d.ts.map +1 -0
- package/build/AcsMeetingModule.js +3 -0
- package/build/AcsMeetingModule.js.map +1 -0
- package/build/AcsMeetingModule.web.d.ts +16 -0
- package/build/AcsMeetingModule.web.d.ts.map +1 -0
- package/build/AcsMeetingModule.web.js +31 -0
- package/build/AcsMeetingModule.web.js.map +1 -0
- package/build/index.d.ts +3 -0
- package/build/index.d.ts.map +1 -0
- package/build/index.js +3 -0
- package/build/index.js.map +1 -0
- package/expo-module.config.json +9 -0
- package/ios/AcsFrameSender.swift +47 -0
- package/ios/AcsMeeting.podspec +27 -0
- package/ios/AcsMeetingModule.swift +558 -0
- package/ios/PcmBridge.swift +117 -0
- package/ios/PhoneMicCapturer.swift +64 -0
- package/ios/PolicyKit/Package.swift +17 -0
- package/ios/PolicyKit/Sources/AcsAudioPolicy/AcsAudioPolicy.swift +102 -0
- package/ios/PolicyKit/Sources/AcsAudioPolicy/AudioPolicyApplier.swift +162 -0
- package/ios/PolicyKit/Sources/AcsAudioPolicy/GlassesMediaSource.swift +68 -0
- package/ios/PolicyKit/Tests/AcsAudioPolicyTests/AcsAudioPolicyTests.swift +273 -0
- package/ios/WhepVideoSource.swift +356 -0
- package/package.json +56 -0
- package/src/AcsMeeting.types.ts +66 -0
- package/src/AcsMeetingModule.ts +16 -0
- package/src/AcsMeetingModule.web.ts +33 -0
- package/src/index.ts +2 -0
package/NOTICE
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
Apache License 2.0 third-party notice
|
|
2
|
+
=====================================
|
|
3
|
+
|
|
4
|
+
This module contains Kotlin adapted from RealWear Collaborate (Apache License 2.0).
|
|
5
|
+
|
|
6
|
+
Upstream: https://github.com/realwear/collaborate
|
|
7
|
+
Commit: 2a143fd18c952c91e1cbae525d8126a858c18edf (HEAD at import)
|
|
8
|
+
Source file: android/apps/rwt/app/src/main/java/com/realwear/acs/util/thermal/VideoFrameSender.kt
|
|
9
|
+
Adapted into: android/src/main/java/com/mentra/acsmeeting/AcsFrameSender.kt
|
|
10
|
+
|
|
11
|
+
Patterns used (not copied verbatim):
|
|
12
|
+
- direct ByteBuffer pooling and reuse
|
|
13
|
+
- format negotiation from RawOutgoingVideoStream constraints
|
|
14
|
+
- frame pacing to the negotiated FPS
|
|
15
|
+
- stream-state handling (start/stop/format-changed) and error paths
|
|
16
|
+
|
|
17
|
+
iOS sources in this module are original Mentra code on the official ACS iOS
|
|
18
|
+
raw-video API (CVPixelBuffer). They do not contain RealWear code.
|
|
19
|
+
|
|
20
|
+
The Apache 2.0 license text is in THIRD_PARTY.
|
package/README.md
ADDED
|
@@ -0,0 +1,412 @@
|
|
|
1
|
+
# @mentra/acs-meeting
|
|
2
|
+
|
|
3
|
+
Phone-native Azure Communication Services client that puts a MentraOS wearer into a
|
|
4
|
+
Microsoft Teams meeting as a guest. The glasses provide the camera and the microphone;
|
|
5
|
+
the phone does all the WebRTC and ACS work.
|
|
6
|
+
|
|
7
|
+
The phone is a **relay, not a capture device**. It subscribes to whatever the glasses
|
|
8
|
+
already published to Cloudflare, decodes it, and re-publishes it into ACS. No frame
|
|
9
|
+
crosses the JavaScript bridge. Production never generates pixels on the phone.
|
|
10
|
+
|
|
11
|
+
An investigation-only synthetic arm can generate packed I420 locally at a fixed 15 Hz
|
|
12
|
+
(`AcsInvestigation.videoArm`). It ships as `WHEP`. Flip it locally, rebuild native, and
|
|
13
|
+
do not commit `SYNTHETIC`.
|
|
14
|
+
|
|
15
|
+
`AcsInvestigation.decoderMode` ships as `TEXTURE` (shared EGL, MediaCodec to Surface).
|
|
16
|
+
`BYTE_BUFFER` skips the GL readback. A 720p hop on `SM_S948U` decoded in hardware
|
|
17
|
+
with `i420P95=0`, but did not clear the campaign gates; keep the flag on TEXTURE.
|
|
18
|
+
`AcsInvestigation.zeroCopy` ships off; when on,
|
|
19
|
+
tight retainable WebRTC planes go straight to ACS, with an automatic copy fallback.
|
|
20
|
+
`AcsInvestigation.pixelFormat` ships as `I420`. `NV12` is the encoder-flip A/B:
|
|
21
|
+
advertise and send biplanar NV12 and read `codec=` on the ladder. Revert unless
|
|
22
|
+
`codecName` leaves `h264 sw`.
|
|
23
|
+
|
|
24
|
+
Mentra Call's persisted default is `720p15` (`VideoProfile.DEFAULT = HD`,
|
|
25
|
+
1280×720@15 / 2.5 Mbps). `540p15` (960×540@15 / 1.5 Mbps) is a user-selectable
|
|
26
|
+
preset on the Home settings picker. It is not migrated onto existing installs
|
|
27
|
+
and is not the native default. Miniapp joins pass width, height, fps, and
|
|
28
|
+
`maxBitrateBps` through to native, so a selected `540p15` reaches both glasses
|
|
29
|
+
WHIP and ACS without changing `VideoProfile.DEFAULT`.
|
|
30
|
+
|
|
31
|
+
## The pipeline in one picture
|
|
32
|
+
|
|
33
|
+
```
|
|
34
|
+
glasses ──WHIP──▶ Cloudflare ──WHEP──▶ │ phone (this module) │ ──▶ ACS ──▶ Teams
|
|
35
|
+
│ │
|
|
36
|
+
video: H.264 ─▶ WebRTC decoder ─▶ I420 ─▶ cropAndScale ─▶ I420 copy or NV12 interleave ─▶ sendRawVideoFrame
|
|
37
|
+
audio: Opus ─▶ WebRTC decoder ─▶ PCM16 48k ─▶ sendRawAudioBuffer
|
|
38
|
+
|
|
39
|
+
return audio: ACS RawIncomingAudioStream ─▶ base64 ─▶ Expo event ─▶ AudioPlaybackService ─▶ A2DP
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Two things about this are load-bearing and easy to get wrong:
|
|
43
|
+
|
|
44
|
+
**ACS encodes for us.** `RawOutgoingVideoStream` accepts raw pixels only — `I420`, `NV12`,
|
|
45
|
+
`RGBA`, and a few others — never an H.264 bitstream. We cannot forward the Cloudflare
|
|
46
|
+
H.264 as-is; it must be decoded and handed over as pixels. ACS then re-encodes to H.264
|
|
47
|
+
internally. The decode is unavoidable; the *conversion* is not, which is why we feed I420
|
|
48
|
+
(what the WebRTC decoder already produces) rather than converting to RGBA.
|
|
49
|
+
|
|
50
|
+
**Only the video path is hot.** At 720p15 the video path moves ~1.4 MB per frame, 15 times
|
|
51
|
+
a second. Everything in `video/` and `source/` is written to avoid per-pixel work in
|
|
52
|
+
Kotlin: scaling is libyuv via `cropAndScale`, packing is a bulk `ByteBuffer` copy, and
|
|
53
|
+
plane buffers are pooled rather than allocated.
|
|
54
|
+
|
|
55
|
+
## Layout
|
|
56
|
+
|
|
57
|
+
```
|
|
58
|
+
android/src/main/java/com/mentra/acsmeeting/
|
|
59
|
+
├── AcsMeetingModule.kt Expo bridge: join/leave/setMuted/getState, onState + onIncomingPcm events
|
|
60
|
+
├── AcsMeetingSession.kt Orchestrator. Owns the ACS Call and wires the four packages together
|
|
61
|
+
│
|
|
62
|
+
├── source/ Upstream — getting glasses media into the process
|
|
63
|
+
│ ├── MediaListeners.kt I420Planes, VideoFrameListener, PcmListener
|
|
64
|
+
│ ├── GlassesMediaSource.kt The transport interface + controller. WHEP is one implementation
|
|
65
|
+
│ ├── CloudflareWhepSource.kt The recvonly WHEP subscriber (PeerConnection, sinks, scaling)
|
|
66
|
+
│ ├── SyntheticFrameFactory.kt Packed I420 generator (CHEAP / MOTION pan / NOISE)
|
|
67
|
+
│ ├── SyntheticI420Source.kt Fixed-rate GlassesMediaSource wrapping the factory
|
|
68
|
+
│ ├── VideoSourceArm.kt Investigation switch. Ships as WHEP / TEXTURE / I420 / zeroCopy=off
|
|
69
|
+
│ └── TrackRegistry.kt Deduplicates track attachment
|
|
70
|
+
│
|
|
71
|
+
├── video/ Downstream — pixels into ACS
|
|
72
|
+
│ ├── AcsFrameSender.kt Owns the RawOutgoingVideoStream and the send executor
|
|
73
|
+
│ ├── I420FormatSpec.kt The I420 format we advertise, as plain values
|
|
74
|
+
│ ├── Nv12FormatSpec.kt The NV12 format we advertise, as plain values
|
|
75
|
+
│ ├── AcsTimestamp.kt 100-ns ticks for RawVideoFrame. Zero means freeze.
|
|
76
|
+
│ ├── I420Packer.kt Stride-aware planar copy; planeMinBytes for malformed guards
|
|
77
|
+
│ ├── Nv12Packer.kt I420 → interleaved UV; used only on the NV12 arm
|
|
78
|
+
│ ├── FrameGeometry.kt Buffer-vs-display coordinates under rotation
|
|
79
|
+
│ └── SendGate.kt Single-in-flight backpressure
|
|
80
|
+
│
|
|
81
|
+
├── audio/ Two microphones, one call
|
|
82
|
+
│ ├── AcsAudioPolicy.kt Pure decision functions: which stream, who gets muted
|
|
83
|
+
│ ├── AudioPolicyApplier.kt Applies a decision to the live call, with retries
|
|
84
|
+
│ └── PcmBridge.kt Downmix/rebuffer to ACS's 48 kHz mono 20 ms frames
|
|
85
|
+
│
|
|
86
|
+
└── telemetry/ The measurement ladder
|
|
87
|
+
├── PipelineStats.kt Every counter, and the 1 Hz "P6 ladder" line
|
|
88
|
+
├── PipelineTicker.kt Emits that line on a timer
|
|
89
|
+
├── RingPercentile.kt p50/p95 over a fixed ring of samples
|
|
90
|
+
└── ChromaProbe.kt Plane averages, to catch a mis-packed frame
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Tests mirror this exactly under `android/src/test/java/com/mentra/acsmeeting/`.
|
|
94
|
+
|
|
95
|
+
The dependency direction is `AcsMeetingSession → {source, video, audio, telemetry}`, with
|
|
96
|
+
`source` and `video` both depending on `telemetry` to report, and `telemetry/ChromaProbe`
|
|
97
|
+
reaching into `video/I420Packer` for stride math. Nothing in `audio/` touches video.
|
|
98
|
+
|
|
99
|
+
## Video path
|
|
100
|
+
|
|
101
|
+
`CloudflareWhepSource` POSTs an SDP offer to the WHEP endpoint, gets an answer, and adds a
|
|
102
|
+
`VideoSink` to the remote track. Per frame, on WebRTC's decode thread:
|
|
103
|
+
|
|
104
|
+
1. If ACS negotiated a different size than the decoder is producing, scale with
|
|
105
|
+
`buffer.cropAndScale(...)` — libyuv, not Kotlin.
|
|
106
|
+
2. `toI420()`, then hand the three planes to `AcsFrameSender.sendPlanes` as `I420Planes`.
|
|
107
|
+
3. The sender copies each plane once into a pooled ACS buffer (`copyP95`), or — when
|
|
108
|
+
`zeroCopy` is on and the planes are tight, direct, and retainable — retains the
|
|
109
|
+
WebRTC buffer and submits those planes without a copy. The NV12 investigation arm
|
|
110
|
+
always converts: Y is copied, U/V are interleaved, and ACS gets two buffers.
|
|
111
|
+
|
|
112
|
+
ACS's I420 contract still requires three separate plane buffers. NV12 is two
|
|
113
|
+
(Y + interleaved UV). The old pack-then-split path is gone. A timed-out zero-copy
|
|
114
|
+
send schedules `release()` after a 1 s grace window so the decoder pool is not starved.
|
|
115
|
+
|
|
116
|
+
Three constraints worth knowing before changing this code:
|
|
117
|
+
|
|
118
|
+
- **No inter-frame pacer.** `SendGate` allows exactly one send in flight and nothing else.
|
|
119
|
+
Any minimum-interarrival gate deletes real frames whenever the decoder delivers in
|
|
120
|
+
bursts, which it does.
|
|
121
|
+
- **Buffer coordinates, never display coordinates.** `cropAndScale` and `toI420` operate on
|
|
122
|
+
the un-rotated buffer. Mixing in `VideoFrame.rotatedWidth/rotatedHeight` overruns the
|
|
123
|
+
plane whenever rotation is 90° or 270°. `FrameGeometry` exists to keep this honest, and
|
|
124
|
+
the ladder prints `rot=` so a non-zero rotation is visible rather than silently corrupting.
|
|
125
|
+
- **A timed-out send does not release its buffers.** `Future.get(timeout)` does not cancel
|
|
126
|
+
the work, so ACS may still be reading those planes. They are deliberately leaked rather
|
|
127
|
+
than recycled, and counted as `abandoned` in the ladder.
|
|
128
|
+
|
|
129
|
+
### Source health and recovery
|
|
130
|
+
|
|
131
|
+
The ACS call and the WHEP subscription fail independently: ICE can drop (phone switched
|
|
132
|
+
Wi-Fi↔cellular) or the WHEP endpoint can 404 (glasses stopped publishing) while ACS stays
|
|
133
|
+
`connected` and Teams holds the last frame. `CloudflareWhepSource` reports every
|
|
134
|
+
`SourceState` transition to the session, which:
|
|
135
|
+
|
|
136
|
+
- carries it on every snapshot as `mediaSource: idle | connecting | live | failed`, so the
|
|
137
|
+
host and miniapp can tell "call up, glasses feed dead" from a healthy call;
|
|
138
|
+
- on `failed`, rebuilds the subscription itself with exponential backoff (1 s → 10 s cap) for
|
|
139
|
+
as long as the call is alive. A host `updateVideoSource` with a new URL cancels the retry.
|
|
140
|
+
|
|
141
|
+
**`LIVE` means a frame reached the sink, not that the WHEP endpoint answered.** The answer
|
|
142
|
+
lands before `setRemoteDescription`, before ICE CHECKING and before the first decode — on an
|
|
143
|
+
S22 reconnect, 3.2 s before the first frame (`whep_answer` 15:56:47.16, `sub=1.0` 15:56:50.36)
|
|
144
|
+
and 6.2 s before a real rate to Teams. Two things followed from promoting on the answer: a
|
|
145
|
+
miniapp reading `mediaSource: live` was still ahead of Teams, and a subscription that answered
|
|
146
|
+
but never delivered read `LIVE` forever behind a frozen frame with nothing able to see it.
|
|
147
|
+
|
|
148
|
+
So the answer arms `FirstFrameGate` and stays `CONNECTING`. The first video frame for that
|
|
149
|
+
peer generation promotes to `LIVE` (`first_frame`); if none arrives within
|
|
150
|
+
`FIRST_FRAME_TIMEOUT_MS` (9 s) the source transitions `FAILED` with reason `no_first_frame`
|
|
151
|
+
and the backoff above rebuilds it. An ICE bounce back to CONNECTED returns the source to
|
|
152
|
+
`CONNECTING` and rearms rather than claiming `LIVE`: a recovered candidate pair is a promise
|
|
153
|
+
of frames, and if media was already flowing the next frame promotes within milliseconds.
|
|
154
|
+
|
|
155
|
+
`canReuseSource` is why a same-URL `restart` during `CONNECTING` is still a no-op: rebuilding
|
|
156
|
+
a subscriber seconds from its first frame only restarts the wait, and `CONNECTING` is bounded
|
|
157
|
+
by the POST timeout and the first-frame deadline, so nothing can strand there.
|
|
158
|
+
|
|
159
|
+
`restartVideoSource()` forces a rebuild on the current URL even when the peer still looks
|
|
160
|
+
healthy; the host calls it from a NetInfo listener when the phone's network identity changes.
|
|
161
|
+
|
|
162
|
+
iOS `WhepVideoSource` mirrors all of this. `sub=` on the P6 ladder is the ground truth for
|
|
163
|
+
"Teams has video"; `first_frame` is the transition that should sit next to it in logcat.
|
|
164
|
+
|
|
165
|
+
## Audio path
|
|
166
|
+
|
|
167
|
+
Two possible microphones — the glasses or the handset — and the wrong answer means either
|
|
168
|
+
silence or the wearer being recorded when they think they are muted. The decision is
|
|
169
|
+
therefore a pure function in `AcsAudioPolicy`, unit-tested in isolation, and separate from
|
|
170
|
+
the code that applies it.
|
|
171
|
+
|
|
172
|
+
- **`audioSource: "glasses"`** arms a `RawOutgoingAudioStream` — which ACS reports as
|
|
173
|
+
`VIRTUAL_OUTGOING` — and feeds it WHEP PCM through `PcmBridge`, which downmixes to mono
|
|
174
|
+
48 kHz (passthrough when WHEP already decoded at 48 kHz) and re-chunks into the 20 ms
|
|
175
|
+
frames ACS expects. The handset mic is never opened. Incoming raw audio stays 16 kHz.
|
|
176
|
+
- **`audioSource: "phone"`** uses ACS's own `LocalOutgoingAudioStream` (`LOCAL_OUTGOING`).
|
|
177
|
+
|
|
178
|
+
`ActiveStreamKind` mirrors that pair, and deliberately reads `NONE` for a stream that is
|
|
179
|
+
attached but not `STARTED` — "attached" is not "live", and treating it as live is how a
|
|
180
|
+
muted wearer ends up audible.
|
|
181
|
+
|
|
182
|
+
`AudioPolicyApplier` reports an `AudioSafety` of `safe`, `degraded`, or `unsafe`. `unsafe`
|
|
183
|
+
means both mute and stop failed and a live mic may be reaching the meeting; it is surfaced
|
|
184
|
+
all the way to JS in `AcsMeetingState.audioSafety`.
|
|
185
|
+
|
|
186
|
+
Return audio does **not** use ACS playback. Incoming PCM is base64'd to JS and played by
|
|
187
|
+
`AudioPlaybackService` / `PcmStreamPlayer`, so it reaches the glasses over A2DP and
|
|
188
|
+
participates in MCU duplex via `setOwnAppAudioPlaying`.
|
|
189
|
+
|
|
190
|
+
## Threads
|
|
191
|
+
|
|
192
|
+
| Thread | What runs on it |
|
|
193
|
+
|---|---|
|
|
194
|
+
| `AcsMeetingSession` single-thread executor | join, leave, and every audio-policy application |
|
|
195
|
+
| `acs-synthetic-i420` | investigation arm only: generate + emit at fixed fps |
|
|
196
|
+
| WebRTC decode thread | the video sink: scale, pack, hand off |
|
|
197
|
+
| `acs-i420-send` | `sendRawVideoFrame` and nothing else |
|
|
198
|
+
| main looper | the 1 Hz ticker and the WebRTC `getStats` poll |
|
|
199
|
+
|
|
200
|
+
The rule that matters: **nothing blocking may run on the WebRTC decode thread.** A
|
|
201
|
+
`PeerConnection.getStats()` call there once stalled video after roughly one second. The
|
|
202
|
+
stats poll is now timer-driven on the main looper, which also means it keeps sampling
|
|
203
|
+
during a freeze instead of going quiet exactly when the data is interesting.
|
|
204
|
+
|
|
205
|
+
## Reading the telemetry
|
|
206
|
+
|
|
207
|
+
```
|
|
208
|
+
adb logcat -s ACS-SPIKE | grep "P6 ladder"
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
One line per second, tracing a frame through every hop:
|
|
212
|
+
|
|
213
|
+
```
|
|
214
|
+
P6 ladder arm=whep 1280x720 recv=14.8 dec=8.1 sink=8.0 dup=4 sub=8.0 wire=1280x720@8.0 kbps=1400 codec=h264_sw rot=0
|
|
215
|
+
drop{size=0 busy=0 notStarted=0 fail=0 nullI420=0 abandoned=0}
|
|
216
|
+
recv{drop=6 lost=0 nack=0 pli=0 freeze=2 freezeSec=3.4 jit=4.0 decMs=3.2 jbMs=210.0 decImpl=OMX.qcom.video.decoder.avc}
|
|
217
|
+
path{mode=texture copy=planes pix=i420} buf{tex=12 i420=0 other=0}
|
|
218
|
+
stride{y=1280 u=768 v=768 tight=0 padded=12} zc{on=0 used=0 fell=0 padded=0 heldMax=0 timeout=0}
|
|
219
|
+
ms{gapP50=68.3 gapP95=523.4 i420P95=12.0 packP95=na scaleP95=na sinkCbP95=20.0 splitP95=na copyP95=4.1 sendP95=2.7}
|
|
220
|
+
alloc{dest=0 plane=3} chroma{y=162 u=132 v=132} cum{sink=1000 sub=1000 drop=0 inFlight=0}
|
|
221
|
+
cpu{proc=112.4 cores=8}
|
|
222
|
+
```
|
|
223
|
+
|
|
224
|
+
The rates form a ladder, and the first place two adjacent numbers diverge is the bottleneck:
|
|
225
|
+
|
|
226
|
+
| Field | Hop |
|
|
227
|
+
|---|---|
|
|
228
|
+
| `recv` | frames WebRTC pulled off the wire (`framesReceived`) |
|
|
229
|
+
| `dec` | frames the decoder produced (`framesDecoded`) |
|
|
230
|
+
| `sink` | frames our `VideoSink` saw |
|
|
231
|
+
| `sub` | frames submitted to ACS |
|
|
232
|
+
| `wire` | frames ACS actually encoded (`Features.MEDIA_STATISTICS`) |
|
|
233
|
+
|
|
234
|
+
So `recv` well under 15 blames the network or Cloudflare; a healthy `recv` with a low `dec`
|
|
235
|
+
blames the decoder or jitter buffer; `sink` below `dec` means frames are being dropped
|
|
236
|
+
before us; and `sub` below `sink` means we are dropping them, with `drop{}` naming which
|
|
237
|
+
reason.
|
|
238
|
+
|
|
239
|
+
The rest:
|
|
240
|
+
|
|
241
|
+
- **`cum{}`** is monotonic and is what pass/fail uses. `inFlight` is *derived* as
|
|
242
|
+
`sink - sub - drop`, which makes conservation an identity — a tracked counter cannot be
|
|
243
|
+
read atomically alongside the others and produced false `CONSERVE_FAIL` alarms on a
|
|
244
|
+
perfectly healthy pipeline.
|
|
245
|
+
- **`ms{}`** is our own cost. At 15 fps the budget is 66 ms. `i420P95` is `toI420()`
|
|
246
|
+
(GL readback when `path.mode=texture`; libyuv when `bytebuf`). `copyP95` is the single
|
|
247
|
+
plane copy into ACS buffers. `packP95`/`splitP95` stay in the line for old captures
|
|
248
|
+
and print `na` on the planes path. `sinkCbP95` is the whole decode-thread callback.
|
|
249
|
+
- **`path{}` / `buf{}` / `stride{}` / `zc{}`** say which arm produced the line. `buf.tex`
|
|
250
|
+
climbing with `mode=texture` is the Surface decoder. `buf.i420` climbing with
|
|
251
|
+
`mode=bytebuf` is the A/B succeeding. `pix=nv12` plus `copy=nv12` is the encoder-flip
|
|
252
|
+
arm. `codec=` is ACS `OutgoingVideoStatistics.codecName` — the NV12 experiment
|
|
253
|
+
succeeds only if this leaves `h264_sw`. `stride.padded` dominating means zero-copy will
|
|
254
|
+
fall back (tight-plane only). `decImpl` turning into a software name is an abort for
|
|
255
|
+
the byte-buffer arm.
|
|
256
|
+
- **`cpu{proc}`** is process CPU% the same way `top` reports it (can exceed 100 on
|
|
257
|
+
multi-core). Pair it with the `ms{}` stages: high `i420`/`sinkCb` + high `proc` supports
|
|
258
|
+
convert-on-decode-thread; high `sendP95` with low `i420` blames ACS encode/submit.
|
|
259
|
+
Preview is JS/UI and is not in this line — toggle it and watch `cpu{proc}`.
|
|
260
|
+
- **`alloc{}`** counts dest/plane direct-buffer allocations. After the first frames these
|
|
261
|
+
should stay flat if pooling holds. Climbing values mean we are still allocating per frame.
|
|
262
|
+
|
|
263
|
+
A/B capture (preview off, ~90 s of steady motion, same room):
|
|
264
|
+
|
|
265
|
+
```bash
|
|
266
|
+
adb logcat -c
|
|
267
|
+
# join the Teams call, hold steady motion
|
|
268
|
+
adb logcat -d -s ACS-SPIKE > /tmp/acs-<arm>.txt
|
|
269
|
+
bun scripts/acs-ladder.ts --compare /tmp/acs-baseline.txt /tmp/acs-<arm>.txt --skip-ms 20000
|
|
270
|
+
```
|
|
271
|
+
|
|
272
|
+
Compare uses medians, not the last tick. A 15% `recv` gap or a 2x `lost` gap prints a
|
|
273
|
+
confound warning: the network differed and the CPU delta is not attributable. Capture
|
|
274
|
+
order is baseline (texture + planes, `zeroCopy=false`, `pixelFormat=I420`) first,
|
|
275
|
+
then one flag at a time. The NV12 encoder-flip is `pixelFormat=NV12`; keep it
|
|
276
|
+
only if `codec=` leaves `h264_sw`.
|
|
277
|
+
|
|
278
|
+
### 540p15 + BYTE_BUFFER campaign (operator steps)
|
|
279
|
+
|
|
280
|
+
`720p15` stays the Home and native default. `540p15` is selectable. Arms 1–2
|
|
281
|
+
were captured on `SM_S906U` (Snapdragon, not the A54) with preview off,
|
|
282
|
+
`pixelFormat=I420`, `zeroCopy=false`. Loss on the 720p window was >2× the 540p
|
|
283
|
+
window, so the CPU delta is **not** attributable. Wire fps is.
|
|
284
|
+
|
|
285
|
+
Hold these constants for arm 3: same phone, same network, same Teams meeting,
|
|
286
|
+
same ~90 s walking/head-turn motion after a 20 s warmup, preview off, picker
|
|
287
|
+
still on `540p · 15 fps`.
|
|
288
|
+
|
|
289
|
+
Confirm the join logs before saving each dump:
|
|
290
|
+
|
|
291
|
+
- glasses WHIP: `whip start (1280x720@15)` or `whip start (960x540@15)`
|
|
292
|
+
- miniapp ACS handoff: `requestedAcs=1280×720 @15` or `requestedAcs=960×540 @15`
|
|
293
|
+
- native: `P5 negotiated ... 1280x720 fps=15` or `P5 negotiated ... 960x540 fps=15`
|
|
294
|
+
- ladder: `wire=1280x720@...` or `wire=960x540@...`
|
|
295
|
+
|
|
296
|
+
**Arm 1 — 720p15 / TEXTURE (baseline).** Leave Glasses video on `720p · 15 fps`.
|
|
297
|
+
Confirm `AcsInvestigation.decoderMode` is still `TEXTURE`. Clear logcat, join,
|
|
298
|
+
hold motion, dump:
|
|
299
|
+
|
|
300
|
+
```bash
|
|
301
|
+
adb logcat -c
|
|
302
|
+
# join Teams on the A54, 90 s of steady motion, preview off
|
|
303
|
+
adb logcat -d -s ACS-SPIKE > /tmp/acs-720p15-texture.txt
|
|
304
|
+
```
|
|
305
|
+
|
|
306
|
+
Leave the call.
|
|
307
|
+
|
|
308
|
+
**Arm 2 — 540p15 / TEXTURE (resolution savings).** On Home → Glasses video, select
|
|
309
|
+
`540p · 15 fps`. The summary under the picker should read
|
|
310
|
+
`960×540 @ 15 · 1.5 Mbps`. Decoder stays `TEXTURE`. Rejoin the same meeting,
|
|
311
|
+
same motion, dump `/tmp/acs-540p15-texture.txt`. Leave.
|
|
312
|
+
|
|
313
|
+
Compare resolution-only savings (still no decoder flip):
|
|
314
|
+
|
|
315
|
+
```bash
|
|
316
|
+
bun scripts/acs-ladder.ts --compare /tmp/acs-720p15-texture.txt /tmp/acs-540p15-texture.txt --skip-ms 20000
|
|
317
|
+
```
|
|
318
|
+
|
|
319
|
+
Recorded on `SM_S906U` 2026-09-01 (medians, 20 s warmup on 540p; 720p is the
|
|
320
|
+
long 15:52 window). `recv` matched (14.9 vs 14.8). `lost` 39 → 7 (confound).
|
|
321
|
+
|
|
322
|
+
| field | 720p15 / TEXTURE | 540p15 / TEXTURE |
|
|
323
|
+
|---|---:|---:|
|
|
324
|
+
| recv | 14.9 | 14.8 |
|
|
325
|
+
| dec / sink / sub | 11 / 11 / 11 | 12.9 / 12 / 12 |
|
|
326
|
+
| wire fps | 8.6 | 13.2 |
|
|
327
|
+
| kbps | 1332 | 1357 |
|
|
328
|
+
| i420P95 / sinkCbP95 / copyP95 | 10.9 / 16.8 / 5.4 | 10.1 / 14.7 / 3.9 |
|
|
329
|
+
| cpu{proc} | 109 | 95 (confounded) |
|
|
330
|
+
| decImpl | `c2.qti.avc.decoder` | `c2.qti.avc.decoder` |
|
|
331
|
+
| codec | `h264_sw` | `h264_sw` |
|
|
332
|
+
|
|
333
|
+
Wire is the win: same ~1.4 Mbps, ~9 → ~13 fps. Encoder stayed software. Keep
|
|
334
|
+
540p15 selectable; do not make it the default.
|
|
335
|
+
|
|
336
|
+
**Arm 3 — 540p15 / BYTE_BUFFER (decoder savings).** Keep the picker on 540p15.
|
|
337
|
+
Flip `AcsInvestigation.decoderMode` to `BYTE_BUFFER` locally. Rebuild native,
|
|
338
|
+
rejoin, dump `/tmp/acs-540p15-bytebuffer.txt`. Leave.
|
|
339
|
+
|
|
340
|
+
```bash
|
|
341
|
+
bun scripts/acs-ladder.ts --compare /tmp/acs-540p15-texture.txt /tmp/acs-540p15-bytebuffer.txt --skip-ms 20000
|
|
342
|
+
```
|
|
343
|
+
|
|
344
|
+
Recorded on `SM_S948U` 2026-09-01 at **720p15** (wrong profile vs the plan;
|
|
345
|
+
different SoC vs arms 1–2). Mechanical decode passed; campaign gates did not.
|
|
346
|
+
|
|
347
|
+
| field | 720p15 / BYTE_BUFFER (`SM_S948U`) |
|
|
348
|
+
|---|---:|
|
|
349
|
+
| path / buf | `bytebuf`, `tex=0`, `i420` climbing |
|
|
350
|
+
| recv / dec / sink | 15 / 15 / 15 |
|
|
351
|
+
| i420P95 / sinkCbP95 / copyP95 | 0.0 / 1.0 / 0.4 |
|
|
352
|
+
| cpu{proc} | ~44 (not comparable to SM_S906U ~109) |
|
|
353
|
+
| decImpl | `c2.qti.avc.decoder` |
|
|
354
|
+
| wire / codec | `na` / `na` |
|
|
355
|
+
| drop.busy | climbing (~18–34) |
|
|
356
|
+
|
|
357
|
+
Do not promote BYTE_BUFFER: no wire fps, no `codec=`, busy backpressure, and
|
|
358
|
+
no same-phone 540p TEXTURE compare. `decoderMode` stays `TEXTURE`.
|
|
359
|
+
|
|
360
|
+
Promote BYTE_BUFFER as the guarded default only when every gate passes:
|
|
361
|
+
|
|
362
|
+
- `path{mode=bytebuf}`, `buf{i420}` rises, and `buf{tex}=0`
|
|
363
|
+
- `recv` differs by at most 15% and packet loss by at most 2× (else confounded)
|
|
364
|
+
- `decImpl` stays hardware (`OMX.qcom`, `c2.qti`, vendor). Abort on
|
|
365
|
+
`c2.android`, `c2.google`, `OMX.google`, or a blank/stalled decoder
|
|
366
|
+
- median process CPU improves by at least 8 percentage points;
|
|
367
|
+
`i420P95`/`sinkCbP95` improve; `wire` does not regress by more than 5%
|
|
368
|
+
- no malformed frames, busy/fail drops, chroma corruption, or remote freezes
|
|
369
|
+
|
|
370
|
+
If it fails, revert only `decoderMode` to `TEXTURE`. 540p15 stays a selectable
|
|
371
|
+
Home preset. Switching Glasses video back to `720p · 15 fps` is the profile
|
|
372
|
+
rollback; it takes effect on the next join.
|
|
373
|
+
|
|
374
|
+
- **`chroma{}`** should sit near `u≈v≈128` on neutral content. Values pinned at 0 or 255
|
|
375
|
+
mean the planes are mis-packed, which is the signature of a stride or geometry bug.
|
|
376
|
+
- **`dup`** counts refused duplicate track attachments. WebRTC delivers the same track
|
|
377
|
+
through three different observer callbacks as three distinct Java wrappers, so object
|
|
378
|
+
identity cannot dedupe them and `TrackRegistry` keys on `track.id()`. Steady `dup=4`
|
|
379
|
+
(two video, two audio) is correct; climbing `dup` means sinks are being re-added.
|
|
380
|
+
|
|
381
|
+
`scripts/acs-ladder.ts` in the Mentra-Call repo parses these lines and prints pass/fail
|
|
382
|
+
over a trailing 10-second window, including the `recv`-vs-`dec` attribution.
|
|
383
|
+
|
|
384
|
+
## Tests
|
|
385
|
+
|
|
386
|
+
```bash
|
|
387
|
+
cd mobile/android && ./gradlew :mentra-acs-meeting:testDebugUnitTest
|
|
388
|
+
```
|
|
389
|
+
|
|
390
|
+
These are plain JVM tests with no Robolectric. That constrains what can be tested: ACS SDK
|
|
391
|
+
types like `VideoStreamFormat` are native-backed and throw `ExceptionInInitializerError`
|
|
392
|
+
outside an Android runtime. The pattern throughout is to keep the logic worth asserting in
|
|
393
|
+
a pure class and let the thin SDK-facing wrapper go untested — `I420FormatSpec` holds the
|
|
394
|
+
stride arithmetic while `AcsFrameSender.i420Format` just copies it onto the SDK object.
|
|
395
|
+
|
|
396
|
+
Anything concurrent has a test that actually races it: `SendGateTest` contends the gate,
|
|
397
|
+
`RingPercentileTest` writes from multiple threads, and `PipelineStatsTest` reproduces the
|
|
398
|
+
counter race that caused the false conservation failures.
|
|
399
|
+
|
|
400
|
+
## Known gaps
|
|
401
|
+
|
|
402
|
+
- **Glasses encoder stats are never reported.** The `encoder-stats` event arrives with
|
|
403
|
+
`reported: false`, so the top of the ladder (`src`) stays dark and we cannot yet tell
|
|
404
|
+
whether the glasses are actually producing 15 fps.
|
|
405
|
+
- **Outgoing glasses PCM reads as silence** (`P4 pcm meanAbs=0`) and needs its own
|
|
406
|
+
investigation.
|
|
407
|
+
- **iOS is foreground-only.** The Swift side under `ios/` does not have the telemetry
|
|
408
|
+
ladder, and Android's audio-routing answers do not transfer — re-verify separately.
|
|
409
|
+
iOS also has no `RemoteRoster` yet (`participants` is never emitted) and does not apply
|
|
410
|
+
`maxBitrateBps` to the outgoing stream.
|
|
411
|
+
|
|
412
|
+
The original spike runbook is preserved at [spike/README.md](spike/README.md).
|
package/THIRD_PARTY
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
Apache License
|
|
2
|
+
Version 2.0, January 2004
|
|
3
|
+
http://www.apache.org/licenses/
|
|
4
|
+
|
|
5
|
+
This product includes software developed by RealWear, Inc.
|
|
6
|
+
(https://github.com/realwear/collaborate), licensed under the Apache License,
|
|
7
|
+
Version 2.0. A copy of the license is available at the URL above.
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
apply plugin: 'com.android.library'
|
|
2
|
+
|
|
3
|
+
group = 'com.mentra.acsmeeting'
|
|
4
|
+
version = '0.1.0'
|
|
5
|
+
|
|
6
|
+
def expoModulesCorePlugin = new File(project(":expo-modules-core").projectDir.absolutePath, "ExpoModulesCorePlugin.gradle")
|
|
7
|
+
apply from: expoModulesCorePlugin
|
|
8
|
+
applyKotlinExpoModulesCorePlugin()
|
|
9
|
+
useCoreDependencies()
|
|
10
|
+
useExpoPublishing()
|
|
11
|
+
|
|
12
|
+
def useManagedAndroidSdkVersions = false
|
|
13
|
+
if (useManagedAndroidSdkVersions) {
|
|
14
|
+
useDefaultAndroidSdkVersions()
|
|
15
|
+
} else {
|
|
16
|
+
buildscript {
|
|
17
|
+
ext.safeExtGet = { prop, fallback ->
|
|
18
|
+
rootProject.ext.has(prop) ? rootProject.ext.get(prop) : fallback
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
project.android {
|
|
22
|
+
compileSdkVersion safeExtGet("compileSdkVersion", 36)
|
|
23
|
+
defaultConfig {
|
|
24
|
+
minSdkVersion safeExtGet("minSdkVersion", 24)
|
|
25
|
+
targetSdkVersion safeExtGet("targetSdkVersion", 36)
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
android {
|
|
31
|
+
namespace "com.mentra.acsmeeting"
|
|
32
|
+
defaultConfig {
|
|
33
|
+
versionCode 1
|
|
34
|
+
versionName "0.1.0"
|
|
35
|
+
}
|
|
36
|
+
compileOptions {
|
|
37
|
+
coreLibraryDesugaringEnabled true
|
|
38
|
+
}
|
|
39
|
+
buildTypes {
|
|
40
|
+
internal {
|
|
41
|
+
initWith release
|
|
42
|
+
matchingFallbacks = ['release']
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
testOptions {
|
|
46
|
+
unitTests {
|
|
47
|
+
includeAndroidResources = true
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
dependencies {
|
|
53
|
+
coreLibraryDesugaring 'com.android.tools:desugar_jdk_libs:2.1.4'
|
|
54
|
+
implementation 'com.squareup.okhttp3:okhttp:4.12.0'
|
|
55
|
+
implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-android:1.8.1'
|
|
56
|
+
// ACS Calling SDK with raw-media APIs (I420 VirtualOutgoingVideoStream).
|
|
57
|
+
implementation 'com.azure.android:azure-communication-calling:2.16.0'
|
|
58
|
+
implementation 'com.azure.android:azure-communication-common:1.3.0'
|
|
59
|
+
// Same libwebrtc AAR as @livekit/react-native-webrtc so Gradle dedupes instead
|
|
60
|
+
// of shipping two WebRTC stacks. P4 confirms remote AudioTrackSink.
|
|
61
|
+
implementation 'io.github.webrtc-sdk:android:137.7151.05'
|
|
62
|
+
testImplementation 'junit:junit:4.13.2'
|
|
63
|
+
testImplementation 'org.assertj:assertj-core:3.24.2'
|
|
64
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
<manifest xmlns:android="http://schemas.android.com/apk/res/android">
|
|
2
|
+
<uses-permission android:name="android.permission.INTERNET" />
|
|
3
|
+
<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
|
|
4
|
+
<uses-permission android:name="android.permission.RECORD_AUDIO" />
|
|
5
|
+
</manifest>
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
package com.mentra.acsmeeting
|
|
2
|
+
|
|
3
|
+
import com.mentra.acsmeeting.video.VideoProfile
|
|
4
|
+
import expo.modules.kotlin.modules.Module
|
|
5
|
+
import expo.modules.kotlin.modules.ModuleDefinition
|
|
6
|
+
|
|
7
|
+
class AcsMeetingModule : Module() {
|
|
8
|
+
private var session: AcsMeetingSession? = null
|
|
9
|
+
|
|
10
|
+
override fun definition() = ModuleDefinition {
|
|
11
|
+
Name("MentraAcsMeeting")
|
|
12
|
+
Events("onState", "onIncomingPcm")
|
|
13
|
+
|
|
14
|
+
AsyncFunction("join") { options: Map<String, Any?> ->
|
|
15
|
+
val token = options["token"] as? String ?: throw IllegalArgumentException("token is required")
|
|
16
|
+
val meetingUrl = options["meetingUrl"] as? String ?: throw IllegalArgumentException("meetingUrl is required")
|
|
17
|
+
val whepUrl = options["whepUrl"] as? String ?: throw IllegalArgumentException("whepUrl is required")
|
|
18
|
+
val displayName = options["displayName"] as? String
|
|
19
|
+
val dumpWav = options["dumpPcmWav"] as? Boolean ?: false
|
|
20
|
+
val audioSource = options["audioSource"] as? String ?: "glasses"
|
|
21
|
+
val video = parseVideo(options["video"])
|
|
22
|
+
val context = appContext.reactContext ?: throw IllegalStateException("no react context")
|
|
23
|
+
val meeting = session ?: AcsMeetingSession(
|
|
24
|
+
context.applicationContext,
|
|
25
|
+
onState = { sendEvent("onState", it) },
|
|
26
|
+
onIncomingPcm = { base64, rate, channels ->
|
|
27
|
+
sendEvent(
|
|
28
|
+
"onIncomingPcm",
|
|
29
|
+
mapOf("base64" to base64, "sampleRate" to rate, "channels" to channels),
|
|
30
|
+
)
|
|
31
|
+
},
|
|
32
|
+
).also { session = it }
|
|
33
|
+
meeting.join(token, meetingUrl, whepUrl, displayName, dumpWav, audioSource, video)
|
|
34
|
+
meeting.getState()
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
AsyncFunction("leave") {
|
|
38
|
+
session?.leave()
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
AsyncFunction("setMuted") { muted: Boolean ->
|
|
42
|
+
session?.setMuted(muted) ?: mapOf("state" to "idle", "muted" to muted)
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
AsyncFunction("setAudioSource") { source: String ->
|
|
46
|
+
session?.setAudioSource(source) ?: mapOf("state" to "idle", "muted" to false, "audioSource" to source)
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
AsyncFunction("updateVideoSource") { whepUrl: String ->
|
|
50
|
+
session?.updateVideoSource(whepUrl)
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
AsyncFunction("restartVideoSource") {
|
|
54
|
+
session?.restartVideoSource()
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
AsyncFunction("getState") {
|
|
58
|
+
session?.getState() ?: mapOf("state" to "idle", "muted" to false)
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
OnDestroy {
|
|
62
|
+
session?.leave()
|
|
63
|
+
session = null
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
private fun parseVideo(raw: Any?): VideoProfile {
|
|
68
|
+
if (raw == null) return VideoProfile.DEFAULT
|
|
69
|
+
val map = raw as? Map<*, *> ?: throw IllegalArgumentException("video must be an object")
|
|
70
|
+
val width = (map["width"] as? Number)?.toInt()
|
|
71
|
+
val height = (map["height"] as? Number)?.toInt()
|
|
72
|
+
val fps = (map["fps"] as? Number)?.toInt()
|
|
73
|
+
val bitrate = (map["maxBitrateBps"] as? Number)?.toInt()
|
|
74
|
+
if (width == null || height == null || fps == null || bitrate == null) {
|
|
75
|
+
throw IllegalArgumentException("video requires width, height, fps, and maxBitrateBps")
|
|
76
|
+
}
|
|
77
|
+
return VideoProfile.parse(width, height, fps, bitrate)
|
|
78
|
+
?: throw IllegalArgumentException("unsupported ACS video ${width}x${height}@${fps}")
|
|
79
|
+
}
|
|
80
|
+
}
|