@mentra/acs-meeting 3.2.0-dev.120
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +20 -0
- package/README.md +412 -0
- package/THIRD_PARTY +7 -0
- package/android/build.gradle +64 -0
- package/android/src/main/AndroidManifest.xml +5 -0
- package/android/src/main/java/com/mentra/acsmeeting/AcsMeetingModule.kt +80 -0
- package/android/src/main/java/com/mentra/acsmeeting/AcsMeetingSession.kt +846 -0
- package/android/src/main/java/com/mentra/acsmeeting/RemoteRoster.kt +123 -0
- package/android/src/main/java/com/mentra/acsmeeting/audio/AcsAudioPolicy.kt +90 -0
- package/android/src/main/java/com/mentra/acsmeeting/audio/AudioPolicyApplier.kt +176 -0
- package/android/src/main/java/com/mentra/acsmeeting/audio/IncomingAudioPump.kt +112 -0
- package/android/src/main/java/com/mentra/acsmeeting/audio/IncomingRateProbe.kt +65 -0
- package/android/src/main/java/com/mentra/acsmeeting/audio/PcmBridge.kt +108 -0
- package/android/src/main/java/com/mentra/acsmeeting/audio/PcmResampler.kt +159 -0
- package/android/src/main/java/com/mentra/acsmeeting/audio/PhoneMicCapturer.kt +115 -0
- package/android/src/main/java/com/mentra/acsmeeting/audio/UplinkPacer.kt +302 -0
- package/android/src/main/java/com/mentra/acsmeeting/audio/UplinkSender.kt +230 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/BitmapFont.kt +99 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/CloudflareWhepSource.kt +578 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/FirstFrameGate.kt +45 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/GlassesMediaSource.kt +101 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/MediaListeners.kt +57 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/SyntheticFrameFactory.kt +205 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/SyntheticI420Source.kt +214 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/TrackRegistry.kt +31 -0
- package/android/src/main/java/com/mentra/acsmeeting/source/VideoSourceArm.kt +41 -0
- package/android/src/main/java/com/mentra/acsmeeting/telemetry/ChromaProbe.kt +78 -0
- package/android/src/main/java/com/mentra/acsmeeting/telemetry/PipelineStats.kt +281 -0
- package/android/src/main/java/com/mentra/acsmeeting/telemetry/PipelineTicker.kt +51 -0
- package/android/src/main/java/com/mentra/acsmeeting/telemetry/ProcessCpu.kt +15 -0
- package/android/src/main/java/com/mentra/acsmeeting/telemetry/RingPercentile.kt +33 -0
- package/android/src/main/java/com/mentra/acsmeeting/video/AcsFrameSender.kt +475 -0
- package/android/src/main/java/com/mentra/acsmeeting/video/AcsTimestamp.kt +26 -0
- package/android/src/main/java/com/mentra/acsmeeting/video/FrameGeometry.kt +22 -0
- package/android/src/main/java/com/mentra/acsmeeting/video/I420FormatSpec.kt +90 -0
- package/android/src/main/java/com/mentra/acsmeeting/video/I420Packer.kt +72 -0
- package/android/src/main/java/com/mentra/acsmeeting/video/Nv12FormatSpec.kt +36 -0
- package/android/src/main/java/com/mentra/acsmeeting/video/Nv12Packer.kt +51 -0
- package/android/src/main/java/com/mentra/acsmeeting/video/SendGate.kt +27 -0
- package/android/src/main/java/com/mentra/acsmeeting/video/VideoProfile.kt +57 -0
- package/android/src/test/java/com/mentra/acsmeeting/RemoteRosterTest.kt +40 -0
- package/android/src/test/java/com/mentra/acsmeeting/audio/AcsAudioPolicyTest.kt +204 -0
- package/android/src/test/java/com/mentra/acsmeeting/audio/AudioPolicyApplierTest.kt +223 -0
- package/android/src/test/java/com/mentra/acsmeeting/audio/CapturePolicyTest.kt +12 -0
- package/android/src/test/java/com/mentra/acsmeeting/audio/IncomingAudioPumpTest.kt +97 -0
- package/android/src/test/java/com/mentra/acsmeeting/audio/IncomingRateProbeTest.kt +78 -0
- package/android/src/test/java/com/mentra/acsmeeting/audio/PcmResamplerTest.kt +125 -0
- package/android/src/test/java/com/mentra/acsmeeting/audio/UplinkPacerTest.kt +201 -0
- package/android/src/test/java/com/mentra/acsmeeting/audio/UplinkSenderTest.kt +166 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/AcsInvestigationTest.kt +26 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/BitmapFontTest.kt +22 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/FirstFrameGateTest.kt +60 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/GlassesMediaSourceTest.kt +133 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/I420PlanesTest.kt +67 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/SyntheticFrameFactoryTest.kt +193 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/SyntheticI420SourceTest.kt +176 -0
- package/android/src/test/java/com/mentra/acsmeeting/source/TrackRegistryTest.kt +36 -0
- package/android/src/test/java/com/mentra/acsmeeting/telemetry/ChromaProbeTest.kt +61 -0
- package/android/src/test/java/com/mentra/acsmeeting/telemetry/PipelineStatsTest.kt +256 -0
- package/android/src/test/java/com/mentra/acsmeeting/telemetry/ProcessCpuTest.kt +18 -0
- package/android/src/test/java/com/mentra/acsmeeting/telemetry/RingPercentileTest.kt +49 -0
- package/android/src/test/java/com/mentra/acsmeeting/video/AcsTimestampTest.kt +43 -0
- package/android/src/test/java/com/mentra/acsmeeting/video/FrameGeometryTest.kt +33 -0
- package/android/src/test/java/com/mentra/acsmeeting/video/I420FormatSpecTest.kt +77 -0
- package/android/src/test/java/com/mentra/acsmeeting/video/I420PackerTest.kt +126 -0
- package/android/src/test/java/com/mentra/acsmeeting/video/Nv12FormatSpecTest.kt +36 -0
- package/android/src/test/java/com/mentra/acsmeeting/video/Nv12PackerTest.kt +58 -0
- package/android/src/test/java/com/mentra/acsmeeting/video/SendGateTest.kt +79 -0
- package/android/src/test/java/com/mentra/acsmeeting/video/VideoProfileTest.kt +68 -0
- package/build/AcsMeeting.types.d.ts +59 -0
- package/build/AcsMeeting.types.d.ts.map +1 -0
- package/build/AcsMeeting.types.js +2 -0
- package/build/AcsMeeting.types.js.map +1 -0
- package/build/AcsMeetingModule.d.ts +15 -0
- package/build/AcsMeetingModule.d.ts.map +1 -0
- package/build/AcsMeetingModule.js +3 -0
- package/build/AcsMeetingModule.js.map +1 -0
- package/build/AcsMeetingModule.web.d.ts +16 -0
- package/build/AcsMeetingModule.web.d.ts.map +1 -0
- package/build/AcsMeetingModule.web.js +31 -0
- package/build/AcsMeetingModule.web.js.map +1 -0
- package/build/index.d.ts +3 -0
- package/build/index.d.ts.map +1 -0
- package/build/index.js +3 -0
- package/build/index.js.map +1 -0
- package/expo-module.config.json +9 -0
- package/ios/AcsFrameSender.swift +47 -0
- package/ios/AcsMeeting.podspec +27 -0
- package/ios/AcsMeetingModule.swift +558 -0
- package/ios/PcmBridge.swift +117 -0
- package/ios/PhoneMicCapturer.swift +64 -0
- package/ios/PolicyKit/Package.swift +17 -0
- package/ios/PolicyKit/Sources/AcsAudioPolicy/AcsAudioPolicy.swift +102 -0
- package/ios/PolicyKit/Sources/AcsAudioPolicy/AudioPolicyApplier.swift +162 -0
- package/ios/PolicyKit/Sources/AcsAudioPolicy/GlassesMediaSource.swift +68 -0
- package/ios/PolicyKit/Tests/AcsAudioPolicyTests/AcsAudioPolicyTests.swift +273 -0
- package/ios/WhepVideoSource.swift +356 -0
- package/package.json +56 -0
- package/src/AcsMeeting.types.ts +66 -0
- package/src/AcsMeetingModule.ts +16 -0
- package/src/AcsMeetingModule.web.ts +33 -0
- package/src/index.ts +2 -0
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
package com.mentra.acsmeeting
|
|
2
|
+
|
|
3
|
+
import android.util.Log
|
|
4
|
+
import com.azure.android.communication.calling.Call
|
|
5
|
+
import com.azure.android.communication.calling.ParticipantState
|
|
6
|
+
import com.azure.android.communication.calling.ParticipantsUpdatedListener
|
|
7
|
+
import com.azure.android.communication.calling.PropertyChangedListener
|
|
8
|
+
import com.azure.android.communication.calling.RemoteParticipant
|
|
9
|
+
import java.util.concurrent.ConcurrentHashMap
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Tracks ACS remote participants so the host can show who is in the room.
|
|
13
|
+
* Listener callbacks arrive on ACS threads; [onChanged] must hop to the
|
|
14
|
+
* session executor before touching session state.
|
|
15
|
+
*/
|
|
16
|
+
class RemoteRoster(
|
|
17
|
+
private val onChanged: () -> Unit,
|
|
18
|
+
) {
|
|
19
|
+
private class Tracked(
|
|
20
|
+
val participant: RemoteParticipant,
|
|
21
|
+
val listener: PropertyChangedListener,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
private var call: Call? = null
|
|
25
|
+
private var updatedListener: ParticipantsUpdatedListener? = null
|
|
26
|
+
private val tracked = ConcurrentHashMap<String, Tracked>()
|
|
27
|
+
|
|
28
|
+
fun attach(joined: Call) {
|
|
29
|
+
detach()
|
|
30
|
+
call = joined
|
|
31
|
+
val listener = ParticipantsUpdatedListener { event ->
|
|
32
|
+
event.addedParticipants.forEach { track(it) }
|
|
33
|
+
event.removedParticipants.forEach { untrack(it) }
|
|
34
|
+
Log.i(TAG, "roster updated added=${event.addedParticipants.size} removed=${event.removedParticipants.size} size=${tracked.size}")
|
|
35
|
+
onChanged()
|
|
36
|
+
}
|
|
37
|
+
joined.addOnRemoteParticipantsUpdatedListener(listener)
|
|
38
|
+
updatedListener = listener
|
|
39
|
+
joined.remoteParticipants.forEach { track(it) }
|
|
40
|
+
Log.i(TAG, "roster attached size=${tracked.size}")
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
fun detach() {
|
|
44
|
+
val joined = call
|
|
45
|
+
val listener = updatedListener
|
|
46
|
+
call = null
|
|
47
|
+
updatedListener = null
|
|
48
|
+
if (joined != null && listener != null) {
|
|
49
|
+
try {
|
|
50
|
+
joined.removeOnRemoteParticipantsUpdatedListener(listener)
|
|
51
|
+
} catch (_: Exception) {
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
tracked.values.forEach { removeListeners(it) }
|
|
55
|
+
tracked.clear()
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
fun snapshot(): List<Map<String, Any?>> =
|
|
59
|
+
tracked.values
|
|
60
|
+
.map { describe(it.participant) }
|
|
61
|
+
.sortedBy { it["id"] as? String ?: "" }
|
|
62
|
+
|
|
63
|
+
private fun track(participant: RemoteParticipant) {
|
|
64
|
+
val id = idOf(participant)
|
|
65
|
+
if (tracked.containsKey(id)) return
|
|
66
|
+
val listener = PropertyChangedListener { onChanged() }
|
|
67
|
+
val entry = Tracked(participant, listener)
|
|
68
|
+
tracked[id] = entry
|
|
69
|
+
try {
|
|
70
|
+
participant.addOnStateChangedListener(listener)
|
|
71
|
+
participant.addOnIsMutedChangedListener(listener)
|
|
72
|
+
participant.addOnIsSpeakingChangedListener(listener)
|
|
73
|
+
participant.addOnDisplayNameChangedListener(listener)
|
|
74
|
+
} catch (error: Exception) {
|
|
75
|
+
Log.w(TAG, "roster listener attach failed", error)
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
private fun untrack(participant: RemoteParticipant) {
|
|
80
|
+
tracked.remove(idOf(participant))?.let { removeListeners(it) }
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
private fun removeListeners(entry: Tracked) {
|
|
84
|
+
try {
|
|
85
|
+
entry.participant.removeOnStateChangedListener(entry.listener)
|
|
86
|
+
entry.participant.removeOnIsMutedChangedListener(entry.listener)
|
|
87
|
+
entry.participant.removeOnIsSpeakingChangedListener(entry.listener)
|
|
88
|
+
entry.participant.removeOnDisplayNameChangedListener(entry.listener)
|
|
89
|
+
} catch (_: Exception) {
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
companion object {
|
|
94
|
+
private const val TAG = "ACS-SPIKE"
|
|
95
|
+
|
|
96
|
+
fun idOf(participant: RemoteParticipant): String =
|
|
97
|
+
try {
|
|
98
|
+
participant.identifier?.rawId ?: "participant-${System.identityHashCode(participant)}"
|
|
99
|
+
} catch (_: Exception) {
|
|
100
|
+
"participant-${System.identityHashCode(participant)}"
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
fun describe(participant: RemoteParticipant): Map<String, Any?> {
|
|
104
|
+
val state = try { participant.state } catch (_: Exception) { null }
|
|
105
|
+
return mapOf(
|
|
106
|
+
"id" to idOf(participant),
|
|
107
|
+
"displayName" to (try { participant.displayName } catch (_: Exception) { null }),
|
|
108
|
+
"state" to stateName(state),
|
|
109
|
+
"isMuted" to (try { participant.isMuted } catch (_: Exception) { false }),
|
|
110
|
+
"isSpeaking" to (try { participant.isSpeaking } catch (_: Exception) { false }),
|
|
111
|
+
)
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
fun stateName(state: ParticipantState?): String = when (state) {
|
|
115
|
+
ParticipantState.CONNECTED -> "connected"
|
|
116
|
+
ParticipantState.CONNECTING, ParticipantState.RINGING, ParticipantState.EARLY_MEDIA -> "connecting"
|
|
117
|
+
ParticipantState.IN_LOBBY -> "lobby"
|
|
118
|
+
ParticipantState.HOLD -> "hold"
|
|
119
|
+
ParticipantState.DISCONNECTED -> "disconnected"
|
|
120
|
+
ParticipantState.IDLE, null -> "idle"
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
package com.mentra.acsmeeting.audio
|
|
2
|
+
|
|
3
|
+
enum class AudioSourceKind { GLASSES, PHONE }
|
|
4
|
+
|
|
5
|
+
/** Mirrors (AudioStreamType, AudioStreamState). Attached-but-STOPPED reads as NONE. */
|
|
6
|
+
enum class ActiveStreamKind { NONE, VIRTUAL, LOCAL }
|
|
7
|
+
|
|
8
|
+
enum class PhysicalMuteAction { MUTE, UNMUTE, LEAVE_ALONE }
|
|
9
|
+
|
|
10
|
+
enum class AudioSafety { SAFE, DEGRADED, UNSAFE }
|
|
11
|
+
|
|
12
|
+
data class AudioPolicyDecision(
|
|
13
|
+
val glassesPcmEnabled: Boolean,
|
|
14
|
+
val phonePcmEnabled: Boolean,
|
|
15
|
+
val physicalMute: PhysicalMuteAction,
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
/** Transport mute is an ACS join concern, distinct from user intent. */
|
|
19
|
+
data class JoinAudioPlan(
|
|
20
|
+
val armVirtual: Boolean,
|
|
21
|
+
val transportMuted: Boolean,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
object AcsAudioPolicy {
|
|
25
|
+
fun decide(
|
|
26
|
+
desired: AudioSourceKind,
|
|
27
|
+
active: ActiveStreamKind,
|
|
28
|
+
userMuted: Boolean,
|
|
29
|
+
): AudioPolicyDecision {
|
|
30
|
+
val glassesPcmEnabled =
|
|
31
|
+
active == ActiveStreamKind.VIRTUAL &&
|
|
32
|
+
desired == AudioSourceKind.GLASSES &&
|
|
33
|
+
!userMuted
|
|
34
|
+
val phonePcmEnabled =
|
|
35
|
+
active == ActiveStreamKind.VIRTUAL &&
|
|
36
|
+
desired == AudioSourceKind.PHONE &&
|
|
37
|
+
!userMuted
|
|
38
|
+
val physicalMute = when (active) {
|
|
39
|
+
ActiveStreamKind.VIRTUAL, ActiveStreamKind.NONE -> PhysicalMuteAction.LEAVE_ALONE
|
|
40
|
+
ActiveStreamKind.LOCAL ->
|
|
41
|
+
if (desired == AudioSourceKind.PHONE && !userMuted) {
|
|
42
|
+
PhysicalMuteAction.UNMUTE
|
|
43
|
+
} else {
|
|
44
|
+
PhysicalMuteAction.MUTE
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
return AudioPolicyDecision(glassesPcmEnabled, phonePcmEnabled, physicalMute)
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
fun planJoin(
|
|
51
|
+
desired: AudioSourceKind,
|
|
52
|
+
userMuted: Boolean,
|
|
53
|
+
glassesRequiresUnmutedTransport: Boolean,
|
|
54
|
+
): JoinAudioPlan {
|
|
55
|
+
// Phone and glasses both feed RawOutgoingAudioStream. Mute is PCM/capturer
|
|
56
|
+
// gating, never a LocalOutgoingAudioStream (that path makes ACS own the route).
|
|
57
|
+
val armVirtual = true
|
|
58
|
+
val transportMuted =
|
|
59
|
+
if (desired == AudioSourceKind.GLASSES && glassesRequiresUnmutedTransport) {
|
|
60
|
+
false
|
|
61
|
+
} else if (desired == AudioSourceKind.PHONE) {
|
|
62
|
+
false
|
|
63
|
+
} else {
|
|
64
|
+
userMuted
|
|
65
|
+
}
|
|
66
|
+
return JoinAudioPlan(armVirtual = armVirtual, transportMuted = transportMuted)
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
@Suppress("UNUSED_PARAMETER")
|
|
70
|
+
fun expectedStream(desired: AudioSourceKind): ActiveStreamKind = ActiveStreamKind.VIRTUAL
|
|
71
|
+
|
|
72
|
+
fun parseSource(raw: String?): AudioSourceKind? =
|
|
73
|
+
when (raw) {
|
|
74
|
+
"glasses" -> AudioSourceKind.GLASSES
|
|
75
|
+
"phone" -> AudioSourceKind.PHONE
|
|
76
|
+
else -> null
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Layer 1: what the glasses encoder captures for this WHIP session. Independent of ACS. */
|
|
81
|
+
object CapturePolicy {
|
|
82
|
+
fun captureGlassesMic(source: AudioSourceKind): Boolean = source == AudioSourceKind.GLASSES
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** Null ACS call handles must fail, never report success via `call?.mute(); Unit`. */
|
|
86
|
+
object CallGuard {
|
|
87
|
+
fun <T : Any> require(value: T?): Result<T> =
|
|
88
|
+
if (value == null) Result.failure(IllegalStateException("no call"))
|
|
89
|
+
else Result.success(value)
|
|
90
|
+
}
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
package com.mentra.acsmeeting.audio
|
|
2
|
+
|
|
3
|
+
interface AudioStreamController {
|
|
4
|
+
fun readActive(): ActiveStreamKind
|
|
5
|
+
fun isPhysicallyMuted(): Boolean?
|
|
6
|
+
fun setGlassesPcmEnabled(enabled: Boolean)
|
|
7
|
+
fun setPhonePcmEnabled(enabled: Boolean)
|
|
8
|
+
fun mutePhysical(): Result<Unit>
|
|
9
|
+
fun unmutePhysical(): Result<Unit>
|
|
10
|
+
fun stopActive(): Result<Unit>
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
interface PolicyScheduler {
|
|
14
|
+
fun schedule(delayMs: Long, task: () -> Unit)
|
|
15
|
+
fun cancelPending()
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
class AudioPolicyApplier(
|
|
19
|
+
private val controller: AudioStreamController,
|
|
20
|
+
private val scheduler: PolicyScheduler,
|
|
21
|
+
private val log: (String) -> Unit = {},
|
|
22
|
+
) {
|
|
23
|
+
private var generation = 0L
|
|
24
|
+
private var lastDecision: AudioPolicyDecision? = null
|
|
25
|
+
private var lastSafety: AudioSafety = AudioSafety.DEGRADED
|
|
26
|
+
|
|
27
|
+
fun lastSafety(): AudioSafety = lastSafety
|
|
28
|
+
|
|
29
|
+
fun apply(desired: AudioSourceKind, userMuted: Boolean, reason: String): AudioSafety {
|
|
30
|
+
generation += 1
|
|
31
|
+
scheduler.cancelPending()
|
|
32
|
+
return runApply(desired, userMuted, reason, generation, force = false, armConvergence = true)
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
fun reset() {
|
|
36
|
+
generation += 1
|
|
37
|
+
scheduler.cancelPending()
|
|
38
|
+
lastDecision = null
|
|
39
|
+
lastSafety = AudioSafety.DEGRADED
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
private fun runApply(
|
|
43
|
+
desired: AudioSourceKind,
|
|
44
|
+
userMuted: Boolean,
|
|
45
|
+
reason: String,
|
|
46
|
+
gen: Long,
|
|
47
|
+
force: Boolean,
|
|
48
|
+
armConvergence: Boolean,
|
|
49
|
+
): AudioSafety {
|
|
50
|
+
if (gen != generation) return lastSafety
|
|
51
|
+
val active = controller.readActive()
|
|
52
|
+
val decision = AcsAudioPolicy.decide(desired, active, userMuted)
|
|
53
|
+
val outcome = applyDecision(decision, force)
|
|
54
|
+
if (gen != generation) return lastSafety
|
|
55
|
+
|
|
56
|
+
val after = controller.readActive()
|
|
57
|
+
val expected = AcsAudioPolicy.expectedStream(desired)
|
|
58
|
+
val safety = safetyOf(desired, after, outcome)
|
|
59
|
+
lastDecision = decision
|
|
60
|
+
lastSafety = safety
|
|
61
|
+
log(
|
|
62
|
+
"audio policy reason=$reason source=$desired userMuted=$userMuted " +
|
|
63
|
+
"active=$after expected=$expected glassesPcm=${decision.glassesPcmEnabled} " +
|
|
64
|
+
"phonePcm=${decision.phonePcmEnabled} mute=${decision.physicalMute} safety=$safety",
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
if (after == expected) {
|
|
68
|
+
scheduler.cancelPending()
|
|
69
|
+
} else if (armConvergence && gen == generation) {
|
|
70
|
+
armConvergence(desired, userMuted, gen, expected)
|
|
71
|
+
}
|
|
72
|
+
return lastSafety
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
private fun armConvergence(
|
|
76
|
+
desired: AudioSourceKind,
|
|
77
|
+
userMuted: Boolean,
|
|
78
|
+
gen: Long,
|
|
79
|
+
expected: ActiveStreamKind,
|
|
80
|
+
) {
|
|
81
|
+
CONVERGE_DELAYS_MS.forEachIndexed { index, delayMs ->
|
|
82
|
+
scheduler.schedule(delayMs) {
|
|
83
|
+
if (gen != generation) return@schedule
|
|
84
|
+
val isLast = index == CONVERGE_DELAYS_MS.lastIndex
|
|
85
|
+
runApply(desired, userMuted, "converge-${delayMs}ms", gen, force = true, armConvergence = false)
|
|
86
|
+
if (isLast && gen == generation && controller.readActive() != expected) {
|
|
87
|
+
log("audio policy convergence timed out source=$desired expected=$expected holding silence")
|
|
88
|
+
lastSafety = AudioSafety.DEGRADED
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
private fun applyDecision(decision: AudioPolicyDecision, force: Boolean): EffectOutcome {
|
|
95
|
+
val previous = lastDecision
|
|
96
|
+
val skipGlassesPcm = !force && previous != null && previous.glassesPcmEnabled == decision.glassesPcmEnabled
|
|
97
|
+
val skipPhonePcm = !force && previous != null && previous.phonePcmEnabled == decision.phonePcmEnabled
|
|
98
|
+
val skipMute = !force && previous != null && previous.physicalMute == decision.physicalMute
|
|
99
|
+
if (!skipGlassesPcm) {
|
|
100
|
+
controller.setGlassesPcmEnabled(decision.glassesPcmEnabled)
|
|
101
|
+
}
|
|
102
|
+
if (!skipPhonePcm) {
|
|
103
|
+
controller.setPhonePcmEnabled(decision.phonePcmEnabled)
|
|
104
|
+
}
|
|
105
|
+
if (skipMute) return EffectOutcome.NONE
|
|
106
|
+
|
|
107
|
+
return when (decision.physicalMute) {
|
|
108
|
+
PhysicalMuteAction.LEAVE_ALONE -> EffectOutcome.NONE
|
|
109
|
+
PhysicalMuteAction.UNMUTE -> {
|
|
110
|
+
val result = controller.unmutePhysical()
|
|
111
|
+
val unmuted = controller.isPhysicallyMuted()
|
|
112
|
+
val failed = result.isFailure || unmuted == true
|
|
113
|
+
EffectOutcome(unmuteAttempted = true, unmuteFailed = failed)
|
|
114
|
+
}
|
|
115
|
+
PhysicalMuteAction.MUTE -> {
|
|
116
|
+
val result = controller.mutePhysical()
|
|
117
|
+
val muted = controller.isPhysicallyMuted()
|
|
118
|
+
val muteFailed = result.isFailure || muted == false
|
|
119
|
+
if (!muteFailed) return EffectOutcome(muteAttempted = true)
|
|
120
|
+
val stop = controller.stopActive()
|
|
121
|
+
val stopped = controller.readActive() == ActiveStreamKind.NONE
|
|
122
|
+
EffectOutcome(
|
|
123
|
+
muteAttempted = true,
|
|
124
|
+
muteFailed = true,
|
|
125
|
+
stopAttempted = true,
|
|
126
|
+
stopFailed = stop.isFailure || !stopped,
|
|
127
|
+
)
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
private fun safetyOf(
|
|
133
|
+
desired: AudioSourceKind,
|
|
134
|
+
active: ActiveStreamKind,
|
|
135
|
+
outcome: EffectOutcome,
|
|
136
|
+
): AudioSafety {
|
|
137
|
+
if (outcome.stopFailed) return AudioSafety.UNSAFE
|
|
138
|
+
if (outcome.stopAttempted) return AudioSafety.SAFE
|
|
139
|
+
if (outcome.unmuteFailed) return AudioSafety.DEGRADED
|
|
140
|
+
val expected = AcsAudioPolicy.expectedStream(desired)
|
|
141
|
+
if (active != expected) return AudioSafety.DEGRADED
|
|
142
|
+
return AudioSafety.SAFE
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
private data class EffectOutcome(
|
|
146
|
+
val muteAttempted: Boolean = false,
|
|
147
|
+
val muteFailed: Boolean = false,
|
|
148
|
+
val unmuteAttempted: Boolean = false,
|
|
149
|
+
val unmuteFailed: Boolean = false,
|
|
150
|
+
val stopAttempted: Boolean = false,
|
|
151
|
+
val stopFailed: Boolean = false,
|
|
152
|
+
) {
|
|
153
|
+
companion object {
|
|
154
|
+
val NONE = EffectOutcome()
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
companion object {
|
|
159
|
+
val CONVERGE_DELAYS_MS = longArrayOf(50, 100, 200, 400, 800, 1600)
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
class ExecutorPolicyScheduler(
|
|
164
|
+
private val executor: java.util.concurrent.ScheduledExecutorService,
|
|
165
|
+
) : PolicyScheduler {
|
|
166
|
+
private val pending = java.util.concurrent.CopyOnWriteArrayList<java.util.concurrent.ScheduledFuture<*>>()
|
|
167
|
+
|
|
168
|
+
override fun schedule(delayMs: Long, task: () -> Unit) {
|
|
169
|
+
pending.add(executor.schedule(task, delayMs, java.util.concurrent.TimeUnit.MILLISECONDS))
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
override fun cancelPending() {
|
|
173
|
+
pending.forEach { it.cancel(false) }
|
|
174
|
+
pending.clear()
|
|
175
|
+
}
|
|
176
|
+
}
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
package com.mentra.acsmeeting.audio
|
|
2
|
+
|
|
3
|
+
import java.io.ByteArrayOutputStream
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* ACS mixed-audio buffers → fixed-format chunks for the host PCM player.
|
|
7
|
+
*
|
|
8
|
+
* - Normalizes to [OUT_RATE] Hz mono regardless of what ACS actually
|
|
9
|
+
* delivers. The requested RawIncomingAudioStreamProperties are a hint;
|
|
10
|
+
* the event carries the real format, and playing 48 kHz data on a 16 kHz
|
|
11
|
+
* AudioTrack is exactly the "slow, low-pitched" symptom.
|
|
12
|
+
* - Pre-rolls [prerollMs] before the first emit and then emits in
|
|
13
|
+
* [batchMs] chunks, so the player always has headroom over bridge jitter
|
|
14
|
+
* instead of underrunning between 20 ms callbacks.
|
|
15
|
+
*
|
|
16
|
+
* Not thread-safe: ACS delivers mixed audio from a single thread.
|
|
17
|
+
*/
|
|
18
|
+
class IncomingAudioPump(
|
|
19
|
+
private val prerollMs: Int = DEFAULT_PREROLL_MS,
|
|
20
|
+
private val batchMs: Int = DEFAULT_BATCH_MS,
|
|
21
|
+
private val emit: (pcm16Le: ByteArray) -> Unit,
|
|
22
|
+
) {
|
|
23
|
+
data class Format(val sampleRate: Int, val channels: Int)
|
|
24
|
+
|
|
25
|
+
private val resampler = PcmResampler(OUT_RATE)
|
|
26
|
+
private val pending = ByteArrayOutputStream()
|
|
27
|
+
private var primed = false
|
|
28
|
+
|
|
29
|
+
var format: Format? = null
|
|
30
|
+
private set
|
|
31
|
+
var formatChanges = 0
|
|
32
|
+
private set
|
|
33
|
+
var eventsIn = 0L
|
|
34
|
+
private set
|
|
35
|
+
var bytesIn = 0L
|
|
36
|
+
private set
|
|
37
|
+
var bytesOut = 0L
|
|
38
|
+
private set
|
|
39
|
+
|
|
40
|
+
fun push(pcm16Le: ByteArray, sampleRate: Int, channels: Int) {
|
|
41
|
+
if (pcm16Le.isEmpty()) return
|
|
42
|
+
val next = Format(sampleRate, channels)
|
|
43
|
+
if (format != next) {
|
|
44
|
+
if (format != null) formatChanges += 1
|
|
45
|
+
format = next
|
|
46
|
+
}
|
|
47
|
+
eventsIn += 1
|
|
48
|
+
bytesIn += pcm16Le.size
|
|
49
|
+
val normalized = resampler.process(pcm16Le, sampleRate, channels)
|
|
50
|
+
pending.write(normalized)
|
|
51
|
+
drain(force = false)
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** Emit whatever is buffered (call on stop so the tail is not lost). */
|
|
55
|
+
fun flush() {
|
|
56
|
+
drain(force = true)
|
|
57
|
+
primed = false
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
fun reset() {
|
|
61
|
+
pending.reset()
|
|
62
|
+
primed = false
|
|
63
|
+
resampler.reset()
|
|
64
|
+
format = null
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
private fun drain(force: Boolean) {
|
|
68
|
+
val threshold = if (primed) bytesFor(batchMs) else bytesFor(prerollMs)
|
|
69
|
+
if (pending.size() == 0) return
|
|
70
|
+
if (!force && pending.size() < threshold) return
|
|
71
|
+
val all = pending.toByteArray()
|
|
72
|
+
pending.reset()
|
|
73
|
+
// Chunk on batch boundaries so the player queue sees a steady cadence;
|
|
74
|
+
// the remainder waits for the next callback unless flushing.
|
|
75
|
+
val batch = bytesFor(batchMs)
|
|
76
|
+
var offset = 0
|
|
77
|
+
while (all.size - offset >= batch) {
|
|
78
|
+
emitChunk(all.copyOfRange(offset, offset + batch))
|
|
79
|
+
offset += batch
|
|
80
|
+
}
|
|
81
|
+
if (offset < all.size) {
|
|
82
|
+
if (force) emitChunk(all.copyOfRange(offset, all.size)) else pending.write(all, offset, all.size - offset)
|
|
83
|
+
}
|
|
84
|
+
primed = true
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
private fun emitChunk(chunk: ByteArray) {
|
|
88
|
+
bytesOut += chunk.size
|
|
89
|
+
emit(chunk)
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
companion object {
|
|
93
|
+
const val OUT_RATE = 16_000
|
|
94
|
+
const val OUT_CHANNELS = 1
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Downlink headroom, and the only knob for the downlink latency A/B.
|
|
98
|
+
*
|
|
99
|
+
* The remote path already stacks Teams/ACS network + ACS receive + RN
|
|
100
|
+
* bridge batching + this preroll + A2DP buffering, so more is not simply
|
|
101
|
+
* better: every millisecond here is conversational delay. Walk
|
|
102
|
+
* [PREROLL_LADDER_MS] on real S22 + Mentra Live hardware and ship the
|
|
103
|
+
* smallest value with no audible underruns. 120 ms underran on the
|
|
104
|
+
* reported jitter; 160 ms is one step up, not the top of the ladder.
|
|
105
|
+
*/
|
|
106
|
+
const val DEFAULT_PREROLL_MS = 160
|
|
107
|
+
val PREROLL_LADDER_MS = listOf(120, 160, 200)
|
|
108
|
+
const val DEFAULT_BATCH_MS = 60
|
|
109
|
+
|
|
110
|
+
fun bytesFor(ms: Int): Int = OUT_RATE * OUT_CHANNELS * 2 * ms / 1000
|
|
111
|
+
}
|
|
112
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
package com.mentra.acsmeeting.audio
|
|
2
|
+
|
|
3
|
+
import kotlin.math.max
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Measures what ACS actually delivers on the incoming mixed-audio callback.
|
|
7
|
+
*
|
|
8
|
+
* Deriving a rate as `bytes / (2 * channels) * 50` only holds if ACS really
|
|
9
|
+
* fires exactly every 20 ms — which is the assumption being diagnosed. This
|
|
10
|
+
* accumulates samples against elapsed monotonic time instead, so a rate ACS
|
|
11
|
+
* mislabels shows up as `measuredRate` disagreeing with `declaredRate` and a
|
|
12
|
+
* cadence that is not 20 ms shows up in `callbackHz`.
|
|
13
|
+
*/
|
|
14
|
+
class IncomingRateProbe(private val windowNanos: Long = DEFAULT_WINDOW_NANOS) {
|
|
15
|
+
data class Reading(
|
|
16
|
+
val declaredRate: Int,
|
|
17
|
+
val channels: Int,
|
|
18
|
+
val samplesPerCallback: Int,
|
|
19
|
+
val callbackHz: Double,
|
|
20
|
+
val measuredRate: Double,
|
|
21
|
+
val events: Long,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
private var windowStartNanos: Long? = null
|
|
25
|
+
private var events = 0L
|
|
26
|
+
private var samples = 0L
|
|
27
|
+
|
|
28
|
+
/** Returns a reading at most once per window; null in between. */
|
|
29
|
+
fun record(nowNanos: Long, bytes: Int, sampleRate: Int, channels: Int): Reading? {
|
|
30
|
+
val samplesPerCallback = bytes / (2 * max(1, channels))
|
|
31
|
+
val start = windowStartNanos
|
|
32
|
+
if (start == null) {
|
|
33
|
+
// This callback only opens the window; it arrived before it.
|
|
34
|
+
windowStartNanos = nowNanos
|
|
35
|
+
return null
|
|
36
|
+
}
|
|
37
|
+
events += 1
|
|
38
|
+
samples += samplesPerCallback
|
|
39
|
+
val elapsedNanos = nowNanos - start
|
|
40
|
+
if (elapsedNanos < windowNanos) return null
|
|
41
|
+
val seconds = elapsedNanos / 1_000_000_000.0
|
|
42
|
+
val reading = Reading(
|
|
43
|
+
declaredRate = sampleRate,
|
|
44
|
+
channels = channels,
|
|
45
|
+
samplesPerCallback = samplesPerCallback,
|
|
46
|
+
callbackHz = events / seconds,
|
|
47
|
+
measuredRate = samples / seconds,
|
|
48
|
+
events = events,
|
|
49
|
+
)
|
|
50
|
+
windowStartNanos = nowNanos
|
|
51
|
+
events = 0
|
|
52
|
+
samples = 0
|
|
53
|
+
return reading
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
fun reset() {
|
|
57
|
+
windowStartNanos = null
|
|
58
|
+
events = 0
|
|
59
|
+
samples = 0
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
companion object {
|
|
63
|
+
const val DEFAULT_WINDOW_NANOS = 1_000_000_000L
|
|
64
|
+
}
|
|
65
|
+
}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
package com.mentra.acsmeeting.audio
|
|
2
|
+
|
|
3
|
+
import android.util.Base64
|
|
4
|
+
import android.util.Log
|
|
5
|
+
import java.io.ByteArrayOutputStream
|
|
6
|
+
import java.io.File
|
|
7
|
+
import java.nio.ByteBuffer
|
|
8
|
+
import java.nio.ByteOrder
|
|
9
|
+
import kotlin.math.abs
|
|
10
|
+
|
|
11
|
+
/** Downmix/resample PCM16 to 48 kHz mono ~20 ms blocks for ACS raw outgoing audio. */
|
|
12
|
+
class PcmBridge(
|
|
13
|
+
private val dumpDir: File?,
|
|
14
|
+
private val dumpWav: Boolean,
|
|
15
|
+
) {
|
|
16
|
+
private var dump: ByteArrayOutputStream? = if (dumpWav) ByteArrayOutputStream() else null
|
|
17
|
+
private var dumpRate = 0
|
|
18
|
+
private var dumpChannels = 0
|
|
19
|
+
private var dumped = false
|
|
20
|
+
private val outgoing = ByteArrayOutputStream()
|
|
21
|
+
private val resampler = PcmResampler(TARGET_RATE)
|
|
22
|
+
private var lastRmsLogMs = 0L
|
|
23
|
+
|
|
24
|
+
fun ingest(pcm16Le: ByteArray, sampleRate: Int, channels: Int): List<ByteArray> {
|
|
25
|
+
if (dump != null && !dumped) {
|
|
26
|
+
dump!!.write(pcm16Le)
|
|
27
|
+
dumpRate = sampleRate
|
|
28
|
+
dumpChannels = channels
|
|
29
|
+
val needed = 30 * sampleRate * channels * 2
|
|
30
|
+
if (dump!!.size() >= needed) finishDump()
|
|
31
|
+
}
|
|
32
|
+
logLevel(pcm16Le, sampleRate, channels)
|
|
33
|
+
// Stateful: filter history and fractional phase carry across WebRTC's
|
|
34
|
+
// 10 ms callbacks, so there is no seam every 480 samples.
|
|
35
|
+
val mono = resampler.process(pcm16Le, sampleRate, channels)
|
|
36
|
+
outgoing.write(mono)
|
|
37
|
+
val frameBytes = TARGET_RATE * 2 / 50 // 20 ms
|
|
38
|
+
val frames = mutableListOf<ByteArray>()
|
|
39
|
+
val buf = outgoing.toByteArray()
|
|
40
|
+
var offset = 0
|
|
41
|
+
while (buf.size - offset >= frameBytes) {
|
|
42
|
+
frames.add(buf.copyOfRange(offset, offset + frameBytes))
|
|
43
|
+
offset += frameBytes
|
|
44
|
+
}
|
|
45
|
+
outgoing.reset()
|
|
46
|
+
if (offset < buf.size) outgoing.write(buf, offset, buf.size - offset)
|
|
47
|
+
return frames
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
@Synchronized
|
|
51
|
+
fun finishDump() {
|
|
52
|
+
if (dumped) return
|
|
53
|
+
val data = dump?.toByteArray() ?: return
|
|
54
|
+
dumped = true
|
|
55
|
+
dump = null
|
|
56
|
+
val dir = dumpDir ?: return
|
|
57
|
+
val file = File(dir, "acs-whep-p4.wav")
|
|
58
|
+
file.writeBytes(wav(data, dumpRate, dumpChannels))
|
|
59
|
+
Log.i(TAG, "P4 wrote ${file.absolutePath} bytes=${data.size} rate=$dumpRate ch=$dumpChannels")
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
private fun logLevel(pcm: ByteArray, sampleRate: Int, channels: Int) {
|
|
63
|
+
val now = System.currentTimeMillis()
|
|
64
|
+
if (now - lastRmsLogMs < 1000) return
|
|
65
|
+
lastRmsLogMs = now
|
|
66
|
+
var acc = 0L
|
|
67
|
+
var n = 0
|
|
68
|
+
var i = 0
|
|
69
|
+
while (i + 1 < pcm.size) {
|
|
70
|
+
val s = (pcm[i].toInt() and 0xff) or (pcm[i + 1].toInt() shl 8)
|
|
71
|
+
val signed = if (s >= 0x8000) s - 0x10000 else s
|
|
72
|
+
acc += abs(signed)
|
|
73
|
+
n++
|
|
74
|
+
i += 2
|
|
75
|
+
}
|
|
76
|
+
val mean = if (n == 0) 0 else acc / n
|
|
77
|
+
Log.i(TAG, "P4 pcm rate=$sampleRate ch=$channels meanAbs=$mean bytes=${pcm.size}")
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
companion object {
|
|
81
|
+
private const val TAG = "ACS-SPIKE"
|
|
82
|
+
const val TARGET_RATE = 48_000
|
|
83
|
+
|
|
84
|
+
/** One-shot convenience (fresh filter state). Streams must use a [PcmResampler] instance. */
|
|
85
|
+
fun toMono(pcm16Le: ByteArray, sampleRate: Int, channels: Int): ByteArray =
|
|
86
|
+
PcmResampler(TARGET_RATE).process(pcm16Le, sampleRate, channels)
|
|
87
|
+
|
|
88
|
+
fun wav(pcm: ByteArray, sampleRate: Int, channels: Int): ByteArray {
|
|
89
|
+
val header = ByteBuffer.allocate(44).order(ByteOrder.LITTLE_ENDIAN)
|
|
90
|
+
header.put("RIFF".toByteArray())
|
|
91
|
+
header.putInt(36 + pcm.size)
|
|
92
|
+
header.put("WAVE".toByteArray())
|
|
93
|
+
header.put("fmt ".toByteArray())
|
|
94
|
+
header.putInt(16)
|
|
95
|
+
header.putShort(1)
|
|
96
|
+
header.putShort(channels.toShort())
|
|
97
|
+
header.putInt(sampleRate)
|
|
98
|
+
header.putInt(sampleRate * channels * 2)
|
|
99
|
+
header.putShort((channels * 2).toShort())
|
|
100
|
+
header.putShort(16)
|
|
101
|
+
header.put("data".toByteArray())
|
|
102
|
+
header.putInt(pcm.size)
|
|
103
|
+
return header.array() + pcm
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
fun encodeBase64(pcm: ByteArray): String = Base64.encodeToString(pcm, Base64.NO_WRAP)
|
|
107
|
+
}
|
|
108
|
+
}
|