@mentra/acs-meeting 3.2.0-dev.120

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. package/NOTICE +20 -0
  2. package/README.md +412 -0
  3. package/THIRD_PARTY +7 -0
  4. package/android/build.gradle +64 -0
  5. package/android/src/main/AndroidManifest.xml +5 -0
  6. package/android/src/main/java/com/mentra/acsmeeting/AcsMeetingModule.kt +80 -0
  7. package/android/src/main/java/com/mentra/acsmeeting/AcsMeetingSession.kt +846 -0
  8. package/android/src/main/java/com/mentra/acsmeeting/RemoteRoster.kt +123 -0
  9. package/android/src/main/java/com/mentra/acsmeeting/audio/AcsAudioPolicy.kt +90 -0
  10. package/android/src/main/java/com/mentra/acsmeeting/audio/AudioPolicyApplier.kt +176 -0
  11. package/android/src/main/java/com/mentra/acsmeeting/audio/IncomingAudioPump.kt +112 -0
  12. package/android/src/main/java/com/mentra/acsmeeting/audio/IncomingRateProbe.kt +65 -0
  13. package/android/src/main/java/com/mentra/acsmeeting/audio/PcmBridge.kt +108 -0
  14. package/android/src/main/java/com/mentra/acsmeeting/audio/PcmResampler.kt +159 -0
  15. package/android/src/main/java/com/mentra/acsmeeting/audio/PhoneMicCapturer.kt +115 -0
  16. package/android/src/main/java/com/mentra/acsmeeting/audio/UplinkPacer.kt +302 -0
  17. package/android/src/main/java/com/mentra/acsmeeting/audio/UplinkSender.kt +230 -0
  18. package/android/src/main/java/com/mentra/acsmeeting/source/BitmapFont.kt +99 -0
  19. package/android/src/main/java/com/mentra/acsmeeting/source/CloudflareWhepSource.kt +578 -0
  20. package/android/src/main/java/com/mentra/acsmeeting/source/FirstFrameGate.kt +45 -0
  21. package/android/src/main/java/com/mentra/acsmeeting/source/GlassesMediaSource.kt +101 -0
  22. package/android/src/main/java/com/mentra/acsmeeting/source/MediaListeners.kt +57 -0
  23. package/android/src/main/java/com/mentra/acsmeeting/source/SyntheticFrameFactory.kt +205 -0
  24. package/android/src/main/java/com/mentra/acsmeeting/source/SyntheticI420Source.kt +214 -0
  25. package/android/src/main/java/com/mentra/acsmeeting/source/TrackRegistry.kt +31 -0
  26. package/android/src/main/java/com/mentra/acsmeeting/source/VideoSourceArm.kt +41 -0
  27. package/android/src/main/java/com/mentra/acsmeeting/telemetry/ChromaProbe.kt +78 -0
  28. package/android/src/main/java/com/mentra/acsmeeting/telemetry/PipelineStats.kt +281 -0
  29. package/android/src/main/java/com/mentra/acsmeeting/telemetry/PipelineTicker.kt +51 -0
  30. package/android/src/main/java/com/mentra/acsmeeting/telemetry/ProcessCpu.kt +15 -0
  31. package/android/src/main/java/com/mentra/acsmeeting/telemetry/RingPercentile.kt +33 -0
  32. package/android/src/main/java/com/mentra/acsmeeting/video/AcsFrameSender.kt +475 -0
  33. package/android/src/main/java/com/mentra/acsmeeting/video/AcsTimestamp.kt +26 -0
  34. package/android/src/main/java/com/mentra/acsmeeting/video/FrameGeometry.kt +22 -0
  35. package/android/src/main/java/com/mentra/acsmeeting/video/I420FormatSpec.kt +90 -0
  36. package/android/src/main/java/com/mentra/acsmeeting/video/I420Packer.kt +72 -0
  37. package/android/src/main/java/com/mentra/acsmeeting/video/Nv12FormatSpec.kt +36 -0
  38. package/android/src/main/java/com/mentra/acsmeeting/video/Nv12Packer.kt +51 -0
  39. package/android/src/main/java/com/mentra/acsmeeting/video/SendGate.kt +27 -0
  40. package/android/src/main/java/com/mentra/acsmeeting/video/VideoProfile.kt +57 -0
  41. package/android/src/test/java/com/mentra/acsmeeting/RemoteRosterTest.kt +40 -0
  42. package/android/src/test/java/com/mentra/acsmeeting/audio/AcsAudioPolicyTest.kt +204 -0
  43. package/android/src/test/java/com/mentra/acsmeeting/audio/AudioPolicyApplierTest.kt +223 -0
  44. package/android/src/test/java/com/mentra/acsmeeting/audio/CapturePolicyTest.kt +12 -0
  45. package/android/src/test/java/com/mentra/acsmeeting/audio/IncomingAudioPumpTest.kt +97 -0
  46. package/android/src/test/java/com/mentra/acsmeeting/audio/IncomingRateProbeTest.kt +78 -0
  47. package/android/src/test/java/com/mentra/acsmeeting/audio/PcmResamplerTest.kt +125 -0
  48. package/android/src/test/java/com/mentra/acsmeeting/audio/UplinkPacerTest.kt +201 -0
  49. package/android/src/test/java/com/mentra/acsmeeting/audio/UplinkSenderTest.kt +166 -0
  50. package/android/src/test/java/com/mentra/acsmeeting/source/AcsInvestigationTest.kt +26 -0
  51. package/android/src/test/java/com/mentra/acsmeeting/source/BitmapFontTest.kt +22 -0
  52. package/android/src/test/java/com/mentra/acsmeeting/source/FirstFrameGateTest.kt +60 -0
  53. package/android/src/test/java/com/mentra/acsmeeting/source/GlassesMediaSourceTest.kt +133 -0
  54. package/android/src/test/java/com/mentra/acsmeeting/source/I420PlanesTest.kt +67 -0
  55. package/android/src/test/java/com/mentra/acsmeeting/source/SyntheticFrameFactoryTest.kt +193 -0
  56. package/android/src/test/java/com/mentra/acsmeeting/source/SyntheticI420SourceTest.kt +176 -0
  57. package/android/src/test/java/com/mentra/acsmeeting/source/TrackRegistryTest.kt +36 -0
  58. package/android/src/test/java/com/mentra/acsmeeting/telemetry/ChromaProbeTest.kt +61 -0
  59. package/android/src/test/java/com/mentra/acsmeeting/telemetry/PipelineStatsTest.kt +256 -0
  60. package/android/src/test/java/com/mentra/acsmeeting/telemetry/ProcessCpuTest.kt +18 -0
  61. package/android/src/test/java/com/mentra/acsmeeting/telemetry/RingPercentileTest.kt +49 -0
  62. package/android/src/test/java/com/mentra/acsmeeting/video/AcsTimestampTest.kt +43 -0
  63. package/android/src/test/java/com/mentra/acsmeeting/video/FrameGeometryTest.kt +33 -0
  64. package/android/src/test/java/com/mentra/acsmeeting/video/I420FormatSpecTest.kt +77 -0
  65. package/android/src/test/java/com/mentra/acsmeeting/video/I420PackerTest.kt +126 -0
  66. package/android/src/test/java/com/mentra/acsmeeting/video/Nv12FormatSpecTest.kt +36 -0
  67. package/android/src/test/java/com/mentra/acsmeeting/video/Nv12PackerTest.kt +58 -0
  68. package/android/src/test/java/com/mentra/acsmeeting/video/SendGateTest.kt +79 -0
  69. package/android/src/test/java/com/mentra/acsmeeting/video/VideoProfileTest.kt +68 -0
  70. package/build/AcsMeeting.types.d.ts +59 -0
  71. package/build/AcsMeeting.types.d.ts.map +1 -0
  72. package/build/AcsMeeting.types.js +2 -0
  73. package/build/AcsMeeting.types.js.map +1 -0
  74. package/build/AcsMeetingModule.d.ts +15 -0
  75. package/build/AcsMeetingModule.d.ts.map +1 -0
  76. package/build/AcsMeetingModule.js +3 -0
  77. package/build/AcsMeetingModule.js.map +1 -0
  78. package/build/AcsMeetingModule.web.d.ts +16 -0
  79. package/build/AcsMeetingModule.web.d.ts.map +1 -0
  80. package/build/AcsMeetingModule.web.js +31 -0
  81. package/build/AcsMeetingModule.web.js.map +1 -0
  82. package/build/index.d.ts +3 -0
  83. package/build/index.d.ts.map +1 -0
  84. package/build/index.js +3 -0
  85. package/build/index.js.map +1 -0
  86. package/expo-module.config.json +9 -0
  87. package/ios/AcsFrameSender.swift +47 -0
  88. package/ios/AcsMeeting.podspec +27 -0
  89. package/ios/AcsMeetingModule.swift +558 -0
  90. package/ios/PcmBridge.swift +117 -0
  91. package/ios/PhoneMicCapturer.swift +64 -0
  92. package/ios/PolicyKit/Package.swift +17 -0
  93. package/ios/PolicyKit/Sources/AcsAudioPolicy/AcsAudioPolicy.swift +102 -0
  94. package/ios/PolicyKit/Sources/AcsAudioPolicy/AudioPolicyApplier.swift +162 -0
  95. package/ios/PolicyKit/Sources/AcsAudioPolicy/GlassesMediaSource.swift +68 -0
  96. package/ios/PolicyKit/Tests/AcsAudioPolicyTests/AcsAudioPolicyTests.swift +273 -0
  97. package/ios/WhepVideoSource.swift +356 -0
  98. package/package.json +56 -0
  99. package/src/AcsMeeting.types.ts +66 -0
  100. package/src/AcsMeetingModule.ts +16 -0
  101. package/src/AcsMeetingModule.web.ts +33 -0
  102. package/src/index.ts +2 -0
@@ -0,0 +1,123 @@
1
+ package com.mentra.acsmeeting
2
+
3
+ import android.util.Log
4
+ import com.azure.android.communication.calling.Call
5
+ import com.azure.android.communication.calling.ParticipantState
6
+ import com.azure.android.communication.calling.ParticipantsUpdatedListener
7
+ import com.azure.android.communication.calling.PropertyChangedListener
8
+ import com.azure.android.communication.calling.RemoteParticipant
9
+ import java.util.concurrent.ConcurrentHashMap
10
+
11
+ /**
12
+ * Tracks ACS remote participants so the host can show who is in the room.
13
+ * Listener callbacks arrive on ACS threads; [onChanged] must hop to the
14
+ * session executor before touching session state.
15
+ */
16
+ class RemoteRoster(
17
+ private val onChanged: () -> Unit,
18
+ ) {
19
+ private class Tracked(
20
+ val participant: RemoteParticipant,
21
+ val listener: PropertyChangedListener,
22
+ )
23
+
24
+ private var call: Call? = null
25
+ private var updatedListener: ParticipantsUpdatedListener? = null
26
+ private val tracked = ConcurrentHashMap<String, Tracked>()
27
+
28
+ fun attach(joined: Call) {
29
+ detach()
30
+ call = joined
31
+ val listener = ParticipantsUpdatedListener { event ->
32
+ event.addedParticipants.forEach { track(it) }
33
+ event.removedParticipants.forEach { untrack(it) }
34
+ Log.i(TAG, "roster updated added=${event.addedParticipants.size} removed=${event.removedParticipants.size} size=${tracked.size}")
35
+ onChanged()
36
+ }
37
+ joined.addOnRemoteParticipantsUpdatedListener(listener)
38
+ updatedListener = listener
39
+ joined.remoteParticipants.forEach { track(it) }
40
+ Log.i(TAG, "roster attached size=${tracked.size}")
41
+ }
42
+
43
+ fun detach() {
44
+ val joined = call
45
+ val listener = updatedListener
46
+ call = null
47
+ updatedListener = null
48
+ if (joined != null && listener != null) {
49
+ try {
50
+ joined.removeOnRemoteParticipantsUpdatedListener(listener)
51
+ } catch (_: Exception) {
52
+ }
53
+ }
54
+ tracked.values.forEach { removeListeners(it) }
55
+ tracked.clear()
56
+ }
57
+
58
+ fun snapshot(): List<Map<String, Any?>> =
59
+ tracked.values
60
+ .map { describe(it.participant) }
61
+ .sortedBy { it["id"] as? String ?: "" }
62
+
63
+ private fun track(participant: RemoteParticipant) {
64
+ val id = idOf(participant)
65
+ if (tracked.containsKey(id)) return
66
+ val listener = PropertyChangedListener { onChanged() }
67
+ val entry = Tracked(participant, listener)
68
+ tracked[id] = entry
69
+ try {
70
+ participant.addOnStateChangedListener(listener)
71
+ participant.addOnIsMutedChangedListener(listener)
72
+ participant.addOnIsSpeakingChangedListener(listener)
73
+ participant.addOnDisplayNameChangedListener(listener)
74
+ } catch (error: Exception) {
75
+ Log.w(TAG, "roster listener attach failed", error)
76
+ }
77
+ }
78
+
79
+ private fun untrack(participant: RemoteParticipant) {
80
+ tracked.remove(idOf(participant))?.let { removeListeners(it) }
81
+ }
82
+
83
+ private fun removeListeners(entry: Tracked) {
84
+ try {
85
+ entry.participant.removeOnStateChangedListener(entry.listener)
86
+ entry.participant.removeOnIsMutedChangedListener(entry.listener)
87
+ entry.participant.removeOnIsSpeakingChangedListener(entry.listener)
88
+ entry.participant.removeOnDisplayNameChangedListener(entry.listener)
89
+ } catch (_: Exception) {
90
+ }
91
+ }
92
+
93
+ companion object {
94
+ private const val TAG = "ACS-SPIKE"
95
+
96
+ fun idOf(participant: RemoteParticipant): String =
97
+ try {
98
+ participant.identifier?.rawId ?: "participant-${System.identityHashCode(participant)}"
99
+ } catch (_: Exception) {
100
+ "participant-${System.identityHashCode(participant)}"
101
+ }
102
+
103
+ fun describe(participant: RemoteParticipant): Map<String, Any?> {
104
+ val state = try { participant.state } catch (_: Exception) { null }
105
+ return mapOf(
106
+ "id" to idOf(participant),
107
+ "displayName" to (try { participant.displayName } catch (_: Exception) { null }),
108
+ "state" to stateName(state),
109
+ "isMuted" to (try { participant.isMuted } catch (_: Exception) { false }),
110
+ "isSpeaking" to (try { participant.isSpeaking } catch (_: Exception) { false }),
111
+ )
112
+ }
113
+
114
+ fun stateName(state: ParticipantState?): String = when (state) {
115
+ ParticipantState.CONNECTED -> "connected"
116
+ ParticipantState.CONNECTING, ParticipantState.RINGING, ParticipantState.EARLY_MEDIA -> "connecting"
117
+ ParticipantState.IN_LOBBY -> "lobby"
118
+ ParticipantState.HOLD -> "hold"
119
+ ParticipantState.DISCONNECTED -> "disconnected"
120
+ ParticipantState.IDLE, null -> "idle"
121
+ }
122
+ }
123
+ }
@@ -0,0 +1,90 @@
1
+ package com.mentra.acsmeeting.audio
2
+
3
+ enum class AudioSourceKind { GLASSES, PHONE }
4
+
5
+ /** Mirrors (AudioStreamType, AudioStreamState). Attached-but-STOPPED reads as NONE. */
6
+ enum class ActiveStreamKind { NONE, VIRTUAL, LOCAL }
7
+
8
+ enum class PhysicalMuteAction { MUTE, UNMUTE, LEAVE_ALONE }
9
+
10
+ enum class AudioSafety { SAFE, DEGRADED, UNSAFE }
11
+
12
+ data class AudioPolicyDecision(
13
+ val glassesPcmEnabled: Boolean,
14
+ val phonePcmEnabled: Boolean,
15
+ val physicalMute: PhysicalMuteAction,
16
+ )
17
+
18
+ /** Transport mute is an ACS join concern, distinct from user intent. */
19
+ data class JoinAudioPlan(
20
+ val armVirtual: Boolean,
21
+ val transportMuted: Boolean,
22
+ )
23
+
24
+ object AcsAudioPolicy {
25
+ fun decide(
26
+ desired: AudioSourceKind,
27
+ active: ActiveStreamKind,
28
+ userMuted: Boolean,
29
+ ): AudioPolicyDecision {
30
+ val glassesPcmEnabled =
31
+ active == ActiveStreamKind.VIRTUAL &&
32
+ desired == AudioSourceKind.GLASSES &&
33
+ !userMuted
34
+ val phonePcmEnabled =
35
+ active == ActiveStreamKind.VIRTUAL &&
36
+ desired == AudioSourceKind.PHONE &&
37
+ !userMuted
38
+ val physicalMute = when (active) {
39
+ ActiveStreamKind.VIRTUAL, ActiveStreamKind.NONE -> PhysicalMuteAction.LEAVE_ALONE
40
+ ActiveStreamKind.LOCAL ->
41
+ if (desired == AudioSourceKind.PHONE && !userMuted) {
42
+ PhysicalMuteAction.UNMUTE
43
+ } else {
44
+ PhysicalMuteAction.MUTE
45
+ }
46
+ }
47
+ return AudioPolicyDecision(glassesPcmEnabled, phonePcmEnabled, physicalMute)
48
+ }
49
+
50
+ fun planJoin(
51
+ desired: AudioSourceKind,
52
+ userMuted: Boolean,
53
+ glassesRequiresUnmutedTransport: Boolean,
54
+ ): JoinAudioPlan {
55
+ // Phone and glasses both feed RawOutgoingAudioStream. Mute is PCM/capturer
56
+ // gating, never a LocalOutgoingAudioStream (that path makes ACS own the route).
57
+ val armVirtual = true
58
+ val transportMuted =
59
+ if (desired == AudioSourceKind.GLASSES && glassesRequiresUnmutedTransport) {
60
+ false
61
+ } else if (desired == AudioSourceKind.PHONE) {
62
+ false
63
+ } else {
64
+ userMuted
65
+ }
66
+ return JoinAudioPlan(armVirtual = armVirtual, transportMuted = transportMuted)
67
+ }
68
+
69
+ @Suppress("UNUSED_PARAMETER")
70
+ fun expectedStream(desired: AudioSourceKind): ActiveStreamKind = ActiveStreamKind.VIRTUAL
71
+
72
+ fun parseSource(raw: String?): AudioSourceKind? =
73
+ when (raw) {
74
+ "glasses" -> AudioSourceKind.GLASSES
75
+ "phone" -> AudioSourceKind.PHONE
76
+ else -> null
77
+ }
78
+ }
79
+
80
+ /** Layer 1: what the glasses encoder captures for this WHIP session. Independent of ACS. */
81
+ object CapturePolicy {
82
+ fun captureGlassesMic(source: AudioSourceKind): Boolean = source == AudioSourceKind.GLASSES
83
+ }
84
+
85
+ /** Null ACS call handles must fail, never report success via `call?.mute(); Unit`. */
86
+ object CallGuard {
87
+ fun <T : Any> require(value: T?): Result<T> =
88
+ if (value == null) Result.failure(IllegalStateException("no call"))
89
+ else Result.success(value)
90
+ }
@@ -0,0 +1,176 @@
1
+ package com.mentra.acsmeeting.audio
2
+
3
+ interface AudioStreamController {
4
+ fun readActive(): ActiveStreamKind
5
+ fun isPhysicallyMuted(): Boolean?
6
+ fun setGlassesPcmEnabled(enabled: Boolean)
7
+ fun setPhonePcmEnabled(enabled: Boolean)
8
+ fun mutePhysical(): Result<Unit>
9
+ fun unmutePhysical(): Result<Unit>
10
+ fun stopActive(): Result<Unit>
11
+ }
12
+
13
+ interface PolicyScheduler {
14
+ fun schedule(delayMs: Long, task: () -> Unit)
15
+ fun cancelPending()
16
+ }
17
+
18
+ class AudioPolicyApplier(
19
+ private val controller: AudioStreamController,
20
+ private val scheduler: PolicyScheduler,
21
+ private val log: (String) -> Unit = {},
22
+ ) {
23
+ private var generation = 0L
24
+ private var lastDecision: AudioPolicyDecision? = null
25
+ private var lastSafety: AudioSafety = AudioSafety.DEGRADED
26
+
27
+ fun lastSafety(): AudioSafety = lastSafety
28
+
29
+ fun apply(desired: AudioSourceKind, userMuted: Boolean, reason: String): AudioSafety {
30
+ generation += 1
31
+ scheduler.cancelPending()
32
+ return runApply(desired, userMuted, reason, generation, force = false, armConvergence = true)
33
+ }
34
+
35
+ fun reset() {
36
+ generation += 1
37
+ scheduler.cancelPending()
38
+ lastDecision = null
39
+ lastSafety = AudioSafety.DEGRADED
40
+ }
41
+
42
+ private fun runApply(
43
+ desired: AudioSourceKind,
44
+ userMuted: Boolean,
45
+ reason: String,
46
+ gen: Long,
47
+ force: Boolean,
48
+ armConvergence: Boolean,
49
+ ): AudioSafety {
50
+ if (gen != generation) return lastSafety
51
+ val active = controller.readActive()
52
+ val decision = AcsAudioPolicy.decide(desired, active, userMuted)
53
+ val outcome = applyDecision(decision, force)
54
+ if (gen != generation) return lastSafety
55
+
56
+ val after = controller.readActive()
57
+ val expected = AcsAudioPolicy.expectedStream(desired)
58
+ val safety = safetyOf(desired, after, outcome)
59
+ lastDecision = decision
60
+ lastSafety = safety
61
+ log(
62
+ "audio policy reason=$reason source=$desired userMuted=$userMuted " +
63
+ "active=$after expected=$expected glassesPcm=${decision.glassesPcmEnabled} " +
64
+ "phonePcm=${decision.phonePcmEnabled} mute=${decision.physicalMute} safety=$safety",
65
+ )
66
+
67
+ if (after == expected) {
68
+ scheduler.cancelPending()
69
+ } else if (armConvergence && gen == generation) {
70
+ armConvergence(desired, userMuted, gen, expected)
71
+ }
72
+ return lastSafety
73
+ }
74
+
75
+ private fun armConvergence(
76
+ desired: AudioSourceKind,
77
+ userMuted: Boolean,
78
+ gen: Long,
79
+ expected: ActiveStreamKind,
80
+ ) {
81
+ CONVERGE_DELAYS_MS.forEachIndexed { index, delayMs ->
82
+ scheduler.schedule(delayMs) {
83
+ if (gen != generation) return@schedule
84
+ val isLast = index == CONVERGE_DELAYS_MS.lastIndex
85
+ runApply(desired, userMuted, "converge-${delayMs}ms", gen, force = true, armConvergence = false)
86
+ if (isLast && gen == generation && controller.readActive() != expected) {
87
+ log("audio policy convergence timed out source=$desired expected=$expected holding silence")
88
+ lastSafety = AudioSafety.DEGRADED
89
+ }
90
+ }
91
+ }
92
+ }
93
+
94
+ private fun applyDecision(decision: AudioPolicyDecision, force: Boolean): EffectOutcome {
95
+ val previous = lastDecision
96
+ val skipGlassesPcm = !force && previous != null && previous.glassesPcmEnabled == decision.glassesPcmEnabled
97
+ val skipPhonePcm = !force && previous != null && previous.phonePcmEnabled == decision.phonePcmEnabled
98
+ val skipMute = !force && previous != null && previous.physicalMute == decision.physicalMute
99
+ if (!skipGlassesPcm) {
100
+ controller.setGlassesPcmEnabled(decision.glassesPcmEnabled)
101
+ }
102
+ if (!skipPhonePcm) {
103
+ controller.setPhonePcmEnabled(decision.phonePcmEnabled)
104
+ }
105
+ if (skipMute) return EffectOutcome.NONE
106
+
107
+ return when (decision.physicalMute) {
108
+ PhysicalMuteAction.LEAVE_ALONE -> EffectOutcome.NONE
109
+ PhysicalMuteAction.UNMUTE -> {
110
+ val result = controller.unmutePhysical()
111
+ val unmuted = controller.isPhysicallyMuted()
112
+ val failed = result.isFailure || unmuted == true
113
+ EffectOutcome(unmuteAttempted = true, unmuteFailed = failed)
114
+ }
115
+ PhysicalMuteAction.MUTE -> {
116
+ val result = controller.mutePhysical()
117
+ val muted = controller.isPhysicallyMuted()
118
+ val muteFailed = result.isFailure || muted == false
119
+ if (!muteFailed) return EffectOutcome(muteAttempted = true)
120
+ val stop = controller.stopActive()
121
+ val stopped = controller.readActive() == ActiveStreamKind.NONE
122
+ EffectOutcome(
123
+ muteAttempted = true,
124
+ muteFailed = true,
125
+ stopAttempted = true,
126
+ stopFailed = stop.isFailure || !stopped,
127
+ )
128
+ }
129
+ }
130
+ }
131
+
132
+ private fun safetyOf(
133
+ desired: AudioSourceKind,
134
+ active: ActiveStreamKind,
135
+ outcome: EffectOutcome,
136
+ ): AudioSafety {
137
+ if (outcome.stopFailed) return AudioSafety.UNSAFE
138
+ if (outcome.stopAttempted) return AudioSafety.SAFE
139
+ if (outcome.unmuteFailed) return AudioSafety.DEGRADED
140
+ val expected = AcsAudioPolicy.expectedStream(desired)
141
+ if (active != expected) return AudioSafety.DEGRADED
142
+ return AudioSafety.SAFE
143
+ }
144
+
145
+ private data class EffectOutcome(
146
+ val muteAttempted: Boolean = false,
147
+ val muteFailed: Boolean = false,
148
+ val unmuteAttempted: Boolean = false,
149
+ val unmuteFailed: Boolean = false,
150
+ val stopAttempted: Boolean = false,
151
+ val stopFailed: Boolean = false,
152
+ ) {
153
+ companion object {
154
+ val NONE = EffectOutcome()
155
+ }
156
+ }
157
+
158
+ companion object {
159
+ val CONVERGE_DELAYS_MS = longArrayOf(50, 100, 200, 400, 800, 1600)
160
+ }
161
+ }
162
+
163
+ class ExecutorPolicyScheduler(
164
+ private val executor: java.util.concurrent.ScheduledExecutorService,
165
+ ) : PolicyScheduler {
166
+ private val pending = java.util.concurrent.CopyOnWriteArrayList<java.util.concurrent.ScheduledFuture<*>>()
167
+
168
+ override fun schedule(delayMs: Long, task: () -> Unit) {
169
+ pending.add(executor.schedule(task, delayMs, java.util.concurrent.TimeUnit.MILLISECONDS))
170
+ }
171
+
172
+ override fun cancelPending() {
173
+ pending.forEach { it.cancel(false) }
174
+ pending.clear()
175
+ }
176
+ }
@@ -0,0 +1,112 @@
1
+ package com.mentra.acsmeeting.audio
2
+
3
+ import java.io.ByteArrayOutputStream
4
+
5
+ /**
6
+ * ACS mixed-audio buffers → fixed-format chunks for the host PCM player.
7
+ *
8
+ * - Normalizes to [OUT_RATE] Hz mono regardless of what ACS actually
9
+ * delivers. The requested RawIncomingAudioStreamProperties are a hint;
10
+ * the event carries the real format, and playing 48 kHz data on a 16 kHz
11
+ * AudioTrack is exactly the "slow, low-pitched" symptom.
12
+ * - Pre-rolls [prerollMs] before the first emit and then emits in
13
+ * [batchMs] chunks, so the player always has headroom over bridge jitter
14
+ * instead of underrunning between 20 ms callbacks.
15
+ *
16
+ * Not thread-safe: ACS delivers mixed audio from a single thread.
17
+ */
18
+ class IncomingAudioPump(
19
+ private val prerollMs: Int = DEFAULT_PREROLL_MS,
20
+ private val batchMs: Int = DEFAULT_BATCH_MS,
21
+ private val emit: (pcm16Le: ByteArray) -> Unit,
22
+ ) {
23
+ data class Format(val sampleRate: Int, val channels: Int)
24
+
25
+ private val resampler = PcmResampler(OUT_RATE)
26
+ private val pending = ByteArrayOutputStream()
27
+ private var primed = false
28
+
29
+ var format: Format? = null
30
+ private set
31
+ var formatChanges = 0
32
+ private set
33
+ var eventsIn = 0L
34
+ private set
35
+ var bytesIn = 0L
36
+ private set
37
+ var bytesOut = 0L
38
+ private set
39
+
40
+ fun push(pcm16Le: ByteArray, sampleRate: Int, channels: Int) {
41
+ if (pcm16Le.isEmpty()) return
42
+ val next = Format(sampleRate, channels)
43
+ if (format != next) {
44
+ if (format != null) formatChanges += 1
45
+ format = next
46
+ }
47
+ eventsIn += 1
48
+ bytesIn += pcm16Le.size
49
+ val normalized = resampler.process(pcm16Le, sampleRate, channels)
50
+ pending.write(normalized)
51
+ drain(force = false)
52
+ }
53
+
54
+ /** Emit whatever is buffered (call on stop so the tail is not lost). */
55
+ fun flush() {
56
+ drain(force = true)
57
+ primed = false
58
+ }
59
+
60
+ fun reset() {
61
+ pending.reset()
62
+ primed = false
63
+ resampler.reset()
64
+ format = null
65
+ }
66
+
67
+ private fun drain(force: Boolean) {
68
+ val threshold = if (primed) bytesFor(batchMs) else bytesFor(prerollMs)
69
+ if (pending.size() == 0) return
70
+ if (!force && pending.size() < threshold) return
71
+ val all = pending.toByteArray()
72
+ pending.reset()
73
+ // Chunk on batch boundaries so the player queue sees a steady cadence;
74
+ // the remainder waits for the next callback unless flushing.
75
+ val batch = bytesFor(batchMs)
76
+ var offset = 0
77
+ while (all.size - offset >= batch) {
78
+ emitChunk(all.copyOfRange(offset, offset + batch))
79
+ offset += batch
80
+ }
81
+ if (offset < all.size) {
82
+ if (force) emitChunk(all.copyOfRange(offset, all.size)) else pending.write(all, offset, all.size - offset)
83
+ }
84
+ primed = true
85
+ }
86
+
87
+ private fun emitChunk(chunk: ByteArray) {
88
+ bytesOut += chunk.size
89
+ emit(chunk)
90
+ }
91
+
92
+ companion object {
93
+ const val OUT_RATE = 16_000
94
+ const val OUT_CHANNELS = 1
95
+
96
+ /**
97
+ * Downlink headroom, and the only knob for the downlink latency A/B.
98
+ *
99
+ * The remote path already stacks Teams/ACS network + ACS receive + RN
100
+ * bridge batching + this preroll + A2DP buffering, so more is not simply
101
+ * better: every millisecond here is conversational delay. Walk
102
+ * [PREROLL_LADDER_MS] on real S22 + Mentra Live hardware and ship the
103
+ * smallest value with no audible underruns. 120 ms underran on the
104
+ * reported jitter; 160 ms is one step up, not the top of the ladder.
105
+ */
106
+ const val DEFAULT_PREROLL_MS = 160
107
+ val PREROLL_LADDER_MS = listOf(120, 160, 200)
108
+ const val DEFAULT_BATCH_MS = 60
109
+
110
+ fun bytesFor(ms: Int): Int = OUT_RATE * OUT_CHANNELS * 2 * ms / 1000
111
+ }
112
+ }
@@ -0,0 +1,65 @@
1
+ package com.mentra.acsmeeting.audio
2
+
3
+ import kotlin.math.max
4
+
5
+ /**
6
+ * Measures what ACS actually delivers on the incoming mixed-audio callback.
7
+ *
8
+ * Deriving a rate as `bytes / (2 * channels) * 50` only holds if ACS really
9
+ * fires exactly every 20 ms — which is the assumption being diagnosed. This
10
+ * accumulates samples against elapsed monotonic time instead, so a rate ACS
11
+ * mislabels shows up as `measuredRate` disagreeing with `declaredRate` and a
12
+ * cadence that is not 20 ms shows up in `callbackHz`.
13
+ */
14
+ class IncomingRateProbe(private val windowNanos: Long = DEFAULT_WINDOW_NANOS) {
15
+ data class Reading(
16
+ val declaredRate: Int,
17
+ val channels: Int,
18
+ val samplesPerCallback: Int,
19
+ val callbackHz: Double,
20
+ val measuredRate: Double,
21
+ val events: Long,
22
+ )
23
+
24
+ private var windowStartNanos: Long? = null
25
+ private var events = 0L
26
+ private var samples = 0L
27
+
28
+ /** Returns a reading at most once per window; null in between. */
29
+ fun record(nowNanos: Long, bytes: Int, sampleRate: Int, channels: Int): Reading? {
30
+ val samplesPerCallback = bytes / (2 * max(1, channels))
31
+ val start = windowStartNanos
32
+ if (start == null) {
33
+ // This callback only opens the window; it arrived before it.
34
+ windowStartNanos = nowNanos
35
+ return null
36
+ }
37
+ events += 1
38
+ samples += samplesPerCallback
39
+ val elapsedNanos = nowNanos - start
40
+ if (elapsedNanos < windowNanos) return null
41
+ val seconds = elapsedNanos / 1_000_000_000.0
42
+ val reading = Reading(
43
+ declaredRate = sampleRate,
44
+ channels = channels,
45
+ samplesPerCallback = samplesPerCallback,
46
+ callbackHz = events / seconds,
47
+ measuredRate = samples / seconds,
48
+ events = events,
49
+ )
50
+ windowStartNanos = nowNanos
51
+ events = 0
52
+ samples = 0
53
+ return reading
54
+ }
55
+
56
+ fun reset() {
57
+ windowStartNanos = null
58
+ events = 0
59
+ samples = 0
60
+ }
61
+
62
+ companion object {
63
+ const val DEFAULT_WINDOW_NANOS = 1_000_000_000L
64
+ }
65
+ }
@@ -0,0 +1,108 @@
1
+ package com.mentra.acsmeeting.audio
2
+
3
+ import android.util.Base64
4
+ import android.util.Log
5
+ import java.io.ByteArrayOutputStream
6
+ import java.io.File
7
+ import java.nio.ByteBuffer
8
+ import java.nio.ByteOrder
9
+ import kotlin.math.abs
10
+
11
+ /** Downmix/resample PCM16 to 48 kHz mono ~20 ms blocks for ACS raw outgoing audio. */
12
+ class PcmBridge(
13
+ private val dumpDir: File?,
14
+ private val dumpWav: Boolean,
15
+ ) {
16
+ private var dump: ByteArrayOutputStream? = if (dumpWav) ByteArrayOutputStream() else null
17
+ private var dumpRate = 0
18
+ private var dumpChannels = 0
19
+ private var dumped = false
20
+ private val outgoing = ByteArrayOutputStream()
21
+ private val resampler = PcmResampler(TARGET_RATE)
22
+ private var lastRmsLogMs = 0L
23
+
24
+ fun ingest(pcm16Le: ByteArray, sampleRate: Int, channels: Int): List<ByteArray> {
25
+ if (dump != null && !dumped) {
26
+ dump!!.write(pcm16Le)
27
+ dumpRate = sampleRate
28
+ dumpChannels = channels
29
+ val needed = 30 * sampleRate * channels * 2
30
+ if (dump!!.size() >= needed) finishDump()
31
+ }
32
+ logLevel(pcm16Le, sampleRate, channels)
33
+ // Stateful: filter history and fractional phase carry across WebRTC's
34
+ // 10 ms callbacks, so there is no seam every 480 samples.
35
+ val mono = resampler.process(pcm16Le, sampleRate, channels)
36
+ outgoing.write(mono)
37
+ val frameBytes = TARGET_RATE * 2 / 50 // 20 ms
38
+ val frames = mutableListOf<ByteArray>()
39
+ val buf = outgoing.toByteArray()
40
+ var offset = 0
41
+ while (buf.size - offset >= frameBytes) {
42
+ frames.add(buf.copyOfRange(offset, offset + frameBytes))
43
+ offset += frameBytes
44
+ }
45
+ outgoing.reset()
46
+ if (offset < buf.size) outgoing.write(buf, offset, buf.size - offset)
47
+ return frames
48
+ }
49
+
50
+ @Synchronized
51
+ fun finishDump() {
52
+ if (dumped) return
53
+ val data = dump?.toByteArray() ?: return
54
+ dumped = true
55
+ dump = null
56
+ val dir = dumpDir ?: return
57
+ val file = File(dir, "acs-whep-p4.wav")
58
+ file.writeBytes(wav(data, dumpRate, dumpChannels))
59
+ Log.i(TAG, "P4 wrote ${file.absolutePath} bytes=${data.size} rate=$dumpRate ch=$dumpChannels")
60
+ }
61
+
62
+ private fun logLevel(pcm: ByteArray, sampleRate: Int, channels: Int) {
63
+ val now = System.currentTimeMillis()
64
+ if (now - lastRmsLogMs < 1000) return
65
+ lastRmsLogMs = now
66
+ var acc = 0L
67
+ var n = 0
68
+ var i = 0
69
+ while (i + 1 < pcm.size) {
70
+ val s = (pcm[i].toInt() and 0xff) or (pcm[i + 1].toInt() shl 8)
71
+ val signed = if (s >= 0x8000) s - 0x10000 else s
72
+ acc += abs(signed)
73
+ n++
74
+ i += 2
75
+ }
76
+ val mean = if (n == 0) 0 else acc / n
77
+ Log.i(TAG, "P4 pcm rate=$sampleRate ch=$channels meanAbs=$mean bytes=${pcm.size}")
78
+ }
79
+
80
+ companion object {
81
+ private const val TAG = "ACS-SPIKE"
82
+ const val TARGET_RATE = 48_000
83
+
84
+ /** One-shot convenience (fresh filter state). Streams must use a [PcmResampler] instance. */
85
+ fun toMono(pcm16Le: ByteArray, sampleRate: Int, channels: Int): ByteArray =
86
+ PcmResampler(TARGET_RATE).process(pcm16Le, sampleRate, channels)
87
+
88
+ fun wav(pcm: ByteArray, sampleRate: Int, channels: Int): ByteArray {
89
+ val header = ByteBuffer.allocate(44).order(ByteOrder.LITTLE_ENDIAN)
90
+ header.put("RIFF".toByteArray())
91
+ header.putInt(36 + pcm.size)
92
+ header.put("WAVE".toByteArray())
93
+ header.put("fmt ".toByteArray())
94
+ header.putInt(16)
95
+ header.putShort(1)
96
+ header.putShort(channels.toShort())
97
+ header.putInt(sampleRate)
98
+ header.putInt(sampleRate * channels * 2)
99
+ header.putShort((channels * 2).toShort())
100
+ header.putShort(16)
101
+ header.put("data".toByteArray())
102
+ header.putInt(pcm.size)
103
+ return header.array() + pcm
104
+ }
105
+
106
+ fun encodeBase64(pcm: ByteArray): String = Base64.encodeToString(pcm, Base64.NO_WRAP)
107
+ }
108
+ }