@mentra/engine 3.2.0-dev.221 → 3.2.0-dev.223
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/generated/releaseMetadata.js +5 -5
- package/build/generated/releaseMetadata.js.map +1 -1
- package/build/index.d.ts +2 -0
- package/build/index.d.ts.map +1 -1
- package/build/index.js +1 -0
- package/build/index.js.map +1 -1
- package/build/services/AcsMeetingService.d.ts +360 -10
- package/build/services/AcsMeetingService.d.ts.map +1 -1
- package/build/services/AcsMeetingService.js +836 -19
- package/build/services/AcsMeetingService.js.map +1 -1
- package/build/services/AudioPlaybackService.d.ts +6 -0
- package/build/services/AudioPlaybackService.d.ts.map +1 -1
- package/build/services/AudioPlaybackService.js +4 -3
- package/build/services/AudioPlaybackService.js.map +1 -1
- package/build/services/GlassesMicProbe.d.ts +78 -0
- package/build/services/GlassesMicProbe.d.ts.map +1 -0
- package/build/services/GlassesMicProbe.js +258 -0
- package/build/services/GlassesMicProbe.js.map +1 -0
- package/build/services/LocalMiniappRuntime.d.ts +83 -0
- package/build/services/LocalMiniappRuntime.d.ts.map +1 -1
- package/build/services/LocalMiniappRuntime.js +714 -39
- package/build/services/LocalMiniappRuntime.js.map +1 -1
- package/build/services/MentraJSLogPipeline.d.ts +39 -0
- package/build/services/MentraJSLogPipeline.d.ts.map +1 -1
- package/build/services/MentraJSLogPipeline.js +59 -3
- package/build/services/MentraJSLogPipeline.js.map +1 -1
- package/build/services/MentraJSRouter.d.ts +1 -1
- package/build/services/MentraJSRouter.d.ts.map +1 -1
- package/build/services/MentraJSRouter.js +2 -2
- package/build/services/MentraJSRouter.js.map +1 -1
- package/build/services/MicStateCoordinator.d.ts +22 -0
- package/build/services/MicStateCoordinator.d.ts.map +1 -1
- package/build/services/MicStateCoordinator.js +34 -3
- package/build/services/MicStateCoordinator.js.map +1 -1
- package/build/services/PhoneStreamCoordinator.d.ts +8 -0
- package/build/services/PhoneStreamCoordinator.d.ts.map +1 -1
- package/build/services/PhoneStreamCoordinator.js +2 -0
- package/build/services/PhoneStreamCoordinator.js.map +1 -1
- package/build/services/SoftapCallTransport.d.ts +365 -0
- package/build/services/SoftapCallTransport.d.ts.map +1 -0
- package/build/services/SoftapCallTransport.js +722 -0
- package/build/services/SoftapCallTransport.js.map +1 -0
- package/build/services/SoftapCleanupBarrier.d.ts +45 -0
- package/build/services/SoftapCleanupBarrier.d.ts.map +1 -0
- package/build/services/SoftapCleanupBarrier.js +51 -0
- package/build/services/SoftapCleanupBarrier.js.map +1 -0
- package/build/utils/pcm16.d.ts +35 -0
- package/build/utils/pcm16.d.ts.map +1 -0
- package/build/utils/pcm16.js +85 -0
- package/build/utils/pcm16.js.map +1 -0
- package/build/utils/softapTrace.d.ts +39 -0
- package/build/utils/softapTrace.d.ts.map +1 -0
- package/build/utils/softapTrace.js +124 -0
- package/build/utils/softapTrace.js.map +1 -0
- package/package.json +7 -7
- package/src/generated/releaseMetadata.ts +5 -5
- package/src/index.ts +2 -0
- package/src/services/AcsMeetingService.ts +956 -22
- package/src/services/AudioPlaybackService.ts +12 -3
- package/src/services/GlassesMicProbe.ts +300 -0
- package/src/services/LocalMiniappRuntime.ts +775 -42
- package/src/services/MentraJSLogPipeline.ts +70 -3
- package/src/services/MentraJSRouter.ts +2 -2
- package/src/services/MicStateCoordinator.ts +35 -3
- package/src/services/PhoneStreamCoordinator.ts +10 -0
- package/src/services/SoftapCallTransport.ts +928 -0
- package/src/services/SoftapCleanupBarrier.ts +72 -0
- package/src/utils/pcm16.ts +93 -0
- package/src/utils/softapTrace.ts +133 -0
|
@@ -4,9 +4,17 @@
|
|
|
4
4
|
* incoming PCM into AudioPlaybackService (A2DP / PcmStreamPlayer).
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
+
import {Platform} from "react-native"
|
|
8
|
+
|
|
9
|
+
import BluetoothSdk from "@mentra/bluetooth-sdk/internal"
|
|
10
|
+
|
|
7
11
|
import audioPlaybackService from "./AudioPlaybackService"
|
|
12
|
+
import micStateCoordinator from "./MicStateCoordinator"
|
|
8
13
|
import {SETTINGS, useSettingsStore} from "../stores/settings"
|
|
9
|
-
import {
|
|
14
|
+
import {Pcm16LevelMeter} from "../utils/pcm16"
|
|
15
|
+
import {softapTrace, softapTraceFailure, softapTraceId} from "../utils/softapTrace"
|
|
16
|
+
import {ACS_CALL_MIC, type AcsAudioSource, type ResolvedAudioSource, type SourceReason} from "./acsAudioSource"
|
|
17
|
+
import type {SoftapProgress} from "./SoftapCallTransport"
|
|
10
18
|
|
|
11
19
|
export {ACS_CALL_MIC}
|
|
12
20
|
export type {ResolvedAudioSource, SourceReason}
|
|
@@ -22,6 +30,19 @@ export type ActiveStream = "none" | "virtual" | "local"
|
|
|
22
30
|
*/
|
|
23
31
|
export type MediaSourceState = "idle" | "connecting" | "live" | "failed"
|
|
24
32
|
|
|
33
|
+
/**
|
|
34
|
+
* How the wearer's voice is reaching ACS.
|
|
35
|
+
*
|
|
36
|
+
* `ble-lc3` is the glasses microphone over Bluetooth, decoded on the phone and pushed into the ACS
|
|
37
|
+
* raw outgoing stream. `whip` is the glasses microphone on the WebRTC track they publish (SoftAP or
|
|
38
|
+
* Cloudflare). `phone` is the phone's own microphone. `none` means the source we committed to went
|
|
39
|
+
* away mid-call, which is a reportable failure rather than a reason to open a different microphone.
|
|
40
|
+
*/
|
|
41
|
+
export type MicTransport = "ble-lc3" | "whip" | "phone" | "none"
|
|
42
|
+
|
|
43
|
+
/** Error surfaced when the pinned glasses microphone stops producing audio during a call. */
|
|
44
|
+
export const GLASSES_MIC_UNAVAILABLE = "GLASSES_MIC_UNAVAILABLE"
|
|
45
|
+
|
|
25
46
|
export type MeetingParticipantState = "idle" | "connecting" | "connected" | "lobby" | "hold" | "disconnected"
|
|
26
47
|
|
|
27
48
|
export interface MeetingParticipant {
|
|
@@ -32,6 +53,36 @@ export interface MeetingParticipant {
|
|
|
32
53
|
isSpeaking: boolean
|
|
33
54
|
}
|
|
34
55
|
|
|
56
|
+
/**
|
|
57
|
+
* One runtime participant capability.
|
|
58
|
+
*
|
|
59
|
+
* `allowed` is nullable because "denied" and "not known yet" are different facts. ACS delivers
|
|
60
|
+
* capabilities asynchronously, so every call is briefly unknown, and collapsing that into `false`
|
|
61
|
+
* would hide controls on calls that do in fact permit them.
|
|
62
|
+
*/
|
|
63
|
+
export interface MeetingCapability {
|
|
64
|
+
allowed: boolean | null
|
|
65
|
+
reason: string | null
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export interface MeetingCapabilities {
|
|
69
|
+
/** Whether this participant may end the Teams group call for everyone. Presenters only. */
|
|
70
|
+
hangUpForEveryone: MeetingCapability
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export function parseMeetingCapabilities(raw: unknown): MeetingCapabilities | undefined {
|
|
74
|
+
if (!raw || typeof raw !== "object") return undefined
|
|
75
|
+
const value = (raw as Record<string, unknown>).hangUpForEveryone
|
|
76
|
+
if (!value || typeof value !== "object") return undefined
|
|
77
|
+
const capability = value as Record<string, unknown>
|
|
78
|
+
return {
|
|
79
|
+
hangUpForEveryone: {
|
|
80
|
+
allowed: typeof capability.allowed === "boolean" ? capability.allowed : null,
|
|
81
|
+
reason: typeof capability.reason === "string" && capability.reason ? capability.reason : null,
|
|
82
|
+
},
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
35
86
|
export interface MeetingState {
|
|
36
87
|
state: MeetingPhase
|
|
37
88
|
muted: boolean
|
|
@@ -44,19 +95,84 @@ export interface MeetingState {
|
|
|
44
95
|
audioSafety?: AudioSafety
|
|
45
96
|
mediaSource?: MediaSourceState
|
|
46
97
|
participants?: MeetingParticipant[]
|
|
98
|
+
/** Runtime capabilities. Omitted by natives that predate them; read that as unknown. */
|
|
99
|
+
capabilities?: MeetingCapabilities
|
|
100
|
+
/** Which microphone path is carrying the wearer's voice, decided at join time. */
|
|
101
|
+
micTransport?: MicTransport
|
|
102
|
+
/**
|
|
103
|
+
* SoftAP join checklist, attached only to the state events the SoftAP orchestrator emits while
|
|
104
|
+
* it walks hotspot → scoped join → ACS join → publish → live. Native ACS events never carry it,
|
|
105
|
+
* so a consumer keeps the last one it saw rather than treating its absence as a reset.
|
|
106
|
+
*/
|
|
107
|
+
softap?: SoftapProgress
|
|
47
108
|
}
|
|
48
109
|
|
|
49
110
|
/**
|
|
50
|
-
* The call microphone is [ACS_CALL_MIC]
|
|
51
|
-
*
|
|
52
|
-
*
|
|
111
|
+
* The call microphone is [ACS_CALL_MIC], and `preferred_mic` governs MentraOS STT capture rather
|
|
112
|
+
* than this call.
|
|
113
|
+
*
|
|
114
|
+
* `"glasses"` selects the wearer's own microphone. Which *transport* carries it is a separate
|
|
115
|
+
* decision made per call by [glassesLc3UplinkSupported]: BLE LC3 into the ACS raw outgoing stream
|
|
116
|
+
* where the host and native both support it, otherwise the glasses' published WebRTC audio track.
|
|
117
|
+
* `"phone"` selects the phone's microphone via `PhoneMicCapturer`; do not route it through ACS
|
|
118
|
+
* `LocalOutgoingAudioStream`, because that hands ACS the audio route
|
|
53
119
|
* (MODE_IN_COMMUNICATION + forced speaker) and opens an echo loop.
|
|
54
|
-
* preferred_mic governs MentraOS STT capture, not this call.
|
|
55
120
|
*/
|
|
56
121
|
export function resolveAcsAudioSource(): ResolvedAudioSource {
|
|
57
122
|
return {source: ACS_CALL_MIC, reason: "explicit"}
|
|
58
123
|
}
|
|
59
124
|
|
|
125
|
+
/**
|
|
126
|
+
* Whether this call can take the wearer's voice off the glasses over BLE LC3.
|
|
127
|
+
*
|
|
128
|
+
* Every term is a hard requirement, and the answer has to be known *before* the glasses are told
|
|
129
|
+
* what to publish — deciding afterwards is how a call ends up with either two copies of the wearer
|
|
130
|
+
* or none. A host whose native module has no `pushOutgoingPcm` keeps the WHIP audio track it has
|
|
131
|
+
* always used rather than silently joining a meeting nobody can be heard in.
|
|
132
|
+
*
|
|
133
|
+
* SoftAP takes BLE LC3 and publishes WHIP video only (`captureAudio=false`), so the glasses SoC
|
|
134
|
+
* microphone is never opened for a call.
|
|
135
|
+
*
|
|
136
|
+
* This was switched off once on a misread: a soak logged `P4 pcm meanAbs` 8–39 and called it
|
|
137
|
+
* digital silence. It was the room. The same LC3 path measured on the bench sits at `meanAbs`
|
|
138
|
+
* ≈30–60 / `peak` ≈80–100 in a quiet room with nobody talking, and that soak had already
|
|
139
|
+
* published with `captureAudio=false`, so the SoC mic was not the thief either. The uplink now
|
|
140
|
+
* logs its own `meanAbs`/`peak` every window ([logMicUplink]) so the next soak reads the level,
|
|
141
|
+
* not the frame rate. Disable only with a level log showing the floor *while the wearer talks*.
|
|
142
|
+
*/
|
|
143
|
+
const SOFTAP_BLE_LC3_UPLINK = true
|
|
144
|
+
/**
|
|
145
|
+
* How long native may take to finish a leave before it reports the cleanup as stuck.
|
|
146
|
+
*
|
|
147
|
+
* Generous on purpose: this is a diagnostic ceiling on a hang-up plus an agent disposal, not a
|
|
148
|
+
* readiness deadline. The caller treats a timeout as "cleanup failed", never as "safe to restart".
|
|
149
|
+
*/
|
|
150
|
+
const ACS_LEAVE_WAIT_MS = 20_000
|
|
151
|
+
/**
|
|
152
|
+
* SoftAP video (camera → H.264 → WHIP → WHEP → ACS) is slower than BLE LC3.
|
|
153
|
+
* Hold glasses PCM this long so the talk track does not lead the picture.
|
|
154
|
+
* Set from SoftAP+LC3 clap correlator (2026-09-10): leads 70/201/173/300/142 ms,
|
|
155
|
+
* median 173, rounded to 170. `AVSYNC clap audioLeadMs` is still the re-cal.
|
|
156
|
+
*/
|
|
157
|
+
export const SOFTAP_LC3_AUDIO_DELAY_MS = 170
|
|
158
|
+
let softapBleLc3UplinkForTests: boolean | null = null
|
|
159
|
+
|
|
160
|
+
export function glassesLc3UplinkSupported(args: {
|
|
161
|
+
videoSource: AcsVideoSource
|
|
162
|
+
audioSource: AcsAudioSource
|
|
163
|
+
hasPushOutgoingPcm: boolean
|
|
164
|
+
platform: string
|
|
165
|
+
}): boolean {
|
|
166
|
+
if (!(softapBleLc3UplinkForTests ?? SOFTAP_BLE_LC3_UPLINK)) return false
|
|
167
|
+
// iOS has no `setMicSourcePin` yet, so it cannot promise the phone microphone stays shut.
|
|
168
|
+
if (args.platform !== "android") return false
|
|
169
|
+
// WHEP audio comes back from Cloudflare already mixed into the subscribed track; there is no
|
|
170
|
+
// captureAudio flag on that path to turn off.
|
|
171
|
+
if (args.videoSource.type !== "softap") return false
|
|
172
|
+
if (args.audioSource !== "glasses") return false
|
|
173
|
+
return args.hasPushOutgoingPcm
|
|
174
|
+
}
|
|
175
|
+
|
|
60
176
|
const PARTICIPANT_STATES = new Set<MeetingParticipantState>([
|
|
61
177
|
"idle",
|
|
62
178
|
"connecting",
|
|
@@ -95,7 +211,7 @@ export type AcsOutgoingVideo = {
|
|
|
95
211
|
}
|
|
96
212
|
|
|
97
213
|
/** ACS VirtualOutgoingVideoStream documented 16:9 sizes. P540 is 960×540, not 540×960. */
|
|
98
|
-
const ACS_VIRTUAL_CAMERA_SIZES = new Set(["1280x720", "960x540"])
|
|
214
|
+
const ACS_VIRTUAL_CAMERA_SIZES = new Set(["1280x720", "960x540", "858x480"])
|
|
99
215
|
|
|
100
216
|
export function parseAcsOutgoingVideo(raw: unknown): AcsOutgoingVideo | undefined {
|
|
101
217
|
if (raw == null) return undefined
|
|
@@ -114,23 +230,127 @@ export function parseAcsOutgoingVideo(raw: unknown): AcsOutgoingVideo | undefine
|
|
|
114
230
|
return {width, height, fps, maxBitrateBps}
|
|
115
231
|
}
|
|
116
232
|
|
|
233
|
+
/**
|
|
234
|
+
* Where the glasses video comes from.
|
|
235
|
+
*
|
|
236
|
+
* A union rather than a nullable URL because the two transports need different inputs: WHEP is
|
|
237
|
+
* given a URL, while SoftAP produces one only after the host binds a local listener. Collapsing
|
|
238
|
+
* them into `whepUrl?: string` is what lets an empty string reach the subscriber and fail seconds
|
|
239
|
+
* later as an opaque HTTP error.
|
|
240
|
+
*/
|
|
241
|
+
export type AcsVideoSource =
|
|
242
|
+
| {type: "whep"; url: string}
|
|
243
|
+
| {type: "softap"; ssid?: string; passphrase?: string; bindAddress?: string}
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* Validates a `videoSource` from a miniapp.
|
|
247
|
+
*
|
|
248
|
+
* Throws rather than defaulting to WHEP. A miniapp that asks for SoftAP and silently gets a
|
|
249
|
+
* Cloudflare call — or vice versa — is a bug that shows up as unexplained latency or a black tile,
|
|
250
|
+
* not as an error anyone can act on.
|
|
251
|
+
*/
|
|
252
|
+
export function parseAcsVideoSource(raw: unknown): AcsVideoSource {
|
|
253
|
+
if (raw == null || typeof raw !== "object") {
|
|
254
|
+
throw new Error(`videoSource must be {type: "whep", url} or {type: "softap"}`)
|
|
255
|
+
}
|
|
256
|
+
const value = raw as Record<string, unknown>
|
|
257
|
+
|
|
258
|
+
if (value.type === "whep") {
|
|
259
|
+
const url = typeof value.url === "string" ? value.url.trim() : ""
|
|
260
|
+
if (!url) throw new Error("videoSource.url is required for a WHEP source")
|
|
261
|
+
return {type: "whep", url}
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
if (value.type === "softap") {
|
|
265
|
+
const ssid = typeof value.ssid === "string" ? value.ssid.trim() : ""
|
|
266
|
+
const passphrase = typeof value.passphrase === "string" ? value.passphrase : ""
|
|
267
|
+
// Half a credential pair would otherwise present as a failed hotspot join much later.
|
|
268
|
+
if (Boolean(ssid) !== Boolean(passphrase)) {
|
|
269
|
+
throw new Error("videoSource.ssid and videoSource.passphrase must be provided together")
|
|
270
|
+
}
|
|
271
|
+
return ssid ? {type: "softap", ssid, passphrase} : {type: "softap"}
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
throw new Error(`unsupported videoSource.type: ${String(value.type)}`)
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
/**
|
|
278
|
+
* The app's default network after a hotspot join.
|
|
279
|
+
*
|
|
280
|
+
* `usable` is the only field to branch on; the rest is for the note the wearer sees and the trace.
|
|
281
|
+
* A held cellular request brings the radio up but does not promise the default route has switched
|
|
282
|
+
* or validated, so this is asked separately rather than inferred from the hold.
|
|
283
|
+
*/
|
|
284
|
+
export interface DefaultNetworkStatus {
|
|
285
|
+
transport: string
|
|
286
|
+
validated: boolean
|
|
287
|
+
present: boolean
|
|
288
|
+
usable: boolean
|
|
289
|
+
detail: string
|
|
290
|
+
}
|
|
291
|
+
|
|
117
292
|
type NativeModule = {
|
|
293
|
+
prepareAgent?(options: {token: string; displayName?: string}): Promise<MeetingState>
|
|
118
294
|
join(options: {
|
|
119
295
|
meetingUrl: string
|
|
120
296
|
token: string
|
|
297
|
+
/** Legacy field, still sent for whep so an older native keeps working. */
|
|
121
298
|
whepUrl: string
|
|
299
|
+
videoSource: AcsVideoSource
|
|
122
300
|
displayName?: string
|
|
123
301
|
dumpPcmWav?: boolean
|
|
124
302
|
audioSource?: "glasses" | "phone"
|
|
303
|
+
audioDelayMs?: number
|
|
125
304
|
video?: AcsOutgoingVideo
|
|
126
|
-
}): Promise<MeetingState>
|
|
305
|
+
}): Promise<MeetingState & {ingestUrl?: string}>
|
|
127
306
|
leave(): Promise<void>
|
|
307
|
+
/**
|
|
308
|
+
* Leave, and resolve only once the hang-up, the agent disposal, and the network releases have
|
|
309
|
+
* actually finished.
|
|
310
|
+
*
|
|
311
|
+
* [leave] queues its work on the session executor and returns immediately, so awaiting it proves
|
|
312
|
+
* nothing about cleanup — which is what let a fast Stop/Start start a second call on top of the
|
|
313
|
+
* first one's teardown. Absent on natives that predate the signal.
|
|
314
|
+
*/
|
|
315
|
+
leaveAndAwait?(options: {timeoutMs: number}): Promise<{completed: boolean}>
|
|
316
|
+
/**
|
|
317
|
+
* End the group call for everyone, then tear this device down. Rejects when the capability is
|
|
318
|
+
* denied or ACS refuses — and has still left the call. Absent on natives that predate End.
|
|
319
|
+
*/
|
|
320
|
+
endForEveryone?(): Promise<MeetingState>
|
|
128
321
|
setMuted(muted: boolean): Promise<MeetingState>
|
|
129
322
|
setAudioSource(source: "glasses" | "phone"): Promise<MeetingState>
|
|
130
323
|
updateVideoSource(whepUrl: string): Promise<void>
|
|
131
324
|
/** Force a WHEP rebuild on the current URL. Absent on natives that predate it. */
|
|
132
325
|
restartVideoSource?(): Promise<void>
|
|
326
|
+
/**
|
|
327
|
+
* Join the glasses hotspot as a scoped, internet-less network; resolves with this phone's
|
|
328
|
+
* address on it. Absent on natives that predate SoftAP, and rejects on iOS.
|
|
329
|
+
*/
|
|
330
|
+
joinScopedNetwork?(ssid: string, passphrase: string): Promise<string>
|
|
331
|
+
beginTrace?(traceId: string): Promise<void>
|
|
332
|
+
leaveScopedNetwork?(): Promise<void>
|
|
333
|
+
/**
|
|
334
|
+
* TCP-probe the hotspot gateway over the scoped network. Absent on natives that predate it.
|
|
335
|
+
* `detail` is a one-line human summary (address, port, latency or the failure).
|
|
336
|
+
*/
|
|
337
|
+
probeScopedGateway?(): Promise<{reachable: boolean; detail: string}>
|
|
338
|
+
/** The joined hotspot: `prefix` is what the media path must stay inside. */
|
|
339
|
+
scopedNetworkInfo?(): Promise<{available: boolean; localIpv4: string | null; prefix: string | null}>
|
|
340
|
+
/**
|
|
341
|
+
* Wait for the app's default network to be validated, after the hotspot join changed it.
|
|
342
|
+
* Absent on natives that predate the cellular transition handling.
|
|
343
|
+
*/
|
|
344
|
+
awaitValidatedDefaultNetwork?(): Promise<DefaultNetworkStatus>
|
|
133
345
|
getState(): Promise<MeetingState>
|
|
346
|
+
/**
|
|
347
|
+
* Hand one chunk of already-decoded microphone PCM to the ACS uplink. Synchronous on purpose:
|
|
348
|
+
* this runs ~100×/s and a promise per chunk would cost more than the copy does.
|
|
349
|
+
*
|
|
350
|
+
* Absent on natives that predate the BLE LC3 uplink, which is exactly what
|
|
351
|
+
* [glassesLc3UplinkSupported] tests for.
|
|
352
|
+
*/
|
|
353
|
+
pushOutgoingPcm?(base64: string, sampleRate: number, channels: number): void
|
|
134
354
|
addListener(event: string, listener: (event: Record<string, unknown>) => void): {remove: () => void}
|
|
135
355
|
}
|
|
136
356
|
|
|
@@ -152,6 +372,11 @@ export function setAcsMeetingNativeForTests(mod: NativeModule | null | undefined
|
|
|
152
372
|
nativeModule = mod
|
|
153
373
|
}
|
|
154
374
|
|
|
375
|
+
/** Test seam: exercise the BLE LC3 path without flipping the production kill switch. */
|
|
376
|
+
export function setSoftapBleLc3UplinkForTests(enabled: boolean | null): void {
|
|
377
|
+
softapBleLc3UplinkForTests = enabled
|
|
378
|
+
}
|
|
379
|
+
|
|
155
380
|
/**
|
|
156
381
|
* Phone connectivity feed. Only the subset of `@react-native-community/netinfo`
|
|
157
382
|
* this service needs, so tests can inject a fake and hosts without the package
|
|
@@ -204,7 +429,57 @@ const MEDIA_RESTART_MIN_INTERVAL_MS = 3000
|
|
|
204
429
|
|
|
205
430
|
/** Formats the native PcmStreamPlayer accepts (mono only). */
|
|
206
431
|
const PCM_SAMPLE_RATES = new Set([16000, 24000, 48000])
|
|
207
|
-
|
|
432
|
+
|
|
433
|
+
/**
|
|
434
|
+
* Playout headroom for the far end's voice.
|
|
435
|
+
*
|
|
436
|
+
* The native player defaults to half a second, which is right for a miniapp clip and wrong here:
|
|
437
|
+
* a streaming track sits at its buffer size, so that default is 500ms of delay before the wearer
|
|
438
|
+
* hears anyone, enough that both sides talk over each other. This trades cushion for immediacy —
|
|
439
|
+
* still several BLE playout beats, but a conversation instead of a broadcast.
|
|
440
|
+
*/
|
|
441
|
+
const PCM_JITTER_MS = 120
|
|
442
|
+
/**
|
|
443
|
+
* Above this the backlog is real congestion rather than the configured cushion.
|
|
444
|
+
*
|
|
445
|
+
* Has to stay clear of [PCM_JITTER_MS], because the reported backlog *includes* the track's own
|
|
446
|
+
* buffer and so never falls below it. A threshold at or under the cushion logs once per interval
|
|
447
|
+
* for the whole call and says nothing.
|
|
448
|
+
*/
|
|
449
|
+
const PCM_BACKLOG_WARN_MS = PCM_JITTER_MS + 400
|
|
450
|
+
|
|
451
|
+
/** The Bluetooth SDK mic identifier the call pins itself to. */
|
|
452
|
+
const GLASSES_MIC_SOURCE = "glasses"
|
|
453
|
+
/**
|
|
454
|
+
* How long the glasses may produce no PCM before the call reports its microphone gone.
|
|
455
|
+
*
|
|
456
|
+
* The pin makes a phone-mic fallback impossible, so the only honest response to a source that
|
|
457
|
+
* stopped is to say so. One second is several BLE mic-beats: long enough that a reconnect blip
|
|
458
|
+
* does not raise an error, short enough that the wearer is not talking into nothing for long.
|
|
459
|
+
*/
|
|
460
|
+
const GLASSES_MIC_GRACE_MS = 1000
|
|
461
|
+
/** Cadence of the uplink health line, matching the native P8 ladder. */
|
|
462
|
+
const MIC_UPLINK_LOG_INTERVAL_MS = 5000
|
|
463
|
+
/** A 50 ms LC3 frame arriving more than this late is a missed beat, not jitter. */
|
|
464
|
+
const MIC_GAP_WARN_MS = 90
|
|
465
|
+
|
|
466
|
+
/**
|
|
467
|
+
* Encode one microphone buffer for `pushOutgoingPcm`.
|
|
468
|
+
*
|
|
469
|
+
* Hermes has no Node `Buffer`. Using it here is how a live SoftAP call selected `ble-lc3`,
|
|
470
|
+
* pinned the glasses, and still sent Teams a minute of silence: every `mic_pcm` event threw
|
|
471
|
+
* `Property 'Buffer' doesn't exist` before native saw a byte. `btoa` is what React Native
|
|
472
|
+
* actually has.
|
|
473
|
+
*/
|
|
474
|
+
export function pcmToBase64(pcm: ArrayBuffer): string {
|
|
475
|
+
const bytes = new Uint8Array(pcm)
|
|
476
|
+
let binary = ""
|
|
477
|
+
const step = 0x8000
|
|
478
|
+
for (let i = 0; i < bytes.length; i += step) {
|
|
479
|
+
binary += String.fromCharCode(...bytes.subarray(i, i + step))
|
|
480
|
+
}
|
|
481
|
+
return btoa(binary)
|
|
482
|
+
}
|
|
208
483
|
|
|
209
484
|
class AcsMeetingService {
|
|
210
485
|
private owner: string | null = null
|
|
@@ -218,9 +493,54 @@ class AcsMeetingService {
|
|
|
218
493
|
private onState: ((packageName: string, state: MeetingState) => void) | null = null
|
|
219
494
|
/** WHEP URL native is (or should be) subscribed to; what a host-triggered restart re-feeds. */
|
|
220
495
|
private whepUrl: string | null = null
|
|
496
|
+
/** Transport for the active call, so recovery picks the right repair. */
|
|
497
|
+
private videoSource: AcsVideoSource | null = null
|
|
498
|
+
/**
|
|
499
|
+
* SoftAP only: the URL the glasses must publish to. An output of the join rather than an input,
|
|
500
|
+
* because it is not known until native has bound a listener and been given a port.
|
|
501
|
+
*/
|
|
502
|
+
private ingestUrl: string | null = null
|
|
221
503
|
private phoneNetworkUnsub: (() => void) | null = null
|
|
222
504
|
private lastPhoneNetworkKey: string | null = null
|
|
223
505
|
private lastMediaRestartAt = 0
|
|
506
|
+
/** Callers parked in [waitForFirstFrame], woken by the next `mediaSource` verdict. */
|
|
507
|
+
private readonly firstFrameWaiters = new Set<(error?: Error) => void>()
|
|
508
|
+
private scopedLostSub: {remove: () => void} | null = null
|
|
509
|
+
private readonly scopedLostListeners = new Set<(error: {code: string; message: string}) => void>()
|
|
510
|
+
/**
|
|
511
|
+
* Set before the scoped network is released, so the loss we are about to cause is not reported as
|
|
512
|
+
* one that happened to us. Android emits `onLost` for a deliberate release, and a naive wiring
|
|
513
|
+
* turns every successful Leave and End into a mid-call network error.
|
|
514
|
+
*/
|
|
515
|
+
private scopedTerminating = false
|
|
516
|
+
/**
|
|
517
|
+
* Bumped by every join and every release, and captured by everything that can outlive a call:
|
|
518
|
+
* the `mic_pcm` listener and the in-flight `native.join`.
|
|
519
|
+
*
|
|
520
|
+
* A leave during a slow join is the case this exists for. Without it the join resolves into a
|
|
521
|
+
* torn-down host, subscribes a microphone listener nobody will ever unsubscribe, and pushes the
|
|
522
|
+
* wearer's voice at a native session that has already left the meeting.
|
|
523
|
+
*/
|
|
524
|
+
private callGeneration = 0
|
|
525
|
+
private micTransport: MicTransport = "whip"
|
|
526
|
+
private micSub: {remove: () => void} | null = null
|
|
527
|
+
/** True between the pin/requirement being taken and released, so release is exactly once. */
|
|
528
|
+
private micUplinkActive = false
|
|
529
|
+
private micFramesForwarded = 0
|
|
530
|
+
/** Frames since the last health line, so the rate and the call total stay separate facts. */
|
|
531
|
+
private micFramesWindow = 0
|
|
532
|
+
private micDropsStale = 0
|
|
533
|
+
private micDropsNonGlasses = 0
|
|
534
|
+
private micGaps = 0
|
|
535
|
+
private micGapMsMax = 0
|
|
536
|
+
private lastGlassesFrameAt = 0
|
|
537
|
+
private lastMicUplinkLogAt = 0
|
|
538
|
+
private lastMicDropLogAt = 0
|
|
539
|
+
/** PCM level over the current health window; read and reset by [logMicUplink]. */
|
|
540
|
+
private micLevel = new Pcm16LevelMeter()
|
|
541
|
+
private lastMicLevel: {meanAbs: number; peak: number} | null = null
|
|
542
|
+
/** Guards [releaseHostState] so a remote hang-up followed by an explicit leave releases once. */
|
|
543
|
+
private hostStateReleased = true
|
|
224
544
|
|
|
225
545
|
setStateHandler(handler: (packageName: string, state: MeetingState) => void): void {
|
|
226
546
|
this.onState = handler
|
|
@@ -234,12 +554,216 @@ class AcsMeetingService {
|
|
|
234
554
|
return this.owner
|
|
235
555
|
}
|
|
236
556
|
|
|
557
|
+
/**
|
|
558
|
+
* The WHIP URL the glasses must POST their offer to, for a SoftAP call. Null for every other
|
|
559
|
+
* transport and until the join has bound a listener; the orchestrator reads it between the ACS
|
|
560
|
+
* join and telling the glasses to publish.
|
|
561
|
+
*/
|
|
562
|
+
softApIngestUrl(): string | null {
|
|
563
|
+
return this.ingestUrl
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
/**
|
|
567
|
+
* Whether this call is taking the wearer's voice over BLE LC3, which means the glasses must
|
|
568
|
+
* publish video only. Read by the SoftAP orchestrator between the meeting join and the publish.
|
|
569
|
+
*/
|
|
570
|
+
glassesLc3UplinkActive(): boolean {
|
|
571
|
+
return this.micTransport === "ble-lc3"
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
/**
|
|
575
|
+
* Join the glasses hotspot as a scoped, internet-less network, returning this phone's address on
|
|
576
|
+
* it. Called before the ACS join, because the local WHIP listener has to bind to that address.
|
|
577
|
+
*
|
|
578
|
+
* A host without the native function is not a host that silently skips the join — the SoftAP call
|
|
579
|
+
* has no network to run on, so this reports the reason instead.
|
|
580
|
+
*/
|
|
581
|
+
async joinScopedNetwork(ssid: string, passphrase: string): Promise<string | undefined> {
|
|
582
|
+
const native = getNative()
|
|
583
|
+
if (!native?.joinScopedNetwork) {
|
|
584
|
+
throw new Error("This host cannot join the glasses hotspot; SoftAP calling is unavailable")
|
|
585
|
+
}
|
|
586
|
+
this.scopedTerminating = false
|
|
587
|
+
this.bindScopedNetworkLost(native)
|
|
588
|
+
await native.beginTrace?.(softapTraceId())
|
|
589
|
+
return await native.joinScopedNetwork(ssid, passphrase)
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
/**
|
|
593
|
+
* Subscribe to "the glasses hotspot went away while we still wanted it".
|
|
594
|
+
*
|
|
595
|
+
* Only unexpected losses arrive here. Native drops the framework callback for a network it
|
|
596
|
+
* released itself, and [leaveScopedNetwork] raises the terminal intent before releasing, so a
|
|
597
|
+
* normal Leave or End cannot manufacture a mid-call failure.
|
|
598
|
+
*/
|
|
599
|
+
onScopedNetworkLost(listener: (error: {code: string; message: string}) => void): () => void {
|
|
600
|
+
this.scopedLostListeners.add(listener)
|
|
601
|
+
return () => this.scopedLostListeners.delete(listener)
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
private bindScopedNetworkLost(native: NativeModule): void {
|
|
605
|
+
if (this.scopedLostSub) return
|
|
606
|
+
try {
|
|
607
|
+
this.scopedLostSub = native.addListener("onScopedNetworkLost", (event) => {
|
|
608
|
+
if (this.scopedTerminating) {
|
|
609
|
+
console.log("[AcsMeeting] phase=scoped-lost-expected", {code: event.code})
|
|
610
|
+
return
|
|
611
|
+
}
|
|
612
|
+
const error = {
|
|
613
|
+
code: typeof event.code === "string" ? event.code : "SOFTAP_NETWORK_LOST",
|
|
614
|
+
message: typeof event.message === "string" ? event.message : "The glasses hotspot went away",
|
|
615
|
+
}
|
|
616
|
+
console.warn("[AcsMeeting] phase=scoped-lost", error)
|
|
617
|
+
for (const listener of [...this.scopedLostListeners]) {
|
|
618
|
+
try {
|
|
619
|
+
listener(error)
|
|
620
|
+
} catch (listenerError) {
|
|
621
|
+
console.warn("[AcsMeeting] scoped-lost listener threw", listenerError)
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
})
|
|
625
|
+
} catch (error) {
|
|
626
|
+
// A native that predates the event simply never reports mid-call loss; that is a smaller
|
|
627
|
+
// problem than failing the join over a missing listener.
|
|
628
|
+
console.warn("[AcsMeeting] scoped network loss events unavailable", error)
|
|
629
|
+
this.scopedLostSub = null
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
/** Raise the terminal intent so the release we are about to do is not read as a failure. */
|
|
634
|
+
beginScopedTeardown(): void {
|
|
635
|
+
this.scopedTerminating = true
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
/**
|
|
639
|
+
* Wait until the phone's default network is validated again after the hotspot join.
|
|
640
|
+
*
|
|
641
|
+
* Joining the glasses hotspot takes the phone off Wi-Fi, and ACS needs the internet for the very
|
|
642
|
+
* next step. Waiting is what turns a 40-second join timeout with device-wide DNS failures into a
|
|
643
|
+
* short, named wait — or an honest failure. Null when the host cannot tell.
|
|
644
|
+
*/
|
|
645
|
+
async awaitValidatedDefaultNetwork(): Promise<DefaultNetworkStatus | null> {
|
|
646
|
+
const native = getNative()
|
|
647
|
+
if (!native?.awaitValidatedDefaultNetwork) return null
|
|
648
|
+
return await native.awaitValidatedDefaultNetwork()
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
/**
|
|
652
|
+
* Can this phone reach the glasses over the hotspot it just joined? Null when the host cannot
|
|
653
|
+
* tell (no native support), so the orchestrator narrates nothing rather than a guess.
|
|
654
|
+
*/
|
|
655
|
+
async probeScopedGateway(): Promise<{reachable: boolean; detail: string} | null> {
|
|
656
|
+
const native = getNative()
|
|
657
|
+
if (!native?.probeScopedGateway) return null
|
|
658
|
+
return await native.probeScopedGateway()
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
/** Safe to call when nothing was joined: teardown runs after failed starts too. */
|
|
662
|
+
async leaveScopedNetwork(): Promise<void> {
|
|
663
|
+
// Intent first, release second. The other order is the false-positive bug: Android reports the
|
|
664
|
+
// loss we asked for, and the call reports a network failure as it is successfully ending.
|
|
665
|
+
this.scopedTerminating = true
|
|
666
|
+
try {
|
|
667
|
+
await getNative()?.leaveScopedNetwork?.()
|
|
668
|
+
} finally {
|
|
669
|
+
this.scopedLostSub?.remove()
|
|
670
|
+
this.scopedLostSub = null
|
|
671
|
+
}
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
/**
|
|
675
|
+
* Resolves once the host reports a frame actually reached ACS, which is the only signal that
|
|
676
|
+
* remote participants can see the camera.
|
|
677
|
+
*
|
|
678
|
+
* Rejects if the feed fails first, and on timeout. A SoftAP call that connects but never paints
|
|
679
|
+
* is the failure this exists to catch: without it the orchestrator would report `live` on the
|
|
680
|
+
* strength of an ACS join that says nothing about video.
|
|
681
|
+
*
|
|
682
|
+
* @param timeoutMs how long to wait before treating the silence as a failure
|
|
683
|
+
*/
|
|
684
|
+
waitForFirstFrame(timeoutMs: number): Promise<void> {
|
|
685
|
+
if (this.lastState.mediaSource === "live") {
|
|
686
|
+
softapTrace("acs_first_frame_already_live", {mediaSource: this.lastState.mediaSource})
|
|
687
|
+
return Promise.resolve()
|
|
688
|
+
}
|
|
689
|
+
const startedAt = Date.now()
|
|
690
|
+
softapTrace("acs_first_frame_wait", {
|
|
691
|
+
timeoutMs,
|
|
692
|
+
mediaSource: this.lastState.mediaSource ?? "unknown",
|
|
693
|
+
state: this.lastState.state,
|
|
694
|
+
})
|
|
695
|
+
return new Promise<void>((resolve, reject) => {
|
|
696
|
+
const settle = (error?: Error) => {
|
|
697
|
+
if (done) return
|
|
698
|
+
done = true
|
|
699
|
+
clearTimeout(timer)
|
|
700
|
+
this.firstFrameWaiters.delete(settle)
|
|
701
|
+
// The step this closes is the one that decides whether the call is usable, so its outcome
|
|
702
|
+
// is named rather than inferred from whichever line happens to follow.
|
|
703
|
+
if (error) {
|
|
704
|
+
softapTraceFailure("acs_first_frame_wait_done", {
|
|
705
|
+
waitedMs: Date.now() - startedAt,
|
|
706
|
+
reason: error.message,
|
|
707
|
+
})
|
|
708
|
+
reject(error)
|
|
709
|
+
} else {
|
|
710
|
+
softapTrace("acs_first_frame_wait_done", {waitedMs: Date.now() - startedAt})
|
|
711
|
+
resolve()
|
|
712
|
+
}
|
|
713
|
+
}
|
|
714
|
+
let done = false
|
|
715
|
+
const timer = setTimeout(
|
|
716
|
+
() => settle(new Error(`No glasses video reached the meeting within ${Math.round(timeoutMs / 1000)}s`)),
|
|
717
|
+
timeoutMs,
|
|
718
|
+
)
|
|
719
|
+
this.firstFrameWaiters.add(settle)
|
|
720
|
+
})
|
|
721
|
+
}
|
|
722
|
+
|
|
723
|
+
/** Wake every `waitForFirstFrame` caller with the outcome the host just reported. */
|
|
724
|
+
private settleFirstFrameWaiters(mediaSource: MediaSourceState | undefined): void {
|
|
725
|
+
if (mediaSource !== "live" && mediaSource !== "failed") return
|
|
726
|
+
const error = mediaSource === "failed" ? new Error("The glasses video feed failed") : undefined
|
|
727
|
+
for (const settle of [...this.firstFrameWaiters]) settle(error)
|
|
728
|
+
}
|
|
729
|
+
|
|
730
|
+
/**
|
|
731
|
+
* Sign in to ACS before the glasses hotspot exists.
|
|
732
|
+
*
|
|
733
|
+
* SoftAP DNS cannot resolve Teams hosts. Doing this on the phone's existing internet is what
|
|
734
|
+
* stops `createCallAgent` from hanging until the hotspot is torn down.
|
|
735
|
+
*/
|
|
736
|
+
async prepareAgent(args: {token: string; displayName?: string}): Promise<void> {
|
|
737
|
+
const native = getNative()
|
|
738
|
+
if (!native?.prepareAgent) {
|
|
739
|
+
// A host that cannot pre-sign-in still joins; it just does the sign-in inside the SoftAP
|
|
740
|
+
// join, which is the 20-second ACS_AGENT_TIMEOUT this step exists to avoid. Worth a line,
|
|
741
|
+
// because from the trace alone that build looks like one whose sign-in was instant.
|
|
742
|
+
softapTrace("acs_prepare_agent_skipped", {reason: native ? "unsupported build" : "no native module"})
|
|
743
|
+
return
|
|
744
|
+
}
|
|
745
|
+
console.log("[AcsMeeting] phase=prepare-agent")
|
|
746
|
+
const startedAt = Date.now()
|
|
747
|
+
softapTrace("acs_native_prepare_agent", {hasDisplayName: Boolean(args.displayName)})
|
|
748
|
+
try {
|
|
749
|
+
await native.prepareAgent({token: args.token, displayName: args.displayName})
|
|
750
|
+
} catch (error) {
|
|
751
|
+
softapTraceFailure("acs_native_prepare_agent_failed", {
|
|
752
|
+
durationMs: Date.now() - startedAt,
|
|
753
|
+
reason: error instanceof Error ? error.message : String(error),
|
|
754
|
+
})
|
|
755
|
+
throw error
|
|
756
|
+
}
|
|
757
|
+
console.log("[AcsMeeting] phase=prepare-agent-ok")
|
|
758
|
+
softapTrace("acs_native_prepare_agent_ok", {durationMs: Date.now() - startedAt})
|
|
759
|
+
}
|
|
760
|
+
|
|
237
761
|
async join(
|
|
238
762
|
packageName: string,
|
|
239
763
|
args: {
|
|
240
764
|
meetingUrl: string
|
|
241
765
|
token: string
|
|
242
|
-
|
|
766
|
+
videoSource: AcsVideoSource
|
|
243
767
|
displayName?: string
|
|
244
768
|
video?: AcsOutgoingVideo
|
|
245
769
|
},
|
|
@@ -255,33 +779,89 @@ class AcsMeetingService {
|
|
|
255
779
|
// Validate before claiming ownership so a bad request cannot leave the slot taken.
|
|
256
780
|
const video = args.video ? parseAcsOutgoingVideo(args.video) : undefined
|
|
257
781
|
const resolved = resolveAcsAudioSource()
|
|
782
|
+
const generation = ++this.callGeneration
|
|
783
|
+
this.hostStateReleased = false
|
|
258
784
|
this.owner = packageName
|
|
259
|
-
|
|
785
|
+
// Only a whep source has a URL to re-feed on recovery; softap rebuilds instead.
|
|
786
|
+
this.whepUrl = args.videoSource.type === "whep" ? args.videoSource.url : null
|
|
787
|
+
this.videoSource = args.videoSource
|
|
788
|
+
// Decided here, before the join, so the SoftAP orchestrator can read it between the join and
|
|
789
|
+
// telling the glasses what to capture.
|
|
790
|
+
const lc3Uplink = glassesLc3UplinkSupported({
|
|
791
|
+
videoSource: args.videoSource,
|
|
792
|
+
audioSource: resolved.source,
|
|
793
|
+
hasPushOutgoingPcm: typeof native.pushOutgoingPcm === "function",
|
|
794
|
+
platform: Platform.OS,
|
|
795
|
+
})
|
|
796
|
+
this.micTransport = lc3Uplink ? "ble-lc3" : resolved.source === "phone" ? "phone" : "whip"
|
|
260
797
|
this.bindNative(native, packageName)
|
|
261
798
|
console.log("[AcsMeeting] phase=join-native", {
|
|
262
799
|
packageName,
|
|
263
800
|
nativeLoaded: true,
|
|
264
801
|
hasToken: Boolean(args.token),
|
|
265
|
-
|
|
802
|
+
transport: args.videoSource.type,
|
|
266
803
|
audioSource: resolved.source,
|
|
267
804
|
audioSourceReason: resolved.reason,
|
|
805
|
+
micTransport: this.micTransport,
|
|
268
806
|
preferredMic: useSettingsStore.getState().getSetting(SETTINGS.preferred_mic.key),
|
|
269
807
|
})
|
|
808
|
+
const joinStartedAt = Date.now()
|
|
809
|
+
softapTrace("acs_native_join", {
|
|
810
|
+
packageName,
|
|
811
|
+
generation,
|
|
812
|
+
transport: args.videoSource.type,
|
|
813
|
+
audioSource: resolved.source,
|
|
814
|
+
micTransport: this.micTransport,
|
|
815
|
+
audioDelayMs: lc3Uplink ? SOFTAP_LC3_AUDIO_DELAY_MS : 0,
|
|
816
|
+
video: video ? `${video.width}x${video.height}@${video.fps}` : "default",
|
|
817
|
+
})
|
|
270
818
|
try {
|
|
271
819
|
const state = await native.join({
|
|
272
820
|
meetingUrl: args.meetingUrl,
|
|
273
821
|
token: args.token,
|
|
274
|
-
whepUrl:
|
|
822
|
+
whepUrl: this.whepUrl ?? "",
|
|
823
|
+
videoSource: args.videoSource,
|
|
275
824
|
displayName: args.displayName,
|
|
276
825
|
audioSource: resolved.source,
|
|
826
|
+
...(lc3Uplink ? {audioDelayMs: SOFTAP_LC3_AUDIO_DELAY_MS} : {}),
|
|
277
827
|
...(video ? {video} : {}),
|
|
278
828
|
})
|
|
829
|
+
softapTrace("acs_native_join_returned", {
|
|
830
|
+
packageName,
|
|
831
|
+
generation,
|
|
832
|
+
state: state.state,
|
|
833
|
+
hasIngestUrl: typeof state.ingestUrl === "string" && state.ingestUrl.length > 0,
|
|
834
|
+
durationMs: Date.now() - joinStartedAt,
|
|
835
|
+
})
|
|
836
|
+
if (generation !== this.callGeneration) {
|
|
837
|
+
// The wearer left while ACS was still joining. Nothing above knows about this call, so
|
|
838
|
+
// hanging it up here is the only thing that takes the device out of the Teams roster.
|
|
839
|
+
console.warn("[AcsMeeting] phase=join-cancelled", {packageName, generation})
|
|
840
|
+
softapTraceFailure("acs_native_join_cancelled", {
|
|
841
|
+
packageName,
|
|
842
|
+
generation,
|
|
843
|
+
current: this.callGeneration,
|
|
844
|
+
durationMs: Date.now() - joinStartedAt,
|
|
845
|
+
})
|
|
846
|
+
await native.leave().catch((leaveError) => {
|
|
847
|
+
console.warn("[AcsMeeting] native leave after a cancelled join failed", leaveError)
|
|
848
|
+
softapTraceFailure("acs_native_leave_after_cancelled_join_failed", {
|
|
849
|
+
packageName,
|
|
850
|
+
reason: leaveError instanceof Error ? leaveError.message : String(leaveError),
|
|
851
|
+
})
|
|
852
|
+
})
|
|
853
|
+
throw new Error("The meeting was cancelled before it finished joining")
|
|
854
|
+
}
|
|
855
|
+
this.ingestUrl = typeof state.ingestUrl === "string" ? state.ingestUrl : null
|
|
856
|
+
const {ingestUrl: _ingestUrl, ...meetingState} = state
|
|
279
857
|
this.lastState = {
|
|
280
|
-
...
|
|
858
|
+
...meetingState,
|
|
281
859
|
audioSource: resolved.source,
|
|
282
860
|
audioSourceReason: resolved.reason,
|
|
861
|
+
micTransport: this.micTransport,
|
|
283
862
|
}
|
|
284
863
|
console.log("[AcsMeeting] phase=join-native-ok", {state: state.state, muted: state.muted})
|
|
864
|
+
this.startGlassesMicUplink(generation)
|
|
285
865
|
} catch (error) {
|
|
286
866
|
// Native never joined (or is unwinding). Release the slot so the same or another
|
|
287
867
|
// miniapp can retry, and make sure nothing half-joined lingers in Teams.
|
|
@@ -289,8 +869,18 @@ class AcsMeetingService {
|
|
|
289
869
|
packageName,
|
|
290
870
|
error: error instanceof Error ? error.message : String(error),
|
|
291
871
|
})
|
|
872
|
+
softapTraceFailure("acs_native_join_failed", {
|
|
873
|
+
packageName,
|
|
874
|
+
generation,
|
|
875
|
+
durationMs: Date.now() - joinStartedAt,
|
|
876
|
+
reason: error instanceof Error ? error.message : String(error),
|
|
877
|
+
})
|
|
292
878
|
await native.leave().catch((leaveError) => {
|
|
293
879
|
console.warn("[AcsMeeting] native leave after failed join also failed", leaveError)
|
|
880
|
+
softapTraceFailure("acs_native_leave_after_failed_join_failed", {
|
|
881
|
+
packageName,
|
|
882
|
+
reason: leaveError instanceof Error ? leaveError.message : String(leaveError),
|
|
883
|
+
})
|
|
294
884
|
})
|
|
295
885
|
await this.releaseHostState()
|
|
296
886
|
throw error
|
|
@@ -311,30 +901,341 @@ class AcsMeetingService {
|
|
|
311
901
|
}
|
|
312
902
|
|
|
313
903
|
async leave(packageName: string): Promise<void> {
|
|
314
|
-
if (this.owner && this.owner !== packageName)
|
|
904
|
+
if (this.owner && this.owner !== packageName) {
|
|
905
|
+
// Not the owner, so this is a no-op rather than a leave. Said out loud because a miniapp
|
|
906
|
+
// that thinks it left and a host that never hung up look identical from the miniapp's side.
|
|
907
|
+
softapTrace("acs_native_leave_ignored", {packageName, owner: this.owner})
|
|
908
|
+
return
|
|
909
|
+
}
|
|
315
910
|
const native = getNative()
|
|
911
|
+
const startedAt = Date.now()
|
|
912
|
+
softapTrace("acs_native_leave", {packageName, nativeLoaded: Boolean(native)})
|
|
316
913
|
try {
|
|
317
914
|
await native?.leave()
|
|
915
|
+
softapTrace("acs_native_leave_ok", {packageName, durationMs: Date.now() - startedAt})
|
|
916
|
+
} catch (error) {
|
|
917
|
+
softapTraceFailure("acs_native_leave_failed", {
|
|
918
|
+
packageName,
|
|
919
|
+
durationMs: Date.now() - startedAt,
|
|
920
|
+
reason: error instanceof Error ? error.message : String(error),
|
|
921
|
+
})
|
|
922
|
+
throw error
|
|
318
923
|
} finally {
|
|
319
924
|
await this.releaseHostState()
|
|
320
925
|
}
|
|
321
926
|
}
|
|
322
927
|
|
|
323
928
|
/**
|
|
324
|
-
*
|
|
325
|
-
*
|
|
326
|
-
*
|
|
929
|
+
* Leave, and do not resolve until native reports its cleanup is finished.
|
|
930
|
+
*
|
|
931
|
+
* The difference from [leave] is the whole point: native's `leave()` queues the hang-up, the
|
|
932
|
+
* agent disposal, and the network releases on its session executor and returns straight away, so
|
|
933
|
+
* a caller that awaits it and then starts the next call is racing the previous one's teardown.
|
|
934
|
+
*
|
|
935
|
+
* Falls back to [leave] on builds that predate the signal and says so, rather than pretending to
|
|
936
|
+
* a guarantee it cannot give.
|
|
937
|
+
*
|
|
938
|
+
* @param timeoutMs how long native may take before it reports the cleanup as stuck
|
|
939
|
+
*/
|
|
940
|
+
async leaveAndAwait(packageName: string, timeoutMs = ACS_LEAVE_WAIT_MS): Promise<{completed: boolean; reason?: string}> {
|
|
941
|
+
if (this.owner && this.owner !== packageName) {
|
|
942
|
+
softapTrace("acs_native_leave_and_await_ignored", {packageName, owner: this.owner})
|
|
943
|
+
return {completed: true}
|
|
944
|
+
}
|
|
945
|
+
const native = getNative()
|
|
946
|
+
if (!native?.leaveAndAwait) {
|
|
947
|
+
// The fallback is the case the cleanup barrier was built for: this build's leave returns
|
|
948
|
+
// before its own teardown has finished, so nothing downstream can treat "left" as "idle".
|
|
949
|
+
softapTraceFailure("acs_native_leave_and_await_unsupported", {
|
|
950
|
+
packageName,
|
|
951
|
+
nativeLoaded: Boolean(native),
|
|
952
|
+
})
|
|
953
|
+
await this.leave(packageName)
|
|
954
|
+
return {completed: false, reason: "unsupported"}
|
|
955
|
+
}
|
|
956
|
+
const startedAt = Date.now()
|
|
957
|
+
softapTrace("acs_native_leave_and_await", {packageName, timeoutMs})
|
|
958
|
+
try {
|
|
959
|
+
const outcome = await native.leaveAndAwait({timeoutMs})
|
|
960
|
+
softapTrace("acs_native_leave_and_await_ok", {
|
|
961
|
+
packageName,
|
|
962
|
+
completed: outcome?.completed !== false,
|
|
963
|
+
durationMs: Date.now() - startedAt,
|
|
964
|
+
})
|
|
965
|
+
return {completed: outcome?.completed !== false}
|
|
966
|
+
} catch (error) {
|
|
967
|
+
softapTraceFailure("acs_native_leave_and_await_failed", {
|
|
968
|
+
packageName,
|
|
969
|
+
durationMs: Date.now() - startedAt,
|
|
970
|
+
reason: error instanceof Error ? error.message : String(error),
|
|
971
|
+
})
|
|
972
|
+
throw error
|
|
973
|
+
} finally {
|
|
974
|
+
await this.releaseHostState()
|
|
975
|
+
}
|
|
976
|
+
}
|
|
977
|
+
|
|
978
|
+
/**
|
|
979
|
+
* End the Teams group call for everyone.
|
|
980
|
+
*
|
|
981
|
+
* Host state is released either way: native tears this device down whatever the hang-up did, so
|
|
982
|
+
* holding the meeting slot open after a rejection would leave the miniapp unable to start another
|
|
983
|
+
* call. The rejection still propagates — the caller has to be able to say "you left, but the
|
|
984
|
+
* meeting may still be active" instead of claiming a clean end.
|
|
985
|
+
*/
|
|
986
|
+
async endForEveryone(packageName: string): Promise<MeetingState> {
|
|
987
|
+
if (this.owner && this.owner !== packageName) {
|
|
988
|
+
throw new Error("This miniapp does not own the active meeting")
|
|
989
|
+
}
|
|
990
|
+
const native = getNative()
|
|
991
|
+
if (!native?.endForEveryone) {
|
|
992
|
+
softapTraceFailure("acs_native_end_unsupported", {packageName, nativeLoaded: Boolean(native)})
|
|
993
|
+
throw new Error("Update the Mentra App to end a meeting for everyone")
|
|
994
|
+
}
|
|
995
|
+
const startedAt = Date.now()
|
|
996
|
+
softapTrace("acs_native_end", {packageName})
|
|
997
|
+
try {
|
|
998
|
+
const state = await native.endForEveryone()
|
|
999
|
+
console.log("[AcsMeeting] phase=end-for-everyone-ok", {state: state.state})
|
|
1000
|
+
softapTrace("acs_native_end_ok", {packageName, state: state.state, durationMs: Date.now() - startedAt})
|
|
1001
|
+
return state
|
|
1002
|
+
} catch (error) {
|
|
1003
|
+
// The meeting may still be live for the others; this device is out regardless. Logged as a
|
|
1004
|
+
// failure of the claim, not of the teardown.
|
|
1005
|
+
softapTraceFailure("acs_native_end_failed", {
|
|
1006
|
+
packageName,
|
|
1007
|
+
durationMs: Date.now() - startedAt,
|
|
1008
|
+
reason: error instanceof Error ? error.message : String(error),
|
|
1009
|
+
})
|
|
1010
|
+
throw error
|
|
1011
|
+
} finally {
|
|
1012
|
+
await this.releaseHostState()
|
|
1013
|
+
}
|
|
1014
|
+
}
|
|
1015
|
+
|
|
1016
|
+
/** Whether the wearer may end this meeting for everyone, as ACS last reported it. */
|
|
1017
|
+
hangUpForEveryoneCapability(): MeetingCapability {
|
|
1018
|
+
return this.lastState.capabilities?.hangUpForEveryone ?? {allowed: null, reason: null}
|
|
1019
|
+
}
|
|
1020
|
+
|
|
1021
|
+
/**
|
|
1022
|
+
* Drop everything the host set up around a session: the microphone uplink, the network watcher,
|
|
1023
|
+
* return audio, native listeners, ownership.
|
|
1024
|
+
*
|
|
1025
|
+
* Idempotent, because there are now several terminal paths into it and they overlap. A remote
|
|
1026
|
+
* hang-up arrives as a native `disconnected`, and the miniapp usually calls `leave` right after
|
|
1027
|
+
* seeing it; releasing the microphone pin twice would clear a pin the *next* call had already
|
|
1028
|
+
* taken. Every release also bumps the call generation, which is what stops a `mic_pcm` event
|
|
1029
|
+
* already queued on the JS thread from reaching a native session that is gone.
|
|
327
1030
|
*/
|
|
328
1031
|
private async releaseHostState(): Promise<void> {
|
|
1032
|
+
if (this.hostStateReleased) return
|
|
1033
|
+
this.hostStateReleased = true
|
|
1034
|
+
this.callGeneration++
|
|
1035
|
+
// Before anything else: a caller parked on a frame that will now never arrive has to be
|
|
1036
|
+
// rejected, or a leave mid-join leaves the orchestrator waiting out its whole timeout.
|
|
1037
|
+
for (const settle of [...this.firstFrameWaiters]) settle(new Error("The meeting ended"))
|
|
1038
|
+
this.firstFrameWaiters.clear()
|
|
1039
|
+
this.stopGlassesMicUplink()
|
|
329
1040
|
this.unwatchPhoneNetwork()
|
|
330
1041
|
await this.stopPcm()
|
|
331
1042
|
this.unbindNative()
|
|
1043
|
+
this.scopedLostSub?.remove()
|
|
1044
|
+
this.scopedLostSub = null
|
|
332
1045
|
this.owner = null
|
|
333
1046
|
this.whepUrl = null
|
|
1047
|
+
this.videoSource = null
|
|
1048
|
+
this.ingestUrl = null
|
|
334
1049
|
this.lastMediaRestartAt = 0
|
|
335
1050
|
this.lastState = {state: "idle", muted: false}
|
|
336
1051
|
}
|
|
337
1052
|
|
|
1053
|
+
/**
|
|
1054
|
+
* Start forwarding the glasses microphone into ACS for this call.
|
|
1055
|
+
*
|
|
1056
|
+
* Order matters and is the whole point: pin the Bluetooth SDK to the glasses *before* asking it
|
|
1057
|
+
* for PCM, so the first frame the mic requirement produces is already from the right source and
|
|
1058
|
+
* the phone microphone is never opened even for one buffer. `generation` is captured by the
|
|
1059
|
+
* listener so a frame that lands after this call ended is dropped rather than pushed at a native
|
|
1060
|
+
* session that has left the meeting.
|
|
1061
|
+
*/
|
|
1062
|
+
private startGlassesMicUplink(generation: number): void {
|
|
1063
|
+
if (this.micTransport !== "ble-lc3") return
|
|
1064
|
+
const native = getNative()
|
|
1065
|
+
const push = native?.pushOutgoingPcm
|
|
1066
|
+
if (!native || !push) return
|
|
1067
|
+
this.micFramesForwarded = 0
|
|
1068
|
+
this.micFramesWindow = 0
|
|
1069
|
+
this.micDropsStale = 0
|
|
1070
|
+
this.micDropsNonGlasses = 0
|
|
1071
|
+
this.micGaps = 0
|
|
1072
|
+
this.micGapMsMax = 0
|
|
1073
|
+
this.lastGlassesFrameAt = Date.now()
|
|
1074
|
+
this.lastMicDropLogAt = 0
|
|
1075
|
+
this.lastMicUplinkLogAt = Date.now()
|
|
1076
|
+
this.micLevel.take()
|
|
1077
|
+
this.lastMicLevel = null
|
|
1078
|
+
try {
|
|
1079
|
+
void Promise.resolve(BluetoothSdk.setMicSourcePin?.(GLASSES_MIC_SOURCE)).catch((error) => {
|
|
1080
|
+
console.warn("[AcsMeeting] pinning the glasses microphone failed", error)
|
|
1081
|
+
})
|
|
1082
|
+
this.micUplinkActive = true
|
|
1083
|
+
micStateCoordinator.setCallRequirement(true)
|
|
1084
|
+
this.micSub = BluetoothSdk.addListener("mic_pcm", (event: {pcm?: ArrayBuffer; sampleRate?: number; source?: string}) => {
|
|
1085
|
+
if (generation !== this.callGeneration) {
|
|
1086
|
+
this.micDropsStale += 1
|
|
1087
|
+
this.logMicDrop("stale", {generation, active: this.callGeneration})
|
|
1088
|
+
return
|
|
1089
|
+
}
|
|
1090
|
+
// The pin makes this unreachable in normal operation, which is exactly why it is checked:
|
|
1091
|
+
// a mic the SDK moved under us would otherwise put the phone's room into a Teams call
|
|
1092
|
+
// that reports the glasses.
|
|
1093
|
+
if (event.source !== GLASSES_MIC_SOURCE) {
|
|
1094
|
+
this.micDropsNonGlasses += 1
|
|
1095
|
+
this.logMicDrop("non-glasses", {source: event.source})
|
|
1096
|
+
this.reportGlassesMicUnavailable(event.source)
|
|
1097
|
+
return
|
|
1098
|
+
}
|
|
1099
|
+
const pcm = event.pcm
|
|
1100
|
+
if (!pcm) return
|
|
1101
|
+
const now = Date.now()
|
|
1102
|
+
if (this.micFramesForwarded > 0) {
|
|
1103
|
+
const gapMs = now - this.lastGlassesFrameAt
|
|
1104
|
+
if (gapMs > MIC_GAP_WARN_MS) {
|
|
1105
|
+
this.micGaps += 1
|
|
1106
|
+
this.micGapMsMax = Math.max(this.micGapMsMax, gapMs)
|
|
1107
|
+
this.logMicDrop("gap", {gapMs})
|
|
1108
|
+
}
|
|
1109
|
+
}
|
|
1110
|
+
this.lastGlassesFrameAt = now
|
|
1111
|
+
this.micFramesForwarded += 1
|
|
1112
|
+
this.micFramesWindow += 1
|
|
1113
|
+
this.micLevel.add(pcm)
|
|
1114
|
+
try {
|
|
1115
|
+
push.call(native, pcmToBase64(pcm), event.sampleRate ?? 16000, 1)
|
|
1116
|
+
} catch (error) {
|
|
1117
|
+
console.warn("[AcsMeeting] pushing glasses PCM to ACS failed", error)
|
|
1118
|
+
}
|
|
1119
|
+
this.logMicUplink()
|
|
1120
|
+
})
|
|
1121
|
+
console.log("[AcsMeeting] phase=glasses-mic-uplink-start", {generation, micTransport: this.micTransport})
|
|
1122
|
+
} catch (error) {
|
|
1123
|
+
// A host that cannot subscribe has no wearer audio at all, and that is worth saying loudly,
|
|
1124
|
+
// but it is not worth failing a call the wearer can still see and hear.
|
|
1125
|
+
console.error("[AcsMeeting] phase=glasses-mic-uplink-unavailable", error)
|
|
1126
|
+
this.stopGlassesMicUplink()
|
|
1127
|
+
this.markMicTransportNone()
|
|
1128
|
+
}
|
|
1129
|
+
}
|
|
1130
|
+
|
|
1131
|
+
/** Release the microphone claims this call took. Safe to call when it never took them. */
|
|
1132
|
+
private stopGlassesMicUplink(): void {
|
|
1133
|
+
this.micSub?.remove()
|
|
1134
|
+
this.micSub = null
|
|
1135
|
+
if (this.micUplinkActive) {
|
|
1136
|
+
this.micUplinkActive = false
|
|
1137
|
+
micStateCoordinator.setCallRequirement(false)
|
|
1138
|
+
// Last, and unconditionally: while the pin is set no other consumer can pick a microphone,
|
|
1139
|
+
// so leaving it behind would leave captions and the cloud uplink stuck on the glasses.
|
|
1140
|
+
void Promise.resolve(BluetoothSdk.setMicSourcePin?.(null)).catch((error) => {
|
|
1141
|
+
console.warn("[AcsMeeting] releasing the glasses microphone pin failed", error)
|
|
1142
|
+
})
|
|
1143
|
+
console.log("[AcsMeeting] phase=glasses-mic-uplink-stop", {
|
|
1144
|
+
frames: this.micFramesForwarded,
|
|
1145
|
+
dropsStale: this.micDropsStale,
|
|
1146
|
+
dropsNonGlasses: this.micDropsNonGlasses,
|
|
1147
|
+
gaps: this.micGaps,
|
|
1148
|
+
gapMsMax: this.micGapMsMax,
|
|
1149
|
+
})
|
|
1150
|
+
}
|
|
1151
|
+
this.micTransport = "whip"
|
|
1152
|
+
}
|
|
1153
|
+
|
|
1154
|
+
/**
|
|
1155
|
+
* Report a call whose pinned microphone stopped delivering. There is deliberately no fallback:
|
|
1156
|
+
* the wearer agreed to be heard from the glasses, and quietly switching to the phone in the
|
|
1157
|
+
* middle of a meeting puts the room they are standing in on the call.
|
|
1158
|
+
*/
|
|
1159
|
+
private reportGlassesMicUnavailable(source: string | undefined): void {
|
|
1160
|
+
if (Date.now() - this.lastGlassesFrameAt < GLASSES_MIC_GRACE_MS) return
|
|
1161
|
+
if (this.micTransport === "none") return
|
|
1162
|
+
console.error("[AcsMeeting] phase=glasses-mic-unavailable", {source, dropsNonGlasses: this.micDropsNonGlasses})
|
|
1163
|
+
this.markMicTransportNone()
|
|
1164
|
+
}
|
|
1165
|
+
|
|
1166
|
+
/**
|
|
1167
|
+
* Record that this call has no wearer audio path.
|
|
1168
|
+
*
|
|
1169
|
+
* The state has to carry it, not just the field: a call that reported `ble-lc3` and then failed
|
|
1170
|
+
* to start the uplink looks identical from the miniapp's side to one that is working, and the
|
|
1171
|
+
* wearer finds out by being asked to repeat themselves.
|
|
1172
|
+
*/
|
|
1173
|
+
private markMicTransportNone(): void {
|
|
1174
|
+
this.micTransport = "none"
|
|
1175
|
+
this.lastState = {...this.lastState, micTransport: "none", error: GLASSES_MIC_UNAVAILABLE}
|
|
1176
|
+
if (this.owner) this.onState?.(this.owner, this.lastState)
|
|
1177
|
+
}
|
|
1178
|
+
|
|
1179
|
+
private logMicUplink(): void {
|
|
1180
|
+
const now = Date.now()
|
|
1181
|
+
const elapsed = now - this.lastMicUplinkLogAt
|
|
1182
|
+
if (elapsed < MIC_UPLINK_LOG_INTERVAL_MS) return
|
|
1183
|
+
this.lastMicUplinkLogAt = now
|
|
1184
|
+
// Level, not just cadence: a 20 Hz stream of the noise floor and a 20 Hz stream of speech
|
|
1185
|
+
// have the same framesPerSecond. Quiet room on Mentra Live LC3 is meanAbs ≈30–60.
|
|
1186
|
+
const level = this.micLevel.take()
|
|
1187
|
+
this.lastMicLevel = {meanAbs: level.meanAbs, peak: level.peak}
|
|
1188
|
+
console.log("[AcsMeeting] phase=glasses-mic-uplink", {
|
|
1189
|
+
framesPerSecond: Math.round((this.micFramesWindow * 1000) / elapsed),
|
|
1190
|
+
frames: this.micFramesForwarded,
|
|
1191
|
+
meanAbs: level.meanAbs,
|
|
1192
|
+
peak: level.peak,
|
|
1193
|
+
dropsStale: this.micDropsStale,
|
|
1194
|
+
dropsNonGlasses: this.micDropsNonGlasses,
|
|
1195
|
+
gaps: this.micGaps,
|
|
1196
|
+
gapMsMax: this.micGapMsMax,
|
|
1197
|
+
})
|
|
1198
|
+
// #region agent log
|
|
1199
|
+
fetch("http://127.0.0.1:7905/ingest/5a9713c9-45ff-4d09-9435-2adc5db5e91d", {
|
|
1200
|
+
method: "POST",
|
|
1201
|
+
headers: {"Content-Type": "application/json", "X-Debug-Session-Id": "828181"},
|
|
1202
|
+
body: JSON.stringify({
|
|
1203
|
+
sessionId: "828181",
|
|
1204
|
+
runId: "run1",
|
|
1205
|
+
hypothesisId: "E",
|
|
1206
|
+
location: "AcsMeetingService.ts:logMicUplink",
|
|
1207
|
+
message: "phone decoded glasses PCM window",
|
|
1208
|
+
data: {
|
|
1209
|
+
fps: Math.round((this.micFramesWindow * 1000) / elapsed),
|
|
1210
|
+
meanAbs: level.meanAbs,
|
|
1211
|
+
peak: level.peak,
|
|
1212
|
+
frames: this.micFramesForwarded,
|
|
1213
|
+
},
|
|
1214
|
+
timestamp: Date.now(),
|
|
1215
|
+
}),
|
|
1216
|
+
}).catch(() => {})
|
|
1217
|
+
// #endregion
|
|
1218
|
+
this.micFramesWindow = 0
|
|
1219
|
+
}
|
|
1220
|
+
|
|
1221
|
+
/** Level of the last uplink window, for tests and dev screens. `null` until a window closed. */
|
|
1222
|
+
lastGlassesMicLevel(): {meanAbs: number; peak: number} | null {
|
|
1223
|
+
return this.lastMicLevel
|
|
1224
|
+
}
|
|
1225
|
+
|
|
1226
|
+
private logMicDrop(reason: string, extra: Record<string, unknown>): void {
|
|
1227
|
+
const now = Date.now()
|
|
1228
|
+
if (now - this.lastMicDropLogAt < 1000) return
|
|
1229
|
+
this.lastMicDropLogAt = now
|
|
1230
|
+
console.warn("[AcsMeeting] phase=glasses-mic-drop", {
|
|
1231
|
+
reason,
|
|
1232
|
+
dropsStale: this.micDropsStale,
|
|
1233
|
+
dropsNonGlasses: this.micDropsNonGlasses,
|
|
1234
|
+
gaps: this.micGaps,
|
|
1235
|
+
...extra,
|
|
1236
|
+
})
|
|
1237
|
+
}
|
|
1238
|
+
|
|
338
1239
|
async setMuted(packageName: string, muted: boolean): Promise<MeetingState> {
|
|
339
1240
|
this.assertOwner(packageName)
|
|
340
1241
|
const native = getNative()
|
|
@@ -344,6 +1245,9 @@ class AcsMeetingService {
|
|
|
344
1245
|
...this.lastState,
|
|
345
1246
|
...state,
|
|
346
1247
|
audioSourceReason: this.lastState.audioSourceReason,
|
|
1248
|
+
// Mute is an ACS-side gate. It does not change which microphone the call is using, and it
|
|
1249
|
+
// deliberately does not turn the glasses microphone off — see the mute chain in native.
|
|
1250
|
+
micTransport: this.micTransport,
|
|
347
1251
|
}
|
|
348
1252
|
return this.lastState
|
|
349
1253
|
}
|
|
@@ -352,6 +1256,11 @@ class AcsMeetingService {
|
|
|
352
1256
|
this.assertOwner(packageName)
|
|
353
1257
|
const native = getNative()
|
|
354
1258
|
if (!native) throw new Error("ACS meeting module is not available on this host")
|
|
1259
|
+
if (this.videoSource?.type === "softap") {
|
|
1260
|
+
// The host owns the softap endpoint, so there is no URL for a caller to change. Failing is
|
|
1261
|
+
// better than accepting it and doing nothing.
|
|
1262
|
+
throw new Error("updateVideoSource is not applicable to a SoftAP call")
|
|
1263
|
+
}
|
|
355
1264
|
this.whepUrl = whepUrl
|
|
356
1265
|
await native.updateVideoSource(whepUrl)
|
|
357
1266
|
}
|
|
@@ -363,8 +1272,8 @@ class AcsMeetingService {
|
|
|
363
1272
|
const native = getNative()
|
|
364
1273
|
if (!native) return {state: "idle", muted: false}
|
|
365
1274
|
const state = await native.getState()
|
|
366
|
-
this.lastState = state
|
|
367
|
-
return
|
|
1275
|
+
this.lastState = {...state, micTransport: this.micTransport}
|
|
1276
|
+
return this.lastState
|
|
368
1277
|
}
|
|
369
1278
|
|
|
370
1279
|
async leaveIfOwner(packageName: string): Promise<void> {
|
|
@@ -397,6 +1306,7 @@ class AcsMeetingService {
|
|
|
397
1306
|
}
|
|
398
1307
|
const participants = parseMeetingParticipants(event.participants)
|
|
399
1308
|
const mediaSource = parseMediaSource(event.mediaSource)
|
|
1309
|
+
const capabilities = parseMeetingCapabilities(event.capabilities)
|
|
400
1310
|
const state: MeetingState = {
|
|
401
1311
|
state: (event.state as MeetingPhase) ?? "idle",
|
|
402
1312
|
muted: Boolean(event.muted),
|
|
@@ -407,8 +1317,11 @@ class AcsMeetingService {
|
|
|
407
1317
|
audioSourceReason: this.lastState.audioSourceReason,
|
|
408
1318
|
activeStream: parseActiveStream(event.activeStream),
|
|
409
1319
|
audioSafety,
|
|
1320
|
+
micTransport: this.micTransport,
|
|
410
1321
|
...(mediaSource ? {mediaSource} : {}),
|
|
411
1322
|
...(participants ? {participants} : {}),
|
|
1323
|
+
// Absent means unknown, so keep the last known verdict rather than clearing it.
|
|
1324
|
+
...((capabilities ?? this.lastState.capabilities) ? {capabilities: capabilities ?? this.lastState.capabilities} : {}),
|
|
412
1325
|
}
|
|
413
1326
|
this.lastState = state
|
|
414
1327
|
console.log("[AcsMeeting] phase=native-state", {
|
|
@@ -419,9 +1332,19 @@ class AcsMeetingService {
|
|
|
419
1332
|
activeStream: state.activeStream,
|
|
420
1333
|
audioSafety: state.audioSafety,
|
|
421
1334
|
mediaSource: state.mediaSource,
|
|
1335
|
+
micTransport: state.micTransport,
|
|
422
1336
|
participants: participants?.length,
|
|
423
1337
|
})
|
|
1338
|
+
this.settleFirstFrameWaiters(mediaSource)
|
|
424
1339
|
this.onState?.(packageName, state)
|
|
1340
|
+
// A remote hang-up, an ACS error or a dropped call never goes through `leave`, so without
|
|
1341
|
+
// this the microphone pin, the PCM requirement and the `mic_pcm` listener would outlive the
|
|
1342
|
+
// meeting — and the next miniapp to start a call would inherit them.
|
|
1343
|
+
if (state.state === "disconnected" || state.state === "error") {
|
|
1344
|
+
void this.releaseHostState().catch((error) => {
|
|
1345
|
+
console.warn("[AcsMeeting] releasing host state after a terminal native state failed", error)
|
|
1346
|
+
})
|
|
1347
|
+
}
|
|
425
1348
|
}),
|
|
426
1349
|
native.addListener("onIncomingPcm", (event) => {
|
|
427
1350
|
const base64 = event.base64 as string | undefined
|
|
@@ -472,7 +1395,16 @@ class AcsMeetingService {
|
|
|
472
1395
|
|
|
473
1396
|
private async restartMediaSource(native: NativeModule, reason: string): Promise<void> {
|
|
474
1397
|
const whepUrl = this.whepUrl
|
|
475
|
-
|
|
1398
|
+
const softap = this.videoSource?.type === "softap"
|
|
1399
|
+
// SoftAP media is bound to the scoped hotspot, not the phone's default route. Joining that
|
|
1400
|
+
// hotspot is what *causes* NetInfo to flap `none:offline → cellular:online` — the default
|
|
1401
|
+
// route looks gone for a beat, then cellular comes back. Rebuilding the WHIP listener on
|
|
1402
|
+
// that flap changes the ingest port after the glasses already have the old URL, and their
|
|
1403
|
+
// POST hits a tombstone (HTTP 410). A real hotspot loss is `onScopedNetworkLost`, not this
|
|
1404
|
+
// default-network watch. WHEP still needs the rebuild: that path rides the default route.
|
|
1405
|
+
if (softap) return
|
|
1406
|
+
if (!this.owner || !MEDIA_ACTIVE_PHASES.has(this.lastState.state)) return
|
|
1407
|
+
if (!whepUrl) return
|
|
476
1408
|
const now = Date.now()
|
|
477
1409
|
if (now - this.lastMediaRestartAt < MEDIA_RESTART_MIN_INTERVAL_MS) return
|
|
478
1410
|
this.lastMediaRestartAt = now
|
|
@@ -481,7 +1413,8 @@ class AcsMeetingService {
|
|
|
481
1413
|
// A same-URL updateVideoSource is a no-op while native still believes the
|
|
482
1414
|
// peer is healthy; after a network switch that belief is exactly what is wrong.
|
|
483
1415
|
if (native.restartVideoSource) await native.restartVideoSource()
|
|
484
|
-
else await native.updateVideoSource(whepUrl)
|
|
1416
|
+
else if (whepUrl) await native.updateVideoSource(whepUrl)
|
|
1417
|
+
else console.warn("[AcsMeeting] softap restart needs a native restartVideoSource", {reason})
|
|
485
1418
|
} catch (error) {
|
|
486
1419
|
console.warn("[AcsMeeting] media restart failed", {reason, error})
|
|
487
1420
|
}
|
|
@@ -529,7 +1462,7 @@ class AcsMeetingService {
|
|
|
529
1462
|
const now = Date.now()
|
|
530
1463
|
if (now - this.lastBacklogWarnAt > 5000) {
|
|
531
1464
|
this.lastBacklogWarnAt = now
|
|
532
|
-
console.warn("[AcsMeeting] incoming PCM backlog high", {bufferedMs})
|
|
1465
|
+
console.warn("[AcsMeeting] incoming PCM backlog high (downlink playback, not glasses mic)", {bufferedMs})
|
|
533
1466
|
}
|
|
534
1467
|
}
|
|
535
1468
|
} catch (error) {
|
|
@@ -546,6 +1479,7 @@ class AcsMeetingService {
|
|
|
546
1479
|
sampleRate,
|
|
547
1480
|
channels,
|
|
548
1481
|
stopOtherAudio: true,
|
|
1482
|
+
jitterMs: PCM_JITTER_MS,
|
|
549
1483
|
onEnded: () => {
|
|
550
1484
|
if (this.pcmStreamId === streamId) {
|
|
551
1485
|
this.pcmStreamId = null
|