@mentra/engine 3.2.0-dev.221 → 3.2.0-dev.223

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/build/generated/releaseMetadata.js +5 -5
  2. package/build/generated/releaseMetadata.js.map +1 -1
  3. package/build/index.d.ts +2 -0
  4. package/build/index.d.ts.map +1 -1
  5. package/build/index.js +1 -0
  6. package/build/index.js.map +1 -1
  7. package/build/services/AcsMeetingService.d.ts +360 -10
  8. package/build/services/AcsMeetingService.d.ts.map +1 -1
  9. package/build/services/AcsMeetingService.js +836 -19
  10. package/build/services/AcsMeetingService.js.map +1 -1
  11. package/build/services/AudioPlaybackService.d.ts +6 -0
  12. package/build/services/AudioPlaybackService.d.ts.map +1 -1
  13. package/build/services/AudioPlaybackService.js +4 -3
  14. package/build/services/AudioPlaybackService.js.map +1 -1
  15. package/build/services/GlassesMicProbe.d.ts +78 -0
  16. package/build/services/GlassesMicProbe.d.ts.map +1 -0
  17. package/build/services/GlassesMicProbe.js +258 -0
  18. package/build/services/GlassesMicProbe.js.map +1 -0
  19. package/build/services/LocalMiniappRuntime.d.ts +83 -0
  20. package/build/services/LocalMiniappRuntime.d.ts.map +1 -1
  21. package/build/services/LocalMiniappRuntime.js +714 -39
  22. package/build/services/LocalMiniappRuntime.js.map +1 -1
  23. package/build/services/MentraJSLogPipeline.d.ts +39 -0
  24. package/build/services/MentraJSLogPipeline.d.ts.map +1 -1
  25. package/build/services/MentraJSLogPipeline.js +59 -3
  26. package/build/services/MentraJSLogPipeline.js.map +1 -1
  27. package/build/services/MentraJSRouter.d.ts +1 -1
  28. package/build/services/MentraJSRouter.d.ts.map +1 -1
  29. package/build/services/MentraJSRouter.js +2 -2
  30. package/build/services/MentraJSRouter.js.map +1 -1
  31. package/build/services/MicStateCoordinator.d.ts +22 -0
  32. package/build/services/MicStateCoordinator.d.ts.map +1 -1
  33. package/build/services/MicStateCoordinator.js +34 -3
  34. package/build/services/MicStateCoordinator.js.map +1 -1
  35. package/build/services/PhoneStreamCoordinator.d.ts +8 -0
  36. package/build/services/PhoneStreamCoordinator.d.ts.map +1 -1
  37. package/build/services/PhoneStreamCoordinator.js +2 -0
  38. package/build/services/PhoneStreamCoordinator.js.map +1 -1
  39. package/build/services/SoftapCallTransport.d.ts +365 -0
  40. package/build/services/SoftapCallTransport.d.ts.map +1 -0
  41. package/build/services/SoftapCallTransport.js +722 -0
  42. package/build/services/SoftapCallTransport.js.map +1 -0
  43. package/build/services/SoftapCleanupBarrier.d.ts +45 -0
  44. package/build/services/SoftapCleanupBarrier.d.ts.map +1 -0
  45. package/build/services/SoftapCleanupBarrier.js +51 -0
  46. package/build/services/SoftapCleanupBarrier.js.map +1 -0
  47. package/build/utils/pcm16.d.ts +35 -0
  48. package/build/utils/pcm16.d.ts.map +1 -0
  49. package/build/utils/pcm16.js +85 -0
  50. package/build/utils/pcm16.js.map +1 -0
  51. package/build/utils/softapTrace.d.ts +39 -0
  52. package/build/utils/softapTrace.d.ts.map +1 -0
  53. package/build/utils/softapTrace.js +124 -0
  54. package/build/utils/softapTrace.js.map +1 -0
  55. package/package.json +7 -7
  56. package/src/generated/releaseMetadata.ts +5 -5
  57. package/src/index.ts +2 -0
  58. package/src/services/AcsMeetingService.ts +956 -22
  59. package/src/services/AudioPlaybackService.ts +12 -3
  60. package/src/services/GlassesMicProbe.ts +300 -0
  61. package/src/services/LocalMiniappRuntime.ts +775 -42
  62. package/src/services/MentraJSLogPipeline.ts +70 -3
  63. package/src/services/MentraJSRouter.ts +2 -2
  64. package/src/services/MicStateCoordinator.ts +35 -3
  65. package/src/services/PhoneStreamCoordinator.ts +10 -0
  66. package/src/services/SoftapCallTransport.ts +928 -0
  67. package/src/services/SoftapCleanupBarrier.ts +72 -0
  68. package/src/utils/pcm16.ts +93 -0
  69. package/src/utils/softapTrace.ts +133 -0
@@ -4,9 +4,17 @@
4
4
  * incoming PCM into AudioPlaybackService (A2DP / PcmStreamPlayer).
5
5
  */
6
6
 
7
+ import {Platform} from "react-native"
8
+
9
+ import BluetoothSdk from "@mentra/bluetooth-sdk/internal"
10
+
7
11
  import audioPlaybackService from "./AudioPlaybackService"
12
+ import micStateCoordinator from "./MicStateCoordinator"
8
13
  import {SETTINGS, useSettingsStore} from "../stores/settings"
9
- import {ACS_CALL_MIC, type ResolvedAudioSource, type SourceReason} from "./acsAudioSource"
14
+ import {Pcm16LevelMeter} from "../utils/pcm16"
15
+ import {softapTrace, softapTraceFailure, softapTraceId} from "../utils/softapTrace"
16
+ import {ACS_CALL_MIC, type AcsAudioSource, type ResolvedAudioSource, type SourceReason} from "./acsAudioSource"
17
+ import type {SoftapProgress} from "./SoftapCallTransport"
10
18
 
11
19
  export {ACS_CALL_MIC}
12
20
  export type {ResolvedAudioSource, SourceReason}
@@ -22,6 +30,19 @@ export type ActiveStream = "none" | "virtual" | "local"
22
30
  */
23
31
  export type MediaSourceState = "idle" | "connecting" | "live" | "failed"
24
32
 
33
+ /**
34
+ * How the wearer's voice is reaching ACS.
35
+ *
36
+ * `ble-lc3` is the glasses microphone over Bluetooth, decoded on the phone and pushed into the ACS
37
+ * raw outgoing stream. `whip` is the glasses microphone on the WebRTC track they publish (SoftAP or
38
+ * Cloudflare). `phone` is the phone's own microphone. `none` means the source we committed to went
39
+ * away mid-call, which is a reportable failure rather than a reason to open a different microphone.
40
+ */
41
+ export type MicTransport = "ble-lc3" | "whip" | "phone" | "none"
42
+
43
+ /** Error surfaced when the pinned glasses microphone stops producing audio during a call. */
44
+ export const GLASSES_MIC_UNAVAILABLE = "GLASSES_MIC_UNAVAILABLE"
45
+
25
46
  export type MeetingParticipantState = "idle" | "connecting" | "connected" | "lobby" | "hold" | "disconnected"
26
47
 
27
48
  export interface MeetingParticipant {
@@ -32,6 +53,36 @@ export interface MeetingParticipant {
32
53
  isSpeaking: boolean
33
54
  }
34
55
 
56
+ /**
57
+ * One runtime participant capability.
58
+ *
59
+ * `allowed` is nullable because "denied" and "not known yet" are different facts. ACS delivers
60
+ * capabilities asynchronously, so every call is briefly unknown, and collapsing that into `false`
61
+ * would hide controls on calls that do in fact permit them.
62
+ */
63
+ export interface MeetingCapability {
64
+ allowed: boolean | null
65
+ reason: string | null
66
+ }
67
+
68
+ export interface MeetingCapabilities {
69
+ /** Whether this participant may end the Teams group call for everyone. Presenters only. */
70
+ hangUpForEveryone: MeetingCapability
71
+ }
72
+
73
+ export function parseMeetingCapabilities(raw: unknown): MeetingCapabilities | undefined {
74
+ if (!raw || typeof raw !== "object") return undefined
75
+ const value = (raw as Record<string, unknown>).hangUpForEveryone
76
+ if (!value || typeof value !== "object") return undefined
77
+ const capability = value as Record<string, unknown>
78
+ return {
79
+ hangUpForEveryone: {
80
+ allowed: typeof capability.allowed === "boolean" ? capability.allowed : null,
81
+ reason: typeof capability.reason === "string" && capability.reason ? capability.reason : null,
82
+ },
83
+ }
84
+ }
85
+
35
86
  export interface MeetingState {
36
87
  state: MeetingPhase
37
88
  muted: boolean
@@ -44,19 +95,84 @@ export interface MeetingState {
44
95
  audioSafety?: AudioSafety
45
96
  mediaSource?: MediaSourceState
46
97
  participants?: MeetingParticipant[]
98
+ /** Runtime capabilities. Omitted by natives that predate them; read that as unknown. */
99
+ capabilities?: MeetingCapabilities
100
+ /** Which microphone path is carrying the wearer's voice, decided at join time. */
101
+ micTransport?: MicTransport
102
+ /**
103
+ * SoftAP join checklist, attached only to the state events the SoftAP orchestrator emits while
104
+ * it walks hotspot → scoped join → ACS join → publish → live. Native ACS events never carry it,
105
+ * so a consumer keeps the last one it saw rather than treating its absence as a reset.
106
+ */
107
+ softap?: SoftapProgress
47
108
  }
48
109
 
49
110
  /**
50
- * The call microphone is [ACS_CALL_MIC]. Flip that constant to `"glasses"`
51
- * to restore glasses WHIP → Cloudflare → WHEP PCM. Do not use ACS
52
- * LocalOutgoingAudioStream for phone: that path makes ACS own the route
111
+ * The call microphone is [ACS_CALL_MIC], and `preferred_mic` governs MentraOS STT capture rather
112
+ * than this call.
113
+ *
114
+ * `"glasses"` selects the wearer's own microphone. Which *transport* carries it is a separate
115
+ * decision made per call by [glassesLc3UplinkSupported]: BLE LC3 into the ACS raw outgoing stream
116
+ * where the host and native both support it, otherwise the glasses' published WebRTC audio track.
117
+ * `"phone"` selects the phone's microphone via `PhoneMicCapturer`; do not route it through ACS
118
+ * `LocalOutgoingAudioStream`, because that hands ACS the audio route
53
119
  * (MODE_IN_COMMUNICATION + forced speaker) and opens an echo loop.
54
- * preferred_mic governs MentraOS STT capture, not this call.
55
120
  */
56
121
  export function resolveAcsAudioSource(): ResolvedAudioSource {
57
122
  return {source: ACS_CALL_MIC, reason: "explicit"}
58
123
  }
59
124
 
125
+ /**
126
+ * Whether this call can take the wearer's voice off the glasses over BLE LC3.
127
+ *
128
+ * Every term is a hard requirement, and the answer has to be known *before* the glasses are told
129
+ * what to publish — deciding afterwards is how a call ends up with either two copies of the wearer
130
+ * or none. A host whose native module has no `pushOutgoingPcm` keeps the WHIP audio track it has
131
+ * always used rather than silently joining a meeting nobody can be heard in.
132
+ *
133
+ * SoftAP takes BLE LC3 and publishes WHIP video only (`captureAudio=false`), so the glasses SoC
134
+ * microphone is never opened for a call.
135
+ *
136
+ * This was switched off once on a misread: a soak logged `P4 pcm meanAbs` 8–39 and called it
137
+ * digital silence. It was the room. The same LC3 path measured on the bench sits at `meanAbs`
138
+ * ≈30–60 / `peak` ≈80–100 in a quiet room with nobody talking, and that soak had already
139
+ * published with `captureAudio=false`, so the SoC mic was not the thief either. The uplink now
140
+ * logs its own `meanAbs`/`peak` every window ([logMicUplink]) so the next soak reads the level,
141
+ * not the frame rate. Disable only with a level log showing the floor *while the wearer talks*.
142
+ */
143
+ const SOFTAP_BLE_LC3_UPLINK = true
144
+ /**
145
+ * How long native may take to finish a leave before it reports the cleanup as stuck.
146
+ *
147
+ * Generous on purpose: this is a diagnostic ceiling on a hang-up plus an agent disposal, not a
148
+ * readiness deadline. The caller treats a timeout as "cleanup failed", never as "safe to restart".
149
+ */
150
+ const ACS_LEAVE_WAIT_MS = 20_000
151
+ /**
152
+ * SoftAP video (camera → H.264 → WHIP → WHEP → ACS) is slower than BLE LC3.
153
+ * Hold glasses PCM this long so the talk track does not lead the picture.
154
+ * Set from SoftAP+LC3 clap correlator (2026-09-10): leads 70/201/173/300/142 ms,
155
+ * median 173, rounded to 170. `AVSYNC clap audioLeadMs` is still the re-cal.
156
+ */
157
+ export const SOFTAP_LC3_AUDIO_DELAY_MS = 170
158
+ let softapBleLc3UplinkForTests: boolean | null = null
159
+
160
+ export function glassesLc3UplinkSupported(args: {
161
+ videoSource: AcsVideoSource
162
+ audioSource: AcsAudioSource
163
+ hasPushOutgoingPcm: boolean
164
+ platform: string
165
+ }): boolean {
166
+ if (!(softapBleLc3UplinkForTests ?? SOFTAP_BLE_LC3_UPLINK)) return false
167
+ // iOS has no `setMicSourcePin` yet, so it cannot promise the phone microphone stays shut.
168
+ if (args.platform !== "android") return false
169
+ // WHEP audio comes back from Cloudflare already mixed into the subscribed track; there is no
170
+ // captureAudio flag on that path to turn off.
171
+ if (args.videoSource.type !== "softap") return false
172
+ if (args.audioSource !== "glasses") return false
173
+ return args.hasPushOutgoingPcm
174
+ }
175
+
60
176
  const PARTICIPANT_STATES = new Set<MeetingParticipantState>([
61
177
  "idle",
62
178
  "connecting",
@@ -95,7 +211,7 @@ export type AcsOutgoingVideo = {
95
211
  }
96
212
 
97
213
  /** ACS VirtualOutgoingVideoStream documented 16:9 sizes. P540 is 960×540, not 540×960. */
98
- const ACS_VIRTUAL_CAMERA_SIZES = new Set(["1280x720", "960x540"])
214
+ const ACS_VIRTUAL_CAMERA_SIZES = new Set(["1280x720", "960x540", "858x480"])
99
215
 
100
216
  export function parseAcsOutgoingVideo(raw: unknown): AcsOutgoingVideo | undefined {
101
217
  if (raw == null) return undefined
@@ -114,23 +230,127 @@ export function parseAcsOutgoingVideo(raw: unknown): AcsOutgoingVideo | undefine
114
230
  return {width, height, fps, maxBitrateBps}
115
231
  }
116
232
 
233
+ /**
234
+ * Where the glasses video comes from.
235
+ *
236
+ * A union rather than a nullable URL because the two transports need different inputs: WHEP is
237
+ * given a URL, while SoftAP produces one only after the host binds a local listener. Collapsing
238
+ * them into `whepUrl?: string` is what lets an empty string reach the subscriber and fail seconds
239
+ * later as an opaque HTTP error.
240
+ */
241
+ export type AcsVideoSource =
242
+ | {type: "whep"; url: string}
243
+ | {type: "softap"; ssid?: string; passphrase?: string; bindAddress?: string}
244
+
245
+ /**
246
+ * Validates a `videoSource` from a miniapp.
247
+ *
248
+ * Throws rather than defaulting to WHEP. A miniapp that asks for SoftAP and silently gets a
249
+ * Cloudflare call — or vice versa — is a bug that shows up as unexplained latency or a black tile,
250
+ * not as an error anyone can act on.
251
+ */
252
+ export function parseAcsVideoSource(raw: unknown): AcsVideoSource {
253
+ if (raw == null || typeof raw !== "object") {
254
+ throw new Error(`videoSource must be {type: "whep", url} or {type: "softap"}`)
255
+ }
256
+ const value = raw as Record<string, unknown>
257
+
258
+ if (value.type === "whep") {
259
+ const url = typeof value.url === "string" ? value.url.trim() : ""
260
+ if (!url) throw new Error("videoSource.url is required for a WHEP source")
261
+ return {type: "whep", url}
262
+ }
263
+
264
+ if (value.type === "softap") {
265
+ const ssid = typeof value.ssid === "string" ? value.ssid.trim() : ""
266
+ const passphrase = typeof value.passphrase === "string" ? value.passphrase : ""
267
+ // Half a credential pair would otherwise present as a failed hotspot join much later.
268
+ if (Boolean(ssid) !== Boolean(passphrase)) {
269
+ throw new Error("videoSource.ssid and videoSource.passphrase must be provided together")
270
+ }
271
+ return ssid ? {type: "softap", ssid, passphrase} : {type: "softap"}
272
+ }
273
+
274
+ throw new Error(`unsupported videoSource.type: ${String(value.type)}`)
275
+ }
276
+
277
+ /**
278
+ * The app's default network after a hotspot join.
279
+ *
280
+ * `usable` is the only field to branch on; the rest is for the note the wearer sees and the trace.
281
+ * A held cellular request brings the radio up but does not promise the default route has switched
282
+ * or validated, so this is asked separately rather than inferred from the hold.
283
+ */
284
+ export interface DefaultNetworkStatus {
285
+ transport: string
286
+ validated: boolean
287
+ present: boolean
288
+ usable: boolean
289
+ detail: string
290
+ }
291
+
117
292
  type NativeModule = {
293
+ prepareAgent?(options: {token: string; displayName?: string}): Promise<MeetingState>
118
294
  join(options: {
119
295
  meetingUrl: string
120
296
  token: string
297
+ /** Legacy field, still sent for whep so an older native keeps working. */
121
298
  whepUrl: string
299
+ videoSource: AcsVideoSource
122
300
  displayName?: string
123
301
  dumpPcmWav?: boolean
124
302
  audioSource?: "glasses" | "phone"
303
+ audioDelayMs?: number
125
304
  video?: AcsOutgoingVideo
126
- }): Promise<MeetingState>
305
+ }): Promise<MeetingState & {ingestUrl?: string}>
127
306
  leave(): Promise<void>
307
+ /**
308
+ * Leave, and resolve only once the hang-up, the agent disposal, and the network releases have
309
+ * actually finished.
310
+ *
311
+ * [leave] queues its work on the session executor and returns immediately, so awaiting it proves
312
+ * nothing about cleanup — which is what let a fast Stop/Start start a second call on top of the
313
+ * first one's teardown. Absent on natives that predate the signal.
314
+ */
315
+ leaveAndAwait?(options: {timeoutMs: number}): Promise<{completed: boolean}>
316
+ /**
317
+ * End the group call for everyone, then tear this device down. Rejects when the capability is
318
+ * denied or ACS refuses — and has still left the call. Absent on natives that predate End.
319
+ */
320
+ endForEveryone?(): Promise<MeetingState>
128
321
  setMuted(muted: boolean): Promise<MeetingState>
129
322
  setAudioSource(source: "glasses" | "phone"): Promise<MeetingState>
130
323
  updateVideoSource(whepUrl: string): Promise<void>
131
324
  /** Force a WHEP rebuild on the current URL. Absent on natives that predate it. */
132
325
  restartVideoSource?(): Promise<void>
326
+ /**
327
+ * Join the glasses hotspot as a scoped, internet-less network; resolves with this phone's
328
+ * address on it. Absent on natives that predate SoftAP, and rejects on iOS.
329
+ */
330
+ joinScopedNetwork?(ssid: string, passphrase: string): Promise<string>
331
+ beginTrace?(traceId: string): Promise<void>
332
+ leaveScopedNetwork?(): Promise<void>
333
+ /**
334
+ * TCP-probe the hotspot gateway over the scoped network. Absent on natives that predate it.
335
+ * `detail` is a one-line human summary (address, port, latency or the failure).
336
+ */
337
+ probeScopedGateway?(): Promise<{reachable: boolean; detail: string}>
338
+ /** The joined hotspot: `prefix` is what the media path must stay inside. */
339
+ scopedNetworkInfo?(): Promise<{available: boolean; localIpv4: string | null; prefix: string | null}>
340
+ /**
341
+ * Wait for the app's default network to be validated, after the hotspot join changed it.
342
+ * Absent on natives that predate the cellular transition handling.
343
+ */
344
+ awaitValidatedDefaultNetwork?(): Promise<DefaultNetworkStatus>
133
345
  getState(): Promise<MeetingState>
346
+ /**
347
+ * Hand one chunk of already-decoded microphone PCM to the ACS uplink. Synchronous on purpose:
348
+ * this runs ~100×/s and a promise per chunk would cost more than the copy does.
349
+ *
350
+ * Absent on natives that predate the BLE LC3 uplink, which is exactly what
351
+ * [glassesLc3UplinkSupported] tests for.
352
+ */
353
+ pushOutgoingPcm?(base64: string, sampleRate: number, channels: number): void
134
354
  addListener(event: string, listener: (event: Record<string, unknown>) => void): {remove: () => void}
135
355
  }
136
356
 
@@ -152,6 +372,11 @@ export function setAcsMeetingNativeForTests(mod: NativeModule | null | undefined
152
372
  nativeModule = mod
153
373
  }
154
374
 
375
+ /** Test seam: exercise the BLE LC3 path without flipping the production kill switch. */
376
+ export function setSoftapBleLc3UplinkForTests(enabled: boolean | null): void {
377
+ softapBleLc3UplinkForTests = enabled
378
+ }
379
+
155
380
  /**
156
381
  * Phone connectivity feed. Only the subset of `@react-native-community/netinfo`
157
382
  * this service needs, so tests can inject a fake and hosts without the package
@@ -204,7 +429,57 @@ const MEDIA_RESTART_MIN_INTERVAL_MS = 3000
204
429
 
205
430
  /** Formats the native PcmStreamPlayer accepts (mono only). */
206
431
  const PCM_SAMPLE_RATES = new Set([16000, 24000, 48000])
207
- const PCM_BACKLOG_WARN_MS = 600
432
+
433
+ /**
434
+ * Playout headroom for the far end's voice.
435
+ *
436
+ * The native player defaults to half a second, which is right for a miniapp clip and wrong here:
437
+ * a streaming track sits at its buffer size, so that default is 500ms of delay before the wearer
438
+ * hears anyone, enough that both sides talk over each other. This trades cushion for immediacy —
439
+ * still several BLE playout beats, but a conversation instead of a broadcast.
440
+ */
441
+ const PCM_JITTER_MS = 120
442
+ /**
443
+ * Above this the backlog is real congestion rather than the configured cushion.
444
+ *
445
+ * Has to stay clear of [PCM_JITTER_MS], because the reported backlog *includes* the track's own
446
+ * buffer and so never falls below it. A threshold at or under the cushion logs once per interval
447
+ * for the whole call and says nothing.
448
+ */
449
+ const PCM_BACKLOG_WARN_MS = PCM_JITTER_MS + 400
450
+
451
+ /** The Bluetooth SDK mic identifier the call pins itself to. */
452
+ const GLASSES_MIC_SOURCE = "glasses"
453
+ /**
454
+ * How long the glasses may produce no PCM before the call reports its microphone gone.
455
+ *
456
+ * The pin makes a phone-mic fallback impossible, so the only honest response to a source that
457
+ * stopped is to say so. One second is several BLE mic-beats: long enough that a reconnect blip
458
+ * does not raise an error, short enough that the wearer is not talking into nothing for long.
459
+ */
460
+ const GLASSES_MIC_GRACE_MS = 1000
461
+ /** Cadence of the uplink health line, matching the native P8 ladder. */
462
+ const MIC_UPLINK_LOG_INTERVAL_MS = 5000
463
+ /** A 50 ms LC3 frame arriving more than this late is a missed beat, not jitter. */
464
+ const MIC_GAP_WARN_MS = 90
465
+
466
+ /**
467
+ * Encode one microphone buffer for `pushOutgoingPcm`.
468
+ *
469
+ * Hermes has no Node `Buffer`. Using it here is how a live SoftAP call selected `ble-lc3`,
470
+ * pinned the glasses, and still sent Teams a minute of silence: every `mic_pcm` event threw
471
+ * `Property 'Buffer' doesn't exist` before native saw a byte. `btoa` is what React Native
472
+ * actually has.
473
+ */
474
+ export function pcmToBase64(pcm: ArrayBuffer): string {
475
+ const bytes = new Uint8Array(pcm)
476
+ let binary = ""
477
+ const step = 0x8000
478
+ for (let i = 0; i < bytes.length; i += step) {
479
+ binary += String.fromCharCode(...bytes.subarray(i, i + step))
480
+ }
481
+ return btoa(binary)
482
+ }
208
483
 
209
484
  class AcsMeetingService {
210
485
  private owner: string | null = null
@@ -218,9 +493,54 @@ class AcsMeetingService {
218
493
  private onState: ((packageName: string, state: MeetingState) => void) | null = null
219
494
  /** WHEP URL native is (or should be) subscribed to; what a host-triggered restart re-feeds. */
220
495
  private whepUrl: string | null = null
496
+ /** Transport for the active call, so recovery picks the right repair. */
497
+ private videoSource: AcsVideoSource | null = null
498
+ /**
499
+ * SoftAP only: the URL the glasses must publish to. An output of the join rather than an input,
500
+ * because it is not known until native has bound a listener and been given a port.
501
+ */
502
+ private ingestUrl: string | null = null
221
503
  private phoneNetworkUnsub: (() => void) | null = null
222
504
  private lastPhoneNetworkKey: string | null = null
223
505
  private lastMediaRestartAt = 0
506
+ /** Callers parked in [waitForFirstFrame], woken by the next `mediaSource` verdict. */
507
+ private readonly firstFrameWaiters = new Set<(error?: Error) => void>()
508
+ private scopedLostSub: {remove: () => void} | null = null
509
+ private readonly scopedLostListeners = new Set<(error: {code: string; message: string}) => void>()
510
+ /**
511
+ * Set before the scoped network is released, so the loss we are about to cause is not reported as
512
+ * one that happened to us. Android emits `onLost` for a deliberate release, and a naive wiring
513
+ * turns every successful Leave and End into a mid-call network error.
514
+ */
515
+ private scopedTerminating = false
516
+ /**
517
+ * Bumped by every join and every release, and captured by everything that can outlive a call:
518
+ * the `mic_pcm` listener and the in-flight `native.join`.
519
+ *
520
+ * A leave during a slow join is the case this exists for. Without it the join resolves into a
521
+ * torn-down host, subscribes a microphone listener nobody will ever unsubscribe, and pushes the
522
+ * wearer's voice at a native session that has already left the meeting.
523
+ */
524
+ private callGeneration = 0
525
+ private micTransport: MicTransport = "whip"
526
+ private micSub: {remove: () => void} | null = null
527
+ /** True between the pin/requirement being taken and released, so release is exactly once. */
528
+ private micUplinkActive = false
529
+ private micFramesForwarded = 0
530
+ /** Frames since the last health line, so the rate and the call total stay separate facts. */
531
+ private micFramesWindow = 0
532
+ private micDropsStale = 0
533
+ private micDropsNonGlasses = 0
534
+ private micGaps = 0
535
+ private micGapMsMax = 0
536
+ private lastGlassesFrameAt = 0
537
+ private lastMicUplinkLogAt = 0
538
+ private lastMicDropLogAt = 0
539
+ /** PCM level over the current health window; read and reset by [logMicUplink]. */
540
+ private micLevel = new Pcm16LevelMeter()
541
+ private lastMicLevel: {meanAbs: number; peak: number} | null = null
542
+ /** Guards [releaseHostState] so a remote hang-up followed by an explicit leave releases once. */
543
+ private hostStateReleased = true
224
544
 
225
545
  setStateHandler(handler: (packageName: string, state: MeetingState) => void): void {
226
546
  this.onState = handler
@@ -234,12 +554,216 @@ class AcsMeetingService {
234
554
  return this.owner
235
555
  }
236
556
 
557
+ /**
558
+ * The WHIP URL the glasses must POST their offer to, for a SoftAP call. Null for every other
559
+ * transport and until the join has bound a listener; the orchestrator reads it between the ACS
560
+ * join and telling the glasses to publish.
561
+ */
562
+ softApIngestUrl(): string | null {
563
+ return this.ingestUrl
564
+ }
565
+
566
+ /**
567
+ * Whether this call is taking the wearer's voice over BLE LC3, which means the glasses must
568
+ * publish video only. Read by the SoftAP orchestrator between the meeting join and the publish.
569
+ */
570
+ glassesLc3UplinkActive(): boolean {
571
+ return this.micTransport === "ble-lc3"
572
+ }
573
+
574
+ /**
575
+ * Join the glasses hotspot as a scoped, internet-less network, returning this phone's address on
576
+ * it. Called before the ACS join, because the local WHIP listener has to bind to that address.
577
+ *
578
+ * A host without the native function is not a host that silently skips the join — the SoftAP call
579
+ * has no network to run on, so this reports the reason instead.
580
+ */
581
+ async joinScopedNetwork(ssid: string, passphrase: string): Promise<string | undefined> {
582
+ const native = getNative()
583
+ if (!native?.joinScopedNetwork) {
584
+ throw new Error("This host cannot join the glasses hotspot; SoftAP calling is unavailable")
585
+ }
586
+ this.scopedTerminating = false
587
+ this.bindScopedNetworkLost(native)
588
+ await native.beginTrace?.(softapTraceId())
589
+ return await native.joinScopedNetwork(ssid, passphrase)
590
+ }
591
+
592
+ /**
593
+ * Subscribe to "the glasses hotspot went away while we still wanted it".
594
+ *
595
+ * Only unexpected losses arrive here. Native drops the framework callback for a network it
596
+ * released itself, and [leaveScopedNetwork] raises the terminal intent before releasing, so a
597
+ * normal Leave or End cannot manufacture a mid-call failure.
598
+ */
599
+ onScopedNetworkLost(listener: (error: {code: string; message: string}) => void): () => void {
600
+ this.scopedLostListeners.add(listener)
601
+ return () => this.scopedLostListeners.delete(listener)
602
+ }
603
+
604
+ private bindScopedNetworkLost(native: NativeModule): void {
605
+ if (this.scopedLostSub) return
606
+ try {
607
+ this.scopedLostSub = native.addListener("onScopedNetworkLost", (event) => {
608
+ if (this.scopedTerminating) {
609
+ console.log("[AcsMeeting] phase=scoped-lost-expected", {code: event.code})
610
+ return
611
+ }
612
+ const error = {
613
+ code: typeof event.code === "string" ? event.code : "SOFTAP_NETWORK_LOST",
614
+ message: typeof event.message === "string" ? event.message : "The glasses hotspot went away",
615
+ }
616
+ console.warn("[AcsMeeting] phase=scoped-lost", error)
617
+ for (const listener of [...this.scopedLostListeners]) {
618
+ try {
619
+ listener(error)
620
+ } catch (listenerError) {
621
+ console.warn("[AcsMeeting] scoped-lost listener threw", listenerError)
622
+ }
623
+ }
624
+ })
625
+ } catch (error) {
626
+ // A native that predates the event simply never reports mid-call loss; that is a smaller
627
+ // problem than failing the join over a missing listener.
628
+ console.warn("[AcsMeeting] scoped network loss events unavailable", error)
629
+ this.scopedLostSub = null
630
+ }
631
+ }
632
+
633
+ /** Raise the terminal intent so the release we are about to do is not read as a failure. */
634
+ beginScopedTeardown(): void {
635
+ this.scopedTerminating = true
636
+ }
637
+
638
+ /**
639
+ * Wait until the phone's default network is validated again after the hotspot join.
640
+ *
641
+ * Joining the glasses hotspot takes the phone off Wi-Fi, and ACS needs the internet for the very
642
+ * next step. Waiting is what turns a 40-second join timeout with device-wide DNS failures into a
643
+ * short, named wait — or an honest failure. Null when the host cannot tell.
644
+ */
645
+ async awaitValidatedDefaultNetwork(): Promise<DefaultNetworkStatus | null> {
646
+ const native = getNative()
647
+ if (!native?.awaitValidatedDefaultNetwork) return null
648
+ return await native.awaitValidatedDefaultNetwork()
649
+ }
650
+
651
+ /**
652
+ * Can this phone reach the glasses over the hotspot it just joined? Null when the host cannot
653
+ * tell (no native support), so the orchestrator narrates nothing rather than a guess.
654
+ */
655
+ async probeScopedGateway(): Promise<{reachable: boolean; detail: string} | null> {
656
+ const native = getNative()
657
+ if (!native?.probeScopedGateway) return null
658
+ return await native.probeScopedGateway()
659
+ }
660
+
661
+ /** Safe to call when nothing was joined: teardown runs after failed starts too. */
662
+ async leaveScopedNetwork(): Promise<void> {
663
+ // Intent first, release second. The other order is the false-positive bug: Android reports the
664
+ // loss we asked for, and the call reports a network failure as it is successfully ending.
665
+ this.scopedTerminating = true
666
+ try {
667
+ await getNative()?.leaveScopedNetwork?.()
668
+ } finally {
669
+ this.scopedLostSub?.remove()
670
+ this.scopedLostSub = null
671
+ }
672
+ }
673
+
674
+ /**
675
+ * Resolves once the host reports a frame actually reached ACS, which is the only signal that
676
+ * remote participants can see the camera.
677
+ *
678
+ * Rejects if the feed fails first, and on timeout. A SoftAP call that connects but never paints
679
+ * is the failure this exists to catch: without it the orchestrator would report `live` on the
680
+ * strength of an ACS join that says nothing about video.
681
+ *
682
+ * @param timeoutMs how long to wait before treating the silence as a failure
683
+ */
684
+ waitForFirstFrame(timeoutMs: number): Promise<void> {
685
+ if (this.lastState.mediaSource === "live") {
686
+ softapTrace("acs_first_frame_already_live", {mediaSource: this.lastState.mediaSource})
687
+ return Promise.resolve()
688
+ }
689
+ const startedAt = Date.now()
690
+ softapTrace("acs_first_frame_wait", {
691
+ timeoutMs,
692
+ mediaSource: this.lastState.mediaSource ?? "unknown",
693
+ state: this.lastState.state,
694
+ })
695
+ return new Promise<void>((resolve, reject) => {
696
+ const settle = (error?: Error) => {
697
+ if (done) return
698
+ done = true
699
+ clearTimeout(timer)
700
+ this.firstFrameWaiters.delete(settle)
701
+ // The step this closes is the one that decides whether the call is usable, so its outcome
702
+ // is named rather than inferred from whichever line happens to follow.
703
+ if (error) {
704
+ softapTraceFailure("acs_first_frame_wait_done", {
705
+ waitedMs: Date.now() - startedAt,
706
+ reason: error.message,
707
+ })
708
+ reject(error)
709
+ } else {
710
+ softapTrace("acs_first_frame_wait_done", {waitedMs: Date.now() - startedAt})
711
+ resolve()
712
+ }
713
+ }
714
+ let done = false
715
+ const timer = setTimeout(
716
+ () => settle(new Error(`No glasses video reached the meeting within ${Math.round(timeoutMs / 1000)}s`)),
717
+ timeoutMs,
718
+ )
719
+ this.firstFrameWaiters.add(settle)
720
+ })
721
+ }
722
+
723
+ /** Wake every `waitForFirstFrame` caller with the outcome the host just reported. */
724
+ private settleFirstFrameWaiters(mediaSource: MediaSourceState | undefined): void {
725
+ if (mediaSource !== "live" && mediaSource !== "failed") return
726
+ const error = mediaSource === "failed" ? new Error("The glasses video feed failed") : undefined
727
+ for (const settle of [...this.firstFrameWaiters]) settle(error)
728
+ }
729
+
730
+ /**
731
+ * Sign in to ACS before the glasses hotspot exists.
732
+ *
733
+ * SoftAP DNS cannot resolve Teams hosts. Doing this on the phone's existing internet is what
734
+ * stops `createCallAgent` from hanging until the hotspot is torn down.
735
+ */
736
+ async prepareAgent(args: {token: string; displayName?: string}): Promise<void> {
737
+ const native = getNative()
738
+ if (!native?.prepareAgent) {
739
+ // A host that cannot pre-sign-in still joins; it just does the sign-in inside the SoftAP
740
+ // join, which is the 20-second ACS_AGENT_TIMEOUT this step exists to avoid. Worth a line,
741
+ // because from the trace alone that build looks like one whose sign-in was instant.
742
+ softapTrace("acs_prepare_agent_skipped", {reason: native ? "unsupported build" : "no native module"})
743
+ return
744
+ }
745
+ console.log("[AcsMeeting] phase=prepare-agent")
746
+ const startedAt = Date.now()
747
+ softapTrace("acs_native_prepare_agent", {hasDisplayName: Boolean(args.displayName)})
748
+ try {
749
+ await native.prepareAgent({token: args.token, displayName: args.displayName})
750
+ } catch (error) {
751
+ softapTraceFailure("acs_native_prepare_agent_failed", {
752
+ durationMs: Date.now() - startedAt,
753
+ reason: error instanceof Error ? error.message : String(error),
754
+ })
755
+ throw error
756
+ }
757
+ console.log("[AcsMeeting] phase=prepare-agent-ok")
758
+ softapTrace("acs_native_prepare_agent_ok", {durationMs: Date.now() - startedAt})
759
+ }
760
+
237
761
  async join(
238
762
  packageName: string,
239
763
  args: {
240
764
  meetingUrl: string
241
765
  token: string
242
- whepUrl: string
766
+ videoSource: AcsVideoSource
243
767
  displayName?: string
244
768
  video?: AcsOutgoingVideo
245
769
  },
@@ -255,33 +779,89 @@ class AcsMeetingService {
255
779
  // Validate before claiming ownership so a bad request cannot leave the slot taken.
256
780
  const video = args.video ? parseAcsOutgoingVideo(args.video) : undefined
257
781
  const resolved = resolveAcsAudioSource()
782
+ const generation = ++this.callGeneration
783
+ this.hostStateReleased = false
258
784
  this.owner = packageName
259
- this.whepUrl = args.whepUrl
785
+ // Only a whep source has a URL to re-feed on recovery; softap rebuilds instead.
786
+ this.whepUrl = args.videoSource.type === "whep" ? args.videoSource.url : null
787
+ this.videoSource = args.videoSource
788
+ // Decided here, before the join, so the SoftAP orchestrator can read it between the join and
789
+ // telling the glasses what to capture.
790
+ const lc3Uplink = glassesLc3UplinkSupported({
791
+ videoSource: args.videoSource,
792
+ audioSource: resolved.source,
793
+ hasPushOutgoingPcm: typeof native.pushOutgoingPcm === "function",
794
+ platform: Platform.OS,
795
+ })
796
+ this.micTransport = lc3Uplink ? "ble-lc3" : resolved.source === "phone" ? "phone" : "whip"
260
797
  this.bindNative(native, packageName)
261
798
  console.log("[AcsMeeting] phase=join-native", {
262
799
  packageName,
263
800
  nativeLoaded: true,
264
801
  hasToken: Boolean(args.token),
265
- hasWhep: Boolean(args.whepUrl),
802
+ transport: args.videoSource.type,
266
803
  audioSource: resolved.source,
267
804
  audioSourceReason: resolved.reason,
805
+ micTransport: this.micTransport,
268
806
  preferredMic: useSettingsStore.getState().getSetting(SETTINGS.preferred_mic.key),
269
807
  })
808
+ const joinStartedAt = Date.now()
809
+ softapTrace("acs_native_join", {
810
+ packageName,
811
+ generation,
812
+ transport: args.videoSource.type,
813
+ audioSource: resolved.source,
814
+ micTransport: this.micTransport,
815
+ audioDelayMs: lc3Uplink ? SOFTAP_LC3_AUDIO_DELAY_MS : 0,
816
+ video: video ? `${video.width}x${video.height}@${video.fps}` : "default",
817
+ })
270
818
  try {
271
819
  const state = await native.join({
272
820
  meetingUrl: args.meetingUrl,
273
821
  token: args.token,
274
- whepUrl: args.whepUrl,
822
+ whepUrl: this.whepUrl ?? "",
823
+ videoSource: args.videoSource,
275
824
  displayName: args.displayName,
276
825
  audioSource: resolved.source,
826
+ ...(lc3Uplink ? {audioDelayMs: SOFTAP_LC3_AUDIO_DELAY_MS} : {}),
277
827
  ...(video ? {video} : {}),
278
828
  })
829
+ softapTrace("acs_native_join_returned", {
830
+ packageName,
831
+ generation,
832
+ state: state.state,
833
+ hasIngestUrl: typeof state.ingestUrl === "string" && state.ingestUrl.length > 0,
834
+ durationMs: Date.now() - joinStartedAt,
835
+ })
836
+ if (generation !== this.callGeneration) {
837
+ // The wearer left while ACS was still joining. Nothing above knows about this call, so
838
+ // hanging it up here is the only thing that takes the device out of the Teams roster.
839
+ console.warn("[AcsMeeting] phase=join-cancelled", {packageName, generation})
840
+ softapTraceFailure("acs_native_join_cancelled", {
841
+ packageName,
842
+ generation,
843
+ current: this.callGeneration,
844
+ durationMs: Date.now() - joinStartedAt,
845
+ })
846
+ await native.leave().catch((leaveError) => {
847
+ console.warn("[AcsMeeting] native leave after a cancelled join failed", leaveError)
848
+ softapTraceFailure("acs_native_leave_after_cancelled_join_failed", {
849
+ packageName,
850
+ reason: leaveError instanceof Error ? leaveError.message : String(leaveError),
851
+ })
852
+ })
853
+ throw new Error("The meeting was cancelled before it finished joining")
854
+ }
855
+ this.ingestUrl = typeof state.ingestUrl === "string" ? state.ingestUrl : null
856
+ const {ingestUrl: _ingestUrl, ...meetingState} = state
279
857
  this.lastState = {
280
- ...state,
858
+ ...meetingState,
281
859
  audioSource: resolved.source,
282
860
  audioSourceReason: resolved.reason,
861
+ micTransport: this.micTransport,
283
862
  }
284
863
  console.log("[AcsMeeting] phase=join-native-ok", {state: state.state, muted: state.muted})
864
+ this.startGlassesMicUplink(generation)
285
865
  } catch (error) {
286
866
  // Native never joined (or is unwinding). Release the slot so the same or another
287
867
  // miniapp can retry, and make sure nothing half-joined lingers in Teams.
@@ -289,8 +869,18 @@ class AcsMeetingService {
289
869
  packageName,
290
870
  error: error instanceof Error ? error.message : String(error),
291
871
  })
872
+ softapTraceFailure("acs_native_join_failed", {
873
+ packageName,
874
+ generation,
875
+ durationMs: Date.now() - joinStartedAt,
876
+ reason: error instanceof Error ? error.message : String(error),
877
+ })
292
878
  await native.leave().catch((leaveError) => {
293
879
  console.warn("[AcsMeeting] native leave after failed join also failed", leaveError)
880
+ softapTraceFailure("acs_native_leave_after_failed_join_failed", {
881
+ packageName,
882
+ reason: leaveError instanceof Error ? leaveError.message : String(leaveError),
883
+ })
294
884
  })
295
885
  await this.releaseHostState()
296
886
  throw error
@@ -311,30 +901,341 @@ class AcsMeetingService {
311
901
  }
312
902
 
313
903
  async leave(packageName: string): Promise<void> {
314
- if (this.owner && this.owner !== packageName) return
904
+ if (this.owner && this.owner !== packageName) {
905
+ // Not the owner, so this is a no-op rather than a leave. Said out loud because a miniapp
906
+ // that thinks it left and a host that never hung up look identical from the miniapp's side.
907
+ softapTrace("acs_native_leave_ignored", {packageName, owner: this.owner})
908
+ return
909
+ }
315
910
  const native = getNative()
911
+ const startedAt = Date.now()
912
+ softapTrace("acs_native_leave", {packageName, nativeLoaded: Boolean(native)})
316
913
  try {
317
914
  await native?.leave()
915
+ softapTrace("acs_native_leave_ok", {packageName, durationMs: Date.now() - startedAt})
916
+ } catch (error) {
917
+ softapTraceFailure("acs_native_leave_failed", {
918
+ packageName,
919
+ durationMs: Date.now() - startedAt,
920
+ reason: error instanceof Error ? error.message : String(error),
921
+ })
922
+ throw error
318
923
  } finally {
319
924
  await this.releaseHostState()
320
925
  }
321
926
  }
322
927
 
323
928
  /**
324
- * Drop everything the host set up around a session: the network watcher, return
325
- * audio, native listeners, ownership. Runs after native has been told to leave
326
- * (or after a join that never produced a native call).
929
+ * Leave, and do not resolve until native reports its cleanup is finished.
930
+ *
931
+ * The difference from [leave] is the whole point: native's `leave()` queues the hang-up, the
932
+ * agent disposal, and the network releases on its session executor and returns straight away, so
933
+ * a caller that awaits it and then starts the next call is racing the previous one's teardown.
934
+ *
935
+ * Falls back to [leave] on builds that predate the signal and says so, rather than pretending to
936
+ * a guarantee it cannot give.
937
+ *
938
+ * @param timeoutMs how long native may take before it reports the cleanup as stuck
939
+ */
940
+ async leaveAndAwait(packageName: string, timeoutMs = ACS_LEAVE_WAIT_MS): Promise<{completed: boolean; reason?: string}> {
941
+ if (this.owner && this.owner !== packageName) {
942
+ softapTrace("acs_native_leave_and_await_ignored", {packageName, owner: this.owner})
943
+ return {completed: true}
944
+ }
945
+ const native = getNative()
946
+ if (!native?.leaveAndAwait) {
947
+ // The fallback is the case the cleanup barrier was built for: this build's leave returns
948
+ // before its own teardown has finished, so nothing downstream can treat "left" as "idle".
949
+ softapTraceFailure("acs_native_leave_and_await_unsupported", {
950
+ packageName,
951
+ nativeLoaded: Boolean(native),
952
+ })
953
+ await this.leave(packageName)
954
+ return {completed: false, reason: "unsupported"}
955
+ }
956
+ const startedAt = Date.now()
957
+ softapTrace("acs_native_leave_and_await", {packageName, timeoutMs})
958
+ try {
959
+ const outcome = await native.leaveAndAwait({timeoutMs})
960
+ softapTrace("acs_native_leave_and_await_ok", {
961
+ packageName,
962
+ completed: outcome?.completed !== false,
963
+ durationMs: Date.now() - startedAt,
964
+ })
965
+ return {completed: outcome?.completed !== false}
966
+ } catch (error) {
967
+ softapTraceFailure("acs_native_leave_and_await_failed", {
968
+ packageName,
969
+ durationMs: Date.now() - startedAt,
970
+ reason: error instanceof Error ? error.message : String(error),
971
+ })
972
+ throw error
973
+ } finally {
974
+ await this.releaseHostState()
975
+ }
976
+ }
977
+
978
+ /**
979
+ * End the Teams group call for everyone.
980
+ *
981
+ * Host state is released either way: native tears this device down whatever the hang-up did, so
982
+ * holding the meeting slot open after a rejection would leave the miniapp unable to start another
983
+ * call. The rejection still propagates — the caller has to be able to say "you left, but the
984
+ * meeting may still be active" instead of claiming a clean end.
985
+ */
986
+ async endForEveryone(packageName: string): Promise<MeetingState> {
987
+ if (this.owner && this.owner !== packageName) {
988
+ throw new Error("This miniapp does not own the active meeting")
989
+ }
990
+ const native = getNative()
991
+ if (!native?.endForEveryone) {
992
+ softapTraceFailure("acs_native_end_unsupported", {packageName, nativeLoaded: Boolean(native)})
993
+ throw new Error("Update the Mentra App to end a meeting for everyone")
994
+ }
995
+ const startedAt = Date.now()
996
+ softapTrace("acs_native_end", {packageName})
997
+ try {
998
+ const state = await native.endForEveryone()
999
+ console.log("[AcsMeeting] phase=end-for-everyone-ok", {state: state.state})
1000
+ softapTrace("acs_native_end_ok", {packageName, state: state.state, durationMs: Date.now() - startedAt})
1001
+ return state
1002
+ } catch (error) {
1003
+ // The meeting may still be live for the others; this device is out regardless. Logged as a
1004
+ // failure of the claim, not of the teardown.
1005
+ softapTraceFailure("acs_native_end_failed", {
1006
+ packageName,
1007
+ durationMs: Date.now() - startedAt,
1008
+ reason: error instanceof Error ? error.message : String(error),
1009
+ })
1010
+ throw error
1011
+ } finally {
1012
+ await this.releaseHostState()
1013
+ }
1014
+ }
1015
+
1016
+ /** Whether the wearer may end this meeting for everyone, as ACS last reported it. */
1017
+ hangUpForEveryoneCapability(): MeetingCapability {
1018
+ return this.lastState.capabilities?.hangUpForEveryone ?? {allowed: null, reason: null}
1019
+ }
1020
+
1021
+ /**
1022
+ * Drop everything the host set up around a session: the microphone uplink, the network watcher,
1023
+ * return audio, native listeners, ownership.
1024
+ *
1025
+ * Idempotent, because there are now several terminal paths into it and they overlap. A remote
1026
+ * hang-up arrives as a native `disconnected`, and the miniapp usually calls `leave` right after
1027
+ * seeing it; releasing the microphone pin twice would clear a pin the *next* call had already
1028
+ * taken. Every release also bumps the call generation, which is what stops a `mic_pcm` event
1029
+ * already queued on the JS thread from reaching a native session that is gone.
327
1030
  */
328
1031
  private async releaseHostState(): Promise<void> {
1032
+ if (this.hostStateReleased) return
1033
+ this.hostStateReleased = true
1034
+ this.callGeneration++
1035
+ // Before anything else: a caller parked on a frame that will now never arrive has to be
1036
+ // rejected, or a leave mid-join leaves the orchestrator waiting out its whole timeout.
1037
+ for (const settle of [...this.firstFrameWaiters]) settle(new Error("The meeting ended"))
1038
+ this.firstFrameWaiters.clear()
1039
+ this.stopGlassesMicUplink()
329
1040
  this.unwatchPhoneNetwork()
330
1041
  await this.stopPcm()
331
1042
  this.unbindNative()
1043
+ this.scopedLostSub?.remove()
1044
+ this.scopedLostSub = null
332
1045
  this.owner = null
333
1046
  this.whepUrl = null
1047
+ this.videoSource = null
1048
+ this.ingestUrl = null
334
1049
  this.lastMediaRestartAt = 0
335
1050
  this.lastState = {state: "idle", muted: false}
336
1051
  }
337
1052
 
1053
+ /**
1054
+ * Start forwarding the glasses microphone into ACS for this call.
1055
+ *
1056
+ * Order matters and is the whole point: pin the Bluetooth SDK to the glasses *before* asking it
1057
+ * for PCM, so the first frame the mic requirement produces is already from the right source and
1058
+ * the phone microphone is never opened even for one buffer. `generation` is captured by the
1059
+ * listener so a frame that lands after this call ended is dropped rather than pushed at a native
1060
+ * session that has left the meeting.
1061
+ */
1062
+ private startGlassesMicUplink(generation: number): void {
1063
+ if (this.micTransport !== "ble-lc3") return
1064
+ const native = getNative()
1065
+ const push = native?.pushOutgoingPcm
1066
+ if (!native || !push) return
1067
+ this.micFramesForwarded = 0
1068
+ this.micFramesWindow = 0
1069
+ this.micDropsStale = 0
1070
+ this.micDropsNonGlasses = 0
1071
+ this.micGaps = 0
1072
+ this.micGapMsMax = 0
1073
+ this.lastGlassesFrameAt = Date.now()
1074
+ this.lastMicDropLogAt = 0
1075
+ this.lastMicUplinkLogAt = Date.now()
1076
+ this.micLevel.take()
1077
+ this.lastMicLevel = null
1078
+ try {
1079
+ void Promise.resolve(BluetoothSdk.setMicSourcePin?.(GLASSES_MIC_SOURCE)).catch((error) => {
1080
+ console.warn("[AcsMeeting] pinning the glasses microphone failed", error)
1081
+ })
1082
+ this.micUplinkActive = true
1083
+ micStateCoordinator.setCallRequirement(true)
1084
+ this.micSub = BluetoothSdk.addListener("mic_pcm", (event: {pcm?: ArrayBuffer; sampleRate?: number; source?: string}) => {
1085
+ if (generation !== this.callGeneration) {
1086
+ this.micDropsStale += 1
1087
+ this.logMicDrop("stale", {generation, active: this.callGeneration})
1088
+ return
1089
+ }
1090
+ // The pin makes this unreachable in normal operation, which is exactly why it is checked:
1091
+ // a mic the SDK moved under us would otherwise put the phone's room into a Teams call
1092
+ // that reports the glasses.
1093
+ if (event.source !== GLASSES_MIC_SOURCE) {
1094
+ this.micDropsNonGlasses += 1
1095
+ this.logMicDrop("non-glasses", {source: event.source})
1096
+ this.reportGlassesMicUnavailable(event.source)
1097
+ return
1098
+ }
1099
+ const pcm = event.pcm
1100
+ if (!pcm) return
1101
+ const now = Date.now()
1102
+ if (this.micFramesForwarded > 0) {
1103
+ const gapMs = now - this.lastGlassesFrameAt
1104
+ if (gapMs > MIC_GAP_WARN_MS) {
1105
+ this.micGaps += 1
1106
+ this.micGapMsMax = Math.max(this.micGapMsMax, gapMs)
1107
+ this.logMicDrop("gap", {gapMs})
1108
+ }
1109
+ }
1110
+ this.lastGlassesFrameAt = now
1111
+ this.micFramesForwarded += 1
1112
+ this.micFramesWindow += 1
1113
+ this.micLevel.add(pcm)
1114
+ try {
1115
+ push.call(native, pcmToBase64(pcm), event.sampleRate ?? 16000, 1)
1116
+ } catch (error) {
1117
+ console.warn("[AcsMeeting] pushing glasses PCM to ACS failed", error)
1118
+ }
1119
+ this.logMicUplink()
1120
+ })
1121
+ console.log("[AcsMeeting] phase=glasses-mic-uplink-start", {generation, micTransport: this.micTransport})
1122
+ } catch (error) {
1123
+ // A host that cannot subscribe has no wearer audio at all, and that is worth saying loudly,
1124
+ // but it is not worth failing a call the wearer can still see and hear.
1125
+ console.error("[AcsMeeting] phase=glasses-mic-uplink-unavailable", error)
1126
+ this.stopGlassesMicUplink()
1127
+ this.markMicTransportNone()
1128
+ }
1129
+ }
1130
+
1131
+ /** Release the microphone claims this call took. Safe to call when it never took them. */
1132
+ private stopGlassesMicUplink(): void {
1133
+ this.micSub?.remove()
1134
+ this.micSub = null
1135
+ if (this.micUplinkActive) {
1136
+ this.micUplinkActive = false
1137
+ micStateCoordinator.setCallRequirement(false)
1138
+ // Last, and unconditionally: while the pin is set no other consumer can pick a microphone,
1139
+ // so leaving it behind would leave captions and the cloud uplink stuck on the glasses.
1140
+ void Promise.resolve(BluetoothSdk.setMicSourcePin?.(null)).catch((error) => {
1141
+ console.warn("[AcsMeeting] releasing the glasses microphone pin failed", error)
1142
+ })
1143
+ console.log("[AcsMeeting] phase=glasses-mic-uplink-stop", {
1144
+ frames: this.micFramesForwarded,
1145
+ dropsStale: this.micDropsStale,
1146
+ dropsNonGlasses: this.micDropsNonGlasses,
1147
+ gaps: this.micGaps,
1148
+ gapMsMax: this.micGapMsMax,
1149
+ })
1150
+ }
1151
+ this.micTransport = "whip"
1152
+ }
1153
+
1154
+ /**
1155
+ * Report a call whose pinned microphone stopped delivering. There is deliberately no fallback:
1156
+ * the wearer agreed to be heard from the glasses, and quietly switching to the phone in the
1157
+ * middle of a meeting puts the room they are standing in on the call.
1158
+ */
1159
+ private reportGlassesMicUnavailable(source: string | undefined): void {
1160
+ if (Date.now() - this.lastGlassesFrameAt < GLASSES_MIC_GRACE_MS) return
1161
+ if (this.micTransport === "none") return
1162
+ console.error("[AcsMeeting] phase=glasses-mic-unavailable", {source, dropsNonGlasses: this.micDropsNonGlasses})
1163
+ this.markMicTransportNone()
1164
+ }
1165
+
1166
+ /**
1167
+ * Record that this call has no wearer audio path.
1168
+ *
1169
+ * The state has to carry it, not just the field: a call that reported `ble-lc3` and then failed
1170
+ * to start the uplink looks identical from the miniapp's side to one that is working, and the
1171
+ * wearer finds out by being asked to repeat themselves.
1172
+ */
1173
+ private markMicTransportNone(): void {
1174
+ this.micTransport = "none"
1175
+ this.lastState = {...this.lastState, micTransport: "none", error: GLASSES_MIC_UNAVAILABLE}
1176
+ if (this.owner) this.onState?.(this.owner, this.lastState)
1177
+ }
1178
+
1179
+ private logMicUplink(): void {
1180
+ const now = Date.now()
1181
+ const elapsed = now - this.lastMicUplinkLogAt
1182
+ if (elapsed < MIC_UPLINK_LOG_INTERVAL_MS) return
1183
+ this.lastMicUplinkLogAt = now
1184
+ // Level, not just cadence: a 20 Hz stream of the noise floor and a 20 Hz stream of speech
1185
+ // have the same framesPerSecond. Quiet room on Mentra Live LC3 is meanAbs ≈30–60.
1186
+ const level = this.micLevel.take()
1187
+ this.lastMicLevel = {meanAbs: level.meanAbs, peak: level.peak}
1188
+ console.log("[AcsMeeting] phase=glasses-mic-uplink", {
1189
+ framesPerSecond: Math.round((this.micFramesWindow * 1000) / elapsed),
1190
+ frames: this.micFramesForwarded,
1191
+ meanAbs: level.meanAbs,
1192
+ peak: level.peak,
1193
+ dropsStale: this.micDropsStale,
1194
+ dropsNonGlasses: this.micDropsNonGlasses,
1195
+ gaps: this.micGaps,
1196
+ gapMsMax: this.micGapMsMax,
1197
+ })
1198
+ // #region agent log
1199
+ fetch("http://127.0.0.1:7905/ingest/5a9713c9-45ff-4d09-9435-2adc5db5e91d", {
1200
+ method: "POST",
1201
+ headers: {"Content-Type": "application/json", "X-Debug-Session-Id": "828181"},
1202
+ body: JSON.stringify({
1203
+ sessionId: "828181",
1204
+ runId: "run1",
1205
+ hypothesisId: "E",
1206
+ location: "AcsMeetingService.ts:logMicUplink",
1207
+ message: "phone decoded glasses PCM window",
1208
+ data: {
1209
+ fps: Math.round((this.micFramesWindow * 1000) / elapsed),
1210
+ meanAbs: level.meanAbs,
1211
+ peak: level.peak,
1212
+ frames: this.micFramesForwarded,
1213
+ },
1214
+ timestamp: Date.now(),
1215
+ }),
1216
+ }).catch(() => {})
1217
+ // #endregion
1218
+ this.micFramesWindow = 0
1219
+ }
1220
+
1221
+ /** Level of the last uplink window, for tests and dev screens. `null` until a window closed. */
1222
+ lastGlassesMicLevel(): {meanAbs: number; peak: number} | null {
1223
+ return this.lastMicLevel
1224
+ }
1225
+
1226
+ private logMicDrop(reason: string, extra: Record<string, unknown>): void {
1227
+ const now = Date.now()
1228
+ if (now - this.lastMicDropLogAt < 1000) return
1229
+ this.lastMicDropLogAt = now
1230
+ console.warn("[AcsMeeting] phase=glasses-mic-drop", {
1231
+ reason,
1232
+ dropsStale: this.micDropsStale,
1233
+ dropsNonGlasses: this.micDropsNonGlasses,
1234
+ gaps: this.micGaps,
1235
+ ...extra,
1236
+ })
1237
+ }
1238
+
338
1239
  async setMuted(packageName: string, muted: boolean): Promise<MeetingState> {
339
1240
  this.assertOwner(packageName)
340
1241
  const native = getNative()
@@ -344,6 +1245,9 @@ class AcsMeetingService {
344
1245
  ...this.lastState,
345
1246
  ...state,
346
1247
  audioSourceReason: this.lastState.audioSourceReason,
1248
+ // Mute is an ACS-side gate. It does not change which microphone the call is using, and it
1249
+ // deliberately does not turn the glasses microphone off — see the mute chain in native.
1250
+ micTransport: this.micTransport,
347
1251
  }
348
1252
  return this.lastState
349
1253
  }
@@ -352,6 +1256,11 @@ class AcsMeetingService {
352
1256
  this.assertOwner(packageName)
353
1257
  const native = getNative()
354
1258
  if (!native) throw new Error("ACS meeting module is not available on this host")
1259
+ if (this.videoSource?.type === "softap") {
1260
+ // The host owns the softap endpoint, so there is no URL for a caller to change. Failing is
1261
+ // better than accepting it and doing nothing.
1262
+ throw new Error("updateVideoSource is not applicable to a SoftAP call")
1263
+ }
355
1264
  this.whepUrl = whepUrl
356
1265
  await native.updateVideoSource(whepUrl)
357
1266
  }
@@ -363,8 +1272,8 @@ class AcsMeetingService {
363
1272
  const native = getNative()
364
1273
  if (!native) return {state: "idle", muted: false}
365
1274
  const state = await native.getState()
366
- this.lastState = state
367
- return state
1275
+ this.lastState = {...state, micTransport: this.micTransport}
1276
+ return this.lastState
368
1277
  }
369
1278
 
370
1279
  async leaveIfOwner(packageName: string): Promise<void> {
@@ -397,6 +1306,7 @@ class AcsMeetingService {
397
1306
  }
398
1307
  const participants = parseMeetingParticipants(event.participants)
399
1308
  const mediaSource = parseMediaSource(event.mediaSource)
1309
+ const capabilities = parseMeetingCapabilities(event.capabilities)
400
1310
  const state: MeetingState = {
401
1311
  state: (event.state as MeetingPhase) ?? "idle",
402
1312
  muted: Boolean(event.muted),
@@ -407,8 +1317,11 @@ class AcsMeetingService {
407
1317
  audioSourceReason: this.lastState.audioSourceReason,
408
1318
  activeStream: parseActiveStream(event.activeStream),
409
1319
  audioSafety,
1320
+ micTransport: this.micTransport,
410
1321
  ...(mediaSource ? {mediaSource} : {}),
411
1322
  ...(participants ? {participants} : {}),
1323
+ // Absent means unknown, so keep the last known verdict rather than clearing it.
1324
+ ...((capabilities ?? this.lastState.capabilities) ? {capabilities: capabilities ?? this.lastState.capabilities} : {}),
412
1325
  }
413
1326
  this.lastState = state
414
1327
  console.log("[AcsMeeting] phase=native-state", {
@@ -419,9 +1332,19 @@ class AcsMeetingService {
419
1332
  activeStream: state.activeStream,
420
1333
  audioSafety: state.audioSafety,
421
1334
  mediaSource: state.mediaSource,
1335
+ micTransport: state.micTransport,
422
1336
  participants: participants?.length,
423
1337
  })
1338
+ this.settleFirstFrameWaiters(mediaSource)
424
1339
  this.onState?.(packageName, state)
1340
+ // A remote hang-up, an ACS error or a dropped call never goes through `leave`, so without
1341
+ // this the microphone pin, the PCM requirement and the `mic_pcm` listener would outlive the
1342
+ // meeting — and the next miniapp to start a call would inherit them.
1343
+ if (state.state === "disconnected" || state.state === "error") {
1344
+ void this.releaseHostState().catch((error) => {
1345
+ console.warn("[AcsMeeting] releasing host state after a terminal native state failed", error)
1346
+ })
1347
+ }
425
1348
  }),
426
1349
  native.addListener("onIncomingPcm", (event) => {
427
1350
  const base64 = event.base64 as string | undefined
@@ -472,7 +1395,16 @@ class AcsMeetingService {
472
1395
 
473
1396
  private async restartMediaSource(native: NativeModule, reason: string): Promise<void> {
474
1397
  const whepUrl = this.whepUrl
475
- if (!this.owner || !whepUrl || !MEDIA_ACTIVE_PHASES.has(this.lastState.state)) return
1398
+ const softap = this.videoSource?.type === "softap"
1399
+ // SoftAP media is bound to the scoped hotspot, not the phone's default route. Joining that
1400
+ // hotspot is what *causes* NetInfo to flap `none:offline → cellular:online` — the default
1401
+ // route looks gone for a beat, then cellular comes back. Rebuilding the WHIP listener on
1402
+ // that flap changes the ingest port after the glasses already have the old URL, and their
1403
+ // POST hits a tombstone (HTTP 410). A real hotspot loss is `onScopedNetworkLost`, not this
1404
+ // default-network watch. WHEP still needs the rebuild: that path rides the default route.
1405
+ if (softap) return
1406
+ if (!this.owner || !MEDIA_ACTIVE_PHASES.has(this.lastState.state)) return
1407
+ if (!whepUrl) return
476
1408
  const now = Date.now()
477
1409
  if (now - this.lastMediaRestartAt < MEDIA_RESTART_MIN_INTERVAL_MS) return
478
1410
  this.lastMediaRestartAt = now
@@ -481,7 +1413,8 @@ class AcsMeetingService {
481
1413
  // A same-URL updateVideoSource is a no-op while native still believes the
482
1414
  // peer is healthy; after a network switch that belief is exactly what is wrong.
483
1415
  if (native.restartVideoSource) await native.restartVideoSource()
484
- else await native.updateVideoSource(whepUrl)
1416
+ else if (whepUrl) await native.updateVideoSource(whepUrl)
1417
+ else console.warn("[AcsMeeting] softap restart needs a native restartVideoSource", {reason})
485
1418
  } catch (error) {
486
1419
  console.warn("[AcsMeeting] media restart failed", {reason, error})
487
1420
  }
@@ -529,7 +1462,7 @@ class AcsMeetingService {
529
1462
  const now = Date.now()
530
1463
  if (now - this.lastBacklogWarnAt > 5000) {
531
1464
  this.lastBacklogWarnAt = now
532
- console.warn("[AcsMeeting] incoming PCM backlog high", {bufferedMs})
1465
+ console.warn("[AcsMeeting] incoming PCM backlog high (downlink playback, not glasses mic)", {bufferedMs})
533
1466
  }
534
1467
  }
535
1468
  } catch (error) {
@@ -546,6 +1479,7 @@ class AcsMeetingService {
546
1479
  sampleRate,
547
1480
  channels,
548
1481
  stopOtherAudio: true,
1482
+ jitterMs: PCM_JITTER_MS,
549
1483
  onEnded: () => {
550
1484
  if (this.pcmStreamId === streamId) {
551
1485
  this.pcmStreamId = null