@mentra/engine 3.2.0-dev.116 → 3.2.0-dev.120

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/build/engine.d.ts.map +1 -1
  2. package/build/engine.js +7 -2
  3. package/build/engine.js.map +1 -1
  4. package/build/generated/releaseMetadata.js +5 -5
  5. package/build/generated/releaseMetadata.js.map +1 -1
  6. package/build/runtime/streamConfig.d.ts +7 -0
  7. package/build/runtime/streamConfig.d.ts.map +1 -1
  8. package/build/runtime/streamConfig.js +7 -0
  9. package/build/runtime/streamConfig.js.map +1 -1
  10. package/build/services/AcsMeetingService.d.ts +154 -0
  11. package/build/services/AcsMeetingService.d.ts.map +1 -0
  12. package/build/services/AcsMeetingService.js +477 -0
  13. package/build/services/AcsMeetingService.js.map +1 -0
  14. package/build/services/DeviceEventRouter.d.ts.map +1 -1
  15. package/build/services/DeviceEventRouter.js +12 -3
  16. package/build/services/DeviceEventRouter.js.map +1 -1
  17. package/build/services/GlassesSettingsSync.d.ts.map +1 -1
  18. package/build/services/GlassesSettingsSync.js +11 -1
  19. package/build/services/GlassesSettingsSync.js.map +1 -1
  20. package/build/services/GlassesStatusProjection.d.ts +12 -0
  21. package/build/services/GlassesStatusProjection.d.ts.map +1 -1
  22. package/build/services/GlassesStatusProjection.js +23 -0
  23. package/build/services/GlassesStatusProjection.js.map +1 -1
  24. package/build/services/LocalMiniappRuntime.d.ts +6 -0
  25. package/build/services/LocalMiniappRuntime.d.ts.map +1 -1
  26. package/build/services/LocalMiniappRuntime.js +130 -1
  27. package/build/services/LocalMiniappRuntime.js.map +1 -1
  28. package/build/services/MentraJSRouter.d.ts.map +1 -1
  29. package/build/services/MentraJSRouter.js +8 -1
  30. package/build/services/MentraJSRouter.js.map +1 -1
  31. package/build/services/PhoneCameraFovCoordinator.d.ts +18 -0
  32. package/build/services/PhoneCameraFovCoordinator.d.ts.map +1 -1
  33. package/build/services/PhoneCameraFovCoordinator.js +36 -0
  34. package/build/services/PhoneCameraFovCoordinator.js.map +1 -1
  35. package/build/services/PhoneStreamCoordinator.d.ts +67 -1
  36. package/build/services/PhoneStreamCoordinator.d.ts.map +1 -1
  37. package/build/services/PhoneStreamCoordinator.js +204 -15
  38. package/build/services/PhoneStreamCoordinator.js.map +1 -1
  39. package/build/services/StreamLifecycleController.d.ts +23 -5
  40. package/build/services/StreamLifecycleController.d.ts.map +1 -1
  41. package/build/services/StreamLifecycleController.js +31 -10
  42. package/build/services/StreamLifecycleController.js.map +1 -1
  43. package/build/services/acsAudioSource.d.ts +24 -0
  44. package/build/services/acsAudioSource.d.ts.map +1 -0
  45. package/build/services/acsAudioSource.js +37 -0
  46. package/build/services/acsAudioSource.js.map +1 -0
  47. package/build/utils/devMiniappLaunch.d.ts.map +1 -1
  48. package/build/utils/devMiniappLaunch.js +4 -0
  49. package/build/utils/devMiniappLaunch.js.map +1 -1
  50. package/package.json +7 -6
  51. package/src/engine.ts +10 -2
  52. package/src/generated/releaseMetadata.ts +5 -5
  53. package/src/runtime/streamConfig.ts +13 -0
  54. package/src/services/AcsMeetingService.ts +570 -0
  55. package/src/services/DeviceEventRouter.ts +12 -3
  56. package/src/services/GlassesSettingsSync.ts +13 -3
  57. package/src/services/GlassesStatusProjection.ts +28 -0
  58. package/src/services/LocalMiniappRuntime.ts +142 -1
  59. package/src/services/MentraJSRouter.ts +7 -1
  60. package/src/services/PhoneCameraFovCoordinator.ts +34 -0
  61. package/src/services/PhoneStreamCoordinator.ts +232 -19
  62. package/src/services/StreamLifecycleController.ts +42 -12
  63. package/src/services/acsAudioSource.ts +53 -0
  64. package/src/utils/devMiniappLaunch.ts +4 -0
@@ -67,7 +67,7 @@ import {
67
67
  } from "../runtime/config"
68
68
  import {getAnalytics, getUiSeams} from "../runtime/bootstrap"
69
69
  import {invokeScanQrSeam} from "../runtime/scanQrSeam"
70
- import {normalizeStreamAudioConfig, normalizeStreamVideoConfig} from "../runtime/streamConfig"
70
+ import {normalizeStreamAudioConfig, normalizeStreamVideoConfig, resolveCaptureAudio} from "../runtime/streamConfig"
71
71
  import {toLanguageHint} from "@mentra/cloud-protocol/languages"
72
72
  import type {AudioSubscription, LanguageSource, TranscriptionData, TranslationData} from "@mentra/cloud-protocol"
73
73
  import {buildMiniappManifestSnapshot, type MiniappRuntimeDiagnosticSnapshot} from "../utils/miniappDiagnostics"
@@ -80,6 +80,7 @@ import {resolveForegroundLocationPermission} from "./ForegroundLocationPermissio
80
80
  import {advanceMiniappPingLiveness} from "./MiniappLiveness"
81
81
  import {listPhoneCalendarEvents, PhoneCalendarError} from "./PhoneCalendarService"
82
82
  import {LocalMiniappStorage} from "./LocalMiniappStorage"
83
+ import acsMeetingService, {parseAcsOutgoingVideo, resolveAcsAudioSource} from "./AcsMeetingService"
83
84
 
84
85
  // =============================================================================
85
86
  // Types
@@ -953,6 +954,9 @@ class LocalMiniappRuntime {
953
954
  void phoneVideoCoordinator.stopForApp(packageName).catch((error) => {
954
955
  console.warn(`${LOG_TAG}: failed to stop video recording for ${packageName} on unregister`, error)
955
956
  })
957
+ void acsMeetingService.leaveIfOwner(packageName).catch((error) => {
958
+ console.warn(`${LOG_TAG}: failed to leave ACS meeting for ${packageName} on unregister`, error)
959
+ })
956
960
 
957
961
  // Detach the per-app nav event forwarder but leave the native nav session
958
962
  // running. The user may have just closed the mini-app UI and will reopen
@@ -1307,6 +1311,21 @@ class LocalMiniappRuntime {
1307
1311
  case MiniappRequestType.MANAGED_STREAM_STOP:
1308
1312
  void this.handleManagedStreamStop(packageName, payload, requestId)
1309
1313
  break
1314
+ case MiniappRequestType.MEETING_JOIN:
1315
+ void this.handleMeetingJoin(packageName, payload, requestId)
1316
+ break
1317
+ case MiniappRequestType.MEETING_LEAVE:
1318
+ void this.handleMeetingLeave(packageName, requestId)
1319
+ break
1320
+ case MiniappRequestType.MEETING_SET_MUTED:
1321
+ void this.handleMeetingSetMuted(packageName, payload, requestId)
1322
+ break
1323
+ case MiniappRequestType.MEETING_UPDATE_VIDEO_SOURCE:
1324
+ void this.handleMeetingUpdateVideoSource(packageName, payload, requestId)
1325
+ break
1326
+ case MiniappRequestType.MEETING_GET_STATE:
1327
+ void this.handleMeetingGetState(packageName, requestId)
1328
+ break
1310
1329
  case REQUEST_WIFI_SETUP_TYPE:
1311
1330
  void this.handleRequestWifiSetup(packageName, payload, requestId)
1312
1331
  break
@@ -1409,6 +1428,7 @@ class LocalMiniappRuntime {
1409
1428
  packageName,
1410
1429
  capabilities,
1411
1430
  permissions: declaredPermissions,
1431
+ hostFeatures: {captureAudio: true},
1412
1432
  ...(initialAuth ? {auth: initialAuth} : {}),
1413
1433
  },
1414
1434
  requestId,
@@ -3406,6 +3426,7 @@ class LocalMiniappRuntime {
3406
3426
  audio: normalizeStreamAudioConfig(payload.audio),
3407
3427
  sound: payload.sound as boolean | undefined,
3408
3428
  authToken: typeof payload.authToken === "string" ? payload.authToken : undefined,
3429
+ captureAudio: resolveCaptureAudio(payload.captureAudio, resolveAcsAudioSource().source),
3409
3430
  })
3410
3431
  this.sendResult(packageName, requestId, true, result)
3411
3432
  } catch (err) {
@@ -3462,6 +3483,7 @@ class LocalMiniappRuntime {
3462
3483
  audio: normalizeStreamAudioConfig(payload.audio),
3463
3484
  sound: payload.sound as boolean | undefined,
3464
3485
  ingest: payload.ingest as "srt" | "whip" | undefined,
3486
+ captureAudio: resolveCaptureAudio(payload.captureAudio, resolveAcsAudioSource().source),
3465
3487
  })
3466
3488
  this.sendResult(packageName, requestId, true, result)
3467
3489
  } catch (err) {
@@ -3482,6 +3504,125 @@ class LocalMiniappRuntime {
3482
3504
  return this.handleStreamStop(packageName, payload, requestId)
3483
3505
  }
3484
3506
 
3507
+ private ensureMeetingStateBridge(): void {
3508
+ acsMeetingService.setStateHandler((owner, state) => {
3509
+ this.sendToMiniapp(owner, {
3510
+ type: MiniappResponseType.MEETING_STATE,
3511
+ ...state,
3512
+ })
3513
+ })
3514
+ }
3515
+
3516
+ private async handleMeetingJoin(
3517
+ packageName: string,
3518
+ payload: Record<string, unknown>,
3519
+ requestId?: string,
3520
+ ): Promise<void> {
3521
+ this.ensureMeetingStateBridge()
3522
+ const meetingUrl = typeof payload.meetingUrl === "string" ? payload.meetingUrl : ""
3523
+ const token = typeof payload.token === "string" ? payload.token : ""
3524
+ const videoSource = payload.videoSource as {type?: string; url?: string} | undefined
3525
+ const whepUrl = videoSource?.type === "whep" ? videoSource.url ?? "" : ""
3526
+ const displayName = typeof payload.displayName === "string" ? payload.displayName : undefined
3527
+ if (!meetingUrl || !token || !whepUrl) {
3528
+ this.sendResult(packageName, requestId, false, undefined, {
3529
+ code: MiniappErrorCode.INVALID_ARGUMENT,
3530
+ message: "meetingUrl, token, and a WHEP videoSource are required",
3531
+ })
3532
+ return
3533
+ }
3534
+ try {
3535
+ let video: ReturnType<typeof parseAcsOutgoingVideo>
3536
+ try {
3537
+ video = parseAcsOutgoingVideo(payload.video)
3538
+ } catch (error) {
3539
+ this.sendResult(packageName, requestId, false, undefined, {
3540
+ code: MiniappErrorCode.INVALID_ARGUMENT,
3541
+ message: error instanceof Error ? error.message : "Invalid meeting video",
3542
+ })
3543
+ return
3544
+ }
3545
+ const state = await acsMeetingService.join(packageName, {
3546
+ meetingUrl,
3547
+ token,
3548
+ whepUrl,
3549
+ displayName,
3550
+ ...(video ? {video} : {}),
3551
+ })
3552
+ this.sendResult(packageName, requestId, true, state)
3553
+ } catch (err) {
3554
+ this.sendResult(packageName, requestId, false, undefined, {
3555
+ code: MiniappErrorCode.INTERNAL,
3556
+ message: err instanceof Error ? err.message : "ACS meeting join failed",
3557
+ })
3558
+ }
3559
+ }
3560
+
3561
+ private async handleMeetingLeave(packageName: string, requestId?: string): Promise<void> {
3562
+ try {
3563
+ await acsMeetingService.leave(packageName)
3564
+ this.sendResult(packageName, requestId, true)
3565
+ } catch (err) {
3566
+ this.sendResult(packageName, requestId, false, undefined, {
3567
+ code: MiniappErrorCode.INTERNAL,
3568
+ message: err instanceof Error ? err.message : "ACS meeting leave failed",
3569
+ })
3570
+ }
3571
+ }
3572
+
3573
+ private async handleMeetingSetMuted(
3574
+ packageName: string,
3575
+ payload: Record<string, unknown>,
3576
+ requestId?: string,
3577
+ ): Promise<void> {
3578
+ try {
3579
+ const state = await acsMeetingService.setMuted(packageName, Boolean(payload.muted))
3580
+ this.sendResult(packageName, requestId, true, state)
3581
+ } catch (err) {
3582
+ this.sendResult(packageName, requestId, false, undefined, {
3583
+ code: MiniappErrorCode.INTERNAL,
3584
+ message: err instanceof Error ? err.message : "ACS mute failed",
3585
+ })
3586
+ }
3587
+ }
3588
+
3589
+ private async handleMeetingUpdateVideoSource(
3590
+ packageName: string,
3591
+ payload: Record<string, unknown>,
3592
+ requestId?: string,
3593
+ ): Promise<void> {
3594
+ const videoSource = payload.videoSource as {type?: string; url?: string} | undefined
3595
+ const whepUrl = videoSource?.type === "whep" ? videoSource.url ?? "" : ""
3596
+ if (!whepUrl) {
3597
+ this.sendResult(packageName, requestId, false, undefined, {
3598
+ code: MiniappErrorCode.INVALID_ARGUMENT,
3599
+ message: "videoSource must be a WHEP URL",
3600
+ })
3601
+ return
3602
+ }
3603
+ try {
3604
+ await acsMeetingService.updateVideoSource(packageName, whepUrl)
3605
+ this.sendResult(packageName, requestId, true)
3606
+ } catch (err) {
3607
+ this.sendResult(packageName, requestId, false, undefined, {
3608
+ code: MiniappErrorCode.INTERNAL,
3609
+ message: err instanceof Error ? err.message : "ACS video source update failed",
3610
+ })
3611
+ }
3612
+ }
3613
+
3614
+ private async handleMeetingGetState(packageName: string, requestId?: string): Promise<void> {
3615
+ try {
3616
+ const state = await acsMeetingService.readState(packageName)
3617
+ this.sendResult(packageName, requestId, true, state)
3618
+ } catch (err) {
3619
+ this.sendResult(packageName, requestId, false, undefined, {
3620
+ code: MiniappErrorCode.INTERNAL,
3621
+ message: err instanceof Error ? err.message : "ACS getState failed",
3622
+ })
3623
+ }
3624
+ }
3625
+
3485
3626
  /**
3486
3627
  * session.glasses.requestWifiSetup — open the phone's glasses Wi-Fi setup
3487
3628
  * flow. The host owns the actual UI via the `engine.configure({ui})` seam;
@@ -469,7 +469,13 @@ export class MentraJSRouter {
469
469
  // dead and let the host surface a "tap to retry" banner.
470
470
  if (iface === "__error") {
471
471
  const payload = this.tryParseArgs(msg.argsJson)
472
- this.logger.error(`[${packageName}] ${method}`, payload)
472
+ // ready_nack is a liveness probe, not a crash. Logging it at error
473
+ // painted WHIP-start queue delay as a Mentra-Call exception.
474
+ if (method === "ready_nack") {
475
+ this.logger.warn(`[${packageName}] ${method}`, payload)
476
+ } else {
477
+ this.logger.error(`[${packageName}] ${method}`, payload)
478
+ }
473
479
  if (this.crashController) {
474
480
  // Only treat "exception" + "unhandledRejection" + "uncaught" as
475
481
  // crash signals. `console.error` calls also flow through the
@@ -90,6 +90,40 @@ export class PhoneCameraFovCoordinator {
90
90
  })
91
91
  }
92
92
 
93
+ /**
94
+ * Re-assert the current override on the glasses.
95
+ *
96
+ * The on-connect device-settings push replays the persistent `camera_fov`,
97
+ * so glasses that connect mid-call come up at the user's saved value and
98
+ * silently drop a miniapp's override — a Mentra Call wearer loses the 102°
99
+ * bottom crop by connecting their glasses after joining.
100
+ *
101
+ * The effective lease is recomputed inside [enqueue], not captured at call
102
+ * time: a `setOverride` or `releaseForApp` already queued ahead of this must
103
+ * win, and it is this coordinator that has to be the authoritative last
104
+ * writer. Sends the complete override — `fov` *and* `roiPosition` — because
105
+ * the scalar alone leaves the ROI wherever the persistent setting put it.
106
+ *
107
+ * No-op in legacy mode: those commands restart the camera HAL, which is far
108
+ * too expensive to fire on every reconnect.
109
+ */
110
+ reapplyEffectiveOverride(): Promise<void> {
111
+ return this.enqueue(async () => {
112
+ if (this.legacyMode) return
113
+ const leaseId = this.effectiveLeaseId
114
+ if (!leaseId) return
115
+ const entry = [...this.overrides.values()].find((candidate) => candidate.leaseId === leaseId)
116
+ if (!entry) return
117
+ await BluetoothSdk.setCameraFovOverride({
118
+ leaseId,
119
+ fov: entry.fov,
120
+ roiPosition: entry.roiPosition,
121
+ ttlMs: CAMERA_FOV_OVERRIDE_TTL_MS,
122
+ })
123
+ this.scheduleRefresh()
124
+ })
125
+ }
126
+
93
127
  releaseForApp(packageName: string): Promise<void> {
94
128
  return this.enqueue(async () => {
95
129
  const removed = this.overrides.get(packageName)
@@ -26,6 +26,18 @@
26
26
  * streams. We use a `phone-` prefix so they're trivially distinguishable from
27
27
  * cloud-minted IDs in logs and from cloud-SDK app streams that flow through
28
28
  * the legacy path.
29
+ *
30
+ * BLE link loss is a SUSPENDED state, not a failure:
31
+ * The glasses publisher keeps pushing over Wi-Fi when the phone's Bluetooth
32
+ * link drops; only our keep-alives stop reaching it. While the store says
33
+ * the glasses are disconnected we pause the heartbeat (so we don't count
34
+ * misses we caused ourselves), fan out `suspended`, and give the link
35
+ * `glassesGraceMs` to come back. On reconnect we resume, heartbeat at once,
36
+ * and fan out `resumed` with the SAME streamId — no re-provision. The grace
37
+ * ends early when Cloudflare reports the publisher gone for two probes in
38
+ * a row (glasses powered off, not a BLE hiccup). If the grace expires the
39
+ * stream is torn down with reason `glasses_disconnected`, and the BLE stop
40
+ * the glasses never received is sent on the next reconnect.
29
41
  */
30
42
 
31
43
  import BluetoothSdk from "@mentra/bluetooth-sdk/internal"
@@ -65,11 +77,40 @@ const DEFAULT_TIMINGS = {
65
77
  hlsReadinessInitialDelayMs: 5_000,
66
78
  hlsReadinessPollMs: 2_000,
67
79
  hlsReadinessMaxAttempts: 30,
80
+ // How long a stream survives a BLE link drop before we give up on it. Must
81
+ // stay under the glasses publisher's own 60s no-keep-alive watchdog with
82
+ // margin for the resume heartbeat's round trip.
83
+ glassesGraceMs: 45_000,
84
+ // Consecutive Cloudflare "publisher disconnected" probes while suspended
85
+ // before we conclude the glasses are off (not just out of BLE range).
86
+ suspendedPublisherGoneProbes: 2,
68
87
  } as const
69
88
 
70
89
  type TimingConfig = {[K in keyof typeof DEFAULT_TIMINGS]: number}
71
90
  export type CoordinatorTimings = Partial<TimingConfig>
72
91
 
92
+ /**
93
+ * Where the coordinator learns whether the phone↔glasses BLE link is up.
94
+ * Injected so tests can drive link transitions without the zustand store.
95
+ */
96
+ export interface GlassesLinkSource {
97
+ isConnected(): boolean
98
+ /** Fires on every connected↔disconnected transition. */
99
+ subscribe(listener: (connected: boolean) => void): () => void
100
+ }
101
+
102
+ const storeLinkSource: GlassesLinkSource = {
103
+ isConnected: () => isGlassesConnected(useGlassesStore.getState().connection),
104
+ subscribe: (listener) => useGlassesStore.subscribe((s) => isGlassesConnected(s.connection), listener),
105
+ }
106
+
107
+ /** Reasons carried on coordinator-sourced `stream_status` fanouts for link events. */
108
+ export const LINK_STATUS = {
109
+ suspended: "suspended",
110
+ resumed: "resumed",
111
+ reason: "glasses_disconnected",
112
+ } as const
113
+
73
114
  // Console-backed minimal logger; replaces pino on the phone.
74
115
  const consoleLogger: LifecycleLogger = {
75
116
  child: (bindings) => ({
@@ -90,6 +131,7 @@ export interface StartUnmanagedOptions {
90
131
  sound?: boolean
91
132
  /** Optional Bearer token for WHIP Authorization (custom authenticated endpoints). */
92
133
  authToken?: string
134
+ captureAudio?: boolean
93
135
  }
94
136
 
95
137
  export interface StartManagedOptions {
@@ -108,6 +150,8 @@ export interface StartManagedOptions {
108
150
  * that drop WHIP/SRT UDP.
109
151
  */
110
152
  ingest?: "srt" | "whip" | "rtmp"
153
+ /** When false, glasses skip encoding their mic for this WHIP session. */
154
+ captureAudio?: boolean
111
155
  }
112
156
 
113
157
  export interface StreamPublisherStartResult {
@@ -184,20 +228,11 @@ export class StreamConflictError extends Error {
184
228
  }
185
229
  }
186
230
 
187
- /**
188
- * Fail fast if glasses aren't connected — BEFORE provisioning. Without this a
189
- * managed start would create a provider live input, fail the BLE command, and
190
- * tear the input down again: a slow, billable no-op with a confusing error.
191
- */
192
- function assertGlassesConnected(): void {
193
- if (!isGlassesConnected(useGlassesStore.getState().connection)) {
194
- throw new StreamConflictError(
195
- "GLASSES_NOT_CONNECTED",
196
- "Glasses are not connected",
197
- "command",
198
- "ble",
199
- )
200
- }
231
+ interface SuspendedState {
232
+ since: number
233
+ graceTimer: ReturnType<typeof setTimeout>
234
+ /** Consecutive Cloudflare probes that saw no publisher during this suspension. */
235
+ publisherGoneProbes: number
201
236
  }
202
237
 
203
238
  export class PhoneStreamCoordinator {
@@ -206,6 +241,16 @@ export class PhoneStreamCoordinator {
206
241
  private statusSubscriber: StatusSubscriber | null = null
207
242
  private idCounter = 0
208
243
  private readonly timings: TimingConfig
244
+ private readonly linkSource: GlassesLinkSource
245
+ private unsubscribeLink: (() => void) | null = null
246
+ private suspended: SuspendedState | null = null
247
+ /**
248
+ * A stream was torn down while the BLE link was down, so the glasses never
249
+ * got `stopStream`. Sent on the next reconnect (if no new stream has claimed
250
+ * the slot) so a publisher that outlived its input does not keep pushing
251
+ * until its own watchdog fires.
252
+ */
253
+ private pendingBleStop: {streamId: string} | null = null
209
254
  /**
210
255
  * Serializes state transitions (start, stop, teardown). Without it, a
211
256
  * second `start*` racing with the first can pass the `this.current === null`
@@ -220,8 +265,30 @@ export class PhoneStreamCoordinator {
220
265
  /** Send full resolvedConfig only once per stream session. */
221
266
  private resolvedConfigForwarded = false
222
267
 
223
- constructor(timings: CoordinatorTimings = {}) {
268
+ constructor(timings: CoordinatorTimings = {}, deps: {linkSource?: GlassesLinkSource} = {}) {
224
269
  this.timings = {...DEFAULT_TIMINGS, ...timings}
270
+ this.linkSource = deps.linkSource ?? storeLinkSource
271
+ }
272
+
273
+ /**
274
+ * Fail fast if glasses aren't connected — BEFORE provisioning. Without this a
275
+ * managed start would create a provider live input, fail the BLE command, and
276
+ * tear the input down again: a slow, billable no-op with a confusing error.
277
+ */
278
+ private assertGlassesConnected(): void {
279
+ if (!this.linkSource.isConnected()) {
280
+ throw new StreamConflictError(
281
+ "GLASSES_NOT_CONNECTED",
282
+ "Glasses are not connected",
283
+ "command",
284
+ "ble",
285
+ )
286
+ }
287
+ }
288
+
289
+ /** True while the active stream is parked on a dropped BLE link. */
290
+ isSuspended(): boolean {
291
+ return this.suspended !== null
225
292
  }
226
293
 
227
294
  /**
@@ -261,7 +328,11 @@ export class PhoneStreamCoordinator {
261
328
 
262
329
  /** Report-safe stream ownership snapshot for incident diagnostics. */
263
330
  getDiagnosticSnapshot(): Record<string, unknown> {
264
- if (!this.current) return {active: false}
331
+ if (!this.current) return {active: false, pendingBleStop: this.pendingBleStop?.streamId ?? null}
332
+ const link = {
333
+ suspended: this.suspended !== null,
334
+ ...(this.suspended ? {suspendedForMs: Date.now() - this.suspended.since} : {}),
335
+ }
265
336
  return this.current.kind === "managed"
266
337
  ? {
267
338
  active: true,
@@ -270,12 +341,14 @@ export class PhoneStreamCoordinator {
270
341
  subscribers: [...this.current.subscribers].sort(),
271
342
  mode: this.current.mode,
272
343
  playbackReady: this.current.hlsReady,
344
+ ...link,
273
345
  }
274
346
  : {
275
347
  active: true,
276
348
  kind: this.current.kind,
277
349
  streamId: this.current.streamId,
278
350
  ownerPackageName: this.current.packageName,
351
+ ...link,
279
352
  }
280
353
  }
281
354
 
@@ -288,7 +361,7 @@ export class PhoneStreamCoordinator {
288
361
  if (!opts.streamUrl || typeof opts.streamUrl !== "string") {
289
362
  throw new StreamConflictError("STREAM_URL_REQUIRED", "streamUrl is required")
290
363
  }
291
- assertGlassesConnected()
364
+ this.assertGlassesConnected()
292
365
  return this.runExclusive(async () => {
293
366
  if (this.current) {
294
367
  throw new StreamConflictError(
@@ -319,6 +392,7 @@ export class PhoneStreamCoordinator {
319
392
  ...(opts.video !== undefined ? {video: opts.video} : {}),
320
393
  ...(opts.audio !== undefined ? {audio: opts.audio} : {}),
321
394
  ...(opts.authToken ? {authToken: opts.authToken} : {}),
395
+ ...(typeof opts.captureAudio === "boolean" ? {captureAudio: opts.captureAudio} : {}),
322
396
  })
323
397
  const result = publisherStartResult(streamId, event)
324
398
  this.startLifecycle(streamId)
@@ -346,7 +420,7 @@ export class PhoneStreamCoordinator {
346
420
  // Two-phase: the entry-claim runs under the transition lock; the wait for
347
421
  // HLS readiness happens AFTER the lock releases so a long warm-up doesn't
348
422
  // block subsequent start/stop transitions on this coordinator.
349
- assertGlassesConnected()
423
+ this.assertGlassesConnected()
350
424
  type JoinDecision =
351
425
  | {kind: "join"; entry: ManagedEntry; immediate: ManagedStartResult | null}
352
426
  | {kind: "fresh"; entry: ManagedEntry}
@@ -422,6 +496,7 @@ export class PhoneStreamCoordinator {
422
496
  // See startUnmanaged: the native bridge rejects explicit `undefined`.
423
497
  ...(opts.video !== undefined ? {video: opts.video} : {}),
424
498
  ...(opts.audio !== undefined ? {audio: opts.audio} : {}),
499
+ ...(typeof opts.captureAudio === "boolean" ? {captureAudio: opts.captureAudio} : {}),
425
500
  })
426
501
  entry.publisherStart = publisherStartResult(streamId, event)
427
502
  console.info("[STREAM_STARTUP]", {
@@ -543,6 +618,109 @@ export class PhoneStreamCoordinator {
543
618
  this.lifecycle?.handleAck(event.ackId)
544
619
  }
545
620
 
621
+ // ===========================================================================
622
+ // BLE link suspension
623
+ // ===========================================================================
624
+
625
+ private attachLink(): void {
626
+ if (this.unsubscribeLink) return
627
+ this.unsubscribeLink = this.linkSource.subscribe((connected) => this.handleLinkChange(connected))
628
+ }
629
+
630
+ private detachLinkIfIdle(): void {
631
+ if (this.current || this.pendingBleStop || !this.unsubscribeLink) return
632
+ this.unsubscribeLink()
633
+ this.unsubscribeLink = null
634
+ }
635
+
636
+ private handleLinkChange(connected: boolean): void {
637
+ if (connected) {
638
+ if (this.current && this.suspended) {
639
+ this.resumeLocked()
640
+ } else if (!this.current && this.pendingBleStop) {
641
+ this.flushPendingBleStop()
642
+ }
643
+ return
644
+ }
645
+ if (this.current && !this.suspended) this.suspend()
646
+ }
647
+
648
+ private suspend(): void {
649
+ const entry = this.current
650
+ if (!entry) return
651
+ const since = Date.now()
652
+ // Pause the heartbeat: misses now would be our own fault, not the
653
+ // publisher's, and counting them would reap a stream that is still live.
654
+ this.lifecycle?.setActive(false)
655
+ const graceTimer = setTimeout(() => this.onGraceExpired(entry.streamId), this.timings.glassesGraceMs)
656
+ this.suspended = {since, graceTimer, publisherGoneProbes: 0}
657
+ console.warn("[STREAM] BLE link lost; stream suspended", {
658
+ streamId: entry.streamId,
659
+ graceMs: this.timings.glassesGraceMs,
660
+ })
661
+ this.fanout({
662
+ streamId: entry.streamId,
663
+ source: "coordinator",
664
+ status: LINK_STATUS.suspended,
665
+ data: {reason: LINK_STATUS.reason, graceMs: this.timings.glassesGraceMs, since},
666
+ })
667
+ }
668
+
669
+ private resumeLocked(): void {
670
+ const entry = this.current
671
+ const suspended = this.suspended
672
+ if (!entry || !suspended) return
673
+ clearTimeout(suspended.graceTimer)
674
+ this.suspended = null
675
+ const suspendedMs = Date.now() - suspended.since
676
+ console.info("[STREAM] BLE link back; stream resumed", {streamId: entry.streamId, suspendedMs})
677
+ // Resume and heartbeat immediately: the glasses watchdog has been running
678
+ // the whole time, so the first keep-alive must not wait another interval.
679
+ this.lifecycle?.setActive(true)
680
+ this.lifecycle?.tickNow()
681
+ // A resumed session is a fresh status baseline for subscribers.
682
+ this.lastFanoutSignature = null
683
+ this.fanout({
684
+ streamId: entry.streamId,
685
+ source: "coordinator",
686
+ status: LINK_STATUS.resumed,
687
+ data: {reason: LINK_STATUS.reason, suspendedMs},
688
+ })
689
+ }
690
+
691
+ private onGraceExpired(streamId: string): void {
692
+ if (this.current?.streamId !== streamId || !this.suspended) return
693
+ this.failSuspended(streamId, "glasses_disconnected", {publisherGone: false})
694
+ }
695
+
696
+ /**
697
+ * End a suspended stream. `publisherGone` distinguishes "glasses are off and
698
+ * Cloudflare confirms nothing is publishing" from "grace ran out with the
699
+ * publisher possibly still alive" — miniapps word the two differently.
700
+ */
701
+ private failSuspended(streamId: string, reason: string, detail: {publisherGone: boolean}): void {
702
+ this.fanout({
703
+ streamId,
704
+ source: "coordinator",
705
+ status: "error",
706
+ data: {reason: LINK_STATUS.reason, teardownReason: reason, ...detail},
707
+ })
708
+ void this.runExclusive(async () => {
709
+ if (this.current?.streamId !== streamId) return
710
+ await this.teardownLocked(reason)
711
+ })
712
+ }
713
+
714
+ private flushPendingBleStop(): void {
715
+ const pending = this.pendingBleStop
716
+ if (!pending) return
717
+ this.pendingBleStop = null
718
+ console.info("[STREAM] BLE link back; sending deferred stopStream", pending)
719
+ void BluetoothSdk.stopStream()
720
+ .catch((err) => console.warn("[STREAM] deferred stopStream failed:", err))
721
+ .finally(() => this.detachLinkIfIdle())
722
+ }
723
+
546
724
  // ===========================================================================
547
725
  // Internal
548
726
  // ===========================================================================
@@ -552,7 +730,11 @@ export class PhoneStreamCoordinator {
552
730
  return `phone-${prefix}-${Date.now().toString(36)}-${this.idCounter}`
553
731
  }
554
732
 
733
+ /** BLE keep-alives. StreamLifecycleController uses BgTimer so they survive MentraOS backgrounding. */
555
734
  private startLifecycle(streamId: string): void {
735
+ // A new stream supersedes any deferred stop for the previous one.
736
+ this.pendingBleStop = null
737
+ this.attachLink()
556
738
  this.lifecycle?.dispose()
557
739
  const ctrl = new StreamLifecycleController(
558
740
  {
@@ -631,6 +813,20 @@ export class PhoneStreamCoordinator {
631
813
  status: status.isConnected ? "connected" : "disconnected",
632
814
  data: status as unknown as Record<string, unknown>,
633
815
  })
816
+ // While the BLE link is down, Cloudflare is the only witness to the
817
+ // publisher. Two consecutive "nobody is publishing" probes mean the
818
+ // glasses are off, not merely out of Bluetooth range — stop waiting.
819
+ if (this.suspended) {
820
+ if (status.isConnected) {
821
+ this.suspended.publisherGoneProbes = 0
822
+ } else {
823
+ this.suspended.publisherGoneProbes += 1
824
+ if (this.suspended.publisherGoneProbes >= this.timings.suspendedPublisherGoneProbes) {
825
+ this.failSuspended(entry.streamId, "glasses_disconnected_publisher_gone", {publisherGone: true})
826
+ keepPolling = false
827
+ }
828
+ }
829
+ }
634
830
  // webrtc mode readiness: first "connected" means WHEP playback is
635
831
  // available (WebRTC playback follows the ingest directly; there is no
636
832
  // manifest to probe).
@@ -799,12 +995,28 @@ export class PhoneStreamCoordinator {
799
995
  this.lastFanoutSignature = null
800
996
  this.resolvedConfigForwarded = false
801
997
 
998
+ if (this.suspended) {
999
+ clearTimeout(this.suspended.graceTimer)
1000
+ this.suspended = null
1001
+ }
1002
+
802
1003
  // Dispose the lifecycle controller immediately so it doesn't fire one
803
1004
  // more keep-alive against a stream we're tearing down. The transition
804
1005
  // lock guarantees no new lifecycle is started concurrently.
805
1006
  this.lifecycle?.dispose()
806
1007
  this.lifecycle = null
807
1008
 
1009
+ // With the link down a BLE write can only fail (and hold the transition
1010
+ // lock for the native timeout). Defer it to the next reconnect instead.
1011
+ const linkUp = this.linkSource.isConnected()
1012
+ if (sendBleStop && !linkUp) {
1013
+ this.pendingBleStop = {streamId: entry.streamId}
1014
+ console.warn("[STREAM] BLE link down during teardown; stopStream deferred", {
1015
+ streamId: entry.streamId,
1016
+ reason,
1017
+ })
1018
+ }
1019
+
808
1020
  if (entry.kind === "managed") {
809
1021
  if (entry.cloudflareTimer) clearTimeout(entry.cloudflareTimer)
810
1022
  if (entry.hlsTimer) clearInterval(entry.hlsTimer)
@@ -816,7 +1028,7 @@ export class PhoneStreamCoordinator {
816
1028
  }
817
1029
 
818
1030
  try {
819
- if (sendBleStop) {
1031
+ if (sendBleStop && linkUp) {
820
1032
  await BluetoothSdk.stopStream()
821
1033
  }
822
1034
  } catch (err) {
@@ -827,6 +1039,7 @@ export class PhoneStreamCoordinator {
827
1039
  // Only clear if we're still the active entry (defensive — runExclusive
828
1040
  // serializes us, so this should always be true).
829
1041
  if (this.current === entry) this.current = null
1042
+ this.detachLinkIfIdle()
830
1043
 
831
1044
  if (entry.kind === "managed") {
832
1045
  // Start remote cleanup only after the publisher has stopped, but do not