@mentra/engine 3.2.0-dev.116 → 3.2.0-dev.121
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/engine.d.ts.map +1 -1
- package/build/engine.js +7 -2
- package/build/engine.js.map +1 -1
- package/build/generated/releaseMetadata.js +5 -5
- package/build/generated/releaseMetadata.js.map +1 -1
- package/build/index.d.ts +1 -0
- package/build/index.d.ts.map +1 -1
- package/build/index.js +1 -0
- package/build/index.js.map +1 -1
- package/build/runtime/streamConfig.d.ts +7 -0
- package/build/runtime/streamConfig.d.ts.map +1 -1
- package/build/runtime/streamConfig.js +7 -0
- package/build/runtime/streamConfig.js.map +1 -1
- package/build/services/AcsMeetingService.d.ts +154 -0
- package/build/services/AcsMeetingService.d.ts.map +1 -0
- package/build/services/AcsMeetingService.js +477 -0
- package/build/services/AcsMeetingService.js.map +1 -0
- package/build/services/AppRegistry.d.ts +9 -0
- package/build/services/AppRegistry.d.ts.map +1 -1
- package/build/services/AppRegistry.js +22 -0
- package/build/services/AppRegistry.js.map +1 -1
- package/build/services/DeviceEventRouter.d.ts.map +1 -1
- package/build/services/DeviceEventRouter.js +12 -3
- package/build/services/DeviceEventRouter.js.map +1 -1
- package/build/services/GlassesSettingsSync.d.ts.map +1 -1
- package/build/services/GlassesSettingsSync.js +11 -1
- package/build/services/GlassesSettingsSync.js.map +1 -1
- package/build/services/GlassesStatusProjection.d.ts +12 -0
- package/build/services/GlassesStatusProjection.d.ts.map +1 -1
- package/build/services/GlassesStatusProjection.js +23 -0
- package/build/services/GlassesStatusProjection.js.map +1 -1
- package/build/services/LocalMiniappRuntime.d.ts +12 -0
- package/build/services/LocalMiniappRuntime.d.ts.map +1 -1
- package/build/services/LocalMiniappRuntime.js +184 -2
- package/build/services/LocalMiniappRuntime.js.map +1 -1
- package/build/services/MentraJSRouter.d.ts.map +1 -1
- package/build/services/MentraJSRouter.js +8 -1
- package/build/services/MentraJSRouter.js.map +1 -1
- package/build/services/MiniappLauncher.d.ts +3 -1
- package/build/services/MiniappLauncher.d.ts.map +1 -1
- package/build/services/MiniappLauncher.js +66 -50
- package/build/services/MiniappLauncher.js.map +1 -1
- package/build/services/PhoneCameraFovCoordinator.d.ts +18 -0
- package/build/services/PhoneCameraFovCoordinator.d.ts.map +1 -1
- package/build/services/PhoneCameraFovCoordinator.js +36 -0
- package/build/services/PhoneCameraFovCoordinator.js.map +1 -1
- package/build/services/PhoneStreamCoordinator.d.ts +67 -1
- package/build/services/PhoneStreamCoordinator.d.ts.map +1 -1
- package/build/services/PhoneStreamCoordinator.js +204 -15
- package/build/services/PhoneStreamCoordinator.js.map +1 -1
- package/build/services/StreamLifecycleController.d.ts +23 -5
- package/build/services/StreamLifecycleController.d.ts.map +1 -1
- package/build/services/StreamLifecycleController.js +31 -10
- package/build/services/StreamLifecycleController.js.map +1 -1
- package/build/services/acsAudioSource.d.ts +24 -0
- package/build/services/acsAudioSource.d.ts.map +1 -0
- package/build/services/acsAudioSource.js +37 -0
- package/build/services/acsAudioSource.js.map +1 -0
- package/build/utils/devMiniappLaunch.d.ts.map +1 -1
- package/build/utils/devMiniappLaunch.js +4 -0
- package/build/utils/devMiniappLaunch.js.map +1 -1
- package/build/utils/devMiniappSnapshot.d.ts +46 -0
- package/build/utils/devMiniappSnapshot.d.ts.map +1 -0
- package/build/utils/devMiniappSnapshot.js +94 -0
- package/build/utils/devMiniappSnapshot.js.map +1 -0
- package/package.json +7 -6
- package/src/engine.ts +10 -2
- package/src/generated/releaseMetadata.ts +5 -5
- package/src/index.ts +4 -0
- package/src/runtime/streamConfig.ts +13 -0
- package/src/services/AcsMeetingService.ts +570 -0
- package/src/services/AppRegistry.ts +23 -0
- package/src/services/DeviceEventRouter.ts +12 -3
- package/src/services/GlassesSettingsSync.ts +13 -3
- package/src/services/GlassesStatusProjection.ts +28 -0
- package/src/services/LocalMiniappRuntime.ts +214 -2
- package/src/services/MentraJSRouter.ts +7 -1
- package/src/services/MiniappLauncher.ts +73 -50
- package/src/services/PhoneCameraFovCoordinator.ts +34 -0
- package/src/services/PhoneStreamCoordinator.ts +232 -19
- package/src/services/StreamLifecycleController.ts +42 -12
- package/src/services/acsAudioSource.ts +53 -0
- package/src/utils/devMiniappLaunch.ts +4 -0
- package/src/utils/devMiniappSnapshot.ts +119 -0
|
@@ -26,6 +26,18 @@
|
|
|
26
26
|
* streams. We use a `phone-` prefix so they're trivially distinguishable from
|
|
27
27
|
* cloud-minted IDs in logs and from cloud-SDK app streams that flow through
|
|
28
28
|
* the legacy path.
|
|
29
|
+
*
|
|
30
|
+
* BLE link loss is a SUSPENDED state, not a failure:
|
|
31
|
+
* The glasses publisher keeps pushing over Wi-Fi when the phone's Bluetooth
|
|
32
|
+
* link drops; only our keep-alives stop reaching it. While the store says
|
|
33
|
+
* the glasses are disconnected we pause the heartbeat (so we don't count
|
|
34
|
+
* misses we caused ourselves), fan out `suspended`, and give the link
|
|
35
|
+
* `glassesGraceMs` to come back. On reconnect we resume, heartbeat at once,
|
|
36
|
+
* and fan out `resumed` with the SAME streamId — no re-provision. The grace
|
|
37
|
+
* ends early when Cloudflare reports the publisher gone for two probes in
|
|
38
|
+
* a row (glasses powered off, not a BLE hiccup). If the grace expires the
|
|
39
|
+
* stream is torn down with reason `glasses_disconnected`, and the BLE stop
|
|
40
|
+
* the glasses never received is sent on the next reconnect.
|
|
29
41
|
*/
|
|
30
42
|
|
|
31
43
|
import BluetoothSdk from "@mentra/bluetooth-sdk/internal"
|
|
@@ -65,11 +77,40 @@ const DEFAULT_TIMINGS = {
|
|
|
65
77
|
hlsReadinessInitialDelayMs: 5_000,
|
|
66
78
|
hlsReadinessPollMs: 2_000,
|
|
67
79
|
hlsReadinessMaxAttempts: 30,
|
|
80
|
+
// How long a stream survives a BLE link drop before we give up on it. Must
|
|
81
|
+
// stay under the glasses publisher's own 60s no-keep-alive watchdog with
|
|
82
|
+
// margin for the resume heartbeat's round trip.
|
|
83
|
+
glassesGraceMs: 45_000,
|
|
84
|
+
// Consecutive Cloudflare "publisher disconnected" probes while suspended
|
|
85
|
+
// before we conclude the glasses are off (not just out of BLE range).
|
|
86
|
+
suspendedPublisherGoneProbes: 2,
|
|
68
87
|
} as const
|
|
69
88
|
|
|
70
89
|
type TimingConfig = {[K in keyof typeof DEFAULT_TIMINGS]: number}
|
|
71
90
|
export type CoordinatorTimings = Partial<TimingConfig>
|
|
72
91
|
|
|
92
|
+
/**
|
|
93
|
+
* Where the coordinator learns whether the phone↔glasses BLE link is up.
|
|
94
|
+
* Injected so tests can drive link transitions without the zustand store.
|
|
95
|
+
*/
|
|
96
|
+
export interface GlassesLinkSource {
|
|
97
|
+
isConnected(): boolean
|
|
98
|
+
/** Fires on every connected↔disconnected transition. */
|
|
99
|
+
subscribe(listener: (connected: boolean) => void): () => void
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
const storeLinkSource: GlassesLinkSource = {
|
|
103
|
+
isConnected: () => isGlassesConnected(useGlassesStore.getState().connection),
|
|
104
|
+
subscribe: (listener) => useGlassesStore.subscribe((s) => isGlassesConnected(s.connection), listener),
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** Reasons carried on coordinator-sourced `stream_status` fanouts for link events. */
|
|
108
|
+
export const LINK_STATUS = {
|
|
109
|
+
suspended: "suspended",
|
|
110
|
+
resumed: "resumed",
|
|
111
|
+
reason: "glasses_disconnected",
|
|
112
|
+
} as const
|
|
113
|
+
|
|
73
114
|
// Console-backed minimal logger; replaces pino on the phone.
|
|
74
115
|
const consoleLogger: LifecycleLogger = {
|
|
75
116
|
child: (bindings) => ({
|
|
@@ -90,6 +131,7 @@ export interface StartUnmanagedOptions {
|
|
|
90
131
|
sound?: boolean
|
|
91
132
|
/** Optional Bearer token for WHIP Authorization (custom authenticated endpoints). */
|
|
92
133
|
authToken?: string
|
|
134
|
+
captureAudio?: boolean
|
|
93
135
|
}
|
|
94
136
|
|
|
95
137
|
export interface StartManagedOptions {
|
|
@@ -108,6 +150,8 @@ export interface StartManagedOptions {
|
|
|
108
150
|
* that drop WHIP/SRT UDP.
|
|
109
151
|
*/
|
|
110
152
|
ingest?: "srt" | "whip" | "rtmp"
|
|
153
|
+
/** When false, glasses skip encoding their mic for this WHIP session. */
|
|
154
|
+
captureAudio?: boolean
|
|
111
155
|
}
|
|
112
156
|
|
|
113
157
|
export interface StreamPublisherStartResult {
|
|
@@ -184,20 +228,11 @@ export class StreamConflictError extends Error {
|
|
|
184
228
|
}
|
|
185
229
|
}
|
|
186
230
|
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
function assertGlassesConnected(): void {
|
|
193
|
-
if (!isGlassesConnected(useGlassesStore.getState().connection)) {
|
|
194
|
-
throw new StreamConflictError(
|
|
195
|
-
"GLASSES_NOT_CONNECTED",
|
|
196
|
-
"Glasses are not connected",
|
|
197
|
-
"command",
|
|
198
|
-
"ble",
|
|
199
|
-
)
|
|
200
|
-
}
|
|
231
|
+
interface SuspendedState {
|
|
232
|
+
since: number
|
|
233
|
+
graceTimer: ReturnType<typeof setTimeout>
|
|
234
|
+
/** Consecutive Cloudflare probes that saw no publisher during this suspension. */
|
|
235
|
+
publisherGoneProbes: number
|
|
201
236
|
}
|
|
202
237
|
|
|
203
238
|
export class PhoneStreamCoordinator {
|
|
@@ -206,6 +241,16 @@ export class PhoneStreamCoordinator {
|
|
|
206
241
|
private statusSubscriber: StatusSubscriber | null = null
|
|
207
242
|
private idCounter = 0
|
|
208
243
|
private readonly timings: TimingConfig
|
|
244
|
+
private readonly linkSource: GlassesLinkSource
|
|
245
|
+
private unsubscribeLink: (() => void) | null = null
|
|
246
|
+
private suspended: SuspendedState | null = null
|
|
247
|
+
/**
|
|
248
|
+
* A stream was torn down while the BLE link was down, so the glasses never
|
|
249
|
+
* got `stopStream`. Sent on the next reconnect (if no new stream has claimed
|
|
250
|
+
* the slot) so a publisher that outlived its input does not keep pushing
|
|
251
|
+
* until its own watchdog fires.
|
|
252
|
+
*/
|
|
253
|
+
private pendingBleStop: {streamId: string} | null = null
|
|
209
254
|
/**
|
|
210
255
|
* Serializes state transitions (start, stop, teardown). Without it, a
|
|
211
256
|
* second `start*` racing with the first can pass the `this.current === null`
|
|
@@ -220,8 +265,30 @@ export class PhoneStreamCoordinator {
|
|
|
220
265
|
/** Send full resolvedConfig only once per stream session. */
|
|
221
266
|
private resolvedConfigForwarded = false
|
|
222
267
|
|
|
223
|
-
constructor(timings: CoordinatorTimings = {}) {
|
|
268
|
+
constructor(timings: CoordinatorTimings = {}, deps: {linkSource?: GlassesLinkSource} = {}) {
|
|
224
269
|
this.timings = {...DEFAULT_TIMINGS, ...timings}
|
|
270
|
+
this.linkSource = deps.linkSource ?? storeLinkSource
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/**
|
|
274
|
+
* Fail fast if glasses aren't connected — BEFORE provisioning. Without this a
|
|
275
|
+
* managed start would create a provider live input, fail the BLE command, and
|
|
276
|
+
* tear the input down again: a slow, billable no-op with a confusing error.
|
|
277
|
+
*/
|
|
278
|
+
private assertGlassesConnected(): void {
|
|
279
|
+
if (!this.linkSource.isConnected()) {
|
|
280
|
+
throw new StreamConflictError(
|
|
281
|
+
"GLASSES_NOT_CONNECTED",
|
|
282
|
+
"Glasses are not connected",
|
|
283
|
+
"command",
|
|
284
|
+
"ble",
|
|
285
|
+
)
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
/** True while the active stream is parked on a dropped BLE link. */
|
|
290
|
+
isSuspended(): boolean {
|
|
291
|
+
return this.suspended !== null
|
|
225
292
|
}
|
|
226
293
|
|
|
227
294
|
/**
|
|
@@ -261,7 +328,11 @@ export class PhoneStreamCoordinator {
|
|
|
261
328
|
|
|
262
329
|
/** Report-safe stream ownership snapshot for incident diagnostics. */
|
|
263
330
|
getDiagnosticSnapshot(): Record<string, unknown> {
|
|
264
|
-
if (!this.current) return {active: false}
|
|
331
|
+
if (!this.current) return {active: false, pendingBleStop: this.pendingBleStop?.streamId ?? null}
|
|
332
|
+
const link = {
|
|
333
|
+
suspended: this.suspended !== null,
|
|
334
|
+
...(this.suspended ? {suspendedForMs: Date.now() - this.suspended.since} : {}),
|
|
335
|
+
}
|
|
265
336
|
return this.current.kind === "managed"
|
|
266
337
|
? {
|
|
267
338
|
active: true,
|
|
@@ -270,12 +341,14 @@ export class PhoneStreamCoordinator {
|
|
|
270
341
|
subscribers: [...this.current.subscribers].sort(),
|
|
271
342
|
mode: this.current.mode,
|
|
272
343
|
playbackReady: this.current.hlsReady,
|
|
344
|
+
...link,
|
|
273
345
|
}
|
|
274
346
|
: {
|
|
275
347
|
active: true,
|
|
276
348
|
kind: this.current.kind,
|
|
277
349
|
streamId: this.current.streamId,
|
|
278
350
|
ownerPackageName: this.current.packageName,
|
|
351
|
+
...link,
|
|
279
352
|
}
|
|
280
353
|
}
|
|
281
354
|
|
|
@@ -288,7 +361,7 @@ export class PhoneStreamCoordinator {
|
|
|
288
361
|
if (!opts.streamUrl || typeof opts.streamUrl !== "string") {
|
|
289
362
|
throw new StreamConflictError("STREAM_URL_REQUIRED", "streamUrl is required")
|
|
290
363
|
}
|
|
291
|
-
assertGlassesConnected()
|
|
364
|
+
this.assertGlassesConnected()
|
|
292
365
|
return this.runExclusive(async () => {
|
|
293
366
|
if (this.current) {
|
|
294
367
|
throw new StreamConflictError(
|
|
@@ -319,6 +392,7 @@ export class PhoneStreamCoordinator {
|
|
|
319
392
|
...(opts.video !== undefined ? {video: opts.video} : {}),
|
|
320
393
|
...(opts.audio !== undefined ? {audio: opts.audio} : {}),
|
|
321
394
|
...(opts.authToken ? {authToken: opts.authToken} : {}),
|
|
395
|
+
...(typeof opts.captureAudio === "boolean" ? {captureAudio: opts.captureAudio} : {}),
|
|
322
396
|
})
|
|
323
397
|
const result = publisherStartResult(streamId, event)
|
|
324
398
|
this.startLifecycle(streamId)
|
|
@@ -346,7 +420,7 @@ export class PhoneStreamCoordinator {
|
|
|
346
420
|
// Two-phase: the entry-claim runs under the transition lock; the wait for
|
|
347
421
|
// HLS readiness happens AFTER the lock releases so a long warm-up doesn't
|
|
348
422
|
// block subsequent start/stop transitions on this coordinator.
|
|
349
|
-
assertGlassesConnected()
|
|
423
|
+
this.assertGlassesConnected()
|
|
350
424
|
type JoinDecision =
|
|
351
425
|
| {kind: "join"; entry: ManagedEntry; immediate: ManagedStartResult | null}
|
|
352
426
|
| {kind: "fresh"; entry: ManagedEntry}
|
|
@@ -422,6 +496,7 @@ export class PhoneStreamCoordinator {
|
|
|
422
496
|
// See startUnmanaged: the native bridge rejects explicit `undefined`.
|
|
423
497
|
...(opts.video !== undefined ? {video: opts.video} : {}),
|
|
424
498
|
...(opts.audio !== undefined ? {audio: opts.audio} : {}),
|
|
499
|
+
...(typeof opts.captureAudio === "boolean" ? {captureAudio: opts.captureAudio} : {}),
|
|
425
500
|
})
|
|
426
501
|
entry.publisherStart = publisherStartResult(streamId, event)
|
|
427
502
|
console.info("[STREAM_STARTUP]", {
|
|
@@ -543,6 +618,109 @@ export class PhoneStreamCoordinator {
|
|
|
543
618
|
this.lifecycle?.handleAck(event.ackId)
|
|
544
619
|
}
|
|
545
620
|
|
|
621
|
+
// ===========================================================================
|
|
622
|
+
// BLE link suspension
|
|
623
|
+
// ===========================================================================
|
|
624
|
+
|
|
625
|
+
private attachLink(): void {
|
|
626
|
+
if (this.unsubscribeLink) return
|
|
627
|
+
this.unsubscribeLink = this.linkSource.subscribe((connected) => this.handleLinkChange(connected))
|
|
628
|
+
}
|
|
629
|
+
|
|
630
|
+
private detachLinkIfIdle(): void {
|
|
631
|
+
if (this.current || this.pendingBleStop || !this.unsubscribeLink) return
|
|
632
|
+
this.unsubscribeLink()
|
|
633
|
+
this.unsubscribeLink = null
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
private handleLinkChange(connected: boolean): void {
|
|
637
|
+
if (connected) {
|
|
638
|
+
if (this.current && this.suspended) {
|
|
639
|
+
this.resumeLocked()
|
|
640
|
+
} else if (!this.current && this.pendingBleStop) {
|
|
641
|
+
this.flushPendingBleStop()
|
|
642
|
+
}
|
|
643
|
+
return
|
|
644
|
+
}
|
|
645
|
+
if (this.current && !this.suspended) this.suspend()
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
private suspend(): void {
|
|
649
|
+
const entry = this.current
|
|
650
|
+
if (!entry) return
|
|
651
|
+
const since = Date.now()
|
|
652
|
+
// Pause the heartbeat: misses now would be our own fault, not the
|
|
653
|
+
// publisher's, and counting them would reap a stream that is still live.
|
|
654
|
+
this.lifecycle?.setActive(false)
|
|
655
|
+
const graceTimer = setTimeout(() => this.onGraceExpired(entry.streamId), this.timings.glassesGraceMs)
|
|
656
|
+
this.suspended = {since, graceTimer, publisherGoneProbes: 0}
|
|
657
|
+
console.warn("[STREAM] BLE link lost; stream suspended", {
|
|
658
|
+
streamId: entry.streamId,
|
|
659
|
+
graceMs: this.timings.glassesGraceMs,
|
|
660
|
+
})
|
|
661
|
+
this.fanout({
|
|
662
|
+
streamId: entry.streamId,
|
|
663
|
+
source: "coordinator",
|
|
664
|
+
status: LINK_STATUS.suspended,
|
|
665
|
+
data: {reason: LINK_STATUS.reason, graceMs: this.timings.glassesGraceMs, since},
|
|
666
|
+
})
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
private resumeLocked(): void {
|
|
670
|
+
const entry = this.current
|
|
671
|
+
const suspended = this.suspended
|
|
672
|
+
if (!entry || !suspended) return
|
|
673
|
+
clearTimeout(suspended.graceTimer)
|
|
674
|
+
this.suspended = null
|
|
675
|
+
const suspendedMs = Date.now() - suspended.since
|
|
676
|
+
console.info("[STREAM] BLE link back; stream resumed", {streamId: entry.streamId, suspendedMs})
|
|
677
|
+
// Resume and heartbeat immediately: the glasses watchdog has been running
|
|
678
|
+
// the whole time, so the first keep-alive must not wait another interval.
|
|
679
|
+
this.lifecycle?.setActive(true)
|
|
680
|
+
this.lifecycle?.tickNow()
|
|
681
|
+
// A resumed session is a fresh status baseline for subscribers.
|
|
682
|
+
this.lastFanoutSignature = null
|
|
683
|
+
this.fanout({
|
|
684
|
+
streamId: entry.streamId,
|
|
685
|
+
source: "coordinator",
|
|
686
|
+
status: LINK_STATUS.resumed,
|
|
687
|
+
data: {reason: LINK_STATUS.reason, suspendedMs},
|
|
688
|
+
})
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
private onGraceExpired(streamId: string): void {
|
|
692
|
+
if (this.current?.streamId !== streamId || !this.suspended) return
|
|
693
|
+
this.failSuspended(streamId, "glasses_disconnected", {publisherGone: false})
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
/**
|
|
697
|
+
* End a suspended stream. `publisherGone` distinguishes "glasses are off and
|
|
698
|
+
* Cloudflare confirms nothing is publishing" from "grace ran out with the
|
|
699
|
+
* publisher possibly still alive" — miniapps word the two differently.
|
|
700
|
+
*/
|
|
701
|
+
private failSuspended(streamId: string, reason: string, detail: {publisherGone: boolean}): void {
|
|
702
|
+
this.fanout({
|
|
703
|
+
streamId,
|
|
704
|
+
source: "coordinator",
|
|
705
|
+
status: "error",
|
|
706
|
+
data: {reason: LINK_STATUS.reason, teardownReason: reason, ...detail},
|
|
707
|
+
})
|
|
708
|
+
void this.runExclusive(async () => {
|
|
709
|
+
if (this.current?.streamId !== streamId) return
|
|
710
|
+
await this.teardownLocked(reason)
|
|
711
|
+
})
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
private flushPendingBleStop(): void {
|
|
715
|
+
const pending = this.pendingBleStop
|
|
716
|
+
if (!pending) return
|
|
717
|
+
this.pendingBleStop = null
|
|
718
|
+
console.info("[STREAM] BLE link back; sending deferred stopStream", pending)
|
|
719
|
+
void BluetoothSdk.stopStream()
|
|
720
|
+
.catch((err) => console.warn("[STREAM] deferred stopStream failed:", err))
|
|
721
|
+
.finally(() => this.detachLinkIfIdle())
|
|
722
|
+
}
|
|
723
|
+
|
|
546
724
|
// ===========================================================================
|
|
547
725
|
// Internal
|
|
548
726
|
// ===========================================================================
|
|
@@ -552,7 +730,11 @@ export class PhoneStreamCoordinator {
|
|
|
552
730
|
return `phone-${prefix}-${Date.now().toString(36)}-${this.idCounter}`
|
|
553
731
|
}
|
|
554
732
|
|
|
733
|
+
/** BLE keep-alives. StreamLifecycleController uses BgTimer so they survive MentraOS backgrounding. */
|
|
555
734
|
private startLifecycle(streamId: string): void {
|
|
735
|
+
// A new stream supersedes any deferred stop for the previous one.
|
|
736
|
+
this.pendingBleStop = null
|
|
737
|
+
this.attachLink()
|
|
556
738
|
this.lifecycle?.dispose()
|
|
557
739
|
const ctrl = new StreamLifecycleController(
|
|
558
740
|
{
|
|
@@ -631,6 +813,20 @@ export class PhoneStreamCoordinator {
|
|
|
631
813
|
status: status.isConnected ? "connected" : "disconnected",
|
|
632
814
|
data: status as unknown as Record<string, unknown>,
|
|
633
815
|
})
|
|
816
|
+
// While the BLE link is down, Cloudflare is the only witness to the
|
|
817
|
+
// publisher. Two consecutive "nobody is publishing" probes mean the
|
|
818
|
+
// glasses are off, not merely out of Bluetooth range — stop waiting.
|
|
819
|
+
if (this.suspended) {
|
|
820
|
+
if (status.isConnected) {
|
|
821
|
+
this.suspended.publisherGoneProbes = 0
|
|
822
|
+
} else {
|
|
823
|
+
this.suspended.publisherGoneProbes += 1
|
|
824
|
+
if (this.suspended.publisherGoneProbes >= this.timings.suspendedPublisherGoneProbes) {
|
|
825
|
+
this.failSuspended(entry.streamId, "glasses_disconnected_publisher_gone", {publisherGone: true})
|
|
826
|
+
keepPolling = false
|
|
827
|
+
}
|
|
828
|
+
}
|
|
829
|
+
}
|
|
634
830
|
// webrtc mode readiness: first "connected" means WHEP playback is
|
|
635
831
|
// available (WebRTC playback follows the ingest directly; there is no
|
|
636
832
|
// manifest to probe).
|
|
@@ -799,12 +995,28 @@ export class PhoneStreamCoordinator {
|
|
|
799
995
|
this.lastFanoutSignature = null
|
|
800
996
|
this.resolvedConfigForwarded = false
|
|
801
997
|
|
|
998
|
+
if (this.suspended) {
|
|
999
|
+
clearTimeout(this.suspended.graceTimer)
|
|
1000
|
+
this.suspended = null
|
|
1001
|
+
}
|
|
1002
|
+
|
|
802
1003
|
// Dispose the lifecycle controller immediately so it doesn't fire one
|
|
803
1004
|
// more keep-alive against a stream we're tearing down. The transition
|
|
804
1005
|
// lock guarantees no new lifecycle is started concurrently.
|
|
805
1006
|
this.lifecycle?.dispose()
|
|
806
1007
|
this.lifecycle = null
|
|
807
1008
|
|
|
1009
|
+
// With the link down a BLE write can only fail (and hold the transition
|
|
1010
|
+
// lock for the native timeout). Defer it to the next reconnect instead.
|
|
1011
|
+
const linkUp = this.linkSource.isConnected()
|
|
1012
|
+
if (sendBleStop && !linkUp) {
|
|
1013
|
+
this.pendingBleStop = {streamId: entry.streamId}
|
|
1014
|
+
console.warn("[STREAM] BLE link down during teardown; stopStream deferred", {
|
|
1015
|
+
streamId: entry.streamId,
|
|
1016
|
+
reason,
|
|
1017
|
+
})
|
|
1018
|
+
}
|
|
1019
|
+
|
|
808
1020
|
if (entry.kind === "managed") {
|
|
809
1021
|
if (entry.cloudflareTimer) clearTimeout(entry.cloudflareTimer)
|
|
810
1022
|
if (entry.hlsTimer) clearInterval(entry.hlsTimer)
|
|
@@ -816,7 +1028,7 @@ export class PhoneStreamCoordinator {
|
|
|
816
1028
|
}
|
|
817
1029
|
|
|
818
1030
|
try {
|
|
819
|
-
if (sendBleStop) {
|
|
1031
|
+
if (sendBleStop && linkUp) {
|
|
820
1032
|
await BluetoothSdk.stopStream()
|
|
821
1033
|
}
|
|
822
1034
|
} catch (err) {
|
|
@@ -827,6 +1039,7 @@ export class PhoneStreamCoordinator {
|
|
|
827
1039
|
// Only clear if we're still the active entry (defensive — runExclusive
|
|
828
1040
|
// serializes us, so this should always be true).
|
|
829
1041
|
if (this.current === entry) this.current = null
|
|
1042
|
+
this.detachLinkIfIdle()
|
|
830
1043
|
|
|
831
1044
|
if (entry.kind === "managed") {
|
|
832
1045
|
// Start remote cleanup only after the publisher has stopped, but do not
|
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Drives
|
|
2
|
+
* Drives stream liveliness checks via a keep-alive heartbeat:
|
|
3
3
|
* `setActive(true)` starts the timer; each tick sends a keep-alive with a
|
|
4
4
|
* fresh ackId; `maxMissedAcks` consecutive timeouts fire `onTimeout`.
|
|
5
5
|
*
|
|
6
6
|
* Phone-owned successor to the retired cloud stream lifecycle controller.
|
|
7
7
|
* It uses the local `LifecycleLogger` interface because pino is not a phone
|
|
8
8
|
* dependency.
|
|
9
|
+
*
|
|
10
|
+
* Timers MUST go through BgTimer. React Native pauses plain `setInterval`
|
|
11
|
+
* while MentraOS is backgrounded; glasses WHIP then auto-stops after 60s
|
|
12
|
+
* without `keep_stream_alive`, and Mentra Call recovers by restarting ingest.
|
|
9
13
|
*/
|
|
10
14
|
|
|
11
15
|
export interface LifecycleLogger {
|
|
@@ -23,6 +27,13 @@ interface StreamLifecycleCallbacks {
|
|
|
23
27
|
onKeepAliveMissed?: (ackId: string, ageMs: number, missedCount: number) => void
|
|
24
28
|
}
|
|
25
29
|
|
|
30
|
+
export interface StreamLifecycleTimerApi {
|
|
31
|
+
setInterval: (callback: () => void, delay: number) => number
|
|
32
|
+
clearInterval: (intervalId: number) => void
|
|
33
|
+
setTimeout: (callback: () => void, delay: number) => number
|
|
34
|
+
clearTimeout: (timeoutId: number) => void
|
|
35
|
+
}
|
|
36
|
+
|
|
26
37
|
export interface StreamLifecycleOptions {
|
|
27
38
|
logger: LifecycleLogger
|
|
28
39
|
streamId: string
|
|
@@ -31,21 +42,27 @@ export interface StreamLifecycleOptions {
|
|
|
31
42
|
maxMissedAcks: number
|
|
32
43
|
shouldSendKeepAlive?: () => boolean
|
|
33
44
|
now?: () => number
|
|
45
|
+
timers?: StreamLifecycleTimerApi
|
|
34
46
|
}
|
|
35
47
|
|
|
36
48
|
interface PendingAckInfo {
|
|
37
49
|
sentAt: number
|
|
38
|
-
timeout:
|
|
50
|
+
timeout: number
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function productionTimers(): StreamLifecycleTimerApi {
|
|
54
|
+
// Lazy so unit tests that inject timers never parse react-native.
|
|
55
|
+
const {BgTimer} = require("../utils/timers") as {BgTimer: StreamLifecycleTimerApi}
|
|
56
|
+
return BgTimer
|
|
39
57
|
}
|
|
40
58
|
|
|
41
59
|
/**
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
* `
|
|
45
|
-
* caller is expected to tear the stream down.
|
|
60
|
+
* Keep-alive heartbeat. Activate to start ticks; each tick sends keep-alive
|
|
61
|
+
* with a fresh ackId and arms a timeout. `maxMissedAcks` consecutive misses
|
|
62
|
+
* fire `onTimeout`, and the caller is expected to tear the stream down.
|
|
46
63
|
*/
|
|
47
64
|
export class StreamLifecycleController {
|
|
48
|
-
private keepAliveTimer?:
|
|
65
|
+
private keepAliveTimer?: number
|
|
49
66
|
private pendingAcks: Map<string, PendingAckInfo> = new Map()
|
|
50
67
|
private missedAcks = 0
|
|
51
68
|
private lastActivityMs: number
|
|
@@ -59,6 +76,7 @@ export class StreamLifecycleController {
|
|
|
59
76
|
private readonly maxMissedAcks: number
|
|
60
77
|
private readonly shouldSendKeepAlive?: () => boolean
|
|
61
78
|
private readonly now: () => number
|
|
79
|
+
private readonly timers: StreamLifecycleTimerApi
|
|
62
80
|
|
|
63
81
|
constructor(
|
|
64
82
|
options: StreamLifecycleOptions,
|
|
@@ -71,6 +89,7 @@ export class StreamLifecycleController {
|
|
|
71
89
|
this.maxMissedAcks = options.maxMissedAcks
|
|
72
90
|
this.shouldSendKeepAlive = options.shouldSendKeepAlive
|
|
73
91
|
this.now = options.now ?? (() => Date.now())
|
|
92
|
+
this.timers = options.timers ?? productionTimers()
|
|
74
93
|
this.lastActivityMs = this.now()
|
|
75
94
|
}
|
|
76
95
|
|
|
@@ -94,6 +113,17 @@ export class StreamLifecycleController {
|
|
|
94
113
|
this.missedAcks = 0
|
|
95
114
|
}
|
|
96
115
|
|
|
116
|
+
/**
|
|
117
|
+
* Send one keep-alive immediately instead of waiting for the next interval.
|
|
118
|
+
* Used when the BLE link comes back after a suspension: the glasses
|
|
119
|
+
* publisher's 60s watchdog has been running the whole time, so the first
|
|
120
|
+
* heartbeat after resume must not wait up to another full interval.
|
|
121
|
+
*/
|
|
122
|
+
tickNow(): void {
|
|
123
|
+
if (this.disposed || !this.active) return
|
|
124
|
+
void this.tick()
|
|
125
|
+
}
|
|
126
|
+
|
|
97
127
|
handleAck(ackId: string): void {
|
|
98
128
|
if (this.disposed) return
|
|
99
129
|
|
|
@@ -103,7 +133,7 @@ export class StreamLifecycleController {
|
|
|
103
133
|
return
|
|
104
134
|
}
|
|
105
135
|
|
|
106
|
-
clearTimeout(ackInfo.timeout)
|
|
136
|
+
this.timers.clearTimeout(ackInfo.timeout)
|
|
107
137
|
this.pendingAcks.delete(ackId)
|
|
108
138
|
this.recordActivity()
|
|
109
139
|
|
|
@@ -125,7 +155,7 @@ export class StreamLifecycleController {
|
|
|
125
155
|
|
|
126
156
|
private startTimer(): void {
|
|
127
157
|
if (this.keepAliveTimer) return
|
|
128
|
-
this.keepAliveTimer = setInterval(() => {
|
|
158
|
+
this.keepAliveTimer = this.timers.setInterval(() => {
|
|
129
159
|
void this.tick()
|
|
130
160
|
}, this.keepAliveIntervalMs)
|
|
131
161
|
this.logger.debug({streamId: this.streamId}, "Keep-alive timer started")
|
|
@@ -133,7 +163,7 @@ export class StreamLifecycleController {
|
|
|
133
163
|
|
|
134
164
|
private stopTimer(): void {
|
|
135
165
|
if (this.keepAliveTimer) {
|
|
136
|
-
clearInterval(this.keepAliveTimer)
|
|
166
|
+
this.timers.clearInterval(this.keepAliveTimer)
|
|
137
167
|
this.keepAliveTimer = undefined
|
|
138
168
|
this.logger.debug({streamId: this.streamId}, "Keep-alive timer stopped")
|
|
139
169
|
}
|
|
@@ -153,7 +183,7 @@ export class StreamLifecycleController {
|
|
|
153
183
|
const ackId = this.createAckId()
|
|
154
184
|
const sentAt = this.now()
|
|
155
185
|
|
|
156
|
-
const timeout = setTimeout(() => {
|
|
186
|
+
const timeout = this.timers.setTimeout(() => {
|
|
157
187
|
this.onAckTimeout(ackId, sentAt)
|
|
158
188
|
}, this.ackTimeoutMs)
|
|
159
189
|
|
|
@@ -202,7 +232,7 @@ export class StreamLifecycleController {
|
|
|
202
232
|
|
|
203
233
|
private clearPendingAcks(): void {
|
|
204
234
|
for (const {timeout} of this.pendingAcks.values()) {
|
|
205
|
-
clearTimeout(timeout)
|
|
235
|
+
this.timers.clearTimeout(timeout)
|
|
206
236
|
}
|
|
207
237
|
this.pendingAcks.clear()
|
|
208
238
|
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
export type SourceReason =
|
|
2
|
+
| "explicit"
|
|
3
|
+
| "current-mic"
|
|
4
|
+
| "ranking"
|
|
5
|
+
| "fallback-glasses-connected"
|
|
6
|
+
| "fallback-no-glasses"
|
|
7
|
+
|
|
8
|
+
export type AcsAudioSource = "glasses" | "phone"
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Single switch for Mentra Call ACS uplink.
|
|
12
|
+
* `"phone"` = AudioRecord / iOS input tap → RawOutgoingAudioStream (no Cloudflare audio).
|
|
13
|
+
* `"glasses"` = today's glasses WHIP → Cloudflare → WHEP PCM → ACS path.
|
|
14
|
+
* preferred_mic does not govern this call.
|
|
15
|
+
*/
|
|
16
|
+
export const ACS_CALL_MIC: AcsAudioSource = "glasses"
|
|
17
|
+
|
|
18
|
+
export type ResolvedAudioSource = {
|
|
19
|
+
source: AcsAudioSource
|
|
20
|
+
reason: SourceReason
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function mapMic(value: string | null | undefined): AcsAudioSource | null {
|
|
24
|
+
if (value === "phone" || value === "bluetooth" || value === "bluetoothClassic") return "phone"
|
|
25
|
+
if (value === "glasses") return "glasses"
|
|
26
|
+
return null
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Pure preferred_mic → ACS source mapping. Store reads stay in the thin wrapper
|
|
31
|
+
* so this table can be unit tested without zustand.
|
|
32
|
+
*/
|
|
33
|
+
export function resolveAudioSource(input: {
|
|
34
|
+
preferred: string
|
|
35
|
+
currentMic: string | null
|
|
36
|
+
micRanking: string[]
|
|
37
|
+
glassesConnected: boolean
|
|
38
|
+
}): ResolvedAudioSource {
|
|
39
|
+
const preferred = input.preferred || "auto"
|
|
40
|
+
if (preferred === "phone" || preferred === "bluetooth") {
|
|
41
|
+
return {source: "phone", reason: "explicit"}
|
|
42
|
+
}
|
|
43
|
+
if (preferred === "glasses") {
|
|
44
|
+
return {source: "glasses", reason: "explicit"}
|
|
45
|
+
}
|
|
46
|
+
const fromCurrent = mapMic(input.currentMic)
|
|
47
|
+
if (fromCurrent) return {source: fromCurrent, reason: "current-mic"}
|
|
48
|
+
const fromRank = mapMic(input.micRanking[0])
|
|
49
|
+
if (fromRank) return {source: fromRank, reason: "ranking"}
|
|
50
|
+
return input.glassesConnected
|
|
51
|
+
? {source: "glasses", reason: "fallback-glasses-connected"}
|
|
52
|
+
: {source: "phone", reason: "fallback-no-glasses"}
|
|
53
|
+
}
|
|
@@ -121,6 +121,10 @@ function collectAlternateHosts(packageName: string, extra?: string[]): string[]
|
|
|
121
121
|
}
|
|
122
122
|
|
|
123
123
|
for (const h of extra ?? []) push(h)
|
|
124
|
+
// USB `adb reverse` publishes the laptop on the phone's loopback. Mentra
|
|
125
|
+
// Call's QR still advertises the Wi-Fi IP, which AP isolation drops. Try
|
|
126
|
+
// loopback last so LAN/mDNS keep winning when they work.
|
|
127
|
+
push("127.0.0.1")
|
|
124
128
|
return hosts
|
|
125
129
|
}
|
|
126
130
|
|