@mentra/engine 3.2.0-dev.115 → 3.2.0-dev.120
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/build/engine.d.ts.map +1 -1
- package/build/engine.js +7 -2
- package/build/engine.js.map +1 -1
- package/build/generated/releaseMetadata.js +5 -5
- package/build/generated/releaseMetadata.js.map +1 -1
- package/build/runtime/streamConfig.d.ts +7 -0
- package/build/runtime/streamConfig.d.ts.map +1 -1
- package/build/runtime/streamConfig.js +7 -0
- package/build/runtime/streamConfig.js.map +1 -1
- package/build/services/AcsMeetingService.d.ts +154 -0
- package/build/services/AcsMeetingService.d.ts.map +1 -0
- package/build/services/AcsMeetingService.js +477 -0
- package/build/services/AcsMeetingService.js.map +1 -0
- package/build/services/DeviceEventRouter.d.ts.map +1 -1
- package/build/services/DeviceEventRouter.js +12 -3
- package/build/services/DeviceEventRouter.js.map +1 -1
- package/build/services/GlassesSettingsSync.d.ts.map +1 -1
- package/build/services/GlassesSettingsSync.js +11 -1
- package/build/services/GlassesSettingsSync.js.map +1 -1
- package/build/services/GlassesStatusProjection.d.ts +12 -0
- package/build/services/GlassesStatusProjection.d.ts.map +1 -1
- package/build/services/GlassesStatusProjection.js +23 -0
- package/build/services/GlassesStatusProjection.js.map +1 -1
- package/build/services/LocalMiniappRuntime.d.ts +6 -0
- package/build/services/LocalMiniappRuntime.d.ts.map +1 -1
- package/build/services/LocalMiniappRuntime.js +130 -1
- package/build/services/LocalMiniappRuntime.js.map +1 -1
- package/build/services/MentraJSRouter.d.ts.map +1 -1
- package/build/services/MentraJSRouter.js +8 -1
- package/build/services/MentraJSRouter.js.map +1 -1
- package/build/services/PhoneCameraFovCoordinator.d.ts +18 -0
- package/build/services/PhoneCameraFovCoordinator.d.ts.map +1 -1
- package/build/services/PhoneCameraFovCoordinator.js +36 -0
- package/build/services/PhoneCameraFovCoordinator.js.map +1 -1
- package/build/services/PhoneStreamCoordinator.d.ts +67 -1
- package/build/services/PhoneStreamCoordinator.d.ts.map +1 -1
- package/build/services/PhoneStreamCoordinator.js +204 -15
- package/build/services/PhoneStreamCoordinator.js.map +1 -1
- package/build/services/StreamLifecycleController.d.ts +23 -5
- package/build/services/StreamLifecycleController.d.ts.map +1 -1
- package/build/services/StreamLifecycleController.js +31 -10
- package/build/services/StreamLifecycleController.js.map +1 -1
- package/build/services/acsAudioSource.d.ts +24 -0
- package/build/services/acsAudioSource.d.ts.map +1 -0
- package/build/services/acsAudioSource.js +37 -0
- package/build/services/acsAudioSource.js.map +1 -0
- package/build/utils/devMiniappLaunch.d.ts.map +1 -1
- package/build/utils/devMiniappLaunch.js +4 -0
- package/build/utils/devMiniappLaunch.js.map +1 -1
- package/package.json +8 -7
- package/src/engine.ts +10 -2
- package/src/generated/releaseMetadata.ts +5 -5
- package/src/runtime/streamConfig.ts +13 -0
- package/src/services/AcsMeetingService.ts +570 -0
- package/src/services/DeviceEventRouter.ts +12 -3
- package/src/services/GlassesSettingsSync.ts +13 -3
- package/src/services/GlassesStatusProjection.ts +28 -0
- package/src/services/LocalMiniappRuntime.ts +142 -1
- package/src/services/MentraJSRouter.ts +7 -1
- package/src/services/PhoneCameraFovCoordinator.ts +34 -0
- package/src/services/PhoneStreamCoordinator.ts +232 -19
- package/src/services/StreamLifecycleController.ts +42 -12
- package/src/services/acsAudioSource.ts +53 -0
- package/src/utils/devMiniappLaunch.ts +4 -0
|
@@ -67,7 +67,7 @@ import {
|
|
|
67
67
|
} from "../runtime/config"
|
|
68
68
|
import {getAnalytics, getUiSeams} from "../runtime/bootstrap"
|
|
69
69
|
import {invokeScanQrSeam} from "../runtime/scanQrSeam"
|
|
70
|
-
import {normalizeStreamAudioConfig, normalizeStreamVideoConfig} from "../runtime/streamConfig"
|
|
70
|
+
import {normalizeStreamAudioConfig, normalizeStreamVideoConfig, resolveCaptureAudio} from "../runtime/streamConfig"
|
|
71
71
|
import {toLanguageHint} from "@mentra/cloud-protocol/languages"
|
|
72
72
|
import type {AudioSubscription, LanguageSource, TranscriptionData, TranslationData} from "@mentra/cloud-protocol"
|
|
73
73
|
import {buildMiniappManifestSnapshot, type MiniappRuntimeDiagnosticSnapshot} from "../utils/miniappDiagnostics"
|
|
@@ -80,6 +80,7 @@ import {resolveForegroundLocationPermission} from "./ForegroundLocationPermissio
|
|
|
80
80
|
import {advanceMiniappPingLiveness} from "./MiniappLiveness"
|
|
81
81
|
import {listPhoneCalendarEvents, PhoneCalendarError} from "./PhoneCalendarService"
|
|
82
82
|
import {LocalMiniappStorage} from "./LocalMiniappStorage"
|
|
83
|
+
import acsMeetingService, {parseAcsOutgoingVideo, resolveAcsAudioSource} from "./AcsMeetingService"
|
|
83
84
|
|
|
84
85
|
// =============================================================================
|
|
85
86
|
// Types
|
|
@@ -953,6 +954,9 @@ class LocalMiniappRuntime {
|
|
|
953
954
|
void phoneVideoCoordinator.stopForApp(packageName).catch((error) => {
|
|
954
955
|
console.warn(`${LOG_TAG}: failed to stop video recording for ${packageName} on unregister`, error)
|
|
955
956
|
})
|
|
957
|
+
void acsMeetingService.leaveIfOwner(packageName).catch((error) => {
|
|
958
|
+
console.warn(`${LOG_TAG}: failed to leave ACS meeting for ${packageName} on unregister`, error)
|
|
959
|
+
})
|
|
956
960
|
|
|
957
961
|
// Detach the per-app nav event forwarder but leave the native nav session
|
|
958
962
|
// running. The user may have just closed the mini-app UI and will reopen
|
|
@@ -1307,6 +1311,21 @@ class LocalMiniappRuntime {
|
|
|
1307
1311
|
case MiniappRequestType.MANAGED_STREAM_STOP:
|
|
1308
1312
|
void this.handleManagedStreamStop(packageName, payload, requestId)
|
|
1309
1313
|
break
|
|
1314
|
+
case MiniappRequestType.MEETING_JOIN:
|
|
1315
|
+
void this.handleMeetingJoin(packageName, payload, requestId)
|
|
1316
|
+
break
|
|
1317
|
+
case MiniappRequestType.MEETING_LEAVE:
|
|
1318
|
+
void this.handleMeetingLeave(packageName, requestId)
|
|
1319
|
+
break
|
|
1320
|
+
case MiniappRequestType.MEETING_SET_MUTED:
|
|
1321
|
+
void this.handleMeetingSetMuted(packageName, payload, requestId)
|
|
1322
|
+
break
|
|
1323
|
+
case MiniappRequestType.MEETING_UPDATE_VIDEO_SOURCE:
|
|
1324
|
+
void this.handleMeetingUpdateVideoSource(packageName, payload, requestId)
|
|
1325
|
+
break
|
|
1326
|
+
case MiniappRequestType.MEETING_GET_STATE:
|
|
1327
|
+
void this.handleMeetingGetState(packageName, requestId)
|
|
1328
|
+
break
|
|
1310
1329
|
case REQUEST_WIFI_SETUP_TYPE:
|
|
1311
1330
|
void this.handleRequestWifiSetup(packageName, payload, requestId)
|
|
1312
1331
|
break
|
|
@@ -1409,6 +1428,7 @@ class LocalMiniappRuntime {
|
|
|
1409
1428
|
packageName,
|
|
1410
1429
|
capabilities,
|
|
1411
1430
|
permissions: declaredPermissions,
|
|
1431
|
+
hostFeatures: {captureAudio: true},
|
|
1412
1432
|
...(initialAuth ? {auth: initialAuth} : {}),
|
|
1413
1433
|
},
|
|
1414
1434
|
requestId,
|
|
@@ -3406,6 +3426,7 @@ class LocalMiniappRuntime {
|
|
|
3406
3426
|
audio: normalizeStreamAudioConfig(payload.audio),
|
|
3407
3427
|
sound: payload.sound as boolean | undefined,
|
|
3408
3428
|
authToken: typeof payload.authToken === "string" ? payload.authToken : undefined,
|
|
3429
|
+
captureAudio: resolveCaptureAudio(payload.captureAudio, resolveAcsAudioSource().source),
|
|
3409
3430
|
})
|
|
3410
3431
|
this.sendResult(packageName, requestId, true, result)
|
|
3411
3432
|
} catch (err) {
|
|
@@ -3462,6 +3483,7 @@ class LocalMiniappRuntime {
|
|
|
3462
3483
|
audio: normalizeStreamAudioConfig(payload.audio),
|
|
3463
3484
|
sound: payload.sound as boolean | undefined,
|
|
3464
3485
|
ingest: payload.ingest as "srt" | "whip" | undefined,
|
|
3486
|
+
captureAudio: resolveCaptureAudio(payload.captureAudio, resolveAcsAudioSource().source),
|
|
3465
3487
|
})
|
|
3466
3488
|
this.sendResult(packageName, requestId, true, result)
|
|
3467
3489
|
} catch (err) {
|
|
@@ -3482,6 +3504,125 @@ class LocalMiniappRuntime {
|
|
|
3482
3504
|
return this.handleStreamStop(packageName, payload, requestId)
|
|
3483
3505
|
}
|
|
3484
3506
|
|
|
3507
|
+
private ensureMeetingStateBridge(): void {
|
|
3508
|
+
acsMeetingService.setStateHandler((owner, state) => {
|
|
3509
|
+
this.sendToMiniapp(owner, {
|
|
3510
|
+
type: MiniappResponseType.MEETING_STATE,
|
|
3511
|
+
...state,
|
|
3512
|
+
})
|
|
3513
|
+
})
|
|
3514
|
+
}
|
|
3515
|
+
|
|
3516
|
+
private async handleMeetingJoin(
|
|
3517
|
+
packageName: string,
|
|
3518
|
+
payload: Record<string, unknown>,
|
|
3519
|
+
requestId?: string,
|
|
3520
|
+
): Promise<void> {
|
|
3521
|
+
this.ensureMeetingStateBridge()
|
|
3522
|
+
const meetingUrl = typeof payload.meetingUrl === "string" ? payload.meetingUrl : ""
|
|
3523
|
+
const token = typeof payload.token === "string" ? payload.token : ""
|
|
3524
|
+
const videoSource = payload.videoSource as {type?: string; url?: string} | undefined
|
|
3525
|
+
const whepUrl = videoSource?.type === "whep" ? videoSource.url ?? "" : ""
|
|
3526
|
+
const displayName = typeof payload.displayName === "string" ? payload.displayName : undefined
|
|
3527
|
+
if (!meetingUrl || !token || !whepUrl) {
|
|
3528
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
3529
|
+
code: MiniappErrorCode.INVALID_ARGUMENT,
|
|
3530
|
+
message: "meetingUrl, token, and a WHEP videoSource are required",
|
|
3531
|
+
})
|
|
3532
|
+
return
|
|
3533
|
+
}
|
|
3534
|
+
try {
|
|
3535
|
+
let video: ReturnType<typeof parseAcsOutgoingVideo>
|
|
3536
|
+
try {
|
|
3537
|
+
video = parseAcsOutgoingVideo(payload.video)
|
|
3538
|
+
} catch (error) {
|
|
3539
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
3540
|
+
code: MiniappErrorCode.INVALID_ARGUMENT,
|
|
3541
|
+
message: error instanceof Error ? error.message : "Invalid meeting video",
|
|
3542
|
+
})
|
|
3543
|
+
return
|
|
3544
|
+
}
|
|
3545
|
+
const state = await acsMeetingService.join(packageName, {
|
|
3546
|
+
meetingUrl,
|
|
3547
|
+
token,
|
|
3548
|
+
whepUrl,
|
|
3549
|
+
displayName,
|
|
3550
|
+
...(video ? {video} : {}),
|
|
3551
|
+
})
|
|
3552
|
+
this.sendResult(packageName, requestId, true, state)
|
|
3553
|
+
} catch (err) {
|
|
3554
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
3555
|
+
code: MiniappErrorCode.INTERNAL,
|
|
3556
|
+
message: err instanceof Error ? err.message : "ACS meeting join failed",
|
|
3557
|
+
})
|
|
3558
|
+
}
|
|
3559
|
+
}
|
|
3560
|
+
|
|
3561
|
+
private async handleMeetingLeave(packageName: string, requestId?: string): Promise<void> {
|
|
3562
|
+
try {
|
|
3563
|
+
await acsMeetingService.leave(packageName)
|
|
3564
|
+
this.sendResult(packageName, requestId, true)
|
|
3565
|
+
} catch (err) {
|
|
3566
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
3567
|
+
code: MiniappErrorCode.INTERNAL,
|
|
3568
|
+
message: err instanceof Error ? err.message : "ACS meeting leave failed",
|
|
3569
|
+
})
|
|
3570
|
+
}
|
|
3571
|
+
}
|
|
3572
|
+
|
|
3573
|
+
private async handleMeetingSetMuted(
|
|
3574
|
+
packageName: string,
|
|
3575
|
+
payload: Record<string, unknown>,
|
|
3576
|
+
requestId?: string,
|
|
3577
|
+
): Promise<void> {
|
|
3578
|
+
try {
|
|
3579
|
+
const state = await acsMeetingService.setMuted(packageName, Boolean(payload.muted))
|
|
3580
|
+
this.sendResult(packageName, requestId, true, state)
|
|
3581
|
+
} catch (err) {
|
|
3582
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
3583
|
+
code: MiniappErrorCode.INTERNAL,
|
|
3584
|
+
message: err instanceof Error ? err.message : "ACS mute failed",
|
|
3585
|
+
})
|
|
3586
|
+
}
|
|
3587
|
+
}
|
|
3588
|
+
|
|
3589
|
+
private async handleMeetingUpdateVideoSource(
|
|
3590
|
+
packageName: string,
|
|
3591
|
+
payload: Record<string, unknown>,
|
|
3592
|
+
requestId?: string,
|
|
3593
|
+
): Promise<void> {
|
|
3594
|
+
const videoSource = payload.videoSource as {type?: string; url?: string} | undefined
|
|
3595
|
+
const whepUrl = videoSource?.type === "whep" ? videoSource.url ?? "" : ""
|
|
3596
|
+
if (!whepUrl) {
|
|
3597
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
3598
|
+
code: MiniappErrorCode.INVALID_ARGUMENT,
|
|
3599
|
+
message: "videoSource must be a WHEP URL",
|
|
3600
|
+
})
|
|
3601
|
+
return
|
|
3602
|
+
}
|
|
3603
|
+
try {
|
|
3604
|
+
await acsMeetingService.updateVideoSource(packageName, whepUrl)
|
|
3605
|
+
this.sendResult(packageName, requestId, true)
|
|
3606
|
+
} catch (err) {
|
|
3607
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
3608
|
+
code: MiniappErrorCode.INTERNAL,
|
|
3609
|
+
message: err instanceof Error ? err.message : "ACS video source update failed",
|
|
3610
|
+
})
|
|
3611
|
+
}
|
|
3612
|
+
}
|
|
3613
|
+
|
|
3614
|
+
private async handleMeetingGetState(packageName: string, requestId?: string): Promise<void> {
|
|
3615
|
+
try {
|
|
3616
|
+
const state = await acsMeetingService.readState(packageName)
|
|
3617
|
+
this.sendResult(packageName, requestId, true, state)
|
|
3618
|
+
} catch (err) {
|
|
3619
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
3620
|
+
code: MiniappErrorCode.INTERNAL,
|
|
3621
|
+
message: err instanceof Error ? err.message : "ACS getState failed",
|
|
3622
|
+
})
|
|
3623
|
+
}
|
|
3624
|
+
}
|
|
3625
|
+
|
|
3485
3626
|
/**
|
|
3486
3627
|
* session.glasses.requestWifiSetup — open the phone's glasses Wi-Fi setup
|
|
3487
3628
|
* flow. The host owns the actual UI via the `engine.configure({ui})` seam;
|
|
@@ -469,7 +469,13 @@ export class MentraJSRouter {
|
|
|
469
469
|
// dead and let the host surface a "tap to retry" banner.
|
|
470
470
|
if (iface === "__error") {
|
|
471
471
|
const payload = this.tryParseArgs(msg.argsJson)
|
|
472
|
-
|
|
472
|
+
// ready_nack is a liveness probe, not a crash. Logging it at error
|
|
473
|
+
// painted WHIP-start queue delay as a Mentra-Call exception.
|
|
474
|
+
if (method === "ready_nack") {
|
|
475
|
+
this.logger.warn(`[${packageName}] ${method}`, payload)
|
|
476
|
+
} else {
|
|
477
|
+
this.logger.error(`[${packageName}] ${method}`, payload)
|
|
478
|
+
}
|
|
473
479
|
if (this.crashController) {
|
|
474
480
|
// Only treat "exception" + "unhandledRejection" + "uncaught" as
|
|
475
481
|
// crash signals. `console.error` calls also flow through the
|
|
@@ -90,6 +90,40 @@ export class PhoneCameraFovCoordinator {
|
|
|
90
90
|
})
|
|
91
91
|
}
|
|
92
92
|
|
|
93
|
+
/**
|
|
94
|
+
* Re-assert the current override on the glasses.
|
|
95
|
+
*
|
|
96
|
+
* The on-connect device-settings push replays the persistent `camera_fov`,
|
|
97
|
+
* so glasses that connect mid-call come up at the user's saved value and
|
|
98
|
+
* silently drop a miniapp's override — a Mentra Call wearer loses the 102°
|
|
99
|
+
* bottom crop by connecting their glasses after joining.
|
|
100
|
+
*
|
|
101
|
+
* The effective lease is recomputed inside [enqueue], not captured at call
|
|
102
|
+
* time: a `setOverride` or `releaseForApp` already queued ahead of this must
|
|
103
|
+
* win, and it is this coordinator that has to be the authoritative last
|
|
104
|
+
* writer. Sends the complete override — `fov` *and* `roiPosition` — because
|
|
105
|
+
* the scalar alone leaves the ROI wherever the persistent setting put it.
|
|
106
|
+
*
|
|
107
|
+
* No-op in legacy mode: those commands restart the camera HAL, which is far
|
|
108
|
+
* too expensive to fire on every reconnect.
|
|
109
|
+
*/
|
|
110
|
+
reapplyEffectiveOverride(): Promise<void> {
|
|
111
|
+
return this.enqueue(async () => {
|
|
112
|
+
if (this.legacyMode) return
|
|
113
|
+
const leaseId = this.effectiveLeaseId
|
|
114
|
+
if (!leaseId) return
|
|
115
|
+
const entry = [...this.overrides.values()].find((candidate) => candidate.leaseId === leaseId)
|
|
116
|
+
if (!entry) return
|
|
117
|
+
await BluetoothSdk.setCameraFovOverride({
|
|
118
|
+
leaseId,
|
|
119
|
+
fov: entry.fov,
|
|
120
|
+
roiPosition: entry.roiPosition,
|
|
121
|
+
ttlMs: CAMERA_FOV_OVERRIDE_TTL_MS,
|
|
122
|
+
})
|
|
123
|
+
this.scheduleRefresh()
|
|
124
|
+
})
|
|
125
|
+
}
|
|
126
|
+
|
|
93
127
|
releaseForApp(packageName: string): Promise<void> {
|
|
94
128
|
return this.enqueue(async () => {
|
|
95
129
|
const removed = this.overrides.get(packageName)
|
|
@@ -26,6 +26,18 @@
|
|
|
26
26
|
* streams. We use a `phone-` prefix so they're trivially distinguishable from
|
|
27
27
|
* cloud-minted IDs in logs and from cloud-SDK app streams that flow through
|
|
28
28
|
* the legacy path.
|
|
29
|
+
*
|
|
30
|
+
* BLE link loss is a SUSPENDED state, not a failure:
|
|
31
|
+
* The glasses publisher keeps pushing over Wi-Fi when the phone's Bluetooth
|
|
32
|
+
* link drops; only our keep-alives stop reaching it. While the store says
|
|
33
|
+
* the glasses are disconnected we pause the heartbeat (so we don't count
|
|
34
|
+
* misses we caused ourselves), fan out `suspended`, and give the link
|
|
35
|
+
* `glassesGraceMs` to come back. On reconnect we resume, heartbeat at once,
|
|
36
|
+
* and fan out `resumed` with the SAME streamId — no re-provision. The grace
|
|
37
|
+
* ends early when Cloudflare reports the publisher gone for two probes in
|
|
38
|
+
* a row (glasses powered off, not a BLE hiccup). If the grace expires the
|
|
39
|
+
* stream is torn down with reason `glasses_disconnected`, and the BLE stop
|
|
40
|
+
* the glasses never received is sent on the next reconnect.
|
|
29
41
|
*/
|
|
30
42
|
|
|
31
43
|
import BluetoothSdk from "@mentra/bluetooth-sdk/internal"
|
|
@@ -65,11 +77,40 @@ const DEFAULT_TIMINGS = {
|
|
|
65
77
|
hlsReadinessInitialDelayMs: 5_000,
|
|
66
78
|
hlsReadinessPollMs: 2_000,
|
|
67
79
|
hlsReadinessMaxAttempts: 30,
|
|
80
|
+
// How long a stream survives a BLE link drop before we give up on it. Must
|
|
81
|
+
// stay under the glasses publisher's own 60s no-keep-alive watchdog with
|
|
82
|
+
// margin for the resume heartbeat's round trip.
|
|
83
|
+
glassesGraceMs: 45_000,
|
|
84
|
+
// Consecutive Cloudflare "publisher disconnected" probes while suspended
|
|
85
|
+
// before we conclude the glasses are off (not just out of BLE range).
|
|
86
|
+
suspendedPublisherGoneProbes: 2,
|
|
68
87
|
} as const
|
|
69
88
|
|
|
70
89
|
type TimingConfig = {[K in keyof typeof DEFAULT_TIMINGS]: number}
|
|
71
90
|
export type CoordinatorTimings = Partial<TimingConfig>
|
|
72
91
|
|
|
92
|
+
/**
|
|
93
|
+
* Where the coordinator learns whether the phone↔glasses BLE link is up.
|
|
94
|
+
* Injected so tests can drive link transitions without the zustand store.
|
|
95
|
+
*/
|
|
96
|
+
export interface GlassesLinkSource {
|
|
97
|
+
isConnected(): boolean
|
|
98
|
+
/** Fires on every connected↔disconnected transition. */
|
|
99
|
+
subscribe(listener: (connected: boolean) => void): () => void
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
const storeLinkSource: GlassesLinkSource = {
|
|
103
|
+
isConnected: () => isGlassesConnected(useGlassesStore.getState().connection),
|
|
104
|
+
subscribe: (listener) => useGlassesStore.subscribe((s) => isGlassesConnected(s.connection), listener),
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** Reasons carried on coordinator-sourced `stream_status` fanouts for link events. */
|
|
108
|
+
export const LINK_STATUS = {
|
|
109
|
+
suspended: "suspended",
|
|
110
|
+
resumed: "resumed",
|
|
111
|
+
reason: "glasses_disconnected",
|
|
112
|
+
} as const
|
|
113
|
+
|
|
73
114
|
// Console-backed minimal logger; replaces pino on the phone.
|
|
74
115
|
const consoleLogger: LifecycleLogger = {
|
|
75
116
|
child: (bindings) => ({
|
|
@@ -90,6 +131,7 @@ export interface StartUnmanagedOptions {
|
|
|
90
131
|
sound?: boolean
|
|
91
132
|
/** Optional Bearer token for WHIP Authorization (custom authenticated endpoints). */
|
|
92
133
|
authToken?: string
|
|
134
|
+
captureAudio?: boolean
|
|
93
135
|
}
|
|
94
136
|
|
|
95
137
|
export interface StartManagedOptions {
|
|
@@ -108,6 +150,8 @@ export interface StartManagedOptions {
|
|
|
108
150
|
* that drop WHIP/SRT UDP.
|
|
109
151
|
*/
|
|
110
152
|
ingest?: "srt" | "whip" | "rtmp"
|
|
153
|
+
/** When false, glasses skip encoding their mic for this WHIP session. */
|
|
154
|
+
captureAudio?: boolean
|
|
111
155
|
}
|
|
112
156
|
|
|
113
157
|
export interface StreamPublisherStartResult {
|
|
@@ -184,20 +228,11 @@ export class StreamConflictError extends Error {
|
|
|
184
228
|
}
|
|
185
229
|
}
|
|
186
230
|
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
function assertGlassesConnected(): void {
|
|
193
|
-
if (!isGlassesConnected(useGlassesStore.getState().connection)) {
|
|
194
|
-
throw new StreamConflictError(
|
|
195
|
-
"GLASSES_NOT_CONNECTED",
|
|
196
|
-
"Glasses are not connected",
|
|
197
|
-
"command",
|
|
198
|
-
"ble",
|
|
199
|
-
)
|
|
200
|
-
}
|
|
231
|
+
interface SuspendedState {
|
|
232
|
+
since: number
|
|
233
|
+
graceTimer: ReturnType<typeof setTimeout>
|
|
234
|
+
/** Consecutive Cloudflare probes that saw no publisher during this suspension. */
|
|
235
|
+
publisherGoneProbes: number
|
|
201
236
|
}
|
|
202
237
|
|
|
203
238
|
export class PhoneStreamCoordinator {
|
|
@@ -206,6 +241,16 @@ export class PhoneStreamCoordinator {
|
|
|
206
241
|
private statusSubscriber: StatusSubscriber | null = null
|
|
207
242
|
private idCounter = 0
|
|
208
243
|
private readonly timings: TimingConfig
|
|
244
|
+
private readonly linkSource: GlassesLinkSource
|
|
245
|
+
private unsubscribeLink: (() => void) | null = null
|
|
246
|
+
private suspended: SuspendedState | null = null
|
|
247
|
+
/**
|
|
248
|
+
* A stream was torn down while the BLE link was down, so the glasses never
|
|
249
|
+
* got `stopStream`. Sent on the next reconnect (if no new stream has claimed
|
|
250
|
+
* the slot) so a publisher that outlived its input does not keep pushing
|
|
251
|
+
* until its own watchdog fires.
|
|
252
|
+
*/
|
|
253
|
+
private pendingBleStop: {streamId: string} | null = null
|
|
209
254
|
/**
|
|
210
255
|
* Serializes state transitions (start, stop, teardown). Without it, a
|
|
211
256
|
* second `start*` racing with the first can pass the `this.current === null`
|
|
@@ -220,8 +265,30 @@ export class PhoneStreamCoordinator {
|
|
|
220
265
|
/** Send full resolvedConfig only once per stream session. */
|
|
221
266
|
private resolvedConfigForwarded = false
|
|
222
267
|
|
|
223
|
-
constructor(timings: CoordinatorTimings = {}) {
|
|
268
|
+
constructor(timings: CoordinatorTimings = {}, deps: {linkSource?: GlassesLinkSource} = {}) {
|
|
224
269
|
this.timings = {...DEFAULT_TIMINGS, ...timings}
|
|
270
|
+
this.linkSource = deps.linkSource ?? storeLinkSource
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/**
|
|
274
|
+
* Fail fast if glasses aren't connected — BEFORE provisioning. Without this a
|
|
275
|
+
* managed start would create a provider live input, fail the BLE command, and
|
|
276
|
+
* tear the input down again: a slow, billable no-op with a confusing error.
|
|
277
|
+
*/
|
|
278
|
+
private assertGlassesConnected(): void {
|
|
279
|
+
if (!this.linkSource.isConnected()) {
|
|
280
|
+
throw new StreamConflictError(
|
|
281
|
+
"GLASSES_NOT_CONNECTED",
|
|
282
|
+
"Glasses are not connected",
|
|
283
|
+
"command",
|
|
284
|
+
"ble",
|
|
285
|
+
)
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
/** True while the active stream is parked on a dropped BLE link. */
|
|
290
|
+
isSuspended(): boolean {
|
|
291
|
+
return this.suspended !== null
|
|
225
292
|
}
|
|
226
293
|
|
|
227
294
|
/**
|
|
@@ -261,7 +328,11 @@ export class PhoneStreamCoordinator {
|
|
|
261
328
|
|
|
262
329
|
/** Report-safe stream ownership snapshot for incident diagnostics. */
|
|
263
330
|
getDiagnosticSnapshot(): Record<string, unknown> {
|
|
264
|
-
if (!this.current) return {active: false}
|
|
331
|
+
if (!this.current) return {active: false, pendingBleStop: this.pendingBleStop?.streamId ?? null}
|
|
332
|
+
const link = {
|
|
333
|
+
suspended: this.suspended !== null,
|
|
334
|
+
...(this.suspended ? {suspendedForMs: Date.now() - this.suspended.since} : {}),
|
|
335
|
+
}
|
|
265
336
|
return this.current.kind === "managed"
|
|
266
337
|
? {
|
|
267
338
|
active: true,
|
|
@@ -270,12 +341,14 @@ export class PhoneStreamCoordinator {
|
|
|
270
341
|
subscribers: [...this.current.subscribers].sort(),
|
|
271
342
|
mode: this.current.mode,
|
|
272
343
|
playbackReady: this.current.hlsReady,
|
|
344
|
+
...link,
|
|
273
345
|
}
|
|
274
346
|
: {
|
|
275
347
|
active: true,
|
|
276
348
|
kind: this.current.kind,
|
|
277
349
|
streamId: this.current.streamId,
|
|
278
350
|
ownerPackageName: this.current.packageName,
|
|
351
|
+
...link,
|
|
279
352
|
}
|
|
280
353
|
}
|
|
281
354
|
|
|
@@ -288,7 +361,7 @@ export class PhoneStreamCoordinator {
|
|
|
288
361
|
if (!opts.streamUrl || typeof opts.streamUrl !== "string") {
|
|
289
362
|
throw new StreamConflictError("STREAM_URL_REQUIRED", "streamUrl is required")
|
|
290
363
|
}
|
|
291
|
-
assertGlassesConnected()
|
|
364
|
+
this.assertGlassesConnected()
|
|
292
365
|
return this.runExclusive(async () => {
|
|
293
366
|
if (this.current) {
|
|
294
367
|
throw new StreamConflictError(
|
|
@@ -319,6 +392,7 @@ export class PhoneStreamCoordinator {
|
|
|
319
392
|
...(opts.video !== undefined ? {video: opts.video} : {}),
|
|
320
393
|
...(opts.audio !== undefined ? {audio: opts.audio} : {}),
|
|
321
394
|
...(opts.authToken ? {authToken: opts.authToken} : {}),
|
|
395
|
+
...(typeof opts.captureAudio === "boolean" ? {captureAudio: opts.captureAudio} : {}),
|
|
322
396
|
})
|
|
323
397
|
const result = publisherStartResult(streamId, event)
|
|
324
398
|
this.startLifecycle(streamId)
|
|
@@ -346,7 +420,7 @@ export class PhoneStreamCoordinator {
|
|
|
346
420
|
// Two-phase: the entry-claim runs under the transition lock; the wait for
|
|
347
421
|
// HLS readiness happens AFTER the lock releases so a long warm-up doesn't
|
|
348
422
|
// block subsequent start/stop transitions on this coordinator.
|
|
349
|
-
assertGlassesConnected()
|
|
423
|
+
this.assertGlassesConnected()
|
|
350
424
|
type JoinDecision =
|
|
351
425
|
| {kind: "join"; entry: ManagedEntry; immediate: ManagedStartResult | null}
|
|
352
426
|
| {kind: "fresh"; entry: ManagedEntry}
|
|
@@ -422,6 +496,7 @@ export class PhoneStreamCoordinator {
|
|
|
422
496
|
// See startUnmanaged: the native bridge rejects explicit `undefined`.
|
|
423
497
|
...(opts.video !== undefined ? {video: opts.video} : {}),
|
|
424
498
|
...(opts.audio !== undefined ? {audio: opts.audio} : {}),
|
|
499
|
+
...(typeof opts.captureAudio === "boolean" ? {captureAudio: opts.captureAudio} : {}),
|
|
425
500
|
})
|
|
426
501
|
entry.publisherStart = publisherStartResult(streamId, event)
|
|
427
502
|
console.info("[STREAM_STARTUP]", {
|
|
@@ -543,6 +618,109 @@ export class PhoneStreamCoordinator {
|
|
|
543
618
|
this.lifecycle?.handleAck(event.ackId)
|
|
544
619
|
}
|
|
545
620
|
|
|
621
|
+
// ===========================================================================
|
|
622
|
+
// BLE link suspension
|
|
623
|
+
// ===========================================================================
|
|
624
|
+
|
|
625
|
+
private attachLink(): void {
|
|
626
|
+
if (this.unsubscribeLink) return
|
|
627
|
+
this.unsubscribeLink = this.linkSource.subscribe((connected) => this.handleLinkChange(connected))
|
|
628
|
+
}
|
|
629
|
+
|
|
630
|
+
private detachLinkIfIdle(): void {
|
|
631
|
+
if (this.current || this.pendingBleStop || !this.unsubscribeLink) return
|
|
632
|
+
this.unsubscribeLink()
|
|
633
|
+
this.unsubscribeLink = null
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
private handleLinkChange(connected: boolean): void {
|
|
637
|
+
if (connected) {
|
|
638
|
+
if (this.current && this.suspended) {
|
|
639
|
+
this.resumeLocked()
|
|
640
|
+
} else if (!this.current && this.pendingBleStop) {
|
|
641
|
+
this.flushPendingBleStop()
|
|
642
|
+
}
|
|
643
|
+
return
|
|
644
|
+
}
|
|
645
|
+
if (this.current && !this.suspended) this.suspend()
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
private suspend(): void {
|
|
649
|
+
const entry = this.current
|
|
650
|
+
if (!entry) return
|
|
651
|
+
const since = Date.now()
|
|
652
|
+
// Pause the heartbeat: misses now would be our own fault, not the
|
|
653
|
+
// publisher's, and counting them would reap a stream that is still live.
|
|
654
|
+
this.lifecycle?.setActive(false)
|
|
655
|
+
const graceTimer = setTimeout(() => this.onGraceExpired(entry.streamId), this.timings.glassesGraceMs)
|
|
656
|
+
this.suspended = {since, graceTimer, publisherGoneProbes: 0}
|
|
657
|
+
console.warn("[STREAM] BLE link lost; stream suspended", {
|
|
658
|
+
streamId: entry.streamId,
|
|
659
|
+
graceMs: this.timings.glassesGraceMs,
|
|
660
|
+
})
|
|
661
|
+
this.fanout({
|
|
662
|
+
streamId: entry.streamId,
|
|
663
|
+
source: "coordinator",
|
|
664
|
+
status: LINK_STATUS.suspended,
|
|
665
|
+
data: {reason: LINK_STATUS.reason, graceMs: this.timings.glassesGraceMs, since},
|
|
666
|
+
})
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
private resumeLocked(): void {
|
|
670
|
+
const entry = this.current
|
|
671
|
+
const suspended = this.suspended
|
|
672
|
+
if (!entry || !suspended) return
|
|
673
|
+
clearTimeout(suspended.graceTimer)
|
|
674
|
+
this.suspended = null
|
|
675
|
+
const suspendedMs = Date.now() - suspended.since
|
|
676
|
+
console.info("[STREAM] BLE link back; stream resumed", {streamId: entry.streamId, suspendedMs})
|
|
677
|
+
// Resume and heartbeat immediately: the glasses watchdog has been running
|
|
678
|
+
// the whole time, so the first keep-alive must not wait another interval.
|
|
679
|
+
this.lifecycle?.setActive(true)
|
|
680
|
+
this.lifecycle?.tickNow()
|
|
681
|
+
// A resumed session is a fresh status baseline for subscribers.
|
|
682
|
+
this.lastFanoutSignature = null
|
|
683
|
+
this.fanout({
|
|
684
|
+
streamId: entry.streamId,
|
|
685
|
+
source: "coordinator",
|
|
686
|
+
status: LINK_STATUS.resumed,
|
|
687
|
+
data: {reason: LINK_STATUS.reason, suspendedMs},
|
|
688
|
+
})
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
private onGraceExpired(streamId: string): void {
|
|
692
|
+
if (this.current?.streamId !== streamId || !this.suspended) return
|
|
693
|
+
this.failSuspended(streamId, "glasses_disconnected", {publisherGone: false})
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
/**
|
|
697
|
+
* End a suspended stream. `publisherGone` distinguishes "glasses are off and
|
|
698
|
+
* Cloudflare confirms nothing is publishing" from "grace ran out with the
|
|
699
|
+
* publisher possibly still alive" — miniapps word the two differently.
|
|
700
|
+
*/
|
|
701
|
+
private failSuspended(streamId: string, reason: string, detail: {publisherGone: boolean}): void {
|
|
702
|
+
this.fanout({
|
|
703
|
+
streamId,
|
|
704
|
+
source: "coordinator",
|
|
705
|
+
status: "error",
|
|
706
|
+
data: {reason: LINK_STATUS.reason, teardownReason: reason, ...detail},
|
|
707
|
+
})
|
|
708
|
+
void this.runExclusive(async () => {
|
|
709
|
+
if (this.current?.streamId !== streamId) return
|
|
710
|
+
await this.teardownLocked(reason)
|
|
711
|
+
})
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
private flushPendingBleStop(): void {
|
|
715
|
+
const pending = this.pendingBleStop
|
|
716
|
+
if (!pending) return
|
|
717
|
+
this.pendingBleStop = null
|
|
718
|
+
console.info("[STREAM] BLE link back; sending deferred stopStream", pending)
|
|
719
|
+
void BluetoothSdk.stopStream()
|
|
720
|
+
.catch((err) => console.warn("[STREAM] deferred stopStream failed:", err))
|
|
721
|
+
.finally(() => this.detachLinkIfIdle())
|
|
722
|
+
}
|
|
723
|
+
|
|
546
724
|
// ===========================================================================
|
|
547
725
|
// Internal
|
|
548
726
|
// ===========================================================================
|
|
@@ -552,7 +730,11 @@ export class PhoneStreamCoordinator {
|
|
|
552
730
|
return `phone-${prefix}-${Date.now().toString(36)}-${this.idCounter}`
|
|
553
731
|
}
|
|
554
732
|
|
|
733
|
+
/** BLE keep-alives. StreamLifecycleController uses BgTimer so they survive MentraOS backgrounding. */
|
|
555
734
|
private startLifecycle(streamId: string): void {
|
|
735
|
+
// A new stream supersedes any deferred stop for the previous one.
|
|
736
|
+
this.pendingBleStop = null
|
|
737
|
+
this.attachLink()
|
|
556
738
|
this.lifecycle?.dispose()
|
|
557
739
|
const ctrl = new StreamLifecycleController(
|
|
558
740
|
{
|
|
@@ -631,6 +813,20 @@ export class PhoneStreamCoordinator {
|
|
|
631
813
|
status: status.isConnected ? "connected" : "disconnected",
|
|
632
814
|
data: status as unknown as Record<string, unknown>,
|
|
633
815
|
})
|
|
816
|
+
// While the BLE link is down, Cloudflare is the only witness to the
|
|
817
|
+
// publisher. Two consecutive "nobody is publishing" probes mean the
|
|
818
|
+
// glasses are off, not merely out of Bluetooth range — stop waiting.
|
|
819
|
+
if (this.suspended) {
|
|
820
|
+
if (status.isConnected) {
|
|
821
|
+
this.suspended.publisherGoneProbes = 0
|
|
822
|
+
} else {
|
|
823
|
+
this.suspended.publisherGoneProbes += 1
|
|
824
|
+
if (this.suspended.publisherGoneProbes >= this.timings.suspendedPublisherGoneProbes) {
|
|
825
|
+
this.failSuspended(entry.streamId, "glasses_disconnected_publisher_gone", {publisherGone: true})
|
|
826
|
+
keepPolling = false
|
|
827
|
+
}
|
|
828
|
+
}
|
|
829
|
+
}
|
|
634
830
|
// webrtc mode readiness: first "connected" means WHEP playback is
|
|
635
831
|
// available (WebRTC playback follows the ingest directly; there is no
|
|
636
832
|
// manifest to probe).
|
|
@@ -799,12 +995,28 @@ export class PhoneStreamCoordinator {
|
|
|
799
995
|
this.lastFanoutSignature = null
|
|
800
996
|
this.resolvedConfigForwarded = false
|
|
801
997
|
|
|
998
|
+
if (this.suspended) {
|
|
999
|
+
clearTimeout(this.suspended.graceTimer)
|
|
1000
|
+
this.suspended = null
|
|
1001
|
+
}
|
|
1002
|
+
|
|
802
1003
|
// Dispose the lifecycle controller immediately so it doesn't fire one
|
|
803
1004
|
// more keep-alive against a stream we're tearing down. The transition
|
|
804
1005
|
// lock guarantees no new lifecycle is started concurrently.
|
|
805
1006
|
this.lifecycle?.dispose()
|
|
806
1007
|
this.lifecycle = null
|
|
807
1008
|
|
|
1009
|
+
// With the link down a BLE write can only fail (and hold the transition
|
|
1010
|
+
// lock for the native timeout). Defer it to the next reconnect instead.
|
|
1011
|
+
const linkUp = this.linkSource.isConnected()
|
|
1012
|
+
if (sendBleStop && !linkUp) {
|
|
1013
|
+
this.pendingBleStop = {streamId: entry.streamId}
|
|
1014
|
+
console.warn("[STREAM] BLE link down during teardown; stopStream deferred", {
|
|
1015
|
+
streamId: entry.streamId,
|
|
1016
|
+
reason,
|
|
1017
|
+
})
|
|
1018
|
+
}
|
|
1019
|
+
|
|
808
1020
|
if (entry.kind === "managed") {
|
|
809
1021
|
if (entry.cloudflareTimer) clearTimeout(entry.cloudflareTimer)
|
|
810
1022
|
if (entry.hlsTimer) clearInterval(entry.hlsTimer)
|
|
@@ -816,7 +1028,7 @@ export class PhoneStreamCoordinator {
|
|
|
816
1028
|
}
|
|
817
1029
|
|
|
818
1030
|
try {
|
|
819
|
-
if (sendBleStop) {
|
|
1031
|
+
if (sendBleStop && linkUp) {
|
|
820
1032
|
await BluetoothSdk.stopStream()
|
|
821
1033
|
}
|
|
822
1034
|
} catch (err) {
|
|
@@ -827,6 +1039,7 @@ export class PhoneStreamCoordinator {
|
|
|
827
1039
|
// Only clear if we're still the active entry (defensive — runExclusive
|
|
828
1040
|
// serializes us, so this should always be true).
|
|
829
1041
|
if (this.current === entry) this.current = null
|
|
1042
|
+
this.detachLinkIfIdle()
|
|
830
1043
|
|
|
831
1044
|
if (entry.kind === "managed") {
|
|
832
1045
|
// Start remote cleanup only after the publisher has stopped, but do not
|