@mentra/engine 3.2.1-dev.277 → 3.2.1-dev.279
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/generated/releaseMetadata.js +5 -5
- package/build/generated/releaseMetadata.js.map +1 -1
- package/build/services/AcsMeetingService.d.ts +22 -6
- package/build/services/AcsMeetingService.d.ts.map +1 -1
- package/build/services/AcsMeetingService.js +39 -20
- package/build/services/AcsMeetingService.js.map +1 -1
- package/build/services/CallGainSweep.d.ts +97 -0
- package/build/services/CallGainSweep.d.ts.map +1 -0
- package/build/services/CallGainSweep.js +249 -0
- package/build/services/CallGainSweep.js.map +1 -0
- package/build/services/GlassesMicProbe.d.ts +2 -0
- package/build/services/GlassesMicProbe.d.ts.map +1 -1
- package/build/services/GlassesMicProbe.js +20 -19
- package/build/services/GlassesMicProbe.js.map +1 -1
- package/build/services/LocalMiniappRuntime.d.ts +18 -0
- package/build/services/LocalMiniappRuntime.d.ts.map +1 -1
- package/build/services/LocalMiniappRuntime.js +131 -1
- package/build/services/LocalMiniappRuntime.js.map +1 -1
- package/build/services/MicSessionManager.d.ts +83 -0
- package/build/services/MicSessionManager.d.ts.map +1 -0
- package/build/services/MicSessionManager.js +198 -0
- package/build/services/MicSessionManager.js.map +1 -0
- package/build/services/MicStateCoordinator.d.ts +48 -5
- package/build/services/MicStateCoordinator.d.ts.map +1 -1
- package/build/services/MicStateCoordinator.js +91 -10
- package/build/services/MicStateCoordinator.js.map +1 -1
- package/build/services/micPolicy.d.ts +84 -0
- package/build/services/micPolicy.d.ts.map +1 -0
- package/build/services/micPolicy.js +82 -0
- package/build/services/micPolicy.js.map +1 -0
- package/build/stores/bluetoothSettingKeys.d.ts.map +1 -1
- package/build/stores/bluetoothSettingKeys.js +3 -0
- package/build/stores/bluetoothSettingKeys.js.map +1 -1
- package/build/stores/settings.d.ts +5 -0
- package/build/stores/settings.d.ts.map +1 -1
- package/build/stores/settings.js +62 -0
- package/build/stores/settings.js.map +1 -1
- package/build/utils/pcm16.d.ts +20 -0
- package/build/utils/pcm16.d.ts.map +1 -1
- package/build/utils/pcm16.js +32 -1
- package/build/utils/pcm16.js.map +1 -1
- package/package.json +8 -8
- package/src/generated/releaseMetadata.ts +5 -5
- package/src/services/AcsMeetingService.ts +45 -20
- package/src/services/CallGainSweep.ts +318 -0
- package/src/services/GlassesMicProbe.ts +20 -17
- package/src/services/LocalMiniappRuntime.ts +152 -3
- package/src/services/MicSessionManager.ts +255 -0
- package/src/services/MicStateCoordinator.ts +91 -10
- package/src/services/micPolicy.ts +129 -0
- package/src/stores/bluetoothSettingKeys.ts +3 -0
- package/src/stores/settings.ts +66 -0
- package/src/utils/pcm16.ts +43 -1
|
@@ -46,6 +46,8 @@ import type {DisplayPayload} from "./LocalDisplayManager"
|
|
|
46
46
|
import headingService from "./HeadingService"
|
|
47
47
|
import localSttFallbackCoordinator from "./LocalSttFallbackCoordinator"
|
|
48
48
|
import micStateCoordinator from "./MicStateCoordinator"
|
|
49
|
+
import micSessionManager, {MIC_SOURCE_CONFLICT, type MicSession} from "./MicSessionManager"
|
|
50
|
+
import {ENGINE_ONLY_USE_CASES, MIC_USE_CASES, VOICE_CALL_PACKAGES, type MicUseCase} from "./micPolicy"
|
|
49
51
|
import {BlobStore} from "./BlobStore"
|
|
50
52
|
import {CloudAudioSubscriptionSync} from "./CloudAudioSubscriptionSync"
|
|
51
53
|
import {phoneCameraFovCoordinator} from "./PhoneCameraFovCoordinator"
|
|
@@ -574,6 +576,14 @@ class LocalMiniappRuntime {
|
|
|
574
576
|
*/
|
|
575
577
|
private transcriptionHintsByApp = new Map<string, string[]>()
|
|
576
578
|
|
|
579
|
+
/**
|
|
580
|
+
* Microphone sessions taken through session.mic.acquire, by session id.
|
|
581
|
+
*
|
|
582
|
+
* Bookkeeping only, so a release can be matched to its owner; MicSessionManager holds the real
|
|
583
|
+
* leases and resolves them into hardware state.
|
|
584
|
+
*/
|
|
585
|
+
private micSessionsByApp = new Map<number, {packageName: string; session: MicSession}>()
|
|
586
|
+
|
|
577
587
|
/** Ping interval handle. */
|
|
578
588
|
private pingIntervalId: number | null = null
|
|
579
589
|
private foregroundProbeTimers: Map<string, number> = new Map()
|
|
@@ -980,6 +990,10 @@ class LocalMiniappRuntime {
|
|
|
980
990
|
public unregisterApp(packageName: string): void {
|
|
981
991
|
console.log(`${LOG_TAG}: unregisterApp(${packageName})`)
|
|
982
992
|
const releasedMicGateOverride = micStateCoordinator.clearMiniappGateOverrides(packageName)
|
|
993
|
+
// Backstop for a miniapp that crashed or was killed mid-call: the microphone profile and the
|
|
994
|
+
// PCM claim must not outlive the app that asked for them.
|
|
995
|
+
this.forgetMicSessions(packageName)
|
|
996
|
+
micSessionManager.releaseOwner(packageName)
|
|
983
997
|
this.clearForegroundProbe(packageName)
|
|
984
998
|
this.clearMiniappAuthRefresh(packageName)
|
|
985
999
|
this.clearMiniappAuthDeliveryRetry(packageName)
|
|
@@ -1259,6 +1273,12 @@ class LocalMiniappRuntime {
|
|
|
1259
1273
|
case MiniappRequestType.MIC_SET_LOUDNESS_GATE_ENABLED:
|
|
1260
1274
|
void this.handleMicSetLoudnessGateEnabled(packageName, payload, requestId)
|
|
1261
1275
|
break
|
|
1276
|
+
case MiniappRequestType.MIC_ACQUIRE:
|
|
1277
|
+
this.handleMicAcquire(packageName, payload, requestId)
|
|
1278
|
+
break
|
|
1279
|
+
case MiniappRequestType.MIC_RELEASE:
|
|
1280
|
+
this.handleMicRelease(packageName, payload, requestId)
|
|
1281
|
+
break
|
|
1262
1282
|
case MiniappRequestType.PING:
|
|
1263
1283
|
// SDK should handle this itself; reply PONG just in case
|
|
1264
1284
|
this.sendToMiniapp(packageName, {type: MiniappResponseType.PONG}, requestId)
|
|
@@ -3220,6 +3240,100 @@ class LocalMiniappRuntime {
|
|
|
3220
3240
|
}
|
|
3221
3241
|
}
|
|
3222
3242
|
|
|
3243
|
+
/**
|
|
3244
|
+
* session.mic.acquire — take a semantic microphone session.
|
|
3245
|
+
*
|
|
3246
|
+
* The miniapp names a use case; MicSessionManager and micPolicy decide what that means for the
|
|
3247
|
+
* hardware. This handler is only the gate: who is allowed to ask for what.
|
|
3248
|
+
*/
|
|
3249
|
+
private handleMicAcquire(packageName: string, payload: Record<string, unknown>, requestId?: string): void {
|
|
3250
|
+
const app = this.connectedApps.get(packageName)
|
|
3251
|
+
const hasMicPermission = app?.installedManifest?.permissions?.some((p) => p.type === "MICROPHONE")
|
|
3252
|
+
if (!hasMicPermission) {
|
|
3253
|
+
logPermissionNotDeclared(packageName, "MICROPHONE", "to acquire a microphone session", `{"type": "MICROPHONE"}`)
|
|
3254
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
3255
|
+
code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
|
|
3256
|
+
message: `MICROPHONE permission not declared in miniapp.json. Add {"type": "MICROPHONE"} to the "permissions" array.`,
|
|
3257
|
+
permission: "MICROPHONE",
|
|
3258
|
+
operation: MiniappRequestType.MIC_ACQUIRE,
|
|
3259
|
+
})
|
|
3260
|
+
return
|
|
3261
|
+
}
|
|
3262
|
+
|
|
3263
|
+
const source = payload.source
|
|
3264
|
+
const useCase = payload.useCase
|
|
3265
|
+
if (source !== "glasses" && source !== "phone") {
|
|
3266
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
3267
|
+
code: MiniappErrorCode.INVALID_ARGUMENT,
|
|
3268
|
+
message: `source must be "glasses" or "phone"`,
|
|
3269
|
+
})
|
|
3270
|
+
return
|
|
3271
|
+
}
|
|
3272
|
+
if (!MIC_USE_CASES.includes(useCase as MicUseCase)) {
|
|
3273
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
3274
|
+
code: MiniappErrorCode.INVALID_ARGUMENT,
|
|
3275
|
+
message: `useCase must be one of ${MIC_USE_CASES.join(", ")}`,
|
|
3276
|
+
})
|
|
3277
|
+
return
|
|
3278
|
+
}
|
|
3279
|
+
|
|
3280
|
+
// A voice call gets a microphone profile tuned for close-talk speech, so which app may ask for
|
|
3281
|
+
// one is a product decision, not a permission a manifest can grant itself.
|
|
3282
|
+
if (useCase === "voice_call" && !VOICE_CALL_PACKAGES.includes(packageName)) {
|
|
3283
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
3284
|
+
code: MiniappErrorCode.PERMISSION_DENIED,
|
|
3285
|
+
message: `${packageName} is not allowed to acquire a voice_call microphone session`,
|
|
3286
|
+
})
|
|
3287
|
+
return
|
|
3288
|
+
}
|
|
3289
|
+
// Diagnostic leases exist so the mic probe measures what users actually get. Handing one to a
|
|
3290
|
+
// miniapp would also hand it a glasses lease it could pair with a meeting join.
|
|
3291
|
+
if (ENGINE_ONLY_USE_CASES.includes(useCase as MicUseCase)) {
|
|
3292
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
3293
|
+
code: MiniappErrorCode.PERMISSION_DENIED,
|
|
3294
|
+
message: `the ${useCase} microphone use case is reserved for the Mentra App`,
|
|
3295
|
+
})
|
|
3296
|
+
return
|
|
3297
|
+
}
|
|
3298
|
+
|
|
3299
|
+
try {
|
|
3300
|
+
const session = micSessionManager.acquire({owner: packageName, source, useCase: useCase as MicUseCase})
|
|
3301
|
+
this.micSessionsByApp.set(session.id, {packageName, session})
|
|
3302
|
+
console.log(`${LOG_TAG}: mic_acquire #${session.id} ${useCase}/${source} (by ${packageName})`)
|
|
3303
|
+
this.sendResult(packageName, requestId, true, {sessionId: session.id})
|
|
3304
|
+
} catch (err) {
|
|
3305
|
+
const message = err instanceof Error ? err.message : "mic acquire error"
|
|
3306
|
+
console.warn(`${LOG_TAG}: mic_acquire failed for ${packageName}:`, message)
|
|
3307
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
3308
|
+
code: message.startsWith(MIC_SOURCE_CONFLICT) ? MiniappErrorCode.MIC_SOURCE_CONFLICT : MiniappErrorCode.INTERNAL,
|
|
3309
|
+
message,
|
|
3310
|
+
})
|
|
3311
|
+
}
|
|
3312
|
+
}
|
|
3313
|
+
|
|
3314
|
+
/** session.mic.acquire(...).release() — drop one session, if the caller owns it. */
|
|
3315
|
+
private handleMicRelease(packageName: string, payload: Record<string, unknown>, requestId?: string): void {
|
|
3316
|
+
const sessionId = payload.sessionId
|
|
3317
|
+
const entry = typeof sessionId === "number" ? this.micSessionsByApp.get(sessionId) : undefined
|
|
3318
|
+
if (!entry || entry.packageName !== packageName) {
|
|
3319
|
+
// Idempotent by design: a miniapp releasing twice, or releasing after unregister already
|
|
3320
|
+
// dropped its sessions, is not an error worth failing a teardown path over.
|
|
3321
|
+
this.sendResult(packageName, requestId, true)
|
|
3322
|
+
return
|
|
3323
|
+
}
|
|
3324
|
+
this.micSessionsByApp.delete(entry.session.id)
|
|
3325
|
+
entry.session.release()
|
|
3326
|
+
console.log(`${LOG_TAG}: mic_release #${entry.session.id} (by ${packageName})`)
|
|
3327
|
+
this.sendResult(packageName, requestId, true)
|
|
3328
|
+
}
|
|
3329
|
+
|
|
3330
|
+
/** Drop the bookkeeping for every session an app owns. The manager is the source of truth. */
|
|
3331
|
+
private forgetMicSessions(packageName: string): void {
|
|
3332
|
+
for (const [id, entry] of this.micSessionsByApp) {
|
|
3333
|
+
if (entry.packageName === packageName) this.micSessionsByApp.delete(id)
|
|
3334
|
+
}
|
|
3335
|
+
}
|
|
3336
|
+
|
|
3223
3337
|
/**
|
|
3224
3338
|
* session.system.scanQr — host camera overlay. Must not clear miniapp
|
|
3225
3339
|
* foreground; the host seam is responsible for presenting a Modal on top.
|
|
@@ -3749,6 +3863,18 @@ class LocalMiniappRuntime {
|
|
|
3749
3863
|
// asked too late to explain itself, and ACS fails the join outright without the camera.
|
|
3750
3864
|
if (!(await this.requireOsPermission(packageName, requestId, PermissionFeatures.CAMERA, "camera"))) return
|
|
3751
3865
|
if (!(await this.requireOsPermission(packageName, requestId, PermissionFeatures.MICROPHONE, "microphone"))) return
|
|
3866
|
+
// The wearer's voice rides a microphone session, not the meeting. An app allowed to make voice
|
|
3867
|
+
// calls must hold one before the sink starts, or it would get a call configured for dictation.
|
|
3868
|
+
// Other joiners are not required to: without a session the BLE LC3 uplink is simply not
|
|
3869
|
+
// selected and the audio rides the WHIP capture path, exactly as it did before.
|
|
3870
|
+
const glassesSession = micSessionManager.hasGlassesSession(packageName)
|
|
3871
|
+
if (VOICE_CALL_PACKAGES.includes(packageName) && !glassesSession) {
|
|
3872
|
+
this.sendResult(packageName, requestId, false, undefined, {
|
|
3873
|
+
code: MiniappErrorCode.MIC_SESSION_REQUIRED,
|
|
3874
|
+
message: "acquire a glasses microphone session before joining a meeting",
|
|
3875
|
+
})
|
|
3876
|
+
return
|
|
3877
|
+
}
|
|
3752
3878
|
const meetingUrl = typeof payload.meetingUrl === "string" ? payload.meetingUrl : ""
|
|
3753
3879
|
const token = typeof payload.token === "string" ? payload.token : ""
|
|
3754
3880
|
const displayName = typeof payload.displayName === "string" ? payload.displayName : undefined
|
|
@@ -3799,7 +3925,14 @@ class LocalMiniappRuntime {
|
|
|
3799
3925
|
})
|
|
3800
3926
|
const startedAt = Date.now()
|
|
3801
3927
|
try {
|
|
3802
|
-
const state = await this.joinSoftapMeeting(packageName, {
|
|
3928
|
+
const state = await this.joinSoftapMeeting(packageName, {
|
|
3929
|
+
meetingUrl,
|
|
3930
|
+
token,
|
|
3931
|
+
displayName,
|
|
3932
|
+
video,
|
|
3933
|
+
origin,
|
|
3934
|
+
glassesSession,
|
|
3935
|
+
})
|
|
3803
3936
|
softapTrace("meeting_join_result", {
|
|
3804
3937
|
packageName,
|
|
3805
3938
|
requestId: requestId ?? "none",
|
|
@@ -3827,6 +3960,7 @@ class LocalMiniappRuntime {
|
|
|
3827
3960
|
videoSource,
|
|
3828
3961
|
displayName,
|
|
3829
3962
|
origin,
|
|
3963
|
+
glassesSession,
|
|
3830
3964
|
...(video ? {video} : {}),
|
|
3831
3965
|
})
|
|
3832
3966
|
this.sendResult(packageName, requestId, true, state)
|
|
@@ -3855,7 +3989,14 @@ class LocalMiniappRuntime {
|
|
|
3855
3989
|
*/
|
|
3856
3990
|
private async joinSoftapMeeting(
|
|
3857
3991
|
packageName: string,
|
|
3858
|
-
args: {
|
|
3992
|
+
args: {
|
|
3993
|
+
meetingUrl: string
|
|
3994
|
+
token: string
|
|
3995
|
+
displayName?: string
|
|
3996
|
+
video?: AcsOutgoingVideo
|
|
3997
|
+
origin?: AcsCallOrigin
|
|
3998
|
+
glassesSession: boolean
|
|
3999
|
+
},
|
|
3859
4000
|
): Promise<MeetingState> {
|
|
3860
4001
|
// Reserve before awaiting retirement: a second Start or Cancel must see this request,
|
|
3861
4002
|
// including while it is waiting for its predecessor's native cleanup.
|
|
@@ -4015,7 +4156,14 @@ class LocalMiniappRuntime {
|
|
|
4015
4156
|
private async runSoftapAttempt(
|
|
4016
4157
|
attempt: SoftapAttempt,
|
|
4017
4158
|
previous: SoftapAttempt | null,
|
|
4018
|
-
args: {
|
|
4159
|
+
args: {
|
|
4160
|
+
meetingUrl: string
|
|
4161
|
+
token: string
|
|
4162
|
+
displayName?: string
|
|
4163
|
+
video?: AcsOutgoingVideo
|
|
4164
|
+
origin?: AcsCallOrigin
|
|
4165
|
+
glassesSession: boolean
|
|
4166
|
+
},
|
|
4019
4167
|
): Promise<MeetingState> {
|
|
4020
4168
|
const packageName = attempt.packageName
|
|
4021
4169
|
if (previous) {
|
|
@@ -4131,6 +4279,7 @@ class LocalMiniappRuntime {
|
|
|
4131
4279
|
videoSource: options.videoSource,
|
|
4132
4280
|
displayName: options.displayName,
|
|
4133
4281
|
origin: args.origin,
|
|
4282
|
+
glassesSession: args.glassesSession,
|
|
4134
4283
|
...(args.video ? {video: args.video} : {}),
|
|
4135
4284
|
}),
|
|
4136
4285
|
leaveMeeting: (pkg) => acsMeetingService.leave(pkg),
|
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MicSessionManager
|
|
3
|
+
*
|
|
4
|
+
* The engine's microphone ownership layer. Applications acquire a lease that
|
|
5
|
+
* says what they are doing; this class tracks the live leases, resolves them
|
|
6
|
+
* through `micPolicy`, and pushes the result to MicStateCoordinator.
|
|
7
|
+
*
|
|
8
|
+
* Everything above it is semantic ("I need a glasses microphone for a voice
|
|
9
|
+
* call"). Everything below it is hardware. Audio *sinks* — AcsMeetingService
|
|
10
|
+
* today, another transport later — are downstream of the PCM this produces and
|
|
11
|
+
* must not reach past it to the coordinator or the Bluetooth SDK.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import {Platform} from "react-native"
|
|
15
|
+
import BluetoothSdk from "@mentra/bluetooth-sdk/internal"
|
|
16
|
+
|
|
17
|
+
import {getCallGainSweep} from "./CallGainSweep"
|
|
18
|
+
import micStateCoordinator from "./MicStateCoordinator"
|
|
19
|
+
import {
|
|
20
|
+
resolveMicPolicy,
|
|
21
|
+
type MicPlatformCaps,
|
|
22
|
+
type MicSessionSpec,
|
|
23
|
+
type MicSource,
|
|
24
|
+
type MicUseCase,
|
|
25
|
+
} from "./micPolicy"
|
|
26
|
+
|
|
27
|
+
const LOG_TAG = "MIC_SESSION"
|
|
28
|
+
|
|
29
|
+
/** Raised when a second source is requested while another is already live. */
|
|
30
|
+
export const MIC_SOURCE_CONFLICT = "MIC_SOURCE_CONFLICT"
|
|
31
|
+
|
|
32
|
+
export interface MicSessionOptions {
|
|
33
|
+
/** Package name for a miniapp, or `engine:<name>` for an engine feature. */
|
|
34
|
+
owner: string
|
|
35
|
+
source: MicSource
|
|
36
|
+
useCase: MicUseCase
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export interface MicSession {
|
|
40
|
+
readonly id: number
|
|
41
|
+
readonly owner: string
|
|
42
|
+
readonly source: MicSource
|
|
43
|
+
readonly useCase: MicUseCase
|
|
44
|
+
/** Idempotent. Releasing a session that is already gone is a no-op. */
|
|
45
|
+
release(): void
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
interface LiveSession extends MicSessionSpec {
|
|
49
|
+
id: number
|
|
50
|
+
owner: string
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function defaultCaps(): MicPlatformCaps {
|
|
54
|
+
return {
|
|
55
|
+
// iOS has no `setMicSourcePin`, so it cannot promise the phone microphone
|
|
56
|
+
// stays shut, and `glassesLc3UplinkSupported` never selects the BLE LC3
|
|
57
|
+
// uplink there.
|
|
58
|
+
glassesPcmUplink: Platform.OS === "android" && typeof BluetoothSdk.setMicSourcePin === "function",
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
class MicSessionManager {
|
|
63
|
+
private static instance: MicSessionManager | null = null
|
|
64
|
+
|
|
65
|
+
private readonly sessions = new Map<number, LiveSession>()
|
|
66
|
+
private nextId = 1
|
|
67
|
+
private caps: MicPlatformCaps = defaultCaps()
|
|
68
|
+
/** Last pin we asked for, so a no-op change does not re-enter native. */
|
|
69
|
+
private pinned = false
|
|
70
|
+
/**
|
|
71
|
+
* One 15→14→15→13 walk per voice-call generation. Cleared when the last
|
|
72
|
+
* glasses voice_call session drops so the next join can measure again.
|
|
73
|
+
*/
|
|
74
|
+
private sweepFinished = false
|
|
75
|
+
/** One-shot A/B walk. Off now that 15/14/13 is measured; Super Mode can still start it. */
|
|
76
|
+
private sweepEnabled = false
|
|
77
|
+
|
|
78
|
+
private constructor() {}
|
|
79
|
+
|
|
80
|
+
public static getInstance(): MicSessionManager {
|
|
81
|
+
if (!MicSessionManager.instance) {
|
|
82
|
+
MicSessionManager.instance = new MicSessionManager()
|
|
83
|
+
}
|
|
84
|
+
return MicSessionManager.instance
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** Test seam. Production reads the platform once at construction. */
|
|
88
|
+
public setPlatformCaps(caps: MicPlatformCaps): void {
|
|
89
|
+
this.caps = caps
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Take a microphone lease.
|
|
94
|
+
*
|
|
95
|
+
* Throws [MIC_SOURCE_CONFLICT] when a different source is already live: only
|
|
96
|
+
* the glasses can be pinned, so silently mixing sources would leave one
|
|
97
|
+
* consumer reading a microphone it did not ask for. That is the failure ACS
|
|
98
|
+
* already guards against frame by frame.
|
|
99
|
+
*/
|
|
100
|
+
public acquire(options: MicSessionOptions): MicSession {
|
|
101
|
+
const conflicting = [...this.sessions.values()].find((s) => s.source !== options.source)
|
|
102
|
+
if (conflicting) {
|
|
103
|
+
throw new Error(
|
|
104
|
+
`${MIC_SOURCE_CONFLICT}: ${options.owner} asked for "${options.source}" while ` +
|
|
105
|
+
`${conflicting.owner} holds "${conflicting.source}"`,
|
|
106
|
+
)
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
const id = this.nextId++
|
|
110
|
+
this.sessions.set(id, {id, owner: options.owner, source: options.source, useCase: options.useCase})
|
|
111
|
+
console.log(`${LOG_TAG}: acquire #${id} ${options.owner} ${options.useCase}/${options.source}`)
|
|
112
|
+
this.recompute()
|
|
113
|
+
|
|
114
|
+
return {
|
|
115
|
+
id,
|
|
116
|
+
owner: options.owner,
|
|
117
|
+
source: options.source,
|
|
118
|
+
useCase: options.useCase,
|
|
119
|
+
release: () => this.releaseId(id),
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** Drop every lease an owner holds. The backstop for unregister / disconnect. */
|
|
124
|
+
public releaseOwner(owner: string): boolean {
|
|
125
|
+
let removed = false
|
|
126
|
+
for (const [id, session] of this.sessions) {
|
|
127
|
+
if (session.owner !== owner) continue
|
|
128
|
+
this.sessions.delete(id)
|
|
129
|
+
removed = true
|
|
130
|
+
}
|
|
131
|
+
if (removed) {
|
|
132
|
+
console.log(`${LOG_TAG}: released all sessions for ${owner}`)
|
|
133
|
+
this.recompute()
|
|
134
|
+
}
|
|
135
|
+
return removed
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* Whether an owner holds a glasses lease.
|
|
140
|
+
*
|
|
141
|
+
* Ownership only. Whether that lease has any hardware effect is
|
|
142
|
+
* [resolveMicPolicy]'s business, so on a platform without a glasses PCM
|
|
143
|
+
* uplink this still answers true while nothing is claimed.
|
|
144
|
+
*/
|
|
145
|
+
public hasGlassesSession(owner: string): boolean {
|
|
146
|
+
for (const session of this.sessions.values()) {
|
|
147
|
+
if (session.owner === owner && session.source === "glasses") return true
|
|
148
|
+
}
|
|
149
|
+
return false
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
public releaseAll(): void {
|
|
153
|
+
if (this.sessions.size === 0) {
|
|
154
|
+
this.stopSweep("released_all")
|
|
155
|
+
return
|
|
156
|
+
}
|
|
157
|
+
this.sessions.clear()
|
|
158
|
+
console.log(`${LOG_TAG}: released all sessions`)
|
|
159
|
+
this.recompute()
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
public cleanup(): void {
|
|
163
|
+
this.stopSweep("cleanup")
|
|
164
|
+
this.sessions.clear()
|
|
165
|
+
this.pinned = false
|
|
166
|
+
this.sweepFinished = false
|
|
167
|
+
MicSessionManager.instance = null
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/** Tests disable the auto-walk so acquire still means "policy 14". */
|
|
171
|
+
public setCallGainSweepEnabled(enabled: boolean): void {
|
|
172
|
+
this.sweepEnabled = enabled
|
|
173
|
+
if (!enabled) this.stopSweep("disabled")
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* Start or restart the 15→14→15→13 comparison on a live voice_call session.
|
|
178
|
+
* Super Mode uses this so a call already in progress can be measured.
|
|
179
|
+
*/
|
|
180
|
+
public startCallGainSweep(): boolean {
|
|
181
|
+
if (!this.hasVoiceCallGlasses()) {
|
|
182
|
+
console.warn(`${LOG_TAG}: CALL_GAIN_SWEEP ignored — no live voice_call glasses session`)
|
|
183
|
+
return false
|
|
184
|
+
}
|
|
185
|
+
if (micStateCoordinator.hasConfiguredMicTuning()) {
|
|
186
|
+
console.warn(
|
|
187
|
+
`${LOG_TAG}: CALL_GAIN_SWEEP Super Mode mic-tuning is set; reset it or the glasses will ignore the sweep`,
|
|
188
|
+
)
|
|
189
|
+
}
|
|
190
|
+
this.sweepFinished = false
|
|
191
|
+
return getCallGainSweep().restart(
|
|
192
|
+
(gain) => micStateCoordinator.setSessionMicTuning({gain}),
|
|
193
|
+
() => {
|
|
194
|
+
this.sweepFinished = true
|
|
195
|
+
this.recompute()
|
|
196
|
+
},
|
|
197
|
+
)
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
private releaseId(id: number): void {
|
|
201
|
+
if (!this.sessions.delete(id)) return
|
|
202
|
+
console.log(`${LOG_TAG}: release #${id}`)
|
|
203
|
+
this.recompute()
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
private hasVoiceCallGlasses(): boolean {
|
|
207
|
+
if (!this.caps.glassesPcmUplink) return false
|
|
208
|
+
for (const session of this.sessions.values()) {
|
|
209
|
+
if (session.useCase === "voice_call" && session.source === "glasses") return true
|
|
210
|
+
}
|
|
211
|
+
return false
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
private stopSweep(reason: string): void {
|
|
215
|
+
if (getCallGainSweep().isActive()) getCallGainSweep().stop(reason)
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
private recompute(): void {
|
|
219
|
+
const policy = resolveMicPolicy([...this.sessions.values()], this.caps)
|
|
220
|
+
const voiceCall = this.hasVoiceCallGlasses()
|
|
221
|
+
if (!voiceCall) {
|
|
222
|
+
this.stopSweep("session_ended")
|
|
223
|
+
this.sweepFinished = false
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
// Pin before claiming PCM, so the first frame the claim produces is
|
|
227
|
+
// already from the right microphone and the phone mic is never opened.
|
|
228
|
+
if (policy.pinGlasses !== this.pinned) {
|
|
229
|
+
this.pinned = policy.pinGlasses
|
|
230
|
+
void Promise.resolve(BluetoothSdk.setMicSourcePin?.(policy.pinGlasses ? "glasses" : null)).catch(
|
|
231
|
+
(error) => {
|
|
232
|
+
console.warn(`${LOG_TAG}: setMicSourcePin failed`, error)
|
|
233
|
+
},
|
|
234
|
+
)
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
micStateCoordinator.setSessionRequirement(policy.rawPcm)
|
|
238
|
+
|
|
239
|
+
if (getCallGainSweep().isActive()) {
|
|
240
|
+
const gain = getCallGainSweep().currentGain()
|
|
241
|
+
micStateCoordinator.setSessionMicTuning(gain == null ? policy.micTuning : {gain})
|
|
242
|
+
return
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
if (this.sweepEnabled && voiceCall && !this.sweepFinished) {
|
|
246
|
+
this.startCallGainSweep()
|
|
247
|
+
return
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
micStateCoordinator.setSessionMicTuning(policy.micTuning)
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
const micSessionManager = MicSessionManager.getInstance()
|
|
255
|
+
export default micSessionManager
|
|
@@ -6,11 +6,17 @@
|
|
|
6
6
|
* Local miniapps subscribe to audio_chunk / transcription streams.
|
|
7
7
|
* This coordinator pushes the aggregate local requirement set to BluetoothSdk
|
|
8
8
|
* so the mic runs whenever at least one local consumer needs it.
|
|
9
|
+
*
|
|
10
|
+
* It also merges what the live microphone sessions require (MicSessionManager,
|
|
11
|
+
* via `setSessionRequirement` / `setSessionMicTuning`) with the OS preferences
|
|
12
|
+
* the settings store derives. Applications go through MicSessionManager; this
|
|
13
|
+
* class is an engine implementation detail.
|
|
9
14
|
*/
|
|
10
15
|
|
|
11
16
|
import BluetoothSdk from "@mentra/bluetooth-sdk/internal"
|
|
12
17
|
|
|
13
18
|
import {createDebouncedPatchFlusher} from "../utils/debouncedPatch"
|
|
19
|
+
import type {MicTuningProfile} from "./micPolicy"
|
|
14
20
|
|
|
15
21
|
const LOG_TAG = "MIC_COORDINATOR"
|
|
16
22
|
|
|
@@ -24,6 +30,7 @@ interface GateOverride {
|
|
|
24
30
|
interface ConfiguredMicGates {
|
|
25
31
|
vadEnabled?: boolean | null
|
|
26
32
|
loudnessGateEnabled?: boolean | null
|
|
33
|
+
micTuning?: Record<string, number> | null
|
|
27
34
|
}
|
|
28
35
|
|
|
29
36
|
/** Mic-requirement flips are debounced (300ms) and merged into one BLE write
|
|
@@ -49,15 +56,28 @@ class MicStateCoordinator {
|
|
|
49
56
|
private localWantsPcm = false
|
|
50
57
|
private localWantsLc3 = false
|
|
51
58
|
/**
|
|
52
|
-
* A live
|
|
59
|
+
* A live microphone session held through MicSessionManager — a voice call today.
|
|
53
60
|
*
|
|
54
61
|
* Tracked separately from the miniapp requirement because the two have independent lifetimes:
|
|
55
62
|
* the call miniapp does not subscribe to `audio_chunk`, and a captions miniapp that stops mid-call
|
|
56
63
|
* must not take the call's microphone with it.
|
|
57
64
|
*/
|
|
58
|
-
private
|
|
65
|
+
private sessionWantsPcm = false
|
|
59
66
|
private configuredVad: boolean | undefined
|
|
60
67
|
private configuredLoudnessGate: boolean | undefined
|
|
68
|
+
/** `mic_tuning` as the settings store last derived it: `super_mode ? desired : {}`. */
|
|
69
|
+
private configuredMicTuning: Record<string, number> = {}
|
|
70
|
+
/** Tuning required by the live sessions, resolved by micPolicy. */
|
|
71
|
+
private sessionMicTuning: MicTuningProfile | null = null
|
|
72
|
+
/**
|
|
73
|
+
* Latched once a session profile has actually been written.
|
|
74
|
+
*
|
|
75
|
+
* Before that, `mic_tuning` stays out of every patch, so a device that never
|
|
76
|
+
* runs a profile does not carry the key on unrelated mic writes. After it,
|
|
77
|
+
* the OS value keeps being restated, which is what stops a reconnect after a
|
|
78
|
+
* call from resurrecting the profile.
|
|
79
|
+
*/
|
|
80
|
+
private sessionMicTuningWritten = false
|
|
61
81
|
private readonly miniappVadOverrides = new Map<string, GateOverride>()
|
|
62
82
|
private readonly miniappLoudnessGateOverrides = new Map<string, GateOverride>()
|
|
63
83
|
private overrideSequence = 0
|
|
@@ -84,26 +104,69 @@ class MicStateCoordinator {
|
|
|
84
104
|
}
|
|
85
105
|
|
|
86
106
|
/**
|
|
87
|
-
* Claim or release raw PCM on behalf of
|
|
107
|
+
* Claim or release raw PCM on behalf of the live microphone sessions.
|
|
88
108
|
*
|
|
89
|
-
*
|
|
109
|
+
* MicSessionManager only. Applications acquire a session; they do not reach past it to here.
|
|
90
110
|
* Releasing is a claim release, not a mic shutdown: if a captions miniapp still wants PCM the
|
|
91
111
|
* microphone stays on, which is the whole reason this is a separate flag rather than a setter on
|
|
92
112
|
* the local requirement.
|
|
93
113
|
*/
|
|
94
|
-
public
|
|
95
|
-
if (this.
|
|
96
|
-
this.
|
|
97
|
-
console.log(`${LOG_TAG}:
|
|
114
|
+
public setSessionRequirement(pcm: boolean): void {
|
|
115
|
+
if (this.sessionWantsPcm === pcm) return
|
|
116
|
+
this.sessionWantsPcm = pcm
|
|
117
|
+
console.log(`${LOG_TAG}: session requirement updated — pcm=${pcm}`)
|
|
98
118
|
this.applyUnion()
|
|
99
119
|
}
|
|
100
120
|
|
|
121
|
+
/**
|
|
122
|
+
* Apply or drop the tuning the live sessions require.
|
|
123
|
+
*
|
|
124
|
+
* MicSessionManager only. Rides `applyUnion` so the profile and the PCM claim land in one
|
|
125
|
+
* debounced write, resolved at flush time: a session released inside the debounce window wins
|
|
126
|
+
* over the value that was queued.
|
|
127
|
+
*/
|
|
128
|
+
public setSessionMicTuning(profile: MicTuningProfile | null): void {
|
|
129
|
+
if (profile?.gain === this.sessionMicTuning?.gain) return
|
|
130
|
+
this.sessionMicTuning = profile
|
|
131
|
+
if (profile) this.sessionMicTuningWritten = true
|
|
132
|
+
// Nothing was ever written, so there is nothing to restore.
|
|
133
|
+
else if (!this.sessionMicTuningWritten) return
|
|
134
|
+
console.log(`${LOG_TAG}: session mic tuning ${profile ? JSON.stringify(profile) : "cleared"}`)
|
|
135
|
+
this.applyUnion()
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/** Last session profile queued, or null when the OS value is in force. */
|
|
139
|
+
public getSessionMicTuning(): MicTuningProfile | null {
|
|
140
|
+
return this.sessionMicTuning
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* Super Mode sliders outrank a session profile. A live override means a gain
|
|
145
|
+
* sweep would write to the coordinator and the glasses would ignore it.
|
|
146
|
+
*/
|
|
147
|
+
public hasConfiguredMicTuning(): boolean {
|
|
148
|
+
return Object.keys(this.configuredMicTuning).length > 0
|
|
149
|
+
}
|
|
150
|
+
|
|
101
151
|
/**
|
|
102
152
|
* Whether anything on this device needs a continuous raw-PCM timeline. Also the condition that
|
|
103
153
|
* forces hardware VAD off: a gate that drops silence turns a call into clipped half-words.
|
|
104
154
|
*/
|
|
105
155
|
private get wantsRawPcm(): boolean {
|
|
106
|
-
return this.localWantsPcm || this.
|
|
156
|
+
return this.localWantsPcm || this.sessionWantsPcm
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* The session profile, but only when it wins.
|
|
161
|
+
*
|
|
162
|
+
* A live Super Mode tuning value outranks it: that screen is how a profile's numbers get found
|
|
163
|
+
* on a real call in the first place. Emptiness is by key count — the settings store hands back a
|
|
164
|
+
* fresh `{}` every time, so reference checks would never match.
|
|
165
|
+
*/
|
|
166
|
+
private winningSessionMicTuning(): Record<string, number> | undefined {
|
|
167
|
+
if (!this.sessionMicTuning) return undefined
|
|
168
|
+
if (Object.keys(this.configuredMicTuning).length > 0) return undefined
|
|
169
|
+
return {...this.sessionMicTuning} as Record<string, number>
|
|
107
170
|
}
|
|
108
171
|
|
|
109
172
|
/**
|
|
@@ -168,6 +231,10 @@ class MicStateCoordinator {
|
|
|
168
231
|
: undefined,
|
|
169
232
|
loudnessGateEnabled:
|
|
170
233
|
typeof settings.loudness_gate_enabled === "boolean" ? settings.loudness_gate_enabled : undefined,
|
|
234
|
+
micTuning:
|
|
235
|
+
settings.mic_tuning && typeof settings.mic_tuning === "object"
|
|
236
|
+
? (settings.mic_tuning as Record<string, number>)
|
|
237
|
+
: undefined,
|
|
171
238
|
})
|
|
172
239
|
|
|
173
240
|
return this.applyActiveRuntimeOverrides(settings)
|
|
@@ -199,6 +266,12 @@ class MicStateCoordinator {
|
|
|
199
266
|
runtimeSettings.loudness_gate_enabled = loudnessOverride.enabled
|
|
200
267
|
}
|
|
201
268
|
|
|
269
|
+
// BES forgets mic_tuning on disconnect, so the on-connect replay is what
|
|
270
|
+
// puts a live session's profile back.
|
|
271
|
+
const sessionTuning = this.winningSessionMicTuning()
|
|
272
|
+
if (sessionTuning) runtimeSettings.mic_tuning = sessionTuning
|
|
273
|
+
else if (this.sessionMicTuningWritten) runtimeSettings.mic_tuning = this.configuredMicTuning
|
|
274
|
+
|
|
202
275
|
return runtimeSettings
|
|
203
276
|
}
|
|
204
277
|
|
|
@@ -235,6 +308,9 @@ class MicStateCoordinator {
|
|
|
235
308
|
if (configured.loudnessGateEnabled !== undefined) {
|
|
236
309
|
this.configuredLoudnessGate = configured.loudnessGateEnabled ?? undefined
|
|
237
310
|
}
|
|
311
|
+
if (configured.micTuning !== undefined) {
|
|
312
|
+
this.configuredMicTuning = configured.micTuning ?? {}
|
|
313
|
+
}
|
|
238
314
|
}
|
|
239
315
|
|
|
240
316
|
private overridesFor(gate: MicGate): Map<string, GateOverride> {
|
|
@@ -265,6 +341,10 @@ class MicStateCoordinator {
|
|
|
265
341
|
patch.loudness_gate_enabled = this.configuredLoudnessGate
|
|
266
342
|
}
|
|
267
343
|
|
|
344
|
+
const sessionTuning = this.winningSessionMicTuning()
|
|
345
|
+
if (sessionTuning) patch.mic_tuning = sessionTuning
|
|
346
|
+
else if (this.sessionMicTuningWritten) patch.mic_tuning = this.configuredMicTuning
|
|
347
|
+
|
|
268
348
|
return patch
|
|
269
349
|
}
|
|
270
350
|
|
|
@@ -274,7 +354,8 @@ class MicStateCoordinator {
|
|
|
274
354
|
public reset(): void {
|
|
275
355
|
this.localWantsPcm = false
|
|
276
356
|
this.localWantsLc3 = false
|
|
277
|
-
this.
|
|
357
|
+
this.sessionWantsPcm = false
|
|
358
|
+
this.sessionMicTuning = null
|
|
278
359
|
this.applyUnion()
|
|
279
360
|
}
|
|
280
361
|
|