@mentra/engine 3.2.1-dev.278 → 3.2.1-dev.282

package/src/services/MicSessionManager.ts

@@ -0,0 +1,258 @@
+/**
+ * MicSessionManager
+ *
+ * The engine's microphone ownership layer. Applications acquire a lease that
+ * says what they are doing; this class tracks the live leases, resolves them
+ * through `micPolicy`, and pushes the result to MicStateCoordinator.
+ *
+ * Everything above it is semantic ("I need a glasses microphone for a voice
+ * call"). Everything below it is hardware. Audio *sinks* — AcsMeetingService
+ * today, another transport later — are downstream of the PCM this produces and
+ * must not reach past it to the coordinator or the Bluetooth SDK.
+ */
+
+import {Platform} from "react-native"
+import BluetoothSdk from "@mentra/bluetooth-sdk/internal"
+
+import {getCallGainSweep} from "./CallGainSweep"
+import micStateCoordinator from "./MicStateCoordinator"
+import {
+  resolveMicPolicy,
+  type MicPlatformCaps,
+  type MicSessionSpec,
+  type MicSource,
+  type MicUseCase,
+} from "./micPolicy"
+
+const LOG_TAG = "MIC_SESSION"
+
+/** Raised when a second source is requested while another is already live. */
+export const MIC_SOURCE_CONFLICT = "MIC_SOURCE_CONFLICT"
+
+export interface MicSessionOptions {
+  /** Package name for a miniapp, or `engine:<name>` for an engine feature. */
+  owner: string
+  source: MicSource
+  useCase: MicUseCase
+}
+
+export interface MicSession {
+  readonly id: number
+  readonly owner: string
+  readonly source: MicSource
+  readonly useCase: MicUseCase
+  /** Idempotent. Releasing a session that is already gone is a no-op. */
+  release(): void
+}
+
+interface LiveSession extends MicSessionSpec {
+  id: number
+  owner: string
+}
+
+function defaultCaps(): MicPlatformCaps {
+  return {
+    // iOS has no `setMicSourcePin`, so it cannot promise the phone microphone
+    // stays shut, and `glassesLc3UplinkSupported` never selects the BLE LC3
+    // uplink there.
+    glassesPcmUplink: Platform.OS === "android" && typeof BluetoothSdk.setMicSourcePin === "function",
+  }
+}
+
+class MicSessionManager {
+  private static instance: MicSessionManager | null = null
+
+  private readonly sessions = new Map<number, LiveSession>()
+  private nextId = 1
+  private caps: MicPlatformCaps = defaultCaps()
+  /** Last pin we asked for, so a no-op change does not re-enter native. */
+  private pinned = false
+  /**
+   * One 15→14→15→13 walk per voice-call generation. Cleared when the last
+   * glasses voice_call session drops so the next join can measure again.
+   */
+  private sweepFinished = false
+  /** One-shot A/B walk. Off now that 15/14/13 is measured; Super Mode can still start it. */
+  private sweepEnabled = false
+
+  private constructor() {}
+
+  public static getInstance(): MicSessionManager {
+    if (!MicSessionManager.instance) {
+      MicSessionManager.instance = new MicSessionManager()
+    }
+    return MicSessionManager.instance
+  }
+
+  /** Test seam. Production reads the platform once at construction. */
+  public setPlatformCaps(caps: MicPlatformCaps): void {
+    this.caps = caps
+  }
+
+  /**
+   * Take a microphone lease.
+   *
+   * Throws [MIC_SOURCE_CONFLICT] when a different source is already live: only
+   * the glasses can be pinned, so silently mixing sources would leave one
+   * consumer reading a microphone it did not ask for. That is the failure ACS
+   * already guards against frame by frame.
+   */
+  public acquire(options: MicSessionOptions): MicSession {
+    const conflicting = [...this.sessions.values()].find((s) => s.source !== options.source)
+    if (conflicting) {
+      throw new Error(
+        `${MIC_SOURCE_CONFLICT}: ${options.owner} asked for "${options.source}" while ` +
+          `${conflicting.owner} holds "${conflicting.source}"`,
+      )
+    }
+
+    const id = this.nextId++
+    this.sessions.set(id, {id, owner: options.owner, source: options.source, useCase: options.useCase})
+    console.log(`${LOG_TAG}: acquire #${id} ${options.owner} ${options.useCase}/${options.source}`)
+    this.recompute()
+
+    return {
+      id,
+      owner: options.owner,
+      source: options.source,
+      useCase: options.useCase,
+      release: () => this.releaseId(id),
+    }
+  }
+
+  /** Drop every lease an owner holds. The backstop for unregister / disconnect. */
+  public releaseOwner(owner: string): boolean {
+    let removed = false
+    for (const [id, session] of this.sessions) {
+      if (session.owner !== owner) continue
+      this.sessions.delete(id)
+      removed = true
+    }
+    if (removed) {
+      console.log(`${LOG_TAG}: released all sessions for ${owner}`)
+      this.recompute()
+    }
+    return removed
+  }
+
+  /**
+   * Whether an owner holds a glasses lease.
+   *
+   * Ownership only. Whether that lease has any hardware effect is
+   * [resolveMicPolicy]'s business, so on a platform without a glasses PCM
+   * uplink this still answers true while nothing is claimed.
+   */
+  public hasGlassesSession(owner: string): boolean {
+    for (const session of this.sessions.values()) {
+      if (session.owner === owner && session.source === "glasses") return true
+    }
+    return false
+  }
+
+  public releaseAll(): void {
+    if (this.sessions.size === 0) {
+      this.stopSweep("released_all")
+      return
+    }
+    this.sessions.clear()
+    console.log(`${LOG_TAG}: released all sessions`)
+    this.recompute()
+  }
+
+  public cleanup(): void {
+    this.stopSweep("cleanup")
+    this.sessions.clear()
+    this.pinned = false
+    this.sweepFinished = false
+    MicSessionManager.instance = null
+  }
+
+  /** Tests disable the auto-walk so acquire still means "policy 14". */
+  public setCallGainSweepEnabled(enabled: boolean): void {
+    this.sweepEnabled = enabled
+    if (!enabled) this.stopSweep("disabled")
+  }
+
+  /**
+   * Start or restart the 15→14→15→13 comparison on a live voice_call session.
+   * Super Mode uses this so a call already in progress can be measured.
+   */
+  public startCallGainSweep(): boolean {
+    if (!this.hasVoiceCallGlasses()) {
+      console.warn(`${LOG_TAG}: CALL_GAIN_SWEEP ignored — no live voice_call glasses session`)
+      return false
+    }
+    if (micStateCoordinator.hasConfiguredMicTuning()) {
+      console.warn(
+        `${LOG_TAG}: CALL_GAIN_SWEEP Super Mode mic-tuning is set; reset it or the glasses will ignore the sweep`,
+      )
+    }
+    this.sweepFinished = false
+    return getCallGainSweep().restart(
+      (gain) => micStateCoordinator.setSessionMicTuning({gain}),
+      () => {
+        this.sweepFinished = true
+        this.recompute()
+      },
+    )
+  }
+
+  private releaseId(id: number): void {
+    if (!this.sessions.delete(id)) return
+    console.log(`${LOG_TAG}: release #${id}`)
+    this.recompute()
+  }
+
+  private hasVoiceCallGlasses(): boolean {
+    if (!this.caps.glassesPcmUplink) return false
+    for (const session of this.sessions.values()) {
+      if (session.useCase === "voice_call" && session.source === "glasses") return true
+    }
+    return false
+  }
+
+  private stopSweep(reason: string): void {
+    if (getCallGainSweep().isActive()) getCallGainSweep().stop(reason)
+  }
+
+  private recompute(): void {
+    const policy = resolveMicPolicy([...this.sessions.values()], this.caps)
+    const voiceCall = this.hasVoiceCallGlasses()
+    if (!voiceCall) {
+      this.stopSweep("session_ended")
+      this.sweepFinished = false
+    }
+
+    // Pin before claiming PCM, so the first frame the claim produces is
+    // already from the right microphone and the phone mic is never opened.
+    if (policy.pinGlasses !== this.pinned) {
+      this.pinned = policy.pinGlasses
+      void Promise.resolve(BluetoothSdk.setMicSourcePin?.(policy.pinGlasses ? "glasses" : null)).catch(
+        (error) => {
+          console.warn(`${LOG_TAG}: setMicSourcePin failed`, error)
+        },
+      )
+    }
+
+    micStateCoordinator.setSessionRequirement(policy.rawPcm)
+    // Barrier is the only echo control left once the PCM claim turns hardware VAD off, so it
+    // rides with the claim rather than with the tuning a sweep may be rewriting.
+    micStateCoordinator.setSessionLoudnessGate(policy.loudnessGate)
+
+    if (getCallGainSweep().isActive()) {
+      const gain = getCallGainSweep().currentGain()
+      micStateCoordinator.setSessionMicTuning(gain == null ? policy.micTuning : {gain})
+      return
+    }
+
+    if (this.sweepEnabled && voiceCall && !this.sweepFinished) {
+      this.startCallGainSweep()
+      return
+    }
+
+    micStateCoordinator.setSessionMicTuning(policy.micTuning)
+  }
+}
+
+const micSessionManager = MicSessionManager.getInstance()
+export default micSessionManager

package/src/services/MicStateCoordinator.ts

@@ -6,14 +6,29 @@
  * Local miniapps subscribe to audio_chunk / transcription streams.
  * This coordinator pushes the aggregate local requirement set to BluetoothSdk
  * so the mic runs whenever at least one local consumer needs it.
+ *
+ * It also merges what the live microphone sessions require (MicSessionManager,
+ * via `setSessionRequirement` / `setSessionMicTuning`) with the OS preferences
+ * the settings store derives. Applications go through MicSessionManager; this
+ * class is an engine implementation detail.
  */
 
 import BluetoothSdk from "@mentra/bluetooth-sdk/internal"
 
 import {createDebouncedPatchFlusher} from "../utils/debouncedPatch"
+import type {MicTuningProfile} from "./micPolicy"
 
 const LOG_TAG = "MIC_COORDINATOR"
 
+/** Field-wise compare, so a profile that moves a threshold without the gain still lands. */
+function sameProfile(a: MicTuningProfile | null, b: MicTuningProfile | null): boolean {
+  if (a === b) return true
+  if (!a || !b) return false
+  const keys = new Set([...Object.keys(a), ...Object.keys(b)]) as Set<keyof MicTuningProfile>
+  for (const key of keys) if (a[key] !== b[key]) return false
+  return true
+}
+
 type MicGate = "vad" | "loudnessGate"
 
 interface GateOverride {
@@ -24,6 +39,7 @@ interface GateOverride {
 interface ConfiguredMicGates {
   vadEnabled?: boolean | null
   loudnessGateEnabled?: boolean | null
+  micTuning?: Record<string, number> | null
 }
 
 /** Mic-requirement flips are debounced (300ms) and merged into one BLE write
@@ -34,6 +50,10 @@ const flushMicRequirementsPatch = createDebouncedPatchFlusher<Record<string, unk
     // gate override during the debounce window, and a captured stale value
     // must not win after that lifecycle transition.
     const runtimePatch = micStateCoordinator.applyEffectiveGatePolicy(patch)
+    // The merged patch, not the intent that produced it. Everything above this line is a
+    // preference; this is the only record of what the glasses were actually told, and the three
+    // keys a call depends on (VAD off, Barrier on, the tuning) are only decided here at flush.
+    console.log(`${LOG_TAG}: write`, runtimePatch)
     void Promise.resolve(BluetoothSdk.updateBluetoothSettings(runtimePatch)).catch((err) => {
       console.error(`${LOG_TAG}: failed to apply mic requirements:`, err)
     })
@@ -49,15 +69,32 @@ class MicStateCoordinator {
   private localWantsPcm = false
   private localWantsLc3 = false
   /**
-   * A live ACS call taking the wearer's voice off the glasses over BLE LC3.
+   * A live microphone session held through MicSessionManager — a voice call today.
    *
    * Tracked separately from the miniapp requirement because the two have independent lifetimes:
    * the call miniapp does not subscribe to `audio_chunk`, and a captions miniapp that stops mid-call
    * must not take the call's microphone with it.
    */
-  private callWantsPcm = false
+  private sessionWantsPcm = false
   private configuredVad: boolean | undefined
   private configuredLoudnessGate: boolean | undefined
+  /** `mic_tuning` as the settings store last derived it: `super_mode ? desired : {}`. */
+  private configuredMicTuning: Record<string, number> = {}
+  /** Tuning required by the live sessions, resolved by micPolicy. */
+  private sessionMicTuning: MicTuningProfile | null = null
+  /**
+   * Latched once a session profile has actually been written.
+   *
+   * Before that, `mic_tuning` stays out of every patch, so a device that never
+   * runs a profile does not carry the key on unrelated mic writes. After it,
+   * the OS value keeps being restated, which is what stops a reconnect after a
+   * call from resurrecting the profile.
+   */
+  private sessionMicTuningWritten = false
+  /** Barrier required by the live sessions, or null when they have no opinion. */
+  private sessionLoudnessGate: boolean | null = null
+  /** Latched like the tuning, so the OS value is restated once a session has moved it. */
+  private sessionLoudnessGateWritten = false
   private readonly miniappVadOverrides = new Map<string, GateOverride>()
   private readonly miniappLoudnessGateOverrides = new Map<string, GateOverride>()
   private overrideSequence = 0
@@ -84,26 +121,104 @@ class MicStateCoordinator {
   }
 
   /**
-   * Claim or release raw PCM on behalf of an active call.
+   * Claim or release raw PCM on behalf of the live microphone sessions.
    *
-   * Called by AcsMeetingService around a call whose uplink is the glasses microphone over BLE LC3.
+   * MicSessionManager only. Applications acquire a session; they do not reach past it to here.
    * Releasing is a claim release, not a mic shutdown: if a captions miniapp still wants PCM the
    * microphone stays on, which is the whole reason this is a separate flag rather than a setter on
    * the local requirement.
    */
-  public setCallRequirement(pcm: boolean): void {
-    if (this.callWantsPcm === pcm) return
-    this.callWantsPcm = pcm
-    console.log(`${LOG_TAG}: call requirement updated — pcm=${pcm}`)
+  public setSessionRequirement(pcm: boolean): void {
+    if (this.sessionWantsPcm === pcm) return
+    this.sessionWantsPcm = pcm
+    console.log(`${LOG_TAG}: session requirement updated — pcm=${pcm}`)
+    this.applyUnion()
+  }
+
+  /**
+   * Apply or drop the tuning the live sessions require.
+   *
+   * MicSessionManager only. Rides `applyUnion` so the profile and the PCM claim land in one
+   * debounced write, resolved at flush time: a session released inside the debounce window wins
+   * over the value that was queued.
+   */
+  public setSessionMicTuning(profile: MicTuningProfile | null): void {
+    if (sameProfile(profile, this.sessionMicTuning)) return
+    this.sessionMicTuning = profile
+    if (profile) this.sessionMicTuningWritten = true
+    // Nothing was ever written, so there is nothing to restore.
+    else if (!this.sessionMicTuningWritten) return
+    console.log(`${LOG_TAG}: session mic tuning ${profile ? JSON.stringify(profile) : "cleared"}`)
+    this.applyUnion()
+  }
+
+  /** Last session profile queued, or null when the OS value is in force. */
+  public getSessionMicTuning(): MicTuningProfile | null {
+    return this.sessionMicTuning
+  }
+
+  /** Last session Barrier queued, or null when the OS value is in force. For call logging. */
+  public getSessionLoudnessGate(): boolean | null {
+    return this.sessionLoudnessGate
+  }
+
+  /**
+   * Run or stop the center-mic loudness gate on behalf of the live sessions.
+   *
+   * MicSessionManager only, and the counterpart to the PCM claim rather than a second opinion on
+   * it: raw PCM turns hardware VAD off, which leaves Barrier as the only thing standing between
+   * the far end and its own echo. Unlike VAD this never stops the stream, so the worst a wrong
+   * threshold costs is a zeroed frame instead of a dropped word.
+   */
+  public setSessionLoudnessGate(enabled: boolean | null): void {
+    if (this.sessionLoudnessGate === enabled) return
+    this.sessionLoudnessGate = enabled
+    if (enabled !== null) this.sessionLoudnessGateWritten = true
+    else if (!this.sessionLoudnessGateWritten) return
+    console.log(`${LOG_TAG}: session loudness gate ${enabled === null ? "cleared" : enabled}`)
     this.applyUnion()
   }
 
+  /**
+   * Super Mode sliders outrank a session profile. A live override means a gain
+   * sweep would write to the coordinator and the glasses would ignore it.
+   */
+  public hasConfiguredMicTuning(): boolean {
+    return Object.keys(this.configuredMicTuning).length > 0
+  }
+
   /**
    * Whether anything on this device needs a continuous raw-PCM timeline. Also the condition that
    * forces hardware VAD off: a gate that drops silence turns a call into clipped half-words.
    */
   private get wantsRawPcm(): boolean {
-    return this.localWantsPcm || this.callWantsPcm
+    return this.localWantsPcm || this.sessionWantsPcm
+  }
+
+  /**
+   * The session profile, but only when it wins.
+   *
+   * A live Super Mode tuning value outranks it: that screen is how a profile's numbers get found
+   * on a real call in the first place. Emptiness is by key count — the settings store hands back a
+   * fresh `{}` every time, so reference checks would never match.
+   */
+  private winningSessionMicTuning(): Record<string, number> | undefined {
+    if (!this.sessionMicTuning) return undefined
+    if (Object.keys(this.configuredMicTuning).length > 0) return undefined
+    return {...this.sessionMicTuning} as Record<string, number>
+  }
+
+  /**
+   * The session's Barrier, but only when it wins.
+   *
+   * Suppressed by a live Super Mode tuning value for the same reason the gain is: that screen is
+   * the manual override, and a gate running against hand-entered thresholds is the one case where
+   * the session's numbers are the wrong ones.
+   */
+  private winningSessionLoudnessGate(): boolean | undefined {
+    if (this.sessionLoudnessGate === null) return undefined
+    if (Object.keys(this.configuredMicTuning).length > 0) return undefined
+    return this.sessionLoudnessGate
   }
 
   /**
@@ -168,6 +283,10 @@ class MicStateCoordinator {
           : undefined,
       loudnessGateEnabled:
         typeof settings.loudness_gate_enabled === "boolean" ? settings.loudness_gate_enabled : undefined,
+      micTuning:
+        settings.mic_tuning && typeof settings.mic_tuning === "object"
+          ? (settings.mic_tuning as Record<string, number>)
+          : undefined,
     })
 
     return this.applyActiveRuntimeOverrides(settings)
@@ -195,10 +314,19 @@ class MicStateCoordinator {
     } else if (vadOverride) {
       runtimeSettings.voice_activity_detection_enabled = vadOverride.enabled
     }
+    const sessionGate = this.winningSessionLoudnessGate()
     if (loudnessOverride) {
       runtimeSettings.loudness_gate_enabled = loudnessOverride.enabled
+    } else if (sessionGate !== undefined) {
+      runtimeSettings.loudness_gate_enabled = sessionGate
     }
 
+    // BES forgets mic_tuning on disconnect, so the on-connect replay is what
+    // puts a live session's profile back.
+    const sessionTuning = this.winningSessionMicTuning()
+    if (sessionTuning) runtimeSettings.mic_tuning = sessionTuning
+    else if (this.sessionMicTuningWritten) runtimeSettings.mic_tuning = this.configuredMicTuning
+
     return runtimeSettings
   }
 
@@ -235,6 +363,9 @@ class MicStateCoordinator {
     if (configured.loudnessGateEnabled !== undefined) {
       this.configuredLoudnessGate = configured.loudnessGateEnabled ?? undefined
     }
+    if (configured.micTuning !== undefined) {
+      this.configuredMicTuning = configured.micTuning ?? {}
+    }
   }
 
   private overridesFor(gate: MicGate): Map<string, GateOverride> {
@@ -260,11 +391,21 @@ class MicStateCoordinator {
     else if (vadOverride) patch.voice_activity_detection_enabled = vadOverride.enabled
     else if (this.configuredVad !== undefined) patch.voice_activity_detection_enabled = this.configuredVad
 
+    const sessionGate = this.winningSessionLoudnessGate()
     if (loudnessOverride) patch.loudness_gate_enabled = loudnessOverride.enabled
+    else if (sessionGate !== undefined) patch.loudness_gate_enabled = sessionGate
     else if (this.configuredLoudnessGate !== undefined) {
       patch.loudness_gate_enabled = this.configuredLoudnessGate
+    } else if (this.sessionLoudnessGateWritten) {
+      // A session moved it and the device carries no preference, so restate the product default
+      // rather than leaving the call's gate running after it ended.
+      patch.loudness_gate_enabled = false
     }
 
+    const sessionTuning = this.winningSessionMicTuning()
+    if (sessionTuning) patch.mic_tuning = sessionTuning
+    else if (this.sessionMicTuningWritten) patch.mic_tuning = this.configuredMicTuning
+
     return patch
   }
 
@@ -274,7 +415,9 @@ class MicStateCoordinator {
   public reset(): void {
     this.localWantsPcm = false
     this.localWantsLc3 = false
-    this.callWantsPcm = false
+    this.sessionWantsPcm = false
+    this.sessionMicTuning = null
+    this.sessionLoudnessGate = null
     this.applyUnion()
   }