@mentra/engine 3.2.1-dev.277 → 3.2.1-dev.279

package/src/services/AcsMeetingService.ts

@@ -9,9 +9,10 @@ import {Platform} from "react-native"
 import BluetoothSdk from "@mentra/bluetooth-sdk/internal"
 
 import audioPlaybackService from "./AudioPlaybackService"
+import {getCallGainSweep} from "./CallGainSweep"
 import micStateCoordinator from "./MicStateCoordinator"
 import {SETTINGS, useSettingsStore} from "../stores/settings"
-import {Pcm16LevelMeter} from "../utils/pcm16"
+import {Pcm16LevelMeter, pcm16WindowStats} from "../utils/pcm16"
 import {softapTrace, softapTraceFailure, softapTraceId} from "../utils/softapTrace"
 import {ACS_CALL_MIC, type AcsAudioSource, type ResolvedAudioSource, type SourceReason} from "./acsAudioSource"
 import type {SoftapProgress} from "./SoftapCallTransport"
@@ -204,8 +205,13 @@ export function glassesLc3UplinkSupported(args: {
   audioSource: AcsAudioSource
   hasPushOutgoingPcm: boolean
   platform: string
+  /** Somebody holds a glasses microphone session through MicSessionManager. */
+  glassesSession: boolean
 }): boolean {
   if (!(softapBleLc3UplinkForTests ?? SOFTAP_BLE_LC3_UPLINK)) return false
+  // The microphone belongs to whoever leased it. Without a lease nothing has pinned the glasses
+  // or claimed raw PCM, so this call has no wearer audio to read off them.
+  if (!args.glassesSession) return false
   // iOS has no `setMicSourcePin` yet, so it cannot promise the phone microphone stays shut.
   if (args.platform !== "android") return false
   // WHEP audio comes back from Cloudflare already mixed into the subscribed track; there is no
@@ -537,6 +543,8 @@ const GLASSES_MIC_SOURCE = "glasses"
 const GLASSES_MIC_GRACE_MS = 1000
 /** Cadence of the uplink health line, matching the native P8 ladder. */
 const MIC_UPLINK_LOG_INTERVAL_MS = 5000
+/** During a gain sweep, one-second windows so each 25 s phase has enough speech samples. */
+const MIC_UPLINK_SWEEP_LOG_INTERVAL_MS = 1000
 /** A 50 ms LC3 frame arriving more than this late is a missed beat, not jitter. */
 const MIC_GAP_WARN_MS = 90
 
@@ -972,6 +980,12 @@ class AcsMeetingService {
       displayName?: string
       video?: AcsOutgoingVideo
       origin?: AcsCallOrigin
+      /**
+       * Whether the caller holds a glasses microphone session. The uplink is the wearer's voice,
+       * so it is only taken off the glasses when somebody has actually claimed that microphone
+       * through MicSessionManager; this class never claims it itself.
+       */
+      glassesSession?: boolean
     },
   ): Promise<MeetingState> {
     const native = getNative()
@@ -998,6 +1012,7 @@ class AcsMeetingService {
       audioSource: resolved.source,
       hasPushOutgoingPcm: typeof native.pushOutgoingPcm === "function",
       platform: Platform.OS,
+      glassesSession: args.glassesSession ?? false,
     })
     this.micTransport = lc3Uplink ? "ble-lc3" : resolved.source === "phone" ? "phone" : "whip"
     this.bindNative(native, packageName)
@@ -1287,11 +1302,13 @@ class AcsMeetingService {
   /**
    * Start forwarding the glasses microphone into ACS for this call.
    *
-   * Order matters and is the whole point: pin the Bluetooth SDK to the glasses *before* asking it
-   * for PCM, so the first frame the mic requirement produces is already from the right source and
-   * the phone microphone is never opened even for one buffer. `generation` is captured by the
-   * listener so a frame that lands after this call ended is dropped rather than pushed at a native
-   * session that has left the meeting.
+   * This class is a sink. The microphone is pinned, claimed and tuned by MicSessionManager on
+   * behalf of whoever leased it; all that happens here is reading the PCM that produces. Reaching
+   * past that to the Bluetooth SDK or MicStateCoordinator would put hardware policy back in the
+   * hands of one audio consumer.
+   *
+   * `generation` is captured by the listener so a frame that lands after this call ended is
+   * dropped rather than pushed at a native session that has left the meeting.
    */
   private startGlassesMicUplink(generation: number): void {
     if (this.micTransport !== "ble-lc3") return
@@ -1310,11 +1327,7 @@ class AcsMeetingService {
     this.micLevel.take()
     this.lastMicLevel = null
     try {
-      void Promise.resolve(BluetoothSdk.setMicSourcePin?.(GLASSES_MIC_SOURCE)).catch((error) => {
-        console.warn("[AcsMeeting] pinning the glasses microphone failed", error)
-      })
       this.micUplinkActive = true
-      micStateCoordinator.setCallRequirement(true)
       this.micSub = BluetoothSdk.addListener("mic_pcm", (event: {pcm?: ArrayBuffer; sampleRate?: number; source?: string}) => {
         if (generation !== this.callGeneration) {
           this.micDropsStale += 1
@@ -1362,18 +1375,18 @@ class AcsMeetingService {
     }
   }
 
-  /** Release the microphone claims this call took. Safe to call when it never took them. */
+  /**
+   * Stop reading the glasses microphone. Safe to call when this call never started.
+   *
+   * Only the subscription is dropped. The microphone itself belongs to whoever leased it through
+   * MicSessionManager, and unsubscribing here before that owner releases is what keeps a normal
+   * hang-up from looking like the wearer's microphone disappearing.
+   */
   private stopGlassesMicUplink(): void {
     this.micSub?.remove()
     this.micSub = null
     if (this.micUplinkActive) {
       this.micUplinkActive = false
-      micStateCoordinator.setCallRequirement(false)
-      // Last, and unconditionally: while the pin is set no other consumer can pick a microphone,
-      // so leaving it behind would leave captions and the cloud uplink stuck on the glasses.
-      void Promise.resolve(BluetoothSdk.setMicSourcePin?.(null)).catch((error) => {
-        console.warn("[AcsMeeting] releasing the glasses microphone pin failed", error)
-      })
       console.log("[AcsMeeting] phase=glasses-mic-uplink-stop", {
         frames: this.micFramesForwarded,
         dropsStale: this.micDropsStale,
@@ -1413,17 +1426,29 @@ class AcsMeetingService {
   private logMicUplink(): void {
     const now = Date.now()
     const elapsed = now - this.lastMicUplinkLogAt
-    if (elapsed < MIC_UPLINK_LOG_INTERVAL_MS) return
+    const sweep = getCallGainSweep()
+    const interval = sweep.isActive() ? MIC_UPLINK_SWEEP_LOG_INTERVAL_MS : MIC_UPLINK_LOG_INTERVAL_MS
+    if (elapsed < interval) return
     this.lastMicUplinkLogAt = now
     // Level, not just cadence: a 20 Hz stream of the noise floor and a 20 Hz stream of speech
     // have the same framesPerSecond. Quiet room on Mentra Live LC3 is meanAbs ≈30–60.
     const level = this.micLevel.take()
+    const stats = pcm16WindowStats(level)
     this.lastMicLevel = {meanAbs: level.meanAbs, peak: level.peak}
+    const gain = sweep.currentGain() ?? micStateCoordinator.getSessionMicTuning()?.gain ?? null
+    if (sweep.isActive()) sweep.ingest(level)
     console.log("[AcsMeeting] phase=glasses-mic-uplink", {
       framesPerSecond: Math.round((this.micFramesWindow * 1000) / elapsed),
       frames: this.micFramesForwarded,
-      meanAbs: level.meanAbs,
-      peak: level.peak,
+      meanAbs: stats.meanAbs,
+      peak: stats.peak,
+      peakPct: stats.peakPct,
+      clipped: stats.clipped,
+      clipPct: stats.clipPct,
+      nearClip: stats.nearClip,
+      nearClipPct: stats.nearClipPct,
+      gain,
+      sweep: sweep.currentLabel(),
       dropsStale: this.micDropsStale,
       dropsNonGlasses: this.micDropsNonGlasses,
       gaps: this.micGaps,

package/src/services/CallGainSweep.ts

@@ -0,0 +1,318 @@
+/**
+ * Timed A/B/A/C gain comparison for a live voice call.
+ *
+ * Mentra Live's last ADC step is +32 dB (index 15); the step below is +26 dB
+ * (14), then 2 dB per index. Close-talk clips at 15 and still rails at 14, so
+ * a call has to be measured at 15 → 14 → 15 → 13 on the same voice, not by
+ * swapping phones or days.
+ *
+ * Production policy stays in micPolicy. This object only writes a temporary
+ * override and logs speech-gated clip stats so a human can pick the index.
+ */
+
+import {PCM16_FULL_SCALE, pcm16WindowStats, type Pcm16Level} from "../utils/pcm16"
+
+const LOG_TAG = "CALL_GAIN_SWEEP"
+
+/** codec_adc_vol[] dB, same table as Super Mode. */
+const GAIN_DB = [-99, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 32]
+
+export const CALL_GAIN_SWEEP_STEPS = [
+  {gain: 15, label: "15a"},
+  {gain: 14, label: "14"},
+  {gain: 15, label: "15b"},
+  {gain: 13, label: "13"},
+] as const
+
+export const CALL_GAIN_SWEEP_STEP_MS = 25_000
+/** Quiet room on this path is meanAbs ≈ 90–120. Talk starts well above this. */
+export const CALL_GAIN_SWEEP_SPEECH_MEAN_ABS = 400
+
+export type CallGainSweepStep = (typeof CALL_GAIN_SWEEP_STEPS)[number]
+
+export type CallGainSweepPhaseSummary = {
+  label: string
+  gain: number
+  db: number
+  windows: number
+  speechWindows: number
+  quietWindows: number
+  clippedWindows: number
+  nearClipWindows: number
+  speechMeanAbs: number
+  speechPeakMax: number
+  speechPeakPct: number
+  speechClipPct: number
+  speechNearClipPct: number
+  samples: number
+  speechSamples: number
+  clipped: number
+  nearClip: number
+}
+
+export type CallGainSweepRecommendation = {
+  gain: number
+  reason: string
+}
+
+export type CallGainSweepClock = {
+  now: () => number
+  setTimeout: (fn: () => void, ms: number) => unknown
+  clearTimeout: (id: unknown) => void
+  log: (...args: unknown[]) => void
+}
+
+type PhaseBucket = {
+  step: CallGainSweepStep
+  windows: number
+  speechWindows: number
+  quietWindows: number
+  clippedWindows: number
+  nearClipWindows: number
+  speechMeanAbsSum: number
+  speechPeakMax: number
+  samples: number
+  speechSamples: number
+  clipped: number
+  nearClip: number
+  speechClipped: number
+  speechNearClip: number
+}
+
+const defaultClock = (): CallGainSweepClock => ({
+  now: () => Date.now(),
+  setTimeout: (fn, ms) => setTimeout(fn, ms),
+  clearTimeout: (id) => clearTimeout(id as ReturnType<typeof setTimeout>),
+  log: (...args) => console.log(...args),
+})
+
+export function gainDb(index: number): number {
+  return GAIN_DB[index] ?? Number.NaN
+}
+
+/**
+ * Highest gain whose speech windows stay under the clip lines.
+ *
+ * Clip is the failure (rail = 32767). Near-clip at 30000 is the warning that
+ * the next syllable will hit the rail. 15 is only legal if both 15a and 15b
+ * agree; otherwise a loud stretch on one of them was luck.
+ */
+export function recommendCallGain(phases: readonly CallGainSweepPhaseSummary[]): CallGainSweepRecommendation {
+  const speech = phases.filter((p) => p.speechWindows > 0)
+  if (speech.length === 0) {
+    return {gain: 13, reason: "no speech windows; stay at 13 until the sweep is talked through"}
+  }
+
+  const fifteen = speech.filter((p) => p.gain === 15)
+  const fourteen = speech.find((p) => p.gain === 14)
+  const thirteen = speech.find((p) => p.gain === 13)
+
+  const clean = (p: CallGainSweepPhaseSummary) => p.speechClipPct < 0.5 && p.speechNearClipPct < 5
+  const fifteenClean = fifteen.length >= 2 && fifteen.every(clean)
+  if (fifteenClean) return {gain: 15, reason: "both 15 phases stayed under 0.5% clip / 5% near-clip"}
+  if (fourteen && clean(fourteen)) {
+    return {gain: 14, reason: "15 clips; 14 stayed under 0.5% clip / 5% near-clip"}
+  }
+  if (thirteen && clean(thirteen)) {
+    return {gain: 13, reason: "15 and 14 clip; 13 stayed under 0.5% clip / 5% near-clip"}
+  }
+  return {gain: 13, reason: "every measured step still clipped; 13 is the floor of this sweep"}
+}
+
+function emptyBucket(step: CallGainSweepStep): PhaseBucket {
+  return {
+    step,
+    windows: 0,
+    speechWindows: 0,
+    quietWindows: 0,
+    clippedWindows: 0,
+    nearClipWindows: 0,
+    speechMeanAbsSum: 0,
+    speechPeakMax: 0,
+    samples: 0,
+    speechSamples: 0,
+    clipped: 0,
+    nearClip: 0,
+    speechClipped: 0,
+    speechNearClip: 0,
+  }
+}
+
+function summarizeBucket(bucket: PhaseBucket): CallGainSweepPhaseSummary {
+  const speechClipPct =
+    bucket.speechSamples > 0 ? Math.round((bucket.speechClipped / bucket.speechSamples) * 1000) / 10 : 0
+  const speechNearClipPct =
+    bucket.speechSamples > 0 ? Math.round((bucket.speechNearClip / bucket.speechSamples) * 1000) / 10 : 0
+  return {
+    label: bucket.step.label,
+    gain: bucket.step.gain,
+    db: gainDb(bucket.step.gain),
+    windows: bucket.windows,
+    speechWindows: bucket.speechWindows,
+    quietWindows: bucket.quietWindows,
+    clippedWindows: bucket.clippedWindows,
+    nearClipWindows: bucket.nearClipWindows,
+    speechMeanAbs:
+      bucket.speechWindows > 0 ? Math.round(bucket.speechMeanAbsSum / bucket.speechWindows) : 0,
+    speechPeakMax: bucket.speechPeakMax,
+    speechPeakPct: Math.round((bucket.speechPeakMax / PCM16_FULL_SCALE) * 1000) / 10,
+    speechClipPct,
+    speechNearClipPct,
+    samples: bucket.samples,
+    speechSamples: bucket.speechSamples,
+    clipped: bucket.clipped,
+    nearClip: bucket.nearClip,
+  }
+}
+
+export class CallGainSweep {
+  private readonly clock: CallGainSweepClock
+  private applyGain: ((gain: number) => void) | null = null
+  private onDone: (() => void) | null = null
+  private index = -1
+  private timer: unknown = null
+  private enteredAt = 0
+  private bucket: PhaseBucket | null = null
+  private readonly finished: CallGainSweepPhaseSummary[] = []
+
+  constructor(clock: CallGainSweepClock = defaultClock()) {
+    this.clock = clock
+  }
+
+  isActive(): boolean {
+    return this.index >= 0
+  }
+
+  currentGain(): number | null {
+    return this.index >= 0 ? CALL_GAIN_SWEEP_STEPS[this.index].gain : null
+  }
+
+  currentLabel(): string | null {
+    return this.index >= 0 ? CALL_GAIN_SWEEP_STEPS[this.index].label : null
+  }
+
+  /**
+   * Begin the 15 → 14 → 15 → 13 walk.
+   * Returns false when a sweep is already mid-phase so a recompute cannot reset the clock.
+   */
+  start(applyGain: (gain: number) => void, onDone?: () => void): boolean {
+    if (this.isActive()) return false
+    this.applyGain = applyGain
+    this.onDone = onDone ?? null
+    this.finished.length = 0
+    this.enter(0)
+    return true
+  }
+
+  /** Super Mode / explicit re-run. Aborts the current phase if one is live. */
+  restart(applyGain: (gain: number) => void, onDone?: () => void): boolean {
+    if (this.isActive()) this.stop("restart")
+    return this.start(applyGain, onDone)
+  }
+
+  stop(reason: string): void {
+    if (!this.isActive() && this.timer == null) return
+    this.closePhase("aborted")
+    this.clearTimer()
+    this.clock.log(`${LOG_TAG} stop`, {reason, phases: this.finished.slice()})
+    this.index = -1
+    this.bucket = null
+    this.applyGain = null
+    this.onDone = null
+  }
+
+  ingest(level: Pcm16Level): void {
+    if (!this.bucket) return
+    const stats = pcm16WindowStats(level)
+    const speech = level.meanAbs >= CALL_GAIN_SWEEP_SPEECH_MEAN_ABS
+    this.bucket.windows++
+    this.bucket.samples += level.samples
+    this.bucket.clipped += level.clipped
+    this.bucket.nearClip += level.nearClip
+    if (speech) {
+      this.bucket.speechWindows++
+      this.bucket.speechSamples += level.samples
+      this.bucket.speechClipped += level.clipped
+      this.bucket.speechNearClip += level.nearClip
+      this.bucket.speechMeanAbsSum += level.meanAbs
+      if (level.peak > this.bucket.speechPeakMax) this.bucket.speechPeakMax = level.peak
+    } else {
+      this.bucket.quietWindows++
+    }
+    if (level.clipped > 0) this.bucket.clippedWindows++
+    if (level.nearClip > 0) this.bucket.nearClipWindows++
+
+    this.clock.log(`${LOG_TAG} window`, {
+      label: this.bucket.step.label,
+      gain: this.bucket.step.gain,
+      db: gainDb(this.bucket.step.gain),
+      speech,
+      meanAbs: stats.meanAbs,
+      peak: stats.peak,
+      peakPct: stats.peakPct,
+      clipped: stats.clipped,
+      clipPct: stats.clipPct,
+      nearClip: stats.nearClip,
+      nearClipPct: stats.nearClipPct,
+      samples: stats.samples,
+    })
+    // RN pauses setTimeout while Mentra is behind Mentra Call. The ACS
+    // uplink still delivers 1 s windows, so the hold is measured here.
+    if (this.clock.now() - this.enteredAt >= CALL_GAIN_SWEEP_STEP_MS) this.advance()
+  }
+
+  private enter(index: number): void {
+    this.index = index
+    const step = CALL_GAIN_SWEEP_STEPS[index]
+    this.bucket = emptyBucket(step)
+    this.enteredAt = this.clock.now()
+    this.applyGain?.(step.gain)
+    this.clock.log(`${LOG_TAG} phase-start`, {
+      label: step.label,
+      gain: step.gain,
+      db: gainDb(step.gain),
+      holdMs: CALL_GAIN_SWEEP_STEP_MS,
+      remaining: CALL_GAIN_SWEEP_STEPS.length - index,
+    })
+    this.timer = this.clock.setTimeout(() => this.advance(), CALL_GAIN_SWEEP_STEP_MS)
+  }
+
+  private advance(): void {
+    this.timer = null
+    this.closePhase("complete")
+    const next = this.index + 1
+    if (next >= CALL_GAIN_SWEEP_STEPS.length) {
+      const pick = recommendCallGain(this.finished)
+      this.clock.log(`${LOG_TAG} done`, {phases: this.finished.slice(), pick})
+      this.index = -1
+      this.bucket = null
+      const done = this.onDone
+      this.applyGain = null
+      this.onDone = null
+      done?.()
+      return
+    }
+    this.enter(next)
+  }
+
+  private closePhase(how: "complete" | "aborted"): void {
+    if (!this.bucket || this.bucket.windows === 0) return
+    const summary = summarizeBucket(this.bucket)
+    this.finished.push(summary)
+    this.clock.log(`${LOG_TAG} phase-${how}`, summary)
+    this.bucket = null
+  }
+
+  private clearTimer(): void {
+    if (this.timer == null) return
+    this.clock.clearTimeout(this.timer)
+    this.timer = null
+  }
+}
+
+const sharedSweep = new CallGainSweep()
+
+/** Process-wide sweep. MicSessionManager writes it; the ACS uplink meters it. */
+export function getCallGainSweep(): CallGainSweep {
+  return sharedSweep
+}

package/src/services/GlassesMicProbe.ts

@@ -12,7 +12,7 @@
 import BluetoothSdk from "@mentra/bluetooth-sdk/internal"
 
 import audioPlaybackService from "./AudioPlaybackService"
-import micStateCoordinator from "./MicStateCoordinator"
+import micSessionManager, {type MicSession} from "./MicSessionManager"
 import {SETTINGS, useSettingsStore} from "../stores/settings"
 import {summarizePcm16} from "../utils/pcm16"
 import {BgTimer} from "../utils/timers"
@@ -85,6 +85,8 @@ export type MicProbeSample = {
 type Listener = (sample: MicProbeSample) => void
 
 const GLASSES = "glasses"
+/** Lease owner for probe runs. Engine features own their sessions; miniapps cannot claim this. */
+const MIC_PROBE_OWNER = "engine:mic-probe"
 const A2DP_RATE = 16000
 const A2DP_CHUNK_MS = 60
 const TONE_HZ = 440
@@ -133,6 +135,8 @@ class GlassesMicProbe {
   private nonGlasses = 0
   /** `preferred_mic` before a `source=phone` control run changed it. */
   private savedPreferredMic: string | null = null
+  /** The probe's microphone lease, which owns the pin and the PCM claim while a run is live. */
+  private micSession: MicSession | null = null
   private listeners = new Set<Listener>()
   private lastSample: MicProbeSample | null = null
 
@@ -164,21 +168,22 @@ class GlassesMicProbe {
     const source = options.source ?? GLASSES
     console.log("[MIC_PROBE] start", options)
 
-    if (source === GLASSES) {
-      // Same order as AcsMeetingService.startGlassesMicUplink: pin first so the first frame the
-      // requirement produces is already from the glasses and the phone mic is never opened.
-      await Promise.resolve(BluetoothSdk.setMicSourcePin?.(GLASSES)).catch((error) => {
-        console.warn("[MIC_PROBE] pin failed", error)
-      })
-    } else {
-      // Control run: no pin (only the glasses can be pinned); steer the ranking with the
-      // user preference and put it back on stop.
+    if (source !== GLASSES) {
+      // Control run: the manager cannot pin a phone (only the glasses can be pinned), so steer the
+      // ranking with the user preference and put it back on stop. That is probe behaviour, not mic
+      // policy, which is why it stays here rather than moving into MicSessionManager.
       const settings = useSettingsStore.getState()
       this.savedPreferredMic = settings.getSetting(SETTINGS.preferred_mic.key) ?? "auto"
       await settings.setSetting(SETTINGS.preferred_mic.key, source, false)
     }
     if (this.generation !== generation) return
-    micStateCoordinator.setCallRequirement(true)
+    // `diagnostic` carries no tuning profile on purpose: the probe has to measure the gain users
+    // actually get, not one it asked for.
+    this.micSession = micSessionManager.acquire({
+      owner: MIC_PROBE_OWNER,
+      source: source === GLASSES ? "glasses" : "phone",
+      useCase: "diagnostic",
+    })
     this.micSub = BluetoothSdk.addListener("mic_pcm", (event: {pcm?: unknown; source?: string}) => {
       this.lastSource = event.source ?? ""
       if (event.source !== source) {
@@ -206,13 +211,11 @@ class GlassesMicProbe {
     this.report()
     this.micSub?.remove()
     this.micSub = null
-    micStateCoordinator.setCallRequirement(false)
+    // Releasing the lease is what unpins the glasses and drops the PCM claim.
+    this.micSession?.release()
+    this.micSession = null
     this.stopping = (async () => {
-      if ((this.options?.source ?? GLASSES) === GLASSES) {
-        await Promise.resolve(BluetoothSdk.setMicSourcePin?.(null)).catch((error) => {
-          console.warn("[MIC_PROBE] unpin failed", error)
-        })
-      } else if (this.savedPreferredMic !== null) {
+      if ((this.options?.source ?? GLASSES) !== GLASSES && this.savedPreferredMic !== null) {
         const restore = this.savedPreferredMic
         this.savedPreferredMic = null
         await useSettingsStore.getState().setSetting(SETTINGS.preferred_mic.key, restore, false)