@mentra/engine 3.2.1-dev.278 → 3.2.1-dev.282

package/src/services/micPolicy.ts

@@ -0,0 +1,198 @@
+/**
+ * Microphone use-case policy.
+ *
+ * Applications say what they are doing ("voice_call"); the engine decides what
+ * that requires of the hardware. Nothing above this file knows that a voice
+ * call means ADC index 14, and no miniapp can ask for a gain directly.
+ *
+ * Pure: every function here is a function of the live session set. The lease
+ * bookkeeping lives in MicSessionManager, the merge with OS preferences lives
+ * in MicStateCoordinator, and the wire write lives below that again.
+ */
+
+/** What an application is doing with the microphone. */
+export type MicUseCase = "voice_call" | "transcription" | "voice_assistant" | "diagnostic"
+
+/** Which microphone the audio comes from. */
+export type MicSource = "glasses" | "phone"
+
+/** Subset of the `mic_tuning` wire payload. Fields left out keep firmware defaults. */
+export type MicTuningProfile = {
+  /** codec_adc_vol index, 0-15. */
+  gain?: number
+  /** Center-mic RMS to open the gate. */
+  open?: number
+  /** Center-mic RMS to close it again. */
+  close?: number
+  /** Open threshold while the speaker is elevated. */
+  sp_open?: number
+  /** Close threshold while the speaker is elevated. */
+  sp_close?: number
+}
+
+/** `codec_adc_vol[]`: index to dB. Index 0 is mute and is never offered. */
+export const GAIN_DB = [-99, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 32]
+
+/**
+ * Mirrored from `center_mic_vad_get_default_config` and `CODEC_SADC_VOL`.
+ *
+ * The four RMS numbers are measured *after* the ADC gain, so they are only
+ * meaningful next to the gain they were calibrated at, which is index 15.
+ */
+export const MIC_TUNING_FIRMWARE_DEFAULTS = {
+  gain: 15,
+  open: 1350,
+  close: 945,
+  sp_open: 2900,
+  sp_close: 1600,
+} as const
+
+/**
+ * Re-express the firmware's thresholds at a different gain.
+ *
+ * The gate compares post-gain RMS, so moving the gain without moving the
+ * thresholds silently changes what the gate means: at -8 dB the wearer has to
+ * be 2.5x louder to clear a number that was chosen for +32 dB. Scaling both by
+ * the same factor keeps the *acoustic* trip point the profile was tuned for,
+ * for the wearer and for speaker leak alike.
+ */
+export function scaleMicTuningToGain(gain: number): Required<MicTuningProfile> {
+  const index = Math.min(GAIN_DB.length - 1, Math.max(1, Math.round(gain)))
+  const factor = Math.pow(10, (GAIN_DB[index]! - GAIN_DB[MIC_TUNING_FIRMWARE_DEFAULTS.gain]!) / 20)
+  return {
+    gain: index,
+    open: Math.round(MIC_TUNING_FIRMWARE_DEFAULTS.open * factor),
+    close: Math.round(MIC_TUNING_FIRMWARE_DEFAULTS.close * factor),
+    sp_open: Math.round(MIC_TUNING_FIRMWARE_DEFAULTS.sp_open * factor),
+    sp_close: Math.round(MIC_TUNING_FIRMWARE_DEFAULTS.sp_close * factor),
+  }
+}
+
+/** A live microphone lease. */
+export type MicSessionSpec = {
+  useCase: MicUseCase
+  source: MicSource
+}
+
+/** What the hardware should do, given every live session. */
+export type ResolvedMicPolicy = {
+  /** Someone needs a continuous raw-PCM timeline, which also forces hardware VAD off. */
+  rawPcm: boolean
+  /** Pin the Bluetooth SDK to the glasses microphone. Only the glasses can be pinned. */
+  pinGlasses: boolean
+  /** Tuning override for the glasses, or null to leave the OS value in force. */
+  micTuning: MicTuningProfile | null
+  /** Run the center-mic loudness gate, or null to leave the OS value in force. */
+  loudnessGate: boolean | null
+}
+
+/** Platform facts the policy cannot infer from the sessions alone. */
+export type MicPlatformCaps = {
+  /**
+   * Whether this platform can take the wearer's voice off the glasses as raw
+   * PCM over BLE LC3. False on iOS, which has no `setMicSourcePin` and never
+   * selects the `ble-lc3` uplink.
+   */
+  glassesPcmUplink: boolean
+}
+
+/**
+ * Mentra Live ships CODEC_SADC_VOL = 15, the last entry of codec_adc_vol[] and
+ * +32 dB. That table steps 2 dB at a time everywhere except the final step,
+ * which jumps 6 dB from +26, so the default sits at the ceiling one oversized
+ * step above everything else. It suits a wearer dictating to a transcription
+ * miniapp across a room; it clips a wearer talking into a Teams call.
+ *
+ * Index 14 is +26 dB: one step down, and the step that removes the table's
+ * anomalous jump. A same-voice sweep showed 15 rails hard (2.5–4.2% clip);
+ * 14/13/12 still kiss the rail on syllable tips. 13 is +24 dB, the middle
+ * of that band, while we listen for loudness vs residual clip.
+ *
+ * The thresholds ride along with the gain rather than being stated here, so
+ * changing the index above cannot leave the gate calibrated for a level the
+ * ADC no longer produces.
+ */
+export const MIC_USE_CASE_PROFILES: Record<MicUseCase, MicTuningProfile> = {
+  voice_call: scaleMicTuningToGain(13),
+  transcription: {},
+  voice_assistant: {},
+  diagnostic: {},
+}
+
+/**
+ * Use cases that run the center-mic loudness gate ("Barrier").
+ *
+ * A voice call is the one place it earns its keep. There is no echo canceller
+ * on the LC3 uplink — `MENTRA_LC3_SPEECH_PROCESSING` is 0 and the capture path
+ * ignores the playback reference — so without a gate the far end hears itself
+ * through the wearer's speaker. Barrier is the safe half of the firmware's two
+ * gates: it zeroes a quiet frame but never stops transmitting, where VAD drops
+ * the stream outright. Paired with the speaker-elevated thresholds above, that
+ * suppresses leak while the far end is talking and costs the wearer nothing.
+ */
+export const MIC_USE_CASE_LOUDNESS_GATE: Record<MicUseCase, boolean> = {
+  voice_call: true,
+  transcription: false,
+  voice_assistant: false,
+  diagnostic: false,
+}
+
+/**
+ * Packages allowed to hold a `voice_call` session.
+ *
+ * Which app may make a voice call is policy, so it lives beside the profiles
+ * rather than in the request handler that enforces it.
+ */
+export const VOICE_CALL_PACKAGES: readonly string[] = ["com.mentra.call"]
+
+/** Every use case, for validating an inbound request. */
+export const MIC_USE_CASES: readonly MicUseCase[] = [
+  "voice_call",
+  "transcription",
+  "voice_assistant",
+  "diagnostic",
+]
+
+/** Owners of engine-internal sessions. Miniapps cannot claim these use cases. */
+export const ENGINE_OWNER_PREFIX = "engine:"
+
+/** Use cases only engine features may acquire. */
+export const ENGINE_ONLY_USE_CASES: readonly MicUseCase[] = ["diagnostic"]
+
+/**
+ * Resolve every live session into one hardware state.
+ *
+ * On a platform without a glasses PCM uplink a glasses session is a lease and
+ * nothing more: it satisfies ownership checks so an iOS call can still be
+ * modelled the same way, but it claims no PCM, pins nothing, and applies no
+ * gain, because the wearer's voice does not reach the call through the BES
+ * BLE path there.
+ */
+export function resolveMicPolicy(
+  sessions: readonly MicSessionSpec[],
+  caps: MicPlatformCaps,
+): ResolvedMicPolicy {
+  const effective = caps.glassesPcmUplink ? sessions : sessions.filter((s) => s.source !== "glasses")
+
+  let pinGlasses = false
+  let loudnessGate = false
+  let winning: MicTuningProfile | null = null
+  for (const session of effective) {
+    if (session.source !== "glasses") continue
+    pinGlasses = true
+    if (MIC_USE_CASE_LOUDNESS_GATE[session.useCase]) loudnessGate = true
+    const profile = MIC_USE_CASE_PROFILES[session.useCase]
+    if (typeof profile?.gain !== "number") continue
+    // Lowest index wins: clipping is the irreversible failure, a slightly
+    // quiet assistant is not. The whole profile travels with it, because its
+    // thresholds are only meaningful at its own gain.
+    if (winning === null || profile.gain < winning.gain!) winning = profile
+  }
+
+  return {
+    rawPcm: effective.length > 0,
+    pinGlasses,
+    micTuning: winning,
+    loudnessGate: pinGlasses ? loudnessGate : null,
+  }
+}

package/src/stores/bluetoothSettingKeys.ts

@@ -7,6 +7,9 @@ export const MENTRA_LIVE_SETTING_KEYS: string[] = [
   "preferred_mic",
   "voice_activity_detection_enabled",
   "loudness_gate_enabled",
+  // Effective tuning only. mic_tuning_desired is engine-side state and must
+  // never appear here, or a persisted super-mode value would reach the glasses.
+  "mic_tuning",
   "core_token",
   "auth_email",
   "button_photo_size",

package/src/stores/settings.ts

@@ -464,6 +464,42 @@ export const SETTINGS: Record<string, Setting> = {
     saveOnServer: true,
     persist: true,
   },
+  /*
+   * Mentra Live mic tuning, split in two on purpose.
+   *
+   * `mic_tuning_desired` is what the super user set. It persists so that
+   * turning super mode off and on again does not lose the calibration, and it
+   * stays out of BLUETOOTH_SETTING_KEYS so it can never reach the glasses on
+   * its own. It is device-local: this is a hardware calibration aid, not
+   * something that should follow an account onto another pair of glasses.
+   *
+   * `mic_tuning` is the effective value, derived in getBluetoothSettings as
+   * `super_mode ? desired : {}`. It is the only one native ever sees. Because
+   * it is not persisted and native seeds no default for it, a process that
+   * connects before the engine has authorized anything can only send a reset
+   * -- the super-mode guarantee is then a property of what native can hold,
+   * not of which initialiser happened to run first.
+   *
+   * `{}` rather than null for "no tuning": the native store drops null writes,
+   * so null would leave a previously applied value in place.
+   *
+   * The empty object is a stable singleton so GlassesSettingsSync's reference
+   * diff does not treat every unrelated settings write as a mic_tuning change.
+   */
+  mic_tuning_desired: {
+    key: "mic_tuning_desired",
+    defaultValue: () => null,
+    writable: true,
+    saveOnServer: false,
+    persist: true,
+  },
+  mic_tuning: {
+    key: "mic_tuning",
+    defaultValue: () => ({}),
+    writable: true,
+    saveOnServer: false,
+    persist: false,
+  },
   always_on_status_bar: {
     key: "always_on_status_bar",
     defaultValue: () => false,
@@ -757,6 +793,8 @@ export const BLUETOOTH_SETTING_KEYS: string[] = [
   SETTINGS.power_saving_mode.key,
   SETTINGS.voice_activity_detection_enabled.key,
   SETTINGS.loudness_gate_enabled.key,
+  // Effective tuning only; mic_tuning_desired deliberately stays engine-side.
+  SETTINGS.mic_tuning.key,
   SETTINGS.lc3_frame_size.key,
   SETTINGS.preferred_mic.key,
   SETTINGS.screen_disabled.key,
@@ -819,6 +857,9 @@ export const PAIRING_IDENTITY_KEYS: string[] = Object.values(SETTINGS)
   .filter((setting) => setting.nativeAuthoritative)
   .map((setting) => setting.key)
 
+/** Stable empty effective tuning. Native reads this as `{"reset":1}`. */
+const EMPTY_MIC_TUNING: Record<string, number> = Object.freeze({})
+
 // const PER_GLASSES_SETTINGS_KEYS: string[] = [SETTINGS.preferred_mic.key]
 
 export interface SettingsState {
@@ -834,6 +875,11 @@ export interface SettingsState {
   loadAllSettings: () => AsyncResult<void, Error>
   // Utility methods
   getBluetoothSettings: () => Record<string, any>
+  /**
+   * The mic tuning native is allowed to send: the desired value while super
+   * mode is on, `{}` (meaning "reset to firmware defaults") otherwise.
+   */
+  getEffectiveMicTuning: () => Record<string, number>
   resetAllSettingsLocally: () => void
 }
 
@@ -1193,10 +1239,30 @@ export const useSettingsStore = create<SettingsState>()(
         if (key === SETTINGS.core_token.key && (typeof value !== "string" || value.trim().length === 0)) {
           continue
         }
+        if (key === SETTINGS.mic_tuning.key) {
+          // Derived here rather than stored, so there is no path that can push
+          // a persisted tuning to the glasses while super mode is off. Every
+          // sync route (change diff, on-connect replay, full seed) reads this.
+          bluetoothSettings[key] = state.getEffectiveMicTuning()
+          continue
+        }
         bluetoothSettings[key] = value
       }
       return bluetoothSettings
     },
+    getEffectiveMicTuning: () => {
+      const state = get()
+      if (!state.getSetting(SETTINGS.super_mode.key)) return EMPTY_MIC_TUNING
+      const desired = state.getSetting(SETTINGS.mic_tuning_desired.key)
+      if (!desired || typeof desired !== "object") return EMPTY_MIC_TUNING
+      const effective: Record<string, number> = {}
+      for (const [key, value] of Object.entries(desired as Record<string, unknown>)) {
+        if (typeof value === "number" && Number.isFinite(value)) {
+          effective[key] = Math.round(value)
+        }
+      }
+      return Object.keys(effective).length === 0 ? EMPTY_MIC_TUNING : effective
+    },
     resetAllSettingsLocally: () => {
       set((_state) => ({
         settings: getDefaultSettings(),

package/src/utils/pcm16.ts

@@ -29,6 +29,11 @@ export function pcmDataView(frame: unknown): DataView | null {
   return null
 }
 
+/** Signed 16-bit full scale. `-32768` abs-counts as 32768 and is also a rail. */
+export const PCM16_FULL_SCALE = 32767
+/** Headroom line: loud speech that is not yet hard-clipped. */
+export const PCM16_NEAR_CLIP = 30000
+
 export type Pcm16Level = {
   /** Mean absolute sample value (16-bit scale). ~30–60 is a quiet room on Mentra Live LC3. */
   meanAbs: number
@@ -36,6 +41,31 @@ export type Pcm16Level = {
   peak: number
   /** Samples counted. */
   samples: number
+  /** Samples at the int16 rail (`>= 32767`). */
+  clipped: number
+  /** Samples at or above [PCM16_NEAR_CLIP]. */
+  nearClip: number
+}
+
+export type Pcm16WindowStats = Pcm16Level & {
+  /** `peak / 32767`, percent. */
+  peakPct: number
+  /** `clipped / samples`, percent. */
+  clipPct: number
+  /** `nearClip / samples`, percent. */
+  nearClipPct: number
+}
+
+/** Percentages for a closed window. Zero samples → all percents 0. */
+export function pcm16WindowStats(level: Pcm16Level): Pcm16WindowStats {
+  const denom = level.samples || 1
+  const pct = (count: number) => Math.round((count / denom) * 1000) / 10
+  return {
+    ...level,
+    peakPct: Math.round((level.peak / PCM16_FULL_SCALE) * 1000) / 10,
+    clipPct: level.samples ? pct(level.clipped) : 0,
+    nearClipPct: level.samples ? pct(level.nearClip) : 0,
+  }
 }
 
 /** Level statistics over a batch of PCM16 frames. Unreadable frames are skipped, never thrown on. */
@@ -43,6 +73,8 @@ export function summarizePcm16(frames: unknown[]): Pcm16Level {
   let sum = 0
   let peak = 0
   let samples = 0
+  let clipped = 0
+  let nearClip = 0
   for (const frame of frames) {
     const view = pcmDataView(frame)
     if (!view) continue
@@ -51,10 +83,12 @@ export function summarizePcm16(frames: unknown[]): Pcm16Level {
       const v = Math.abs(view.getInt16(i * 2, true))
       sum += v
       if (v > peak) peak = v
+      if (v >= PCM16_FULL_SCALE) clipped++
+      if (v >= PCM16_NEAR_CLIP) nearClip++
     }
     samples += n
   }
-  return {meanAbs: samples ? Math.round(sum / samples) : 0, peak, samples}
+  return {meanAbs: samples ? Math.round(sum / samples) : 0, peak, samples, clipped, nearClip}
 }
 
 /**
@@ -65,6 +99,8 @@ export class Pcm16LevelMeter {
   private sum = 0
   private peakValue = 0
   private count = 0
+  private clipped = 0
+  private nearClip = 0
 
   add(frame: unknown): void {
     const view = pcmDataView(frame)
@@ -74,6 +110,8 @@ export class Pcm16LevelMeter {
       const v = Math.abs(view.getInt16(i * 2, true))
       this.sum += v
       if (v > this.peakValue) this.peakValue = v
+      if (v >= PCM16_FULL_SCALE) this.clipped++
+      if (v >= PCM16_NEAR_CLIP) this.nearClip++
     }
     this.count += n
   }
@@ -84,10 +122,14 @@ export class Pcm16LevelMeter {
       meanAbs: this.count ? Math.round(this.sum / this.count) : 0,
       peak: this.peakValue,
       samples: this.count,
+      clipped: this.clipped,
+      nearClip: this.nearClip,
     }
     this.sum = 0
     this.peakValue = 0
     this.count = 0
+    this.clipped = 0
+    this.nearClip = 0
     return level
   }
 }