@mentra/engine 3.2.1-dev.288 → 3.2.1-dev.289

package/src/services/ManagedWebRtcRelay.ts

@@ -1,6 +1,7 @@
 import BluetoothSdk from "@mentra/bluetooth-sdk/internal"
 import type {StreamStartRequest, StreamStatusEvent} from "@mentra/bluetooth-sdk/internal"
 import {acquireGlassesHotspot} from "./GlassesHotspotLease"
+import {pcmToBase64} from "../utils/pcmToBase64"
 
 export interface RelayOptions {
   streamId: string
@@ -24,9 +25,11 @@ interface NativeRelay {
     password: string
     gatewayAddress?: string
     captureAudio: boolean
+    audioTransport: "whip" | "ble-lc3" | "none"
     bitrate: number
   }): Promise<string>
   stop(attemptId: string): Promise<void>
+  pushOutgoingPcm?(attemptId: string, base64: string, sampleRate: number, channels: number): boolean
   addListener(event: "onRelayState", listener: (event: NativeRelayEvent) => void): {remove(): void}
 }
 
@@ -40,6 +43,11 @@ export interface RelayDependencies {
   sleep(ms: number): Promise<void>
   now(): number
   acquire(): () => void
+  /** Present only on hosts that can pin and decode the glasses' BLE microphone, as ACS does. */
+  microphone?: {
+    acquire(): () => void
+    subscribe(listener: (event: {pcm?: ArrayBuffer; sampleRate?: number; source?: string}) => void): {remove(): void}
+  }
 }
 
 export interface ManagedRelay {
@@ -50,7 +58,7 @@ export interface ManagedRelay {
   handleGlassesStatus(event: StreamStatusEvent): void
 }
 
-/** One attempt at a time; media stays native. All retries unwind both peers and the hotspot. */
+/** One attempt at a time; video stays native. All retries unwind both peers, mic and hotspot. */
 export class ManagedWebRtcRelay implements ManagedRelay {
   private cancelled = false
   private started = false
@@ -67,6 +75,8 @@ export class ManagedWebRtcRelay implements ManagedRelay {
   private retries = 0
   private readyAt: number | null = null
   private stopping: Promise<void> | null = null
+  private releaseMic: (() => void) | null = null
+  private micListener: {remove(): void} | null = null
 
   constructor(
     private readonly options: RelayOptions,
@@ -114,6 +124,10 @@ export class ManagedWebRtcRelay implements ManagedRelay {
     this.attemptId = `${this.options.streamId}-relay-${++this.attempt}`
     this.attemptError = null
     this.checkpoint()
+    const captureAudio = this.options.captureAudio !== false
+    const lc3 = captureAudio && !!this.deps.microphone && typeof this.deps.native.pushOutgoingPcm === "function"
+    // Claim through the same policy/ownership layer as ACS before turning off WHIP audio.
+    if (lc3) this.releaseMic = this.deps.microphone!.acquire()
     this.hotspotTouched = true // A timed-out BLE command may still have enabled the AP.
     const hotspot = await this.deps.hotspot(true)
     this.checkpoint()
@@ -128,10 +142,35 @@ export class ManagedWebRtcRelay implements ManagedRelay {
       ssid: hotspot.ssid,
       password: hotspot.password,
       gatewayAddress: hotspot.localIp,
-      captureAudio: this.options.captureAudio !== false,
+      captureAudio,
+      audioTransport: lc3 ? "ble-lc3" : captureAudio ? "whip" : "none",
       bitrate: this.options.video?.bitrate ?? 2_000_000,
     })
     this.checkpoint()
+    if (lc3) {
+      const id = this.attemptId
+      let received = false
+      this.micListener = this.deps.microphone!.subscribe((event) => {
+        if (this.cancelled || this.attemptId !== id || this.attemptError) return
+        if (event.source !== "glasses") {
+          this.failed(new Error("Managed stream lost the glasses microphone"))
+          return
+        }
+        if (!event.pcm?.byteLength) return
+        try {
+          const accepted = this.deps.native.pushOutgoingPcm!(id, pcmToBase64(event.pcm), event.sampleRate ?? 16000, 1)
+          if (accepted && !received) {
+            received = true
+            console.log("[ManagedRelay] First BLE LC3 audio frame", {
+              attemptId: id,
+              sampleRate: event.sampleRate ?? 16000,
+            })
+          }
+        } catch (error) {
+          this.failed(asError(error))
+        }
+      })
+    }
     this.glassesTouched = true
     const result = await this.deps.startGlasses({
       type: "start_stream",
@@ -139,7 +178,7 @@ export class ManagedWebRtcRelay implements ManagedRelay {
       streamUrl: url,
       ice: {stun: ""},
       sound: this.options.sound ?? true,
-      captureAudio: this.options.captureAudio !== false,
+      captureAudio: captureAudio && !lc3,
       ...(this.options.video !== undefined ? {video: this.options.video} : {}),
       ...(this.options.audio !== undefined ? {audio: this.options.audio} : {}),
     })
@@ -223,6 +262,12 @@ export class ManagedWebRtcRelay implements ManagedRelay {
     // Invalidate callbacks before intentionally stopping the glasses or dropping the network.
     const id = this.attemptId
     this.attemptId = null
+    await step(async () => {
+      this.micListener?.remove()
+      this.micListener = null
+      this.releaseMic?.()
+      this.releaseMic = null
+    })
     if (this.glassesTouched)
       await step(async () => {
         if (this.deps.connected()) await this.deps.stopGlasses()
@@ -264,6 +309,8 @@ export function createManagedWebRtcRelay(
   const {requireNativeModule} = require("expo-modules-core") as typeof import("expo-modules-core")
   const {BgTimer} = require("../utils/timers") as typeof import("../utils/timers")
   const native = requireNativeModule<NativeRelay>("MentraGlassesMediaRelay")
+  const {Platform} = require("react-native") as typeof import("react-native")
+  const {default: micSessionManager} = require("./MicSessionManager") as typeof import("./MicSessionManager")
   return new ManagedWebRtcRelay(options, onStatus, onFailure, {
     native,
     hotspot: (enabled) => BluetoothSdk.setHotspotState(enabled),
@@ -274,5 +321,18 @@ export function createManagedWebRtcRelay(
     now: Date.now,
     sleep: (ms) => new Promise((resolve) => BgTimer.setTimeout(resolve, ms)),
     acquire: acquireGlassesHotspot,
+    ...(Platform.OS === "android" && typeof BluetoothSdk.setMicSourcePin === "function"
+      ? {
+          microphone: {
+            acquire: () =>
+              micSessionManager.acquire({
+                owner: `engine:managed-relay:${options.streamId}`,
+                source: "glasses",
+                useCase: "livestream",
+              }).release,
+            subscribe: (listener) => BluetoothSdk.addListener("mic_pcm", listener),
+          },
+        }
+      : {}),
   })
 }

package/src/services/micPolicy.ts

@@ -11,7 +11,7 @@
  */
 
 /** What an application is doing with the microphone. */
-export type MicUseCase = "voice_call" | "transcription" | "voice_assistant" | "diagnostic"
+export type MicUseCase = "voice_call" | "transcription" | "voice_assistant" | "diagnostic" | "livestream"
 
 /** Which microphone the audio comes from. */
 export type MicSource = "glasses" | "phone"
@@ -117,6 +117,7 @@ export const MIC_USE_CASE_PROFILES: Record<MicUseCase, MicTuningProfile> = {
   transcription: {},
   voice_assistant: {},
   diagnostic: {},
+  livestream: {}, // Preserve the configured capture gain; there is no call return audio.
 }
 
 /**
@@ -135,6 +136,7 @@ export const MIC_USE_CASE_LOUDNESS_GATE: Record<MicUseCase, boolean> = {
   transcription: false,
   voice_assistant: false,
   diagnostic: false,
+  livestream: false,
 }
 
 /**
@@ -151,13 +153,14 @@ export const MIC_USE_CASES: readonly MicUseCase[] = [
   "transcription",
   "voice_assistant",
   "diagnostic",
+  "livestream",
 ]
 
 /** Owners of engine-internal sessions. Miniapps cannot claim these use cases. */
 export const ENGINE_OWNER_PREFIX = "engine:"
 
 /** Use cases only engine features may acquire. */
-export const ENGINE_ONLY_USE_CASES: readonly MicUseCase[] = ["diagnostic"]
+export const ENGINE_ONLY_USE_CASES: readonly MicUseCase[] = ["diagnostic", "livestream"]
 
 /**
  * Resolve every live session into one hardware state.

package/src/utils/pcmToBase64.ts

@@ -0,0 +1,10 @@
+/** Encode decoded LC3 PCM for native sinks. Hermes has btoa, but no Node Buffer. */
+export function pcmToBase64(pcm: ArrayBuffer): string {
+  const bytes = new Uint8Array(pcm)
+  let binary = ""
+  const step = 0x8000
+  for (let i = 0; i < bytes.length; i += step) {
+    binary += String.fromCharCode(...bytes.subarray(i, i + step))
+  }
+  return btoa(binary)
+}