@mentra/engine 3.2.0-dev.226 → 3.2.0-dev.232

package/src/services/LocalMiniappRuntime.ts

@@ -9,6 +9,7 @@
  * management (connect/disconnect/ping).
  */
 
+import {acquireGlassesHotspot} from "./GlassesHotspotLease"
 import {AppState, Linking} from "react-native"
 import Share from "react-native-share"
 import * as Battery from "expo-battery"
@@ -68,7 +69,12 @@ import {
 } from "../runtime/config"
 import {getAnalytics, getMiniappConfiguration, getUiSeams, isFeatureEnabled} from "../runtime/bootstrap"
 import {invokeScanQrSeam} from "../runtime/scanQrSeam"
-import {normalizeStreamAudioConfig, normalizeStreamVideoConfig, resolveCaptureAudio} from "../runtime/streamConfig"
+import {
+  normalizeStreamAudioConfig,
+  normalizeStreamVideoConfig,
+  normalizeCaptureAudio,
+  resolveCaptureAudio,
+} from "../runtime/streamConfig"
 import {toLanguageHint} from "@mentra/cloud-protocol/languages"
 import type {AudioSubscription, LanguageSource, TranscriptionData, TranslationData} from "@mentra/cloud-protocol"
 import {buildMiniappManifestSnapshot, type MiniappRuntimeDiagnosticSnapshot} from "../utils/miniappDiagnostics"
@@ -399,6 +405,7 @@ type SoftapAttempt = {
   /** Set by leave, end, or a superseding join. Checked after every await the join performs. */
   cancelled: boolean
   /** Set only after the preceding attempt has fully settled. */
+  releaseHotspot?: () => void
   ownsResources: boolean
   transport: SoftapCallTransport | null
   /** The checklist as the miniapp last saw it; preflight rows live here until `start()` takes over. */
@@ -991,13 +998,14 @@ class LocalMiniappRuntime {
     // remain active for another subscriber; a rejected FOV release then stops
     // renewing the departed owner's lease and ASG restores it once idle.
     const cameraCleanup = Promise.allSettled([
+      this.leaveMeetingForApp(packageName),
       phonePhotoCoordinator.stopWarmUpForApp(packageName),
       phoneStreamCoordinator.stop(packageName),
       phoneVideoCoordinator.stopForApp(packageName),
     ]).then((results) => {
       for (const result of results) {
         if (result.status === "rejected") {
-          console.warn(`${LOG_TAG}: failed to stop camera consumer for ${packageName} on unregister`, result.reason)
+          console.warn(`${LOG_TAG}: failed to stop miniapp resource for ${packageName} on unregister`, result.reason)
         }
       }
     })
@@ -1031,10 +1039,6 @@ class LocalMiniappRuntime {
     // so a crashed/closed miniapp doesn't leak partial files or file handles.
     this.blobStore.onAppGone(packageName)
 
-    void acsMeetingService.leaveIfOwner(packageName).catch((error) => {
-      console.warn(`${LOG_TAG}: failed to leave ACS meeting for ${packageName} on unregister`, error)
-    })
-
     // Detach the per-app nav event forwarder but leave the native nav session
     // running. The user may have just closed the mini-app UI and will reopen
     // it; stopping the session here would kill an active trip mid-route.
@@ -2327,11 +2331,11 @@ class LocalMiniappRuntime {
       typeof rawText === "string"
         ? [rawText.trim()].filter(Boolean)
         : Array.isArray(rawText)
-        ? rawText
-            .filter((sentence): sentence is string => typeof sentence === "string")
-            .map((sentence) => sentence.trim())
-            .filter(Boolean)
-        : []
+          ? rawText
+              .filter((sentence): sentence is string => typeof sentence === "string")
+              .map((sentence) => sentence.trim())
+              .filter(Boolean)
+          : []
     const enableSanitization = payload.enableSanitization !== false
     const sentences = prepareTtsSentences(rawSentences, enableSanitization)
     if (sentences.length === 0) {
@@ -3627,13 +3631,18 @@ class LocalMiniappRuntime {
       return
     }
     try {
+      if (payload.ingest === "whip" && !(await permissions.check(PermissionFeatures.LOCAL_WIFI))) {
+        if (!(await permissions.request(PermissionFeatures.LOCAL_WIFI))) {
+          throw new Error("Nearby devices permission is required to join the glasses hotspot")
+        }
+      }
       const result = await streaming.startManaged(packageName, {
         restreamDestinations: payload.restreamDestinations as Array<string | {url: string; name?: string}> | undefined,
         video: normalizeStreamVideoConfig(payload.video),
         audio: normalizeStreamAudioConfig(payload.audio),
         sound: payload.sound as boolean | undefined,
-        ingest: payload.ingest as "srt" | "whip" | undefined,
-        captureAudio: resolveCaptureAudio(payload.captureAudio, resolveAcsAudioSource().source),
+        ingest: payload.ingest as "srt" | "whip" | "rtmp" | undefined,
+        captureAudio: normalizeCaptureAudio(payload.captureAudio),
       })
       this.sendResult(packageName, requestId, true, result)
     } catch (err) {
@@ -3656,6 +3665,9 @@ class LocalMiniappRuntime {
 
   private ensureMeetingStateBridge(): void {
     acsMeetingService.setStateHandler((owner, state) => {
+      const attempt = this.softapAttempt
+      // While a replacement waits for cleanup, native events still describe its predecessor.
+      if (attempt?.packageName === owner && !attempt.ownsResources) return
       this.sendToMiniapp(owner, {
         type: MiniappResponseType.MEETING_STATE,
         ...state,
@@ -3663,6 +3675,16 @@ class LocalMiniappRuntime {
     })
   }
 
+  /** Startup owns the hotspot before ACS has an owner. Closing the app must retire both. */
+  private async leaveMeetingForApp(packageName: string): Promise<void> {
+    const attempt = this.softapAttempt
+    if (attempt?.packageName === packageName) {
+      await this.retireSoftapAttempt({attempt})
+    } else {
+      await acsMeetingService.leaveIfOwner(packageName)
+    }
+  }
+
   private async handleMeetingJoin(
     packageName: string,
     payload: Record<string, unknown>,
@@ -3891,7 +3913,8 @@ class LocalMiniappRuntime {
       return
     }
     attempt.progress = progress
-    const current = acsMeetingService.getState()
+    // A reservation waiting behind another call must not inherit that call's connected state.
+    const current = attempt.transport ? acsMeetingService.getState() : {state: "connecting", muted: false}
     this.sendToMiniapp(attempt.packageName, {
       type: MiniappResponseType.MEETING_STATE,
       ...current,
@@ -3925,6 +3948,7 @@ class LocalMiniappRuntime {
   ): Promise<MeetingState> {
     const packageName = attempt.packageName
     if (previous) {
+      this.narrateSoftapPreflight(attempt, "Finishing the previous call’s cleanup…")
       softapTrace("softap_join_superseding", {
         previousAttempt: previous.id,
         previousAgeMs: Date.now() - previous.startedAt,
@@ -3949,6 +3973,7 @@ class LocalMiniappRuntime {
       softapTraceFailure("softap_join_refused", {packageName, reason: cleanupError})
       throw new Error(`Previous call cleanup failed: ${cleanupError}. Power-cycle the glasses hotspot and try again.`)
     }
+    attempt.releaseHotspot = acquireGlassesHotspot()
     attempt.ownsResources = true
     this.narrateSoftapPreflight(attempt, "Checking the Nearby devices permission…")
     const permissionStartedAt = Date.now()
@@ -3980,12 +4005,14 @@ class LocalMiniappRuntime {
           setHotspotState: async (enabled) => {
             const status = await BluetoothSdk.setHotspotState(enabled)
             if (status.state === "enabled") {
-              return {state: status.state, ssid: status.ssid, password: status.password}
+              return {state: status.state, ssid: status.ssid, password: status.password, localIp: status.localIp}
             }
             return {state: status.state}
           },
-          joinScopedNetwork: (ssid, passphrase) => acsMeetingService.joinScopedNetwork(ssid, passphrase),
+          joinScopedNetwork: (ssid, passphrase, gateway) =>
+            acsMeetingService.joinScopedNetwork(ssid, passphrase, gateway),
           leaveScopedNetwork: () => acsMeetingService.leaveScopedNetwork(),
+          cancelScopedNetworkJoin: () => acsMeetingService.cancelScopedNetworkJoin(),
           probeGateway: async () => {
             const verdict = await acsMeetingService.probeScopedGateway()
             // No native probe means no verdict, and the orchestrator must not read that as "down".
@@ -4237,7 +4264,7 @@ class LocalMiniappRuntime {
     // Teams configuration problem.
     const defaultNetworkStartedAt = Date.now()
     try {
-      const network = await acsMeetingService.awaitValidatedDefaultNetwork()
+      const network = await acsMeetingService.awaitDefaultNetworkAfterHotspot()
       softapTrace("softap_teardown_step", {
         attempt: attempt.id,
         step: "awaitValidatedDefaultNetwork",
@@ -4310,6 +4337,8 @@ class LocalMiniappRuntime {
       failures: failures.join("; "),
       endRefused: Boolean(endFailure),
     })
+    attempt.releaseHotspot?.()
+    attempt.releaseHotspot = undefined
     if (endFailure) throw endFailure
   }
 
@@ -4437,7 +4466,7 @@ class LocalMiniappRuntime {
     requestId?: string,
   ): Promise<void> {
     const videoSource = payload.videoSource as {type?: string; url?: string} | undefined
-    const whepUrl = videoSource?.type === "whep" ? videoSource.url ?? "" : ""
+    const whepUrl = videoSource?.type === "whep" ? (videoSource.url ?? "") : ""
     if (!whepUrl) {
       this.sendResult(packageName, requestId, false, undefined, {
         code: MiniappErrorCode.INVALID_ARGUMENT,

package/src/services/ManagedWebRtcRelay.ts

@@ -0,0 +1,278 @@
+import BluetoothSdk from "@mentra/bluetooth-sdk/internal"
+import type {StreamStartRequest, StreamStatusEvent} from "@mentra/bluetooth-sdk/internal"
+import {acquireGlassesHotspot} from "./GlassesHotspotLease"
+
+export interface RelayOptions {
+  streamId: string
+  ingestUrl: string
+  video?: StreamStartRequest["video"]
+  audio?: StreamStartRequest["audio"]
+  captureAudio?: boolean
+  sound?: boolean
+}
+
+interface NativeRelayEvent {
+  attemptId: string
+  state: string
+  reason: string
+}
+interface NativeRelay {
+  prepare(options: {
+    attemptId: string
+    ingestUrl: string
+    ssid: string
+    password: string
+    gatewayAddress?: string
+    captureAudio: boolean
+    bitrate: number
+  }): Promise<string>
+  stop(attemptId: string): Promise<void>
+  addListener(event: "onRelayState", listener: (event: NativeRelayEvent) => void): {remove(): void}
+}
+
+export interface RelayDependencies {
+  native: NativeRelay
+  hotspot(enabled: boolean): Promise<{state: string; ssid?: string; password?: string; localIp?: string}>
+  startGlasses(request: StreamStartRequest): Promise<StreamStatusEvent | undefined>
+  stopGlasses(): Promise<unknown>
+  deferredStop(): void
+  connected(): boolean
+  sleep(ms: number): Promise<void>
+  now(): number
+  acquire(): () => void
+}
+
+export interface ManagedRelay {
+  start(): Promise<StreamStatusEvent | undefined>
+  cancel(): void
+  stop(): Promise<void>
+  owns(streamId: string): boolean
+  handleGlassesStatus(event: StreamStatusEvent): void
+}
+
+/** One attempt at a time; media stays native. All retries unwind both peers and the hotspot. */
+export class ManagedWebRtcRelay implements ManagedRelay {
+  private cancelled = false
+  private started = false
+  private attempt = 0
+  private attemptId: string | null = null
+  private attemptError: Error | null = null
+  private hotspotTouched = false
+  private nativeTouched = false
+  private glassesTouched = false
+  private release: (() => void) | null = null
+  private listener: {remove(): void} | null = null
+  private operation: Promise<unknown> = Promise.resolve()
+  private restarting = false
+  private retries = 0
+  private readyAt: number | null = null
+  private stopping: Promise<void> | null = null
+
+  constructor(
+    private readonly options: RelayOptions,
+    private readonly onStatus: (status: string, reason: string) => void,
+    private readonly onFailure: (error: Error) => void,
+    private readonly deps: RelayDependencies,
+  ) {}
+
+  start(): Promise<StreamStatusEvent | undefined> {
+    if (this.started) return Promise.reject(new Error("Relay already started"))
+    this.started = true
+    this.release = this.deps.acquire()
+    this.listener = this.deps.native.addListener("onRelayState", (event) => {
+      if (event.attemptId !== this.attemptId || this.cancelled) return
+      if (event.state === "failed") this.failed(new Error(event.reason))
+      else this.onStatus(event.state, event.reason)
+    })
+    const start = this.startAttempt()
+    this.operation = start
+    return start
+  }
+
+  cancel(): void {
+    this.cancelled = true
+  }
+
+  owns(streamId: string): boolean {
+    // Consume late status from all attempts without allowing it to affect the current attempt.
+    return streamId.startsWith(`${this.options.streamId}-relay-`)
+  }
+
+  handleGlassesStatus(event: StreamStatusEvent): void {
+    if (event.streamId !== this.attemptId || this.cancelled) return
+    if (event.status === "stopped" || event.terminal) {
+      this.failed(new Error("Glasses publisher stopped"))
+    }
+  }
+
+  private checkpoint(): void {
+    if (this.cancelled) throw new Error("Relay cancelled")
+    if (this.attemptError) throw this.attemptError
+  }
+
+  private async startAttempt(): Promise<StreamStatusEvent | undefined> {
+    this.attemptId = `${this.options.streamId}-relay-${++this.attempt}`
+    this.attemptError = null
+    this.checkpoint()
+    this.hotspotTouched = true // A timed-out BLE command may still have enabled the AP.
+    const hotspot = await this.deps.hotspot(true)
+    this.checkpoint()
+    if (hotspot.state !== "enabled" || !hotspot.ssid || !hotspot.password)
+      throw new Error("Glasses hotspot did not start")
+    await this.deps.sleep(3_000) // Same beacon/DHCP startup allowance as ACS.
+    this.checkpoint()
+    this.nativeTouched = true
+    const url = await this.deps.native.prepare({
+      attemptId: this.attemptId,
+      ingestUrl: this.options.ingestUrl,
+      ssid: hotspot.ssid,
+      password: hotspot.password,
+      gatewayAddress: hotspot.localIp,
+      captureAudio: this.options.captureAudio !== false,
+      bitrate: this.options.video?.bitrate ?? 2_000_000,
+    })
+    this.checkpoint()
+    this.glassesTouched = true
+    const result = await this.deps.startGlasses({
+      type: "start_stream",
+      streamId: this.attemptId,
+      streamUrl: url,
+      ice: {stun: ""},
+      sound: this.options.sound ?? true,
+      captureAudio: this.options.captureAudio !== false,
+      ...(this.options.video !== undefined ? {video: this.options.video} : {}),
+      ...(this.options.audio !== undefined ? {audio: this.options.audio} : {}),
+    })
+    this.checkpoint()
+    this.readyAt = this.deps.now()
+    return result
+  }
+
+  private failed(error: Error): void {
+    if (this.cancelled || this.attemptError) return
+    this.attemptError = error
+    // Bound consecutive trouble, not the lifetime number of recoveries in a long stream.
+    if (this.readyAt !== null && this.deps.now() - this.readyAt >= 60_000) this.retries = 0
+    this.readyAt = null
+    if (this.restarting) return
+    this.restarting = true
+    // Await startup before cleanup: late native/BLE success still belongs to this attempt.
+    this.operation = this.operation
+      .catch(() => undefined)
+      .then(async () => {
+        while (!this.cancelled) {
+          try {
+            await this.cleanupAttempt()
+            if (this.cancelled) return
+            if (this.retries >= 3) throw error
+            this.onStatus("reconnecting", error.message)
+            await this.deps.sleep(1_000 * 2 ** this.retries++)
+            if (this.cancelled) return
+            await this.startAttempt()
+            this.onStatus("reconnected", "Glasses relay restarted")
+            return
+          } catch (failure) {
+            // Cleanup failure retains ownership; never create another stream on uncertain state.
+            if (this.hotspotTouched || this.nativeTouched || this.glassesTouched) {
+              try {
+                await this.cleanupAttempt()
+              } catch (cleanupError) {
+                if (!this.cancelled) this.onFailure(asError(cleanupError))
+                return
+              }
+            }
+            if (this.retries >= 3) {
+              if (!this.cancelled) this.onFailure(asError(failure))
+              return
+            }
+          }
+        }
+      })
+      .finally(() => {
+        this.restarting = false
+      })
+  }
+
+  stop(): Promise<void> {
+    this.cancel()
+    if (this.stopping) return this.stopping
+    this.listener?.remove()
+    this.listener = null
+    this.stopping = this.operation
+      .catch(() => undefined)
+      .then(async () => {
+        await this.cleanupAttempt()
+        this.release?.()
+        this.release = null
+      })
+      .finally(() => {
+        this.stopping = null
+      })
+    return this.stopping
+  }
+
+  private async cleanupAttempt(): Promise<void> {
+    const failures: unknown[] = []
+    const step = async (action: () => Promise<void>) => {
+      try {
+        await action()
+      } catch (error) {
+        failures.push(error)
+      }
+    }
+    // Invalidate callbacks before intentionally stopping the glasses or dropping the network.
+    const id = this.attemptId
+    this.attemptId = null
+    if (this.glassesTouched)
+      await step(async () => {
+        if (this.deps.connected()) await this.deps.stopGlasses()
+        else this.deps.deferredStop()
+        this.glassesTouched = false
+      })
+    if (this.nativeTouched && id)
+      await step(async () => {
+        await this.deps.native.stop(id)
+        this.nativeTouched = false
+      })
+    if (this.hotspotTouched)
+      await step(async () => {
+        if (this.deps.connected()) {
+          const result = await this.deps.hotspot(false)
+          if (result.state !== "disabled") throw new Error("Glasses hotspot shutdown was not confirmed")
+        } else this.deps.deferredStop()
+        this.hotspotTouched = false
+      })
+    if (failures.length) {
+      this.attemptId = id // Retain the native attempt token for a subsequent cleanup retry.
+      throw new Error(`Relay cleanup failed: ${failures.map((error) => asError(error).message).join("; ")}`)
+    }
+  }
+}
+
+function asError(error: unknown): Error {
+  return error instanceof Error ? error : new Error(String(error))
+}
+
+export function createManagedWebRtcRelay(
+  options: RelayOptions,
+  onStatus: (status: string, reason: string) => void,
+  onFailure: (error: Error) => void,
+  connected: () => boolean,
+  deferredStop: () => void,
+): ManagedRelay {
+  // Load only for managed WHIP. Hosts using direct SRT/RTMP need no relay module.
+  const {requireNativeModule} = require("expo-modules-core") as typeof import("expo-modules-core")
+  const {BgTimer} = require("../utils/timers") as typeof import("../utils/timers")
+  const native = requireNativeModule<NativeRelay>("MentraGlassesMediaRelay")
+  return new ManagedWebRtcRelay(options, onStatus, onFailure, {
+    native,
+    hotspot: (enabled) => BluetoothSdk.setHotspotState(enabled),
+    startGlasses: (request) => BluetoothSdk.startStream(request),
+    stopGlasses: () => BluetoothSdk.stopStream(),
+    connected,
+    deferredStop,
+    now: Date.now,
+    sleep: (ms) => new Promise((resolve) => BgTimer.setTimeout(resolve, ms)),
+    acquire: acquireGlassesHotspot,
+  })
+}