@mentra/engine 0.1.0-dev.0 → 3.1.0-dev.4

package/src/services/LocalMiniappStorage.ts

@@ -0,0 +1,82 @@
+/**
+ * Phone-local SimpleStorage for bundled miniapps.
+ *
+ * Keys are scoped to the stable Mentra user id and package name. Access tokens
+ * must never be part of the namespace: Core access tokens rotate and are held
+ * in memory only, so using one makes every app restart look like a new user.
+ */
+
+export interface LocalMiniappStorageBackend {
+  get(key: string): unknown | null
+  has(key: string): boolean
+  set(key: string, value: unknown): void
+  remove(key: string): void
+  keys(): string[]
+}
+
+export interface LocalMiniappStorageHooks {
+  getUserId: () => Promise<string>
+  backend: LocalMiniappStorageBackend
+}
+
+const KEY_ROOT = "mentraos_localstorage_"
+
+export class LocalMiniappStorage {
+  constructor(private readonly hooks: LocalMiniappStorageHooks) {}
+
+  async get(packageName: string, key: string): Promise<unknown | null> {
+    return this.hooks.backend.get((await this.prefix(packageName)) + key)
+  }
+
+  async set(packageName: string, key: string, value: unknown): Promise<void> {
+    this.hooks.backend.set((await this.prefix(packageName)) + key, value)
+  }
+
+  async delete(packageName: string, key: string): Promise<void> {
+    this.hooks.backend.remove((await this.prefix(packageName)) + key)
+  }
+
+  async keys(packageName: string): Promise<string[]> {
+    const prefix = await this.prefix(packageName)
+    return this.hooks.backend
+      .keys()
+      .filter((key) => key.startsWith(prefix))
+      .map((key) => key.slice(prefix.length))
+  }
+
+  async clear(packageName: string): Promise<void> {
+    const prefix = await this.prefix(packageName)
+    for (const key of this.hooks.backend.keys()) {
+      if (key.startsWith(prefix)) this.hooks.backend.remove(key)
+    }
+  }
+
+  async has(packageName: string, key: string): Promise<boolean> {
+    return this.hooks.backend.has((await this.prefix(packageName)) + key)
+  }
+
+  async getAll(packageName: string): Promise<Record<string, string>> {
+    const prefix = await this.prefix(packageName)
+    const values: Record<string, string> = {}
+    for (const key of this.hooks.backend.keys()) {
+      if (!key.startsWith(prefix)) continue
+      if (!this.hooks.backend.has(key)) continue
+      const value = this.hooks.backend.get(key)
+      values[key.slice(prefix.length)] = typeof value === "string" ? value : String(value ?? "")
+    }
+    return values
+  }
+
+  async setMultiple(packageName: string, values: Record<string, unknown>): Promise<void> {
+    const prefix = await this.prefix(packageName)
+    for (const [key, value] of Object.entries(values)) {
+      this.hooks.backend.set(prefix + key, value ?? null)
+    }
+  }
+
+  private async prefix(packageName: string): Promise<string> {
+    const userId = (await this.hooks.getUserId()).trim()
+    if (!userId) throw new Error("Mentra user identity is unavailable")
+    return `${KEY_ROOT}${userId}_${packageName}_`
+  }
+}

package/src/services/LocalMiniappUserIdentity.ts

@@ -0,0 +1,46 @@
+export interface LocalMiniappUserIdentityBackend {
+  get(): string | null
+  set(userId: string): void
+  remove(): void
+}
+
+/**
+ * Keeps the stable Mentra user id available to local miniapps across process
+ * restarts. The id is non-secret and is deliberately stored separately from
+ * the short-lived Core access token.
+ */
+export class LocalMiniappUserIdentity {
+  private current: string | null = null
+
+  constructor(private readonly backend: LocalMiniappUserIdentityBackend) {}
+
+  async resolve(resolveOnline: () => Promise<string>): Promise<string> {
+    const available = this.get()
+    if (available) return available
+
+    const resolved = (await resolveOnline()).trim()
+    if (!resolved) throw new Error("Mentra user identity is unavailable")
+    this.remember(resolved)
+    return resolved
+  }
+
+  get(): string | null {
+    if (this.current) return this.current
+    const cached = this.backend.get()?.trim() ?? ""
+    if (!cached) return null
+    this.current = cached
+    return cached
+  }
+
+  remember(userId: string): void {
+    const resolved = userId.trim()
+    if (!resolved) throw new Error("Mentra user identity is unavailable")
+    this.backend.set(resolved)
+    this.current = resolved
+  }
+
+  forget(): void {
+    this.current = null
+    this.backend.remove()
+  }
+}

package/src/services/LocalSttFallbackCoordinator.ts

@@ -6,22 +6,24 @@ import sttModelManager from "./STTModelManager"
 
 /**
  * Decides whether on-device Sherpa STT should produce transcripts for local
- * miniapps, instead of (or in addition to) cloud transcription. The chooser
- * is connection-state-driven: when the WebSocket to cloud is down AND a
- * local miniapp is subscribed to transcription, we activate on-device STT.
+ * miniapps, instead of (or in addition to) cloud transcription. Local STT is
+ * active when the cloud is unavailable or any subscription requires
+ * `forceLocal`.
  *
  * Activation flips `localSttFallbackActive` in the settings store, which
  * the bluetooth-sdk native side reads to decide whether to feed PCM into
  * the SherpaOnnxTranscriber. Local transcripts then flow back through the
  * `local_transcription` Bridge event; the host wires that event to
  * `localMiniappRuntime.forwardEvent`, with this coordinator providing the
- * subscription-active gate so we never publish transcripts to miniapps
- * that aren't asking for them.
+ * subscription-active gate so we never publish transcripts to miniapps that
+ * aren't asking for them. LocalMiniappRuntime filters mixed local/cloud
+ * delivery per miniapp.
  */
 class LocalSttFallbackCoordinator {
   private static instance: LocalSttFallbackCoordinator
 
   private hasTranscriptionSubscription = false
+  private hasForceLocalSubscription = false
   private activeLanguage: string | null = null
   /**
    * Default to "cloud is up" so we never accidentally activate local STT before
@@ -74,9 +76,10 @@ class LocalSttFallbackCoordinator {
     return this.activeLanguage
   }
 
-  onSubscriptionChange(hasTranscription: boolean, language: string | null): void {
-    this.log(`onSubscriptionChange(hasTx=${hasTranscription}, lang=${language})`)
+  onSubscriptionChange(hasTranscription: boolean, language: string | null, hasForceLocal = false): void {
+    this.log(`onSubscriptionChange(hasTx=${hasTranscription}, lang=${language}, forceLocal=${hasForceLocal})`)
     this.hasTranscriptionSubscription = hasTranscription
+    this.hasForceLocalSubscription = hasTranscription && hasForceLocal
     this.activeLanguage = hasTranscription ? language : null
     void this.reconcile()
   }
@@ -92,7 +95,7 @@ class LocalSttFallbackCoordinator {
 
   private async reconcile(): Promise<void> {
     this.attachCloudAdapterIfReady()
-    const shouldBeActive = this.hasTranscriptionSubscription && !this.cloudConnected
+    const shouldBeActive = this.shouldUseLocalStt()
     if (shouldBeActive && !this.localActive) {
       await this.startLocalStt()
     } else if (!shouldBeActive && this.localActive) {
@@ -107,8 +110,8 @@ class LocalSttFallbackCoordinator {
       this.log("local stt model is not available yet — skipping activation")
       return
     }
-    if (!this.hasTranscriptionSubscription || this.cloudConnected) {
-      this.log("local stt activation skipped: cloud recovered before start completed")
+    if (!this.shouldUseLocalStt()) {
+      this.log("local stt activation skipped: routing changed before start completed")
       return
     }
     try {
@@ -117,8 +120,8 @@ class LocalSttFallbackCoordinator {
     } catch (err) {
       this.log(`restartTranscriber failed: ${err}`)
     }
-    if (!this.hasTranscriptionSubscription || this.cloudConnected) {
-      this.log("local stt activation skipped: cloud recovered during transcriber restart")
+    if (!this.shouldUseLocalStt()) {
+      this.log("local stt activation skipped: routing changed during transcriber restart")
       return
     }
     useSettingsStore.getState().setSetting(ISLAND_SETTINGS_KEYS.localSttFallbackActive, true)
@@ -131,6 +134,10 @@ class LocalSttFallbackCoordinator {
     this.localActive = false
   }
 
+  private shouldUseLocalStt(): boolean {
+    return this.hasTranscriptionSubscription && (this.hasForceLocalSubscription || !this.cloudConnected)
+  }
+
   private log(msg: string): void {
     console.log(`[LocalSttFallback] ${msg}`)
   }

package/src/services/MicStateCoordinator.ts

@@ -14,11 +14,27 @@ import {createDebouncedPatchFlusher} from "../utils/debouncedPatch"
 
 const LOG_TAG = "MIC_COORDINATOR"
 
+type MicGate = "vad" | "loudnessGate"
+
+interface GateOverride {
+  enabled: boolean
+  order: number
+}
+
+interface ConfiguredMicGates {
+  vadEnabled?: boolean | null
+  loudnessGateEnabled?: boolean | null
+}
+
 /** Mic-requirement flips are debounced (300ms) and merged into one BLE write
  *  (wire v2 keeps BLE JSON small and infrequent). */
 const flushMicRequirementsPatch = createDebouncedPatchFlusher<Record<string, unknown>>((patch) => {
   try {
-    void Promise.resolve(BluetoothSdk.updateBluetoothSettings(patch)).catch((err) => {
+    // Resolve runtime overrides at flush time. A miniapp can acquire/release a
+    // gate override during the debounce window, and a captured stale value
+    // must not win after that lifecycle transition.
+    const runtimePatch = micStateCoordinator.applyEffectiveGatePolicy(patch)
+    void Promise.resolve(BluetoothSdk.updateBluetoothSettings(runtimePatch)).catch((err) => {
       console.error(`${LOG_TAG}: failed to apply mic requirements:`, err)
     })
   } catch (err) {
@@ -32,6 +48,11 @@ class MicStateCoordinator {
   // Local miniapp requirements (set when miniapps subscribe to audio streams)
   private localWantsPcm = false
   private localWantsLc3 = false
+  private configuredVad: boolean | undefined
+  private configuredLoudnessGate: boolean | undefined
+  private readonly miniappVadOverrides = new Map<string, GateOverride>()
+  private readonly miniappLoudnessGateOverrides = new Map<string, GateOverride>()
+  private overrideSequence = 0
 
   private constructor() {}
 
@@ -46,13 +67,110 @@ class MicStateCoordinator {
    * Update local miniapp requirements. Called by LocalMiniappRuntime when
    * the aggregated set of local subscriptions changes.
    */
-  public setLocalRequirements(req: {pcm: boolean; lc3: boolean}): void {
+  public setLocalRequirements(req: {pcm: boolean; lc3: boolean} & ConfiguredMicGates): void {
     this.localWantsPcm = req.pcm
     this.localWantsLc3 = req.lc3
+    this.rememberConfiguredGates(req)
     console.log(`${LOG_TAG}: local requirements updated — pcm=${req.pcm} lc3=${req.lc3}`)
     this.applyUnion()
   }
 
+  /**
+   * Apply a miniapp-owned gate override without changing the OS preference.
+   * Overrides are lifecycle-scoped and last-live-owner-wins independently for
+   * VAD and Barrier.
+   */
+  public async setMiniappGateOverride(
+    packageName: string,
+    gate: MicGate,
+    enabled: boolean,
+    configured: ConfiguredMicGates = {},
+  ): Promise<void> {
+    this.rememberConfiguredGates(configured)
+    const overrides = this.overridesFor(gate)
+    const previous = overrides.get(packageName)
+    const next = {enabled, order: ++this.overrideSequence}
+    overrides.set(packageName, next)
+
+    try {
+      await BluetoothSdk.updateBluetoothSettings(this.effectiveGatePatch())
+    } catch (error) {
+      // Do not roll back a newer request from the same package that landed
+      // while this native write was in flight.
+      if (overrides.get(packageName) === next) {
+        if (previous) overrides.set(packageName, previous)
+        else overrides.delete(packageName)
+      }
+      throw error
+    }
+  }
+
+  /**
+   * Remove every gate override owned by a miniapp. This is synchronous so
+   * unregister can drop ownership before recomputing aggregate mic state.
+   */
+  public clearMiniappGateOverrides(packageName: string): boolean {
+    const removedVad = this.miniappVadOverrides.delete(packageName)
+    const removedLoudness = this.miniappLoudnessGateOverrides.delete(packageName)
+    return removedVad || removedLoudness
+  }
+
+  /** Re-apply the current runtime policy after an owner is released. */
+  public async syncEffectiveGatePolicy(configured: ConfiguredMicGates = {}): Promise<void> {
+    this.rememberConfiguredGates(configured)
+    const patch = this.effectiveGatePatch()
+    if (Object.keys(patch).length === 0) return
+    await BluetoothSdk.updateBluetoothSettings(patch)
+  }
+
+  /**
+   * Preserve the runtime microphone contract when the persisted device
+   * settings are replayed (for example after a glasses reconnect). Active
+   * miniapp gate overrides replace the OS values, and raw PCM keeps VAD off
+   * until the last raw-audio consumer unsubscribes.
+   */
+  public applyRuntimeOverrides(settings: Record<string, unknown>): Record<string, unknown> {
+    this.rememberConfiguredGates({
+      vadEnabled:
+        typeof settings.voice_activity_detection_enabled === "boolean"
+          ? settings.voice_activity_detection_enabled
+          : undefined,
+      loudnessGateEnabled:
+        typeof settings.loudness_gate_enabled === "boolean" ? settings.loudness_gate_enabled : undefined,
+    })
+
+    return this.applyActiveRuntimeOverrides(settings)
+  }
+
+  /**
+   * Apply the complete current gate policy without treating the input as an OS
+   * preference update. Used by debounced mic-requirement writes, whose queued
+   * gate values may be stale after an override is acquired or released.
+   */
+  public applyEffectiveGatePolicy(settings: Record<string, unknown>): Record<string, unknown> {
+    return {
+      ...settings,
+      ...this.effectiveGatePatch(),
+    }
+  }
+
+  private applyActiveRuntimeOverrides(settings: Record<string, unknown>): Record<string, unknown> {
+    const runtimeSettings = {...settings}
+    const vadOverride = this.latestOverride(this.miniappVadOverrides)
+    const loudnessOverride = this.latestOverride(this.miniappLoudnessGateOverrides)
+
+    if (this.localWantsPcm) {
+      runtimeSettings.voice_activity_detection_enabled = false
+    } else if (vadOverride) {
+      runtimeSettings.voice_activity_detection_enabled = vadOverride.enabled
+    }
+    if (loudnessOverride) {
+      runtimeSettings.loudness_gate_enabled = loudnessOverride.enabled
+    }
+
+    return runtimeSettings
+  }
+
   /**
    * Push local requirements to BluetoothSdk. `should_send_pcm` is strictly for
    * on-device PCM consumers; cloud audio uses LC3 through AudioCloudUplink.
@@ -67,11 +185,56 @@ class MicStateCoordinator {
 
     // The mic control plane is a direct btsdk call now (was a host setMicRequirements
     // hook) so a bare OEM streams audio without wiring it.
-    flushMicRequirementsPatch({
+    const patch: Record<string, unknown> = {
       should_send_pcm: shouldSendPcm,
       should_send_lc3: shouldSendLc3,
       should_send_transcript: false,
-    })
+      ...this.effectiveGatePatch(),
+    }
+
+    flushMicRequirementsPatch(patch)
+  }
+
+  private rememberConfiguredGates(configured: ConfiguredMicGates): void {
+    // null means the connected device intentionally omits that setting.
+    // undefined means the caller is not updating the remembered preference.
+    if (configured.vadEnabled !== undefined) {
+      this.configuredVad = configured.vadEnabled ?? undefined
+    }
+    if (configured.loudnessGateEnabled !== undefined) {
+      this.configuredLoudnessGate = configured.loudnessGateEnabled ?? undefined
+    }
+  }
+
+  private overridesFor(gate: MicGate): Map<string, GateOverride> {
+    return gate === "vad" ? this.miniappVadOverrides : this.miniappLoudnessGateOverrides
+  }
+
+  private latestOverride(overrides: Map<string, GateOverride>): GateOverride | undefined {
+    let latest: GateOverride | undefined
+    for (const entry of overrides.values()) {
+      if (!latest || entry.order > latest.order) latest = entry
+    }
+    return latest
+  }
+
+  private effectiveGatePatch(): Record<string, unknown> {
+    const patch: Record<string, unknown> = {}
+    const vadOverride = this.latestOverride(this.miniappVadOverrides)
+    const loudnessOverride = this.latestOverride(this.miniappLoudnessGateOverrides)
+
+    // Hardware VAD suppresses silence. Raw-audio consumers need a continuous
+    // timeline, so their requirement wins over both OS and miniapp VAD values.
+    if (this.localWantsPcm) patch.voice_activity_detection_enabled = false
+    else if (vadOverride) patch.voice_activity_detection_enabled = vadOverride.enabled
+    else if (this.configuredVad !== undefined) patch.voice_activity_detection_enabled = this.configuredVad
+
+    if (loudnessOverride) patch.loudness_gate_enabled = loudnessOverride.enabled
+    else if (this.configuredLoudnessGate !== undefined) {
+      patch.loudness_gate_enabled = this.configuredLoudnessGate
+    }
+
+    return patch
   }
 
   /**
@@ -85,6 +248,8 @@ class MicStateCoordinator {
 
   public cleanup(): void {
     console.log(`${LOG_TAG}: cleanup()`)
+    this.miniappVadOverrides.clear()
+    this.miniappLoudnessGateOverrides.clear()
     this.reset()
     MicStateCoordinator.instance = null
   }

package/src/services/MiniappLauncher.ts

@@ -109,7 +109,9 @@ class MiniappLauncher {
       const entry = manifest.entry as {background?: string; ui?: string} | undefined
       if (!entry?.background) return null
 
-      const base = devUrl.replace(/\/$/, "")
+      // Use the host that actually answered — may differ from the stored IP
+      // after a laptop Wi-Fi change (mDNS / Metro failover inside decideDevLaunchRoute).
+      const base = route.resolvedUrl.replace(/\/$/, "")
       // entry.* are bundle-root paths (dist/ stripped); the dev server serves
       // files relative to cwd, so prepend dist/.
       const bgUrl = `${base}/dist/${entry.background.replace(/^\.?\/+/, "")}`
@@ -120,9 +122,16 @@ class MiniappLauncher {
         .map((p) => (typeof p === "string" ? p : p?.type))
         .filter((t): t is string => typeof t === "string")
       const installedManifest: InstalledMiniappManifest = {
+        packageName: typeof manifest.packageName === "string" ? manifest.packageName : packageName,
         name: manifest.name,
+        version: typeof manifest.version === "string" ? manifest.version : undefined,
+        sdkVersion: typeof manifest.sdkVersion === "string" ? manifest.sdkVersion : undefined,
+        minHostVersion: typeof manifest.minHostVersion === "string" ? manifest.minHostVersion : undefined,
+        type: typeof manifest.type === "string" ? manifest.type : undefined,
+        entry: manifest.entry as InstalledMiniappManifest["entry"],
         permissions: manifest.permissions as InstalledMiniappManifest["permissions"],
         hardwareRequirements: manifest.hardwareRequirements as InstalledMiniappManifest["hardwareRequirements"],
+        actions: manifest.actions as InstalledMiniappManifest["actions"],
       }
 
       let bgSource: string
@@ -140,7 +149,7 @@ class MiniappLauncher {
         uiBaseDir: uiUri ? uiUri.replace(/\/[^/]+$/, "/") : null,
         declaredPermissions,
         installedManifest,
-        devUrl,
+        devUrl: route.resolvedUrl,
         devPort: this.resolveDevPort(hints?.devPort, packageName),
       }
     }
@@ -152,15 +161,33 @@ class MiniappLauncher {
     if (!entryPaths?.background) return null
 
     const manifest = appRegistry.getMiniappManifest(packageName, version) as {
+      packageName?: string
       name?: string
+      version?: string
+      sdkVersion?: string
+      minHostVersion?: string
+      type?: string
+      entry?: {background?: string; ui?: string}
       permissions?: Array<{type: string; required?: boolean; description?: string}>
       hardwareRequirements?: Array<{type: string; level: string; description?: string}>
+      actions?: Array<{id?: unknown; description?: unknown; parameters?: unknown; outputSchema?: unknown}>
     } | null
     const declaredPermissions = (manifest?.permissions ?? [])
       .map((p) => p.type)
       .filter((t): t is string => typeof t === "string")
     const installedManifest: InstalledMiniappManifest | undefined = manifest
-      ? {name: manifest.name, permissions: manifest.permissions, hardwareRequirements: manifest.hardwareRequirements}
+      ? {
+          packageName: manifest.packageName ?? packageName,
+          name: manifest.name,
+          version: manifest.version ?? version,
+          sdkVersion: manifest.sdkVersion,
+          minHostVersion: manifest.minHostVersion,
+          type: manifest.type,
+          entry: manifest.entry,
+          permissions: manifest.permissions,
+          hardwareRequirements: manifest.hardwareRequirements,
+          actions: manifest.actions,
+        }
       : undefined
 
     let bgSource: string

package/src/services/MiniappLiveness.ts

@@ -0,0 +1,24 @@
+export interface MiniappPingLivenessDecision {
+  shouldUnregister: boolean
+  unansweredPingRounds: number
+}
+
+/**
+ * Advance liveness by one scheduled ping round.
+ *
+ * Counting actual rounds avoids treating time while React Native's scheduler
+ * was paused in the background as pings the miniapp failed to answer.
+ */
+export function advanceMiniappPingLiveness(
+  unansweredPingRounds: number,
+  timeoutThreshold: number,
+): MiniappPingLivenessDecision {
+  if (unansweredPingRounds >= timeoutThreshold) {
+    return {shouldUnregister: true, unansweredPingRounds}
+  }
+
+  return {
+    shouldUnregister: false,
+    unansweredPingRounds: unansweredPingRounds + 1,
+  }
+}

package/src/services/NativeHttpResponse.ts

@@ -0,0 +1,6 @@
+const BODY_FORBIDDEN_STATUSES = new Set([204, 205, 304])
+
+/** Fetch forbids a non-null Response body for these HTTP statuses. */
+export function nativeHttpResponseBody(status: number, body: string): string | null {
+  return BODY_FORBIDDEN_STATUSES.has(status) ? null : body
+}