@mentra/engine 0.1.0-beta.0

package/src/services/MentraUIRouter.ts

@@ -0,0 +1,244 @@
+/**
+ * MentraUIRouter — bidirectional bus between a per-miniapp UI WebView
+ * and its bound background JSContext.
+ *
+ * The flow:
+ *
+ *   WebView (mentra.send/on)                Background (session.ui.send/on)
+ *      │                                       │
+ *      │   window.ReactNativeWebView          │
+ *      │   .postMessage(jsonEnvelope)         │  session.sendOneShot({
+ *      │                                      │     type: "UI_SEND", channel, payload, seq
+ *      │                                      │  })
+ *      │                                      │
+ *      ▼                                      ▼
+ *   MiniappHost.onMessage              MentraJSRouter (recognises payload.type==="UI_SEND")
+ *      │                                      │
+ *      ▼                                      ▼
+ *      ──── MentraUIRouter.routeFromWebView   ──── MentraUIRouter.routeFromBackground
+ *           ──► Crust.mentraJsDispatchToJs        ──► webView.injectJavaScript("window.__mentra.recv(...)")
+ *                {kind:"bridge", raw: JSON({
+ *                  payload:{
+ *                    type:"EVENT", streamType:"_ui",
+ *                    data:{type:"UI_MESSAGE", channel, payload, seq}
+ *                  }})}
+ *
+ * The router is a singleton — one instance for the whole host process —
+ * because only one WebView is allowed open at a time per spec. The
+ * binding map is therefore N entries (one per "currently has a UI
+ * WebView mounted" miniapp), but in practice it's 0 or 1.
+ *
+ * UI_OPEN / UI_CLOSE lifecycle envelopes are pushed by the WebView's
+ * `mentra.ready()` and the host's WebView teardown handler; the router
+ * forwards them to background via the same EVENT/_ui envelope shape.
+ */
+
+interface MentraUIInjectFn {
+  (jsSource: string): void
+}
+
+/**
+ * Subset of the Crust native module that MentraUIRouter touches.
+ * Loose binding keeps tests free of Expo imports.
+ */
+export interface MentraUICrustBinding {
+  mentraJsDispatchToJs(packageName: string, envelope: Record<string, unknown>): Promise<void> | void
+}
+
+interface BoundWebView {
+  inject: MentraUIInjectFn
+}
+
+/**
+ * NOTE: the WebView ↔ host heartbeat was removed when the lifecycle
+ * inversion landed. The current model is **at most one WebView at a
+ * time, foreground UI with an always-on background JSContext** (same
+ * shape as the cloud-WebView miniapps). User navigation closes it
+ * explicitly and `onContentProcessDidTerminate` catches OS-level
+ * crashes, but the host may still re-announce UI_OPEN for an already
+ * mounted WebView after app resume/dev respawn so the background can
+ * push a fresh authoritative snapshot.
+ */
+
+export class MentraUIRouter {
+  private readonly bindings: Map<string, BoundWebView> = new Map()
+  private readonly crust: MentraUICrustBinding
+
+  constructor(crust: MentraUICrustBinding) {
+    this.crust = crust
+  }
+
+  /**
+   * Called when a UI WebView mounts. `injectFn` wraps
+   * `webView.injectJavaScript` — the router calls it with prebuilt
+   * `window.__mentra.recv(...)` strings to deliver background-side
+   * frames into the WebView's mentra global.
+   *
+   * The router also pushes a UI_OPEN envelope to the background side
+   * once the WebView signals ready (see {@link routeFromWebView}).
+   * Calling bindWebView on a packageName that already has a binding
+   * replaces the previous one (defensive — should not happen in
+   * practice because only one WebView is open at a time).
+   */
+  bindWebView(packageName: string, injectFn: MentraUIInjectFn): void {
+    this.bindings.set(packageName, {inject: injectFn})
+  }
+
+  /**
+   * Called when the WebView unmounts (user navigated away or the host
+   * tore it down on heartbeat timeout). Pushes UI_CLOSE to background
+   * so handlers can flush state, then drops the binding.
+   */
+  unbindWebView(packageName: string): void {
+    if (!this.bindings.has(packageName)) return
+    this.bindings.delete(packageName)
+    this.deliverToBackground(packageName, {type: "UI_CLOSE"})
+  }
+
+  /** True iff a WebView is currently bound to the named package. */
+  isBound(packageName: string): boolean {
+    return this.bindings.has(packageName)
+  }
+
+  /**
+   * Re-announce the already-mounted WebView to the background JSContext.
+   * This covers dev background hot-reload, app resume, and thawed WebViews
+   * whose injected UI_SEND frames may have been missed while the host was
+   * suspended. Re-emitting UI_OPEN flips `ui.bound` back to true if needed
+   * and lets background code push a fresh snapshot via session.ui.onOpen.
+   *
+   * No-op if no WebView is currently bound for the package.
+   */
+  notifyReopen(packageName: string): void {
+    if (!this.bindings.has(packageName)) return
+    this.deliverToBackground(packageName, {type: "UI_OPEN"})
+  }
+
+  /**
+   * Forward a WebView-originated postMessage envelope to the JSContext.
+   * `rawJson` is the raw string from `event.nativeEvent.data` — the
+   * caller hasn't parsed it yet so the router controls the wire format
+   * end-to-end.
+   *
+   * Recognised envelope types from the WebView shim:
+   *   - {type: "ready"}                              → fire UI_OPEN
+   *   - {type: "msg", seq, channel, payload}         → fire UI_MESSAGE
+   *   - {type: "heartbeat", seq}                     → silently ack
+   */
+  routeFromWebView(packageName: string, rawJson: string): void {
+    let env: {
+      type?: string
+      seq?: number
+      channel?: string
+      payload?: unknown
+      requestId?: string
+    }
+    try {
+      env = JSON.parse(rawJson)
+    } catch {
+      return
+    }
+    if (typeof env.type !== "string") return
+
+    if (env.type === "ready") {
+      this.deliverToBackground(packageName, {type: "UI_OPEN"})
+      return
+    }
+    if (env.type === "msg" && typeof env.channel === "string") {
+      const out: Record<string, unknown> = {
+        type: "UI_MESSAGE",
+        channel: env.channel,
+        payload: env.payload,
+        seq: env.seq,
+      }
+      if (typeof env.requestId === "string") out.requestId = env.requestId
+      this.deliverToBackground(packageName, out)
+      return
+    }
+    if (env.type === "cancel" && typeof env.requestId === "string") {
+      this.deliverToBackground(packageName, {type: "UI_CANCEL", requestId: env.requestId})
+      return
+    }
+    // NOTE: console.* logs do NOT come through this router. The
+    // miniappGlobals console-tap shim posts {payload:{type:"dev_log"}}
+    // envelopes which LocalMiniappRuntime handles and forwards to the
+    // dev sidecar tagged source:"ui". Keeping that path one-way means
+    // we don't have two console-capture pipelines competing.
+    //
+    // Unknown envelope — drop silently.
+  }
+
+  /**
+   * Called by MentraJSRouter when it sees a `__bridge.send` outbound
+   * frame whose parsed payload is a UI_SEND envelope. The router
+   * forwards it as a `msg` frame to the bound WebView (or drops if no
+   * WebView is currently mounted).
+   *
+   * `uiSendPayload` is the raw `{type:"UI_SEND", channel, payload, seq}`
+   * shape produced by `session.ui.send` on the background side.
+   */
+  routeFromBackground(
+    packageName: string,
+    uiSendPayload: {
+      type: string
+      channel?: string
+      payload?: unknown
+      seq?: number
+      requestId?: string
+    },
+  ): void {
+    const binding = this.bindings.get(packageName)
+    if (!binding) return
+    if (uiSendPayload.type === "UI_CANCEL" && typeof uiSendPayload.requestId === "string") {
+      const cancel = {type: "cancel", requestId: uiSendPayload.requestId}
+      const literal = JSON.stringify(cancel)
+      const escaped = JSON.stringify(literal)
+      binding.inject(`if (window.__mentra && window.__mentra.recv) window.__mentra.recv(JSON.parse(${escaped})); true;`)
+      return
+    }
+    const outbound: Record<string, unknown> = {
+      type: "msg",
+      seq: uiSendPayload.seq ?? 0,
+      channel: uiSendPayload.channel,
+      payload: uiSendPayload.payload,
+    }
+    if (typeof uiSendPayload.requestId === "string") outbound.requestId = uiSendPayload.requestId
+    const literal = JSON.stringify(outbound)
+    const escaped = JSON.stringify(literal)
+    binding.inject(`if (window.__mentra && window.__mentra.recv) window.__mentra.recv(JSON.parse(${escaped})); true;`)
+  }
+
+  /**
+   * Send a synthetic frame to the WebView. Used for host-initiated
+   * lifecycle events (`open`, `close`, `ack`). The wire shape is the
+   * same as the WebView shim's recv handler expects.
+   */
+  pushLifecycleFrame(packageName: string, frame: {type: "open" | "close" | "ack"; seq?: number}): void {
+    const binding = this.bindings.get(packageName)
+    if (!binding) return
+    const literal = JSON.stringify(frame)
+    const escaped = JSON.stringify(literal)
+    binding.inject(`if (window.__mentra && window.__mentra.recv) window.__mentra.recv(JSON.parse(${escaped})); true;`)
+  }
+
+  /**
+   * Wrap an EVENT envelope and push it to background via the bridge.
+   * Background's UIModule listens for stream `_ui`. The `type` is the
+   * wire value `miniapp_event` — must match `MiniappResponseType.EVENT`
+   * exactly, otherwise the SDK's session switch falls through and the
+   * `_ui` fan-out never fires (bound stays false; `ui.send` drops).
+   */
+  private deliverToBackground(packageName: string, data: Record<string, unknown>): void {
+    const innerEnvelope = {
+      payload: {
+        type: "miniapp_event",
+        streamType: "_ui",
+        data,
+      },
+    }
+    void this.crust.mentraJsDispatchToJs(packageName, {
+      kind: "bridge",
+      raw: JSON.stringify(innerEnvelope),
+    })
+  }
+}

package/src/services/MicStateCoordinator.ts

@@ -0,0 +1,94 @@
+/**
+ * MicStateCoordinator
+ *
+ * Owns local-miniapp-driven microphone requirements.
+ *
+ * Local miniapps subscribe to audio_chunk / transcription streams.
+ * This coordinator pushes the aggregate local requirement set to BluetoothSdk
+ * so the mic runs whenever at least one local consumer needs it.
+ */
+
+import BluetoothSdk from "@mentra/bluetooth-sdk/internal"
+
+import {createDebouncedPatchFlusher} from "../utils/debouncedPatch"
+
+const LOG_TAG = "MIC_COORDINATOR"
+
+/** Mic-requirement flips are debounced (300ms) and merged into one BLE write
+ *  (wire v2 keeps BLE JSON small and infrequent). */
+const flushMicRequirementsPatch = createDebouncedPatchFlusher<Record<string, unknown>>((patch) => {
+  try {
+    void Promise.resolve(BluetoothSdk.updateBluetoothSettings(patch)).catch((err) => {
+      console.error(`${LOG_TAG}: failed to apply mic requirements:`, err)
+    })
+  } catch (err) {
+    console.error(`${LOG_TAG}: failed to apply mic requirements:`, err)
+  }
+}, 300)
+
+class MicStateCoordinator {
+  private static instance: MicStateCoordinator | null = null
+
+  // Local miniapp requirements (set when miniapps subscribe to audio streams)
+  private localWantsPcm = false
+  private localWantsLc3 = false
+
+  private constructor() {}
+
+  public static getInstance(): MicStateCoordinator {
+    if (!MicStateCoordinator.instance) {
+      MicStateCoordinator.instance = new MicStateCoordinator()
+    }
+    return MicStateCoordinator.instance
+  }
+
+  /**
+   * Update local miniapp requirements. Called by LocalMiniappRuntime when
+   * the aggregated set of local subscriptions changes.
+   */
+  public setLocalRequirements(req: {pcm: boolean; lc3: boolean}): void {
+    this.localWantsPcm = req.pcm
+    this.localWantsLc3 = req.lc3
+    console.log(`${LOG_TAG}: local requirements updated — pcm=${req.pcm} lc3=${req.lc3}`)
+    this.applyUnion()
+  }
+
+  /**
+   * Push local requirements to BluetoothSdk. `should_send_pcm` is strictly for
+   * on-device PCM consumers; cloud audio uses LC3 through AudioCloudUplink.
+   */
+  private applyUnion(): void {
+    const shouldSendPcm = this.localWantsPcm
+    const shouldSendLc3 = this.localWantsLc3
+
+    // console.log(
+    //   `${LOG_TAG}: applying requirements — pcm=${shouldSendPcm} lc3=${shouldSendLc3}`,
+    // )
+
+    // The mic control plane is a direct btsdk call now (was a host setMicRequirements
+    // hook) so a bare OEM streams audio without wiring it.
+    flushMicRequirementsPatch({
+      should_send_pcm: shouldSendPcm,
+      should_send_lc3: shouldSendLc3,
+      should_send_transcript: false,
+    })
+  }
+
+  /**
+   * Reset all requirements to off. Called during cleanup.
+   */
+  public reset(): void {
+    this.localWantsPcm = false
+    this.localWantsLc3 = false
+    this.applyUnion()
+  }
+
+  public cleanup(): void {
+    console.log(`${LOG_TAG}: cleanup()`)
+    this.reset()
+    MicStateCoordinator.instance = null
+  }
+}
+
+const micStateCoordinator = MicStateCoordinator.getInstance()
+export default micStateCoordinator

package/src/services/MiniappEngine.ts

@@ -0,0 +1,143 @@
+/**
+ * MiniappEngine — constructs the MentraJS router singletons (crash controller,
+ * UI router, JS router), binds them to the native Crust module + the launcher,
+ * wires the dev-server background-respawn signal, and starts the message pump.
+ *
+ * Engine owns this so a bare OEM's `engine.start()` brings up the local-miniapp
+ * engine with no host construction step. Host-only concerns that remain outside
+ * engine, such as Mentra-app crashloop telemetry, are attached by the host after
+ * this returns via `router.onCrashloop` / `router.onRestartToast`.
+ *
+ * Idempotent — `ensureMiniappEngine()` returns the same singletons on every call,
+ * so it's safe to call from both `engine.start()` and the host bootstrap.
+ */
+
+import CrustModule from "@mentra/crust"
+
+import devServerBridge from "./DevServerBridge"
+import localMiniappRuntime from "./LocalMiniappRuntime"
+import {miniappLauncher} from "./MiniappLauncher"
+import {MentraJSCrashController} from "./MentraJSCrashController"
+import {MentraJSRouter, type MentraJSCrustBinding} from "./MentraJSRouter"
+import {MentraUIRouter} from "./MentraUIRouter"
+
+export interface MiniappEngine {
+  router: MentraJSRouter
+  uiRouter: MentraUIRouter
+  crashController: MentraJSCrashController
+}
+
+let engine: MiniappEngine | null = null
+
+function wireDevServerRespawnBackground(router: MentraJSRouter, uiRouter: MentraUIRouter): void {
+  devServerBridge.onRespawnBackground(async (packageName) => {
+    try {
+      // Re-resolve the freshly built bundle (dev: HTTP off the dev server;
+      // released: file:// snapshot) via the launcher's shared recipe. Carries
+      // the declared permissions + manifest across the respawn — omitting them
+      // would respawn the JSContext with no permissions, so SUBSCRIBE gates and
+      // per-call dispatch checks would start rejecting after a background
+      // hot-reload.
+      const resolved = await miniappLauncher.resolveBundle(packageName)
+      if (!resolved) {
+        console.warn(`MentraJS: respawn-bg could not resolve bundle for ${packageName}`)
+        return
+      }
+      // Force a respawn (kill + spawn) rather than launcher.ensureRunning,
+      // which is idempotent and would no-op an already-registered context.
+      await router.unregister(packageName)
+      const ok = await router.spawnAndRegister(packageName, resolved.bgSource, {
+        permissions: resolved.declaredPermissions,
+        installedManifest: resolved.installedManifest,
+      })
+      if (!ok) {
+        console.warn(`MentraJS: respawn-bg failed for ${packageName}`)
+        return
+      }
+      // The respawned JSContext is a fresh MiniappSession with ui.bound false.
+      // The mounted WebView won't re-fire mentra.ready() (it's latched), so
+      // re-announce it so RPC replies (mentra.request) reach the UI again
+      // instead of being dropped while unbound.
+      uiRouter.notifyReopen(packageName)
+    } catch (e) {
+      console.warn(`MentraJS: respawn-bg threw for ${packageName}:`, e)
+    }
+  })
+}
+
+/**
+ * Construct (once) and return the MentraJS engine singletons. Starts the router's
+ * message pump on first call. Subsequent calls return the same instances.
+ */
+export function ensureMiniappEngine(): MiniappEngine {
+  if (engine) {
+    wireDevServerRespawnBackground(engine.router, engine.uiRouter)
+    engine.router.start()
+    return engine
+  }
+
+  // The Crust native module's published TS typing doesn't include the new
+  // mentraJs* functions until expo prebuild regenerates it; cast to the binding
+  // shapes the routers expect.
+  const crust = CrustModule as unknown as MentraJSCrustBinding
+
+  const crashController = new MentraJSCrashController({
+    maxRetries: 3,
+  })
+  const uiRouter = new MentraUIRouter({
+    mentraJsDispatchToJs: (packageName: string, envelope: Record<string, unknown>) =>
+      (
+        CrustModule as unknown as {
+          mentraJsDispatchToJs: (p: string, e: Record<string, unknown>) => Promise<void>
+        }
+      ).mentraJsDispatchToJs(packageName, envelope),
+  })
+  const router = new MentraJSRouter(localMiniappRuntime, crust)
+  router.crashController = crashController
+  router.uiRouter = uiRouter
+
+  // Hand the router to the engine MiniappLauncher so headless launch/teardown
+  // (apps.ts start/stop, the action broker, the WebView mount path) all spawn
+  // through one place.
+  miniappLauncher.configure({router})
+
+  // Wire up the dev server's "respawn-bg" signal so a touch under
+  // src/background/ kills + re-spawns the JSContext with the latest bundle.
+  // The WebView reload path stays separate (devServerBridge.onReload).
+  wireDevServerRespawnBackground(router, uiRouter)
+
+  router.start()
+
+  engine = {router, uiRouter, crashController}
+  return engine
+}
+
+/** Stop the MentraJS message pump and tear down router-managed JSContexts. */
+export async function stopMiniappEngine(): Promise<void> {
+  const current = engine
+  if (!current) return
+  // Null the singleton BEFORE the async teardown below: a concurrent
+  // ensureMiniappEngine() must construct fresh instead of re-starting (and
+  // re-subscribing) the router that is mid-teardown, which would leak a zombie
+  // Crust subscription once this function finishes.
+  engine = null
+
+  devServerBridge.clearRespawnBackgroundHandler()
+  current.router.stop()
+
+  const packages = current.router.registeredPackages()
+  await Promise.all(
+    packages.map(async (packageName) => {
+      try {
+        await current.router.unregister(packageName)
+      } catch (err) {
+        console.warn(`MentraJS: failed to unregister ${packageName} during stop:`, err)
+      }
+    }),
+  )
+}
+
+/** Returns the engine singletons if already constructed, else null. */
+export function getMiniappEngine(): MiniappEngine | null {
+  return engine
+}