@mentra/engine 3.2.0-dev.120 → 3.2.0-dev.122

package/src/services/LocalMiniappRuntime.ts

@@ -34,6 +34,7 @@ import {BgTimer} from "../utils/timers"
 import devServerBridge from "./DevServerBridge"
 import {islandNotifications} from "./NotificationsEmitter"
 import {isGlassesConnected} from "./GlassesReadiness"
+import {toMiniappConnectionData} from "./GlassesStatusProjection"
 import audioPlaybackService from "./AudioPlaybackService"
 import {phoneLocationService} from "./PhoneLocationService"
 import {useGlassesStore} from "../stores/glasses"
@@ -759,6 +760,22 @@ class LocalMiniappRuntime {
       this.replaceStreamSubscribers(packageName, existing.subscriptions, [])
       this.recomputeMicRequirements()
       this.updateCloudSubscriptions()
+      // Managed streams and ACS meetings deliberately survive a respawn: the new
+      // incarnation re-adopts them on restore (Mentra Call does exactly this) so a
+      // WebView reload does not drop the wearer out of a live call. An unmanaged
+      // stream cannot be re-adopted — startUnmanaged rejects while any stream is
+      // active, even for the same owner — so release it or the respawned miniapp
+      // is stuck behind STREAM_ALREADY_ACTIVE until the process restarts.
+      const streamSnapshot = phoneStreamCoordinator.getDiagnosticSnapshot()
+      if (
+        streamSnapshot.active === true &&
+        streamSnapshot.kind === "unmanaged" &&
+        streamSnapshot.ownerPackageName === packageName
+      ) {
+        void phoneStreamCoordinator.stop(packageName).catch((error) => {
+          console.warn(`${LOG_TAG}: failed to stop unmanaged stream while replacing ${packageName}`, error)
+        })
+      }
     }
     this.connectedApps.set(packageName, {
       subscriptions: new Set(),
@@ -1817,7 +1834,10 @@ class LocalMiniappRuntime {
         this.sendToMiniapp(packageName, {
           type: MiniappResponseType.EVENT,
           streamType: "glasses_connection",
-          data: glassesState,
+          data: {
+            connected: glassesState.connected,
+            modelName: glassesState.deviceModel || undefined,
+          },
         })
       } else if (stream === "glasses_wifi") {
         // Snapshot the current glasses Wi-Fi state on subscribe (like battery).
@@ -1836,6 +1856,7 @@ class LocalMiniappRuntime {
           streamType: "glasses_wifi",
           data: {
             connected,
+            linkConnected: glassesState.connected === true,
             ssid: connected ? wifi?.ssid : undefined,
             localIp: connected ? wifi?.localIp : undefined,
             timestamp: Date.now(),
@@ -3261,6 +3282,30 @@ class LocalMiniappRuntime {
   // Photo + streaming handlers (cloud-coordinated)
   // ===========================================================================
 
+  /**
+   * Manifest permission gate shared by the camera-and-mic RPCs (streams, meetings).
+   * Sends PERMISSION_NOT_DECLARED and returns false when the miniapp did not declare
+   * `permission`; the caller must return without doing the work.
+   */
+  private requireManifestPermission(
+    packageName: string,
+    requestId: string | undefined,
+    permission: "CAMERA" | "MICROPHONE",
+    purpose: string,
+    operation: MiniappRequestType,
+  ): boolean {
+    const app = this.connectedApps.get(packageName)
+    if (app?.installedManifest?.permissions?.some((p) => p.type === permission)) return true
+    logPermissionNotDeclared(packageName, permission, purpose, `{"type": "${permission}"}`)
+    this.sendResult(packageName, requestId, false, undefined, {
+      code: MiniappErrorCode.PERMISSION_NOT_DECLARED,
+      message: `${permission} permission not declared in miniapp.json. Add {"type": "${permission}"} to the "permissions" array.`,
+      permission,
+      operation,
+    })
+    return false
+  }
+
   private async handlePhoto(packageName: string, payload: Record<string, unknown>, requestId?: string): Promise<void> {
     // Manifest CAMERA permission gate.
     const app = this.connectedApps.get(packageName)
@@ -3419,6 +3464,10 @@ class LocalMiniappRuntime {
       })
       return
     }
+    // A stream is the glasses camera (and mic) leaving the device; gate it like a photo.
+    if (!this.requireManifestPermission(packageName, requestId, "CAMERA", "to stream the camera", MiniappRequestType.STREAM_START)) {
+      return
+    }
     try {
       const result = await streaming.startUnmanaged(packageName, {
         streamUrl: payload.streamUrl as string,
@@ -3476,6 +3525,17 @@ class LocalMiniappRuntime {
       })
       return
     }
+    if (
+      !this.requireManifestPermission(
+        packageName,
+        requestId,
+        "CAMERA",
+        "to stream the camera",
+        MiniappRequestType.MANAGED_STREAM_START,
+      )
+    ) {
+      return
+    }
     try {
       const result = await streaming.startManaged(packageName, {
         restreamDestinations: payload.restreamDestinations as Array<string | {url: string; name?: string}> | undefined,
@@ -3519,6 +3579,14 @@ class LocalMiniappRuntime {
     requestId?: string,
   ): Promise<void> {
     this.ensureMeetingStateBridge()
+    // A meeting puts the glasses camera and mic in front of remote strangers; it
+    // needs both declared, same as photo/stream and mic capture do individually.
+    if (
+      !this.requireManifestPermission(packageName, requestId, "CAMERA", "to join a meeting", MiniappRequestType.MEETING_JOIN) ||
+      !this.requireManifestPermission(packageName, requestId, "MICROPHONE", "to join a meeting", MiniappRequestType.MEETING_JOIN)
+    ) {
+      return
+    }
     const meetingUrl = typeof payload.meetingUrl === "string" ? payload.meetingUrl : ""
     const token = typeof payload.token === "string" ? payload.token : ""
     const videoSource = payload.videoSource as {type?: string; url?: string} | undefined
@@ -4045,6 +4113,9 @@ class LocalMiniappRuntime {
     // The bare `touch_event` stream above still catches `onTouch(handler)`.
     let perGestureStream: string | null = null
     let outboundData = data
+    if (normalizedStream === MiniappStreamType.GLASSES_CONNECTION) {
+      outboundData = toMiniappConnectionData(data) ?? data
+    }
     if (normalizedStream === MiniappStreamType.TOUCH_EVENT) {
       // The Bluetooth SDK delivers the gesture under `gestureName` (single_tap /
       // double_tap / triple_tap / long_press / swipe_up / swipe_down). Surface it

package/src/services/MiniappLauncher.ts

@@ -25,7 +25,7 @@
 
 import {File} from "expo-file-system"
 
-import {decideDevLaunchRoute} from "../utils/devMiniappLaunch"
+import {resolveDevBundleSource} from "../utils/devMiniappSnapshot"
 import {storage} from "../utils/storage/storage"
 import appRegistry, {getLocalAppRunningState, saveLocalAppRunningState} from "./AppRegistry"
 import devServerBridge from "./DevServerBridge"
@@ -96,67 +96,89 @@ class MiniappLauncher {
    * package. Handles both dev (HTTP off the running dev server) and released
    * (file:// from the installed snapshot). Reads disk/network/storage; does
    * NOT spawn. Returns null when the bundle can't be resolved (dev server
-   * unreachable, missing entry, no installed version).
+   * unreachable with no on-disk snapshot, missing entry, no installed version).
    */
   async resolveBundle(packageName: string, hints?: LaunchHints): Promise<ResolvedBundle | null> {
     const devUrl = hints?.devUrl ?? this.storedDevUrl(packageName)
 
-    // --- Dev: load directly off the local dev server over HTTP. ---
+    // --- Dev: live HTTP, then the last on-disk snapshot if the laptop is gone. ---
     if (devUrl) {
-      const route = await decideDevLaunchRoute(packageName, devUrl)
-      if (route.decision === "offline" || !route.manifest) return null
-      const manifest = route.manifest
-      const entry = manifest.entry as {background?: string; ui?: string} | undefined
-      if (!entry?.background) return null
-
-      // Use the host that actually answered — may differ from the stored IP
-      // after a laptop Wi-Fi change (mDNS / Metro failover inside decideDevLaunchRoute).
-      const base = route.resolvedUrl.replace(/\/$/, "")
-      // entry.* are bundle-root paths (dist/ stripped); the dev server serves
-      // files relative to cwd, so prepend dist/.
-      const bgUrl = `${base}/dist/${entry.background.replace(/^\.?\/+/, "")}`
-      const uiUri = entry.ui ? `${base}/dist/${entry.ui.replace(/^\.?\/+/, "")}` : null
-
-      const perms = manifest.permissions as Array<{type?: string} | string> | undefined
-      const declaredPermissions = (perms ?? [])
-        .map((p) => (typeof p === "string" ? p : p?.type))
-        .filter((t): t is string => typeof t === "string")
-      const installedManifest: InstalledMiniappManifest = {
-        packageName: typeof manifest.packageName === "string" ? manifest.packageName : packageName,
-        name: manifest.name,
-        version: typeof manifest.version === "string" ? manifest.version : undefined,
-        sdkVersion: typeof manifest.sdkVersion === "string" ? manifest.sdkVersion : undefined,
-        minHostVersion: typeof manifest.minHostVersion === "string" ? manifest.minHostVersion : undefined,
-        type: typeof manifest.type === "string" ? manifest.type : undefined,
-        entry: manifest.entry as InstalledMiniappManifest["entry"],
-        permissions: manifest.permissions as InstalledMiniappManifest["permissions"],
-        hardwareRequirements: manifest.hardwareRequirements as InstalledMiniappManifest["hardwareRequirements"],
-        actions: manifest.actions as InstalledMiniappManifest["actions"],
+      const source = await resolveDevBundleSource(packageName, devUrl)
+      if (source.kind === "live") {
+        const live = await this.resolveLiveHttp(packageName, source.resolvedUrl, source.manifest, hints)
+        if (live) return live
+        // Live probe succeeded but the entry fetch failed — still try disk.
       }
-
-      let bgSource: string
-      try {
-        const res = await fetch(bgUrl)
-        if (!res.ok) return null
-        bgSource = await res.text()
-      } catch {
-        return null
-      }
-
-      return {
-        bgSource,
-        uiUri,
-        uiBaseDir: uiUri ? uiUri.replace(/\/[^/]+$/, "/") : null,
-        declaredPermissions,
-        installedManifest,
-        devUrl: route.resolvedUrl,
-        devPort: this.resolveDevPort(hints?.devPort, packageName),
+      const snapshotVersion =
+        source.kind === "snapshot" ? source.version : appRegistry.getLatestDevSnapshotVersion(packageName)
+      if (snapshotVersion) {
+        const snapshot = await this.resolveInstalledBundle(packageName, snapshotVersion)
+        if (snapshot) return snapshot
       }
+      return null
     }
 
     // --- Released: resolve from the installed file:// snapshot. ---
     const version = hints?.version ?? (await appRegistry.getActiveVersion(packageName))
     if (!version) return null
+    return this.resolveInstalledBundle(packageName, version)
+  }
+
+  private async resolveLiveHttp(
+    packageName: string,
+    resolvedUrl: string,
+    manifest: {entry?: unknown; permissions?: unknown; [key: string]: unknown},
+    hints?: LaunchHints,
+  ): Promise<ResolvedBundle | null> {
+    const entry = manifest.entry as {background?: string; ui?: string} | undefined
+    if (!entry?.background) return null
+
+    // Use the host that actually answered — may differ from the stored IP
+    // after a laptop Wi-Fi change (mDNS / Metro failover inside decideDevLaunchRoute).
+    const base = resolvedUrl.replace(/\/$/, "")
+    // entry.* are bundle-root paths (dist/ stripped); the dev server serves
+    // files relative to cwd, so prepend dist/.
+    const bgUrl = `${base}/dist/${entry.background.replace(/^\.?\/+/, "")}`
+    const uiUri = entry.ui ? `${base}/dist/${entry.ui.replace(/^\.?\/+/, "")}` : null
+
+    const perms = manifest.permissions as Array<{type?: string} | string> | undefined
+    const declaredPermissions = (perms ?? [])
+      .map((p) => (typeof p === "string" ? p : p?.type))
+      .filter((t): t is string => typeof t === "string")
+    const installedManifest: InstalledMiniappManifest = {
+      packageName: typeof manifest.packageName === "string" ? manifest.packageName : packageName,
+      name: typeof manifest.name === "string" ? manifest.name : undefined,
+      version: typeof manifest.version === "string" ? manifest.version : undefined,
+      sdkVersion: typeof manifest.sdkVersion === "string" ? manifest.sdkVersion : undefined,
+      minHostVersion: typeof manifest.minHostVersion === "string" ? manifest.minHostVersion : undefined,
+      type: typeof manifest.type === "string" ? manifest.type : undefined,
+      entry: manifest.entry as InstalledMiniappManifest["entry"],
+      permissions: manifest.permissions as InstalledMiniappManifest["permissions"],
+      hardwareRequirements: manifest.hardwareRequirements as InstalledMiniappManifest["hardwareRequirements"],
+      actions: manifest.actions as InstalledMiniappManifest["actions"],
+    }
+
+    let bgSource: string
+    try {
+      const res = await fetch(bgUrl)
+      if (!res.ok) return null
+      bgSource = await res.text()
+    } catch {
+      return null
+    }
+
+    return {
+      bgSource,
+      uiUri,
+      uiBaseDir: uiUri ? uiUri.replace(/\/[^/]+$/, "/") : null,
+      declaredPermissions,
+      installedManifest,
+      devUrl: resolvedUrl,
+      devPort: this.resolveDevPort(hints?.devPort, packageName),
+    }
+  }
+
+  private async resolveInstalledBundle(packageName: string, version: string): Promise<ResolvedBundle | null> {
     const entryPaths = appRegistry.getMiniappEntryPaths(packageName, version)
     if (!entryPaths?.background) return null
 
@@ -203,6 +225,7 @@ class MiniappLauncher {
       uiBaseDir: entryPaths.ui ? entryPaths.ui.replace(/\/[^/]+$/, "/") : null,
       declaredPermissions,
       installedManifest,
+      // Snapshot fallback is file:// — do not wire the sidecar; the laptop is gone.
       devUrl: null,
       devPort: null,
     }

package/src/utils/devMiniappSnapshot.ts

@@ -0,0 +1,119 @@
+/**
+ * Dev miniapp snapshot — keep a local copy of the last live-dev bundle so
+ * walking away from the laptop still opens the miniapp while Mentra App is
+ * running (and after, as long as the on-disk `dev-*` snapshot remains).
+ *
+ * Live launches load HTTP off `mentra-miniapp dev` for hot reload. After a
+ * successful reachability probe we fire-and-forget `${devUrl}/bundle.zip`
+ * into `lmas/<pkg>/dev-<ms>/`. When the laptop is unreachable, launch
+ * routes and MiniappLauncher fall back to that snapshot instead of the
+ * "Dev server offline" dead-end.
+ */
+
+import appRegistry from "../services/AppRegistry"
+import {storage} from "./storage/storage"
+import {
+  decideDevLaunchRoute,
+  type DecideDevLaunchOptions,
+  type DevLaunchResult,
+  type DevManifest,
+} from "./devMiniappLaunch"
+
+export type DevOpenDecision =
+  | {decision: "live"; manifest: DevManifest; resolvedUrl: string}
+  | {decision: "cached"}
+  | {decision: "offline"}
+
+export type DevBundleSource =
+  | {kind: "live"; resolvedUrl: string; manifest: DevManifest}
+  | {kind: "snapshot"; version: string}
+  | {kind: "none"}
+
+const snapshotInFlight = new Map<string, Promise<void>>()
+
+/** Candidate zip URLs: same-origin `/bundle.zip`, then the sidecar path. */
+export function snapshotCandidateUrls(baseUrl: string, sidecarPort?: number | null): string[] {
+  const trimmed = baseUrl.replace(/\/$/, "")
+  const urls = [`${trimmed}/bundle.zip`]
+  try {
+    const url = new URL(trimmed)
+    const userPort = Number(url.port) || (url.protocol === "https:" ? 443 : 80)
+    const sidecar = sidecarPort && sidecarPort > 0 ? sidecarPort : userPort + 1
+    url.port = String(sidecar)
+    const sidecarZip = `${url.origin}/__mentra_dev/bundle.zip`
+    if (!urls.includes(sidecarZip)) urls.push(sidecarZip)
+  } catch {
+    /* ignore malformed base */
+  }
+  return urls
+}
+
+function storedSidecarPort(packageName: string): number | undefined {
+  const stored = storage.load<number>(`${packageName}_dev_port`)
+  return stored.is_ok() && Number.isFinite(stored.value) ? stored.value : undefined
+}
+
+/**
+ * Download the current live-dev zip into `lmas/<pkg>/dev-<ms>/` and keep
+ * only the newest snapshot. Coalesces concurrent calls per package.
+ */
+export function queueDevSnapshot(packageName: string, baseUrl: string, sidecarPort?: number | null): void {
+  if (!packageName || !baseUrl) return
+  if (snapshotInFlight.has(packageName)) return
+  const promise = installDevSnapshot(packageName, baseUrl, sidecarPort ?? storedSidecarPort(packageName)).finally(() => {
+    snapshotInFlight.delete(packageName)
+  })
+  snapshotInFlight.set(packageName, promise)
+}
+
+async function installDevSnapshot(packageName: string, baseUrl: string, sidecarPort?: number | null): Promise<void> {
+  let lastError: unknown
+  for (const url of snapshotCandidateUrls(baseUrl, sidecarPort)) {
+    const res = await appRegistry.installFromUrl(url, {
+      versionOverride: `dev-${Date.now()}`,
+      releaseIdentity: {source: "dev_snapshot"},
+    })
+    if (res.is_ok()) {
+      appRegistry.gcDevVersions(packageName, 1)
+      return
+    }
+    lastError = res.error
+  }
+  console.warn(`Dev snapshot failed for ${packageName}:`, lastError)
+}
+
+/** Reachability first; if the laptop is down, open the last local snapshot. */
+export async function decideDevOpenRoute(
+  packageName: string,
+  devUrl: string,
+  options?: DecideDevLaunchOptions,
+): Promise<DevOpenDecision> {
+  const route = await decideDevLaunchRoute(packageName, devUrl, options)
+  if (route.decision === "live") {
+    queueDevSnapshot(packageName, route.resolvedUrl)
+    return route
+  }
+  if (packageName && appRegistry.hasDevSnapshot(packageName)) {
+    return {decision: "cached"}
+  }
+  return {decision: "offline"}
+}
+
+/**
+ * Same live-vs-snapshot choice MiniappLauncher uses to resolve a bundle.
+ * Live also kicks a background snapshot so the next offline open has a copy.
+ */
+export async function resolveDevBundleSource(
+  packageName: string,
+  devUrl: string,
+  options?: DecideDevLaunchOptions,
+): Promise<DevBundleSource> {
+  const route: DevLaunchResult = await decideDevLaunchRoute(packageName, devUrl, options)
+  if (route.decision === "live" && route.manifest) {
+    queueDevSnapshot(packageName, route.resolvedUrl)
+    return {kind: "live", resolvedUrl: route.resolvedUrl, manifest: route.manifest}
+  }
+  const version = appRegistry.getLatestDevSnapshotVersion(packageName)
+  if (version) return {kind: "snapshot", version}
+  return {kind: "none"}
+}