@mentra/engine 0.1.0-dev.0 → 0.1.0-dev.1

package/src/utils/devMiniappLaunch.ts

@@ -12,12 +12,27 @@
  *      get it in the same round trip — no second fetch.
  *
  * Pre-flighting at the call site keeps /applet/local a pure mount route.
+ *
+ * When the laptop's Wi-Fi IP changes, the URL stored from the last QR scan
+ * goes stale. We then retry alternate hosts (Metro / EXPO_PUBLIC_LOCAL_MINIAPP_HOST,
+ * plus an mDNS name captured at scan time) and persist the working URL so
+ * the next launch doesn't need a re-scan.
  */
 
+import {configuredDevHost} from "./configuredDevHost"
 import {storage} from "./storage/storage"
 
 const REACHABILITY_TIMEOUT_MS = 1500
 
+// A single dropped DNS lookup or momentary Wi-Fi blip (AP roam, DHCP renewal,
+// phone hopping to cellular for a beat) looks identical to a dead dev server —
+// both throw out of fetch(). Retrying once absorbs that transient case instead
+// of bouncing straight to the offline screen. We do NOT retry a real HTTP
+// response (res.ok === false): that's the server telling us something
+// definitive, not a network-layer hiccup.
+const REACHABILITY_MAX_ATTEMPTS = 2
+const REACHABILITY_RETRY_DELAY_MS = 400
+
 export type DevManifest = {
   packageName?: string
   name?: string
@@ -29,40 +44,153 @@ export type DevManifest = {
 }
 
 export type DevLaunchResult =
-  | {decision: "live"; manifest: DevManifest}
-  | {decision: "offline"; manifest: null}
+  | {decision: "live"; manifest: DevManifest; resolvedUrl: string}
+  | {decision: "offline"; manifest: null; resolvedUrl?: undefined}
+
+export type DecideDevLaunchOptions = {
+  /** Extra hostnames / IPs to try if `devUrl` is unreachable (no scheme/port). */
+  alternateHosts?: string[]
+}
+
+function delay(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms))
+}
+
+/** Single GET <devUrl>/miniapp.json attempt with a hard timeout. */
+async function fetchManifestOnce(devUrl: string): Promise<DevManifest | null> {
+  const controller = new AbortController()
+  const timer = setTimeout(() => controller.abort(), REACHABILITY_TIMEOUT_MS)
+  try {
+    const res = await fetch(`${devUrl.replace(/\/$/, "")}/miniapp.json`, {
+      method: "GET",
+      signal: controller.signal,
+    })
+    if (!res.ok) return null
+    return (await res.json()) as DevManifest
+  } finally {
+    clearTimeout(timer)
+  }
+}
+
+function rewriteHost(devUrl: string, host: string): string | null {
+  const trimmed = host.trim()
+  if (!trimmed) return null
+  try {
+    const url = new URL(devUrl)
+    if (url.hostname === trimmed) return null
+    url.hostname = trimmed
+    return url.toString().replace(/\/$/, "")
+  } catch {
+    return null
+  }
+}
+
+/**
+ * Point an http(s) icon at the host that actually answered. Cached dev icons are
+ * local `file://` paths and third-party icons live on unrelated hosts, so only
+ * icons still served by the stale dev host are rewritten.
+ */
+function rewriteIconHost(iconUrl: string, staleDevUrl: string, resolvedDevUrl: string): string | null {
+  try {
+    const icon = new URL(iconUrl)
+    if (icon.protocol !== "http:" && icon.protocol !== "https:") return null
+    const stale = new URL(staleDevUrl)
+    const resolved = new URL(resolvedDevUrl)
+    // Compare host (hostname+port) so an icon on the same hostname but a
+    // different service port is left alone.
+    if (icon.host !== stale.host) return null
+    icon.host = resolved.host
+    return icon.toString()
+  } catch {
+    return null
+  }
+}
+
+function collectAlternateHosts(packageName: string, extra?: string[]): string[] {
+  const hosts: string[] = []
+  const push = (h: string | undefined | null) => {
+    const v = h?.trim()
+    if (v && !hosts.includes(v)) hosts.push(v)
+  }
+
+  push(configuredDevHost())
+
+  if (packageName) {
+    const mdns = storage.load<string>(`${packageName}_dev_mdns`)
+    if (mdns.is_ok()) push(mdns.value)
+  }
+
+  for (const h of extra ?? []) push(h)
+  return hosts
+}
+
+async function probeUrl(devUrl: string): Promise<DevManifest | null | "network-error"> {
+  for (let attempt = 1; attempt <= REACHABILITY_MAX_ATTEMPTS; attempt++) {
+    try {
+      const manifest = await fetchManifestOnce(devUrl)
+      // Definitive HTTP failure — do not retry this URL further.
+      if (!manifest) return null
+      return manifest
+    } catch {
+      if (attempt < REACHABILITY_MAX_ATTEMPTS) {
+        await delay(REACHABILITY_RETRY_DELAY_MS)
+        continue
+      }
+      return "network-error"
+    }
+  }
+  return "network-error"
+}
 
 /**
- * GET <devUrl>/miniapp.json with a hard timeout. Returns the parsed
+ * GET <devUrl>/miniapp.json with a hard timeout, retrying once on a
+ * network-layer failure (DNS/connection/abort) before trying alternate
+ * hosts (Metro laptop IP, mDNS name from the QR). Returns the parsed
  * manifest on success ("live") or null ("offline").
  *
- * Side effect: on success, writes <packageName>_dev_last_reachable so
- * the dev-offline screen can show "Last reached: N min ago" the next
- * time the user lands there.
- *
- * The fetch doubles as the reachability probe AND the manifest source —
- * one request per launch attempt instead of two.
+ * Side effects on success:
+ *   - writes <packageName>_dev_last_reachable
+ *   - if an alternate host worked, rewrites <packageName>_dev_url so the
+ *     next launch skips the stale IP
  */
 export async function decideDevLaunchRoute(
   packageName: string,
   devUrl: string,
+  options?: DecideDevLaunchOptions,
 ): Promise<DevLaunchResult> {
-  const controller = new AbortController()
-  const timer = setTimeout(() => controller.abort(), REACHABILITY_TIMEOUT_MS)
-  try {
-    const res = await fetch(`${devUrl.replace(/\/$/, "")}/miniapp.json`, {
-      method: "GET",
-      signal: controller.signal,
-    })
-    if (!res.ok) return {decision: "offline", manifest: null}
-    const manifest = (await res.json()) as DevManifest
+  const candidates: string[] = [devUrl]
+  for (const host of collectAlternateHosts(packageName, options?.alternateHosts)) {
+    const rewritten = rewriteHost(devUrl, host)
+    if (rewritten && !candidates.includes(rewritten)) candidates.push(rewritten)
+  }
+
+  for (const candidate of candidates) {
+    const result = await probeUrl(candidate)
+    if (result === "network-error" || result === null) continue
+
     if (packageName) {
       storage.save(`${packageName}_dev_last_reachable`, Date.now())
+      if (candidate !== devUrl) {
+        storage.save(`${packageName}_dev_url`, candidate)
+        // Keep _dev_meta.devUrl in sync when present so home-tile reads match.
+        const metaRes = storage.load<string>(`${packageName}_dev_meta`)
+        if (metaRes.is_ok()) {
+          try {
+            const meta = JSON.parse(metaRes.value) as {devUrl?: string; iconUrl?: string}
+            meta.devUrl = candidate
+            if (typeof meta.iconUrl === "string") {
+              const rewritten = rewriteIconHost(meta.iconUrl, devUrl, candidate)
+              if (rewritten) meta.iconUrl = rewritten
+            }
+            storage.save(`${packageName}_dev_meta`, JSON.stringify(meta))
+          } catch {
+            /* ignore malformed meta */
+          }
+        }
+      }
     }
-    return {decision: "live", manifest}
-  } catch {
-    return {decision: "offline", manifest: null}
-  } finally {
-    clearTimeout(timer)
+    return {decision: "live", manifest: result, resolvedUrl: candidate}
   }
+
+  return {decision: "offline", manifest: null}
 }

package/src/utils/diagnosticContext.ts

@@ -4,11 +4,14 @@ import * as Location from "expo-location"
 import {Platform} from "react-native"
 import type {ReportContext} from "@mentra/cloud-client"
 
-import {useAppStatusStore} from "../stores/apps"
+import appRegistry from "../services/AppRegistry"
+import localMiniappRuntime from "../services/LocalMiniappRuntime"
+import {getLastOpenTime, useAppStatusStore} from "../stores/apps"
 import {useConnectionStore} from "../stores/connection"
 import {useCoreStore} from "../stores/core"
 import {useGlassesStore} from "../stores/glasses"
 import {SETTINGS, useSettingsStore} from "../stores/settings"
+import {buildMiniappDiagnosticContext} from "./miniappDiagnostics"
 
 const SENSITIVE_SETTINGS_KEYS = ["core_token", "auth_token", "auth_email"] as const
 const SENSITIVE_GLASSES_KEYS = ["hotspotPassword"] as const
@@ -108,17 +111,17 @@ export async function collectDiagnosticContext(extra?: Partial<ReportContext>):
 
   const offlineMode = await useSettingsStore.getState().getSetting(SETTINGS.offline_mode.key)
   const defaultWearable = await useSettingsStore.getState().getSetting(SETTINGS.default_wearable.key)
-  const apps = appletState.apps.map((app) => ({
-    packageName: app.packageName,
-    name: app.name,
-    running: app.running,
-    loading: app.loading,
-    healthy: app.healthy,
-    hidden: app.hidden,
-    type: app.type,
-    offline: app.offline,
-    local: app.local,
-  }))
+  const miniappRuntime = localMiniappRuntime.getDiagnosticSnapshot()
+  const miniapps = buildMiniappDiagnosticContext({
+    apps: appletState.apps,
+    foregroundedPackage: appletState.foregroundedPackage,
+    runtime: miniappRuntime,
+    getLastOpenedAtMs: (app) => getLastOpenTime(app.packageName),
+    getManifest: (app) =>
+      app.local && app.version ? appRegistry.getMiniappManifest(app.packageName, app.version) : undefined,
+    getReleaseIdentity: (app) =>
+      app.local && app.version ? appRegistry.getReleaseIdentity(app.packageName, app.version) : undefined,
+  })
 
   return {
     app: {
@@ -139,12 +142,9 @@ export async function collectDiagnosticContext(extra?: Partial<ReportContext>):
     runtime: {
       core: coreState,
       connection: connectionState,
+      miniapps: miniappRuntime,
     },
-    apps: {
-      apps,
-      installed: apps.map((app) => app.packageName),
-      running: apps.filter((app) => app.running).map((app) => app.packageName),
-    },
+    apps: miniapps,
     settings: {
       ...filteredSettings,
       offlineMode: !!offlineMode,

package/src/utils/hardware/hardware.ts

@@ -1,11 +1,4 @@
-import {
-  Capabilities,
-  HardwareRequirement,
-  HardwareType,
-  HardwareRequirementLevel,
-  DeviceTypes,
-} from "../../types"
-import {simulatedGlasses} from "../../types"
+import {Capabilities, DeviceTypes, HardwareRequirement, HardwareRequirementLevel, HardwareType, simulatedGlasses} from "../../types"
 
 /**
  * Result of a hardware compatibility check
@@ -86,7 +79,12 @@ export class HardwareCompatibility {
         return capabilities.hasDisplay
 
       case HardwareType.MICROPHONE:
-        return capabilities.hasMicrophone
+        // MentraOS always provides an audio-input path. When the connected
+        // glasses do not have an onboard microphone, the engine falls back to
+        // the phone microphone. Keep `capabilities.hasMicrophone` reserved for
+        // physical-device/source selection; app compatibility is about whether
+        // the requested input is available to the miniapp.
+        return true
 
       case HardwareType.SPEAKER:
         return capabilities.hasSpeaker

package/src/utils/miniappDiagnostics.ts

@@ -0,0 +1,226 @@
+import type {ClientApp} from "../types/applet"
+
+const MAX_DIAGNOSTIC_STRING_LENGTH = 512
+const MAX_DIAGNOSTIC_LIST_ITEMS = 100
+const MAX_RECENT_MINIAPPS = 20
+
+export interface MiniappManifestSnapshot {
+  packageName: string
+  name?: string
+  version?: string
+  sdkVersion?: string
+  minHostVersion?: string
+  type?: string
+  entry?: {
+    background?: string
+    ui?: string
+  }
+  permissions: Array<{type: string; required?: boolean}>
+  hardwareRequirements: Array<{type: string; level: string}>
+  actionIds: string[]
+}
+
+export interface MiniappRuntimeAppSnapshot {
+  packageName: string
+  handshakeComplete: boolean
+  subscriptions: string[]
+  lastMessageAgeMs: number
+  requestedLocationRate: string | null
+  hasActiveSpeakerStream: boolean
+  manifest?: MiniappManifestSnapshot
+}
+
+export interface MiniappRuntimeDiagnosticSnapshot {
+  connectedApps: MiniappRuntimeAppSnapshot[]
+  subscriptionsByStream: Record<string, string[]>
+  resources: {
+    display: Record<string, unknown>
+    camera: Record<string, unknown>
+    video: Record<string, unknown>
+    stream: Record<string, unknown>
+    cameraFov: Record<string, unknown>
+  }
+}
+
+type ManifestFallback = {
+  packageName: string
+  name?: string
+  version?: string
+  type?: string
+}
+
+function boundedString(value: unknown): string | undefined {
+  if (typeof value !== "string") return undefined
+  const trimmed = value.trim()
+  if (!trimmed) return undefined
+  return trimmed.slice(0, MAX_DIAGNOSTIC_STRING_LENGTH)
+}
+
+function recordValue(value: unknown): Record<string, unknown> | null {
+  return value && typeof value === "object" && !Array.isArray(value) ? (value as Record<string, unknown>) : null
+}
+
+function diagnosticPermissions(value: unknown): Array<{type: string; required?: boolean}> {
+  if (!Array.isArray(value)) return []
+  const permissions: Array<{type: string; required?: boolean}> = []
+  for (const raw of value) {
+    if (typeof raw === "string") {
+      const type = boundedString(raw)
+      if (type) permissions.push({type})
+      continue
+    }
+    const permission = recordValue(raw)
+    const type = boundedString(permission?.type)
+    if (!type) continue
+    permissions.push({
+      type,
+      ...(typeof permission?.required === "boolean" ? {required: permission.required} : {}),
+    })
+  }
+  return permissions.slice(0, MAX_DIAGNOSTIC_LIST_ITEMS)
+}
+
+function diagnosticHardwareRequirements(value: unknown): Array<{type: string; level: string}> {
+  if (!Array.isArray(value)) return []
+  const requirements: Array<{type: string; level: string}> = []
+  for (const raw of value) {
+    const requirement = recordValue(raw)
+    const type = boundedString(requirement?.type)
+    const level = boundedString(requirement?.level)
+    if (type && level) requirements.push({type, level})
+  }
+  return requirements.slice(0, MAX_DIAGNOSTIC_LIST_ITEMS)
+}
+
+function diagnosticActionIds(value: unknown): string[] {
+  if (!Array.isArray(value)) return []
+  return value
+    .map((raw) => boundedString(typeof raw === "string" ? raw : recordValue(raw)?.id))
+    .filter((id): id is string => id !== undefined)
+    .sort()
+    .slice(0, MAX_DIAGNOSTIC_LIST_ITEMS)
+}
+
+/**
+ * Keep the routing/build fields from miniapp.json while excluding arbitrary
+ * free-form manifest content from reports. Descriptions, URLs, and unknown
+ * extension fields are intentionally omitted.
+ */
+export function buildMiniappManifestSnapshot(raw: unknown, fallback: ManifestFallback): MiniappManifestSnapshot {
+  const manifest = recordValue(raw)
+  const entry = recordValue(manifest?.entry)
+  const packageName = boundedString(manifest?.packageName) ?? boundedString(fallback.packageName) ?? "unknown"
+  const name = boundedString(manifest?.name) ?? boundedString(fallback.name)
+  const version = boundedString(manifest?.version) ?? boundedString(fallback.version)
+  const type = boundedString(manifest?.type) ?? boundedString(fallback.type)
+  const background = boundedString(entry?.background)
+  const ui = boundedString(entry?.ui)
+
+  return {
+    packageName,
+    ...(name ? {name} : {}),
+    ...(version ? {version} : {}),
+    ...(boundedString(manifest?.sdkVersion) ? {sdkVersion: boundedString(manifest?.sdkVersion)} : {}),
+    ...(boundedString(manifest?.minHostVersion) ? {minHostVersion: boundedString(manifest?.minHostVersion)} : {}),
+    ...(type ? {type} : {}),
+    ...(background || ui ? {entry: {...(background ? {background} : {}), ...(ui ? {ui} : {})}} : {}),
+    permissions: diagnosticPermissions(manifest?.permissions),
+    hardwareRequirements: diagnosticHardwareRequirements(manifest?.hardwareRequirements),
+    actionIds: diagnosticActionIds(manifest?.actions ?? manifest?.actionIds),
+  }
+}
+
+function executionMode(app: ClientApp): "built_in" | "dev_server" | "dev_snapshot" | "installed_bundle" {
+  if (app.offline) return "built_in"
+  if (app.isMiniappDev && app.devUrl && !app.version) return "dev_server"
+  if (app.isMiniappDev) return "dev_snapshot"
+  return "installed_bundle"
+}
+
+function safeReleaseIdentity(value: unknown): Record<string, string> | undefined {
+  const identity = recordValue(value)
+  if (!identity) return undefined
+  const out: Record<string, string> = {}
+  for (const key of ["source", "releaseId", "bundleSha256", "channel"] as const) {
+    const field = boundedString(identity[key])
+    if (field) out[key] = field
+  }
+  return Object.keys(out).length > 0 ? out : undefined
+}
+
+export function buildMiniappDiagnosticContext(input: {
+  apps: ClientApp[]
+  foregroundedPackage: string | null
+  runtime: MiniappRuntimeDiagnosticSnapshot
+  getLastOpenedAtMs: (app: ClientApp) => number
+  getManifest: (app: ClientApp) => unknown
+  getReleaseIdentity: (app: ClientApp) => unknown
+}): Record<string, unknown> {
+  const runtimeByPackage = new Map(input.runtime.connectedApps.map((app) => [app.packageName, app]))
+  const appSnapshots = input.apps.map((app) => {
+    const lastOpenedAtMs = input.getLastOpenedAtMs(app)
+    const packageName = boundedString(app.packageName) ?? "unknown"
+    return {
+      packageName,
+      name: boundedString(app.name) ?? packageName,
+      version: boundedString(app.version) ?? null,
+      executionMode: executionMode(app),
+      running: app.running,
+      foregrounded: app.packageName === input.foregroundedPackage,
+      loading: app.loading,
+      healthy: app.healthy,
+      hidden: app.hidden,
+      type: app.type,
+      offline: app.offline,
+      local: app.local,
+      lastOpenedAtMs: Number.isFinite(lastOpenedAtMs) ? lastOpenedAtMs : 0,
+    }
+  })
+
+  const recentlyInteracted = [...appSnapshots]
+    .filter((app) => app.lastOpenedAtMs > 0)
+    .sort((a, b) => b.lastOpenedAtMs - a.lastOpenedAtMs)
+    .slice(0, MAX_RECENT_MINIAPPS)
+    .map((app) => ({packageName: app.packageName, lastOpenedAtMs: app.lastOpenedAtMs}))
+
+  const runningMiniapps = input.apps
+    .filter((app) => app.running)
+    .map((app) => {
+      const runtime = runtimeByPackage.get(app.packageName)
+      const releaseIdentity = safeReleaseIdentity(input.getReleaseIdentity(app))
+      const packageName = boundedString(app.packageName) ?? "unknown"
+      const manifest = buildMiniappManifestSnapshot(input.getManifest(app) ?? runtime?.manifest, {
+        packageName,
+        name: app.name,
+        version: app.version,
+        type: app.type,
+      })
+      return {
+        packageName,
+        version: boundedString(app.version) ?? manifest.version ?? null,
+        executionMode: executionMode(app),
+        foregrounded: app.packageName === input.foregroundedPackage,
+        manifest,
+        ...(releaseIdentity ? {releaseIdentity} : {}),
+        runtime: runtime
+          ? {
+              handshakeComplete: runtime.handshakeComplete,
+              subscriptions: runtime.subscriptions,
+              lastMessageAgeMs: runtime.lastMessageAgeMs,
+              requestedLocationRate: runtime.requestedLocationRate,
+              hasActiveSpeakerStream: runtime.hasActiveSpeakerStream,
+            }
+          : null,
+      }
+    })
+
+  return {
+    apps: appSnapshots,
+    installed: appSnapshots.map((app) => app.packageName),
+    running: runningMiniapps.map((app) => app.packageName),
+    foregroundedPackage: boundedString(input.foregroundedPackage) ?? null,
+    mostRecentlyInteractedPackage: recentlyInteracted[0]?.packageName ?? null,
+    recentlyInteracted,
+    runningMiniapps,
+  }
+}

package/src/utils/permissions/MediaLibraryPermissions.ts

@@ -5,6 +5,14 @@ import CrustModule from "@mentra/crust"
 
 import {deriveGalleryDisplayName} from "./galleryDisplayName"
 
+export interface MediaLibrarySaveReceipt {
+  success: boolean
+  platform: "ios" | "android" | "unknown"
+  uri?: string
+  identifier?: string
+  error?: string
+}
+
 /**
  * MediaLibraryPermissions - Handles save-only permissions for camera roll
  *
@@ -14,6 +22,18 @@ import {deriveGalleryDisplayName} from "./galleryDisplayName"
  * - Android 9-: Uses WRITE_EXTERNAL_STORAGE (legacy)
  */
 export class MediaLibraryPermissions {
+  /** Limited iOS access cannot see historical, unselected assets for duplicate reconciliation. */
+  static async hasLimitedAccess(): Promise<boolean> {
+    if (Platform.OS !== "ios") return false
+    try {
+      return (await check(PERMISSIONS.IOS.PHOTO_LIBRARY)) === RESULTS.LIMITED
+    } catch (error) {
+      console.error("[MediaLibrary] Error checking limited-library access:", error)
+      // Conservatively block legacy reconciliation when the scope cannot be verified.
+      return true
+    }
+  }
+
   /**
    * Check if we have permission to save to the camera roll
    * Note: On Android 10+, this always returns true since no permission is needed
@@ -85,6 +105,12 @@ export class MediaLibraryPermissions {
    * @param creationTime - Optional creation/capture time in milliseconds (Unix timestamp)
    */
   static async saveToLibrary(filePath: string, creationTime?: number): Promise<boolean> {
+    return (await this.saveToLibraryWithReceipt(filePath, creationTime)).success
+  }
+
+  /** Save and return the durable MediaStore/PhotoKit identifier used as the export receipt. */
+  static async saveToLibraryWithReceipt(filePath: string, creationTime?: number): Promise<MediaLibrarySaveReceipt> {
+    const platform = Platform.OS === "ios" || Platform.OS === "android" ? Platform.OS : "unknown"
     try {
       // On Android 10+, we can save without permission
       // On iOS and older Android, check permission first
@@ -95,7 +121,7 @@ export class MediaLibraryPermissions {
           const granted = await this.requestPermission()
           if (!granted) {
             console.warn("[MediaLibrary] No permission to save to library - photos saved to app storage only")
-            return false
+            return {success: false, platform, error: "permission denied"}
           }
         }
       }
@@ -117,14 +143,19 @@ export class MediaLibraryPermissions {
         } else {
           console.log(`[MediaLibrary] Saved to camera roll: ${cleanPath}`)
         }
-        return true
+        return {
+          success: true,
+          platform,
+          uri: result.uri,
+          identifier: result.identifier,
+        }
       } else {
         console.error(`[MediaLibrary] Failed to save to library: ${result.error}`)
-        return false
+        return {success: false, platform, error: result.error || "native export failed"}
       }
     } catch (error) {
       console.error("[MediaLibrary] Error saving to library:", error)
-      return false
+      return {success: false, platform, error: error instanceof Error ? error.message : String(error)}
     }
   }
 }