@mentra/engine 0.1.0-dev.0

package/src/types/capabilities/vuzix-z100.ts

@@ -0,0 +1,64 @@
+/**
+ * @fileoverview Vuzix Z100 Hardware Capabilities
+ *
+ * Capability profile for the Vuzix Z100 smart glasses model.
+ * Defines the hardware and software features available on this device.
+ */
+
+import type { Capabilities } from "../hardware";
+
+/**
+ * Vuzix Z100 capability profile
+ */
+export const vuzixZ100: Capabilities = {
+  modelName: "Vuzix Z100",
+
+  // Camera capabilities - does not have a camera
+  hasCamera: false,
+  camera: null,
+
+  // Display capabilities - has a green monochrome display
+  hasDisplay: true,
+  display: {
+    count: 1,
+    isColor: false,
+    color: "green",
+    canDisplayBitmap: false,
+    resolution: { width: 640, height: 480 },
+    fieldOfView: { horizontal: 30 },
+    maxTextLines: 7,
+    adjustBrightness: true,
+
+    // Scene display API — no positioned elements; scenes degrade to text walls.
+    canPosition: false,
+    maxImageElements: 0,
+  },
+
+  // Microphone capabilities - does not have a microphone
+  hasMicrophone: false,
+  microphone: null,
+
+  // Speaker capabilities - does not have a speaker
+  hasSpeaker: false,
+  speaker: null,
+
+  // IMU capabilities - does not have an IMU
+  hasIMU: false,
+  imu: null,
+
+  // Button capabilities - does not have buttons
+  hasButton: false,
+  button: null,
+
+  // Light capabilities - does not have lights
+  hasLight: false,
+  light: null,
+
+  // Power capabilities - does not have external battery
+  power: {
+    hasExternalBattery: false,
+  },
+
+  // WiFi capabilities - does not support WiFi
+  hasWifi: false,
+};

package/src/types/enums.ts

@@ -0,0 +1,43 @@
+/**
+ * @mentra/types - Shared enums for MentraOS
+ * These are runtime values (not pure types)
+ */
+
+/**
+ * Hardware component types that apps can require
+ */
+export enum HardwareType {
+  CAMERA = "CAMERA",
+  DISPLAY = "DISPLAY",
+  MICROPHONE = "MICROPHONE",
+  SPEAKER = "SPEAKER",
+  IMU = "IMU",
+  BUTTON = "BUTTON",
+  LIGHT = "LIGHT",
+  WIFI = "WIFI",
+  EXIST = "EXIST",
+}
+
+/**
+ * Levels of hardware requirements
+ */
+export enum HardwareRequirementLevel {
+  REQUIRED = "REQUIRED", // App cannot function without this hardware
+  OPTIONAL = "OPTIONAL", // App has enhanced features with this hardware
+}
+
+export enum DeviceTypes {
+  NONE = "None",
+  SIMULATED = "Simulated Glasses",
+  G1 = "Even Realities G1",
+  G2 = "Even Realities G2",
+  LIVE = "Mentra Live",
+  MACH1 = "Mentra Mach1",
+  Z100 = "Vuzix Z100",
+  NEX = "Mentra Display",
+  FRAME = "Brilliant Frame",
+}
+
+export enum ControllerTypes {
+  R1 = "Even Realities R1",
+}

package/src/types/hardware.ts

@@ -0,0 +1,203 @@
+/**
+ * @mentra/types - Hardware capability types
+ */
+
+import { evenRealitiesG1 } from "./capabilities/even-realities-g1";
+import { evenRealitiesG2 } from "./capabilities/even-realities-g2";
+import { mentraDisplay } from "./capabilities/mentra-display";
+import { mentraLive } from "./capabilities/mentra-live";
+import { simulatedGlasses } from "./capabilities/simulated-glasses";
+import { vuzixZ100 } from "./capabilities/vuzix-z100";
+import { none } from "./capabilities/none";
+import { DeviceTypes, HardwareRequirementLevel, HardwareType } from "./enums";
+
+/**
+ * Hardware requirement for an app
+ * Specifies what hardware components an app needs
+ */
+export interface HardwareRequirement {
+  type: HardwareType;
+  level: HardwareRequirementLevel;
+  description?: string; // Why this hardware is needed
+}
+
+/**
+ * Camera capabilities
+ */
+export interface CameraCapabilities {
+  resolution?: { width: number; height: number };
+  hasHDR?: boolean;
+  hasFocus?: boolean;
+  video: {
+    canRecord: boolean;
+    canStream: boolean;
+    supportedStreamTypes?: string[];
+    supportedResolutions?: { width: number; height: number }[];
+  };
+}
+
+/**
+ * Display capabilities
+ */
+export interface DisplayCapabilities {
+  count?: number;
+  isColor?: boolean;
+  color?: string; // e.g., "green", "full_color", "pallet"
+  canDisplayBitmap?: boolean;
+  resolution?: { width: number; height: number };
+  fieldOfView?: { horizontal?: number; vertical?: number };
+  maxTextLines?: number;
+  adjustBrightness?: boolean;
+
+  // --- Scene display API (display.render()) — typed capabilities ---
+  // Pure DATA the host scene pipeline acts on generically; a device without
+  // canPosition renders scenes via the host degrade path instead.
+  /** Public drawable canvas in px — the raw coordinate space for render() boxes. */
+  width?: number;
+  height?: number;
+  /** True ⇒ the device renders positioned elements (containers / canvas components). */
+  canPosition?: boolean;
+  /** Element budgets. Rects share the text pool on container-based devices. */
+  maxTextElements?: number;
+  maxImageElements?: number;
+  /** Per-image dimension cap (box-level), when the firmware has one. */
+  maxImagePx?: { width: number; height: number };
+  shapes?: "rect"[];
+  intensityLevels?: number;
+  partialUpdate?: boolean;
+}
+
+/**
+ * Microphone capabilities
+ */
+export interface MicrophoneCapabilities {
+  count?: number;
+  hasVAD?: boolean; // Voice Activity Detection
+}
+
+/**
+ * Speaker capabilities
+ */
+export interface SpeakerCapabilities {
+  count?: number;
+  isPrivate?: boolean; // e.g., bone conduction
+}
+
+/**
+ * IMU (Inertial Measurement Unit) capabilities
+ */
+export interface IMUCapabilities {
+  axisCount?: number;
+  hasAccelerometer?: boolean;
+  hasCompass?: boolean;
+  hasGyroscope?: boolean;
+}
+
+/**
+ * Button capabilities
+ */
+export interface ButtonCapabilities {
+  count?: number;
+  buttons?: {
+    type: "press" | "swipe1d" | "swipe2d";
+    events: string[]; // e.g., "press", "double_press", "long_press", "swipe_up", "swipe_down"
+    isCapacitive?: boolean;
+  }[];
+}
+
+/**
+ * Light capabilities
+ */
+export interface LightCapabilities {
+  count?: number;
+  lights?: {
+    id: string; // Unique identifier for the LED (e.g., "privacy", "user_feedback")
+    purpose: "privacy" | "user_feedback" | "general"; // LED purpose/function
+    isFullColor: boolean;
+    color?: string; // e.g., "white", "rgb"
+    position?: "front_facing" | "user_facing" | "side" | "unknown"; // LED physical position
+  }[];
+}
+
+/**
+ * Power capabilities
+ */
+export interface PowerCapabilities {
+  hasExternalBattery: boolean; // e.g., a case or puck
+}
+
+/**
+ * Device hardware capabilities
+ * Complete information about what hardware a device has
+ */
+export interface Capabilities {
+  modelName: string;
+
+  // Camera capabilities
+  hasCamera: boolean;
+  camera: CameraCapabilities | null;
+
+  // Display capabilities
+  hasDisplay: boolean;
+  display: DisplayCapabilities | null;
+
+  // Microphone capabilities
+  hasMicrophone: boolean;
+  microphone: MicrophoneCapabilities | null;
+
+  // Speaker capabilities
+  hasSpeaker: boolean;
+  speaker: SpeakerCapabilities | null;
+
+  // IMU capabilities
+  hasIMU: boolean;
+  imu: IMUCapabilities | null;
+
+  // Button capabilities
+  hasButton: boolean;
+  button: ButtonCapabilities | null;
+
+  // Light capabilities
+  hasLight: boolean;
+  light: LightCapabilities | null;
+
+  // Power capabilities
+  power: PowerCapabilities;
+
+  // WiFi capability
+  hasWifi: boolean;
+
+  // Native dashboard capability
+  // True when the device renders its own dashboard in firmware (e.g. Even
+  // Realities' built-in dashboard on G2). MentraOS does not manage or expose
+  // dashboard settings for these devices. Absent/false means MentraOS owns the
+  // dashboard.
+  hasNativeDashboard?: boolean;
+}
+
+/**
+ * Hardware capability profiles for supported glasses models
+ * Key: model_name string (e.g., "Even Realities G1", "Mentra Live")
+ * Value: Capabilities object defining device features
+ */
+export const HARDWARE_CAPABILITIES: Record<string, Capabilities> = {
+  [evenRealitiesG1.modelName]: evenRealitiesG1,
+  [evenRealitiesG2.modelName]: evenRealitiesG2,
+  [mentraDisplay.modelName]: mentraDisplay,
+  [mentraLive.modelName]: mentraLive,
+  [simulatedGlasses.modelName]: simulatedGlasses,
+  [vuzixZ100.modelName]: vuzixZ100,
+  [DeviceTypes.MACH1]: vuzixZ100, // Mach1 uses same Vuzix Ultralite hardware as Z100
+  [none.modelName]: none,
+};
+
+export const getModelCapabilities = (deviceType: DeviceTypes): Capabilities => {
+  const modelName = deviceType as string;
+  if (!HARDWARE_CAPABILITIES[modelName]) {
+    return HARDWARE_CAPABILITIES[DeviceTypes.NONE];
+  }
+  return HARDWARE_CAPABILITIES[modelName];
+};
+
+// export * from "./capabilities"
+export { simulatedGlasses, evenRealitiesG1, evenRealitiesG2, mentraLive, vuzixZ100, mentraDisplay };

package/src/types/index.ts

@@ -0,0 +1,37 @@
+/**
+ * Engine shared types — copied verbatim from @mentra/types so the module is
+ * self-contained. Keep these in sync with cloud/packages/types/src when the
+ * canonical types change. The capability profiles also live here under
+ * ./capabilities/.
+ */
+
+// Enums (runtime values)
+export {HardwareType, HardwareRequirementLevel, DeviceTypes, ControllerTypes} from "./enums"
+
+// Hardware types
+export type {
+  HardwareRequirement,
+  CameraCapabilities,
+  DisplayCapabilities,
+  MicrophoneCapabilities,
+  SpeakerCapabilities,
+  IMUCapabilities,
+  ButtonCapabilities,
+  LightCapabilities,
+  PowerCapabilities,
+  Capabilities,
+} from "./hardware"
+
+export {
+  HARDWARE_CAPABILITIES,
+  getModelCapabilities,
+  simulatedGlasses,
+  evenRealitiesG1,
+  evenRealitiesG2,
+  mentraLive,
+  vuzixZ100,
+  mentraDisplay,
+} from "./hardware"
+
+// Applet types
+export type {AppletType, AppPermissionType, AppletPermission, AppletInterface, ClientApp} from "./applet"

package/src/utils/GlobalEventEmitter.ts

@@ -0,0 +1,14 @@
+// @ts-nocheck — `events` (node builtin, RN-polyfilled) ships no types in engine's
+// standalone build; the host build resolves them fine.
+import {EventEmitter} from "events"
+
+/**
+ * Process-wide event bus — moved into engine so engine-owned services (RestComms)
+ * and the host share ONE emitter instance across the engine↔host boundary.
+ * Re-exported through the host's `@/utils/GlobalEventEmitter` shim.
+ *
+ * @deprecated Use BluetoothSDK subscriptions directly instead.
+ */
+const GlobalEventEmitter = new EventEmitter()
+
+export default GlobalEventEmitter

package/src/utils/cloudClient/RnUdpAdapter.ts

@@ -0,0 +1,47 @@
+/**
+ * @fileoverview react-native-udp backed UdpSocketLike for @mentra/cloud-client.
+ *
+ * The cloud-client encrypts each audio frame in shared code (NaCl secretbox) and
+ * hands the raw bytes here; this adapter only owns the native dgram socket and
+ * sends them. Audio bytes never round-trip through extra JS here beyond the
+ * Buffer wrap the native module needs.
+ */
+import dgram from "react-native-udp"
+import type {UdpSocketLike} from "@mentra/cloud-client"
+
+type CloudUdpSocket = ReturnType<typeof dgram.createSocket> & {
+  on(event: "error", cb: (err: Error) => void): void
+  on(event: "message", cb: (msg: Uint8Array) => void): void
+}
+
+export function createCloudUdpSocket(): UdpSocketLike {
+  const socket = dgram.createSocket({type: "udp4"}) as CloudUdpSocket
+  let onBytes: ((bytes: Uint8Array) => void) | null = null
+
+  socket.on("error", (err: Error) => {
+    console.warn(`[cloud-client udp] socket error: ${err.message}`)
+  })
+  socket.on("message", (msg: Uint8Array) => {
+    onBytes?.(new Uint8Array(msg))
+  })
+  // Bind to any available port so the socket is ready to send.
+  socket.bind(0)
+
+  return {
+    send(bytes: Uint8Array, host: string, port: number): void {
+      socket.send(bytes, 0, bytes.length, port, host, (err?: Error) => {
+        if (err) console.warn(`[cloud-client udp] send failed: ${err.message}`)
+      })
+    },
+    onMessage(cb: (bytes: Uint8Array) => void): void {
+      onBytes = cb
+    },
+    close(): void {
+      try {
+        socket.close()
+      } catch {
+        /* already closed */
+      }
+    },
+  }
+}

package/src/utils/cloudClient/cloudSecureStore.ts

@@ -0,0 +1,28 @@
+/**
+ * MMKV-backed KeyValueStore for @mentra/cloud-client — moved into engine so
+ * engine owns the cloud-v2 credential storage (the v2 refresh token, so a
+ * relaunch does not force a new login). A dedicated MMKV instance keeps
+ * cloud-v2 credentials out of the app's general store. Re-exported through the
+ * host's `@/utils/cloudClient/MmkvSecureStore` shim.
+ *
+ * NOTE: MMKV is not hardware-backed. Before production handling of the
+ * long-lived refresh token, swap this for a Keychain/Keystore-backed store
+ * (react-native-keychain). It is fine for dev / simulator e2e.
+ */
+import {createMMKV, type MMKV} from "react-native-mmkv"
+import type {KeyValueStore} from "@mentra/cloud-client"
+
+const store: MMKV = createMMKV({id: "mentra-cloud-v2"})
+
+export const cloudSecureStore: KeyValueStore = {
+  async get(key: string): Promise<string | null> {
+    return store.getString(key) ?? null
+  },
+  async set(key: string, value: string): Promise<void> {
+    store.set(key, value)
+  },
+  async delete(key: string): Promise<void> {
+    // react-native-mmkv v4's instance method is `remove`, not `delete`.
+    store.remove(key)
+  },
+}

package/src/utils/debouncedPatch.ts

@@ -0,0 +1,28 @@
+/**
+ * Merge partial patches and flush once after `delayMs` of quiet time.
+ * Uses global setTimeout so Jest fake timers can advance debounced writes.
+ */
+export function createDebouncedPatchFlusher<T extends Record<string, unknown>>(
+  flush: (patch: T) => void,
+  delayMs: number,
+) {
+  let pending = {} as T
+  let timer: ReturnType<typeof setTimeout> | null = null
+
+  const schedule = (callback: () => void) => {
+    timer = setTimeout(callback, delayMs) as ReturnType<typeof setTimeout>
+  }
+
+  return (patch: Partial<T>) => {
+    pending = {...pending, ...patch}
+    if (timer != null) {
+      clearTimeout(timer)
+    }
+    schedule(() => {
+      timer = null
+      const batch = pending
+      pending = {} as T
+      if (Object.keys(batch).length > 0) flush(batch)
+    })
+  }
+}

package/src/utils/devLogging.ts

@@ -0,0 +1,140 @@
+import {LogBox} from "react-native"
+import {configureReanimatedLogger, ReanimatedLogLevel} from "react-native-reanimated"
+
+export interface LogEntry {
+  timestamp: number
+  level: "debug" | "info" | "warn" | "error"
+  message: string
+  source?: string
+  metadata?: Record<string, unknown>
+}
+
+class LogRingBuffer {
+  private logs: LogEntry[] = []
+  private maxAgeMs = 10 * 60 * 1000
+  private maxEntries = 10000
+  private isIntercepting = false
+
+  append(entry: Omit<LogEntry, "timestamp">) {
+    this.logs.push({...entry, timestamp: Date.now()})
+    this.prune()
+  }
+
+  getRecentLogs(): LogEntry[] {
+    this.prune()
+    return [...this.logs]
+  }
+
+  clear() {
+    this.logs = []
+  }
+
+  getStats(): {count: number; oldestTimestamp: number | null; newestTimestamp: number | null} {
+    this.prune()
+    return {
+      count: this.logs.length,
+      oldestTimestamp: this.logs.length > 0 ? this.logs[0].timestamp : null,
+      newestTimestamp: this.logs.length > 0 ? this.logs[this.logs.length - 1].timestamp : null,
+    }
+  }
+
+  private prune() {
+    const cutoff = Date.now() - this.maxAgeMs
+    this.logs = this.logs.filter((l) => l.timestamp > cutoff)
+    if (this.logs.length > this.maxEntries) {
+      this.logs = this.logs.slice(-this.maxEntries)
+    }
+  }
+
+  startConsoleInterception() {
+    if (this.isIntercepting) return
+    this.isIntercepting = true
+
+    const ignoredLogs = [
+      /Failed to open debugger. Please check that the dev server is running and reload the app./,
+      /Require cycle:/,
+      /is missing the required default export./,
+      /Attempted to import the module/,
+      /The action 'RESET' with payload/,
+      /The action 'POP_TO_TOP' was not handled/,
+      /socket-0 binding/,
+      /socket-0 bound to/,
+      /Error while flushing PostHog/,
+      /backTitleFontFamily prop is not available on Android/,
+      /disableBackButtonMenu prop is not available on Android/,
+      /backTitleVisible prop is not available on Android/,
+      /topInsetEnabled prop is not available on Android/,
+      /largeTitleHideShadow prop is not available on Android/,
+      /largeTitleFontWeight prop is not available on Android/,
+      /userInterfaceStyle prop is not available on Android/,
+      /largeTitleFontFamily prop is not available on Android/,
+      /Points are coincident/,
+    ]
+
+    LogBox.ignoreLogs(ignoredLogs)
+
+    if (__DEV__) {
+      const withoutIgnored =
+        (logger: (...args: unknown[]) => void) =>
+        (...args: unknown[]) => {
+          const output = args.join(" ")
+          if (!ignoredLogs.some((log) => log.test(output))) {
+            logger(...args)
+          }
+        }
+
+      console.log = withoutIgnored(console.log)
+      console.info = withoutIgnored(console.info)
+      console.warn = withoutIgnored(console.warn)
+      console.error = withoutIgnored(console.error)
+    }
+
+    configureReanimatedLogger({
+      level: ReanimatedLogLevel.warn,
+      strict: false,
+    })
+
+    const originalConsole = {
+      log: console.log,
+      info: console.info,
+      warn: console.warn,
+      error: console.error,
+      debug: console.debug,
+    }
+
+    const appendToBuffer = this.append.bind(this)
+
+    const createInterceptor =
+      (level: "debug" | "info" | "warn" | "error", originalFn: (...args: unknown[]) => void) =>
+      (...args: unknown[]) => {
+        originalFn.apply(console, args)
+        appendToBuffer({
+          level,
+          message: args
+            .map((a) => {
+              if (a === null) return "null"
+              if (a === undefined) return "undefined"
+              if (typeof a === "object") {
+                try {
+                  return JSON.stringify(a)
+                } catch {
+                  return String(a)
+                }
+              }
+              return String(a)
+            })
+            .join(" "),
+          source: "console",
+        })
+      }
+
+    console.log = createInterceptor("info", originalConsole.log)
+    console.info = createInterceptor("info", originalConsole.info)
+    console.warn = createInterceptor("warn", originalConsole.warn)
+    console.error = createInterceptor("error", originalConsole.error)
+    console.debug = createInterceptor("debug", originalConsole.debug)
+  }
+}
+
+export const logBuffer = new LogRingBuffer()
+export default logBuffer

package/src/utils/devMiniappLaunch.ts

@@ -0,0 +1,68 @@
+/**
+ * decideDevLaunchRoute — pre-flight reachability + manifest fetch for dev
+ * miniapp launches.
+ *
+ * Every entry point that wants to launch a dev miniapp (home tile, QR
+ * scan, URL screen, dev-offline "Try again" button) calls this BEFORE
+ * navigating, so:
+ *
+ *   1. We land on the right destination in a single transition
+ *      (live mount vs offline takeover) — no flash through /applet/local.
+ *   2. Callers that need the manifest (permission gate, name/icon read)
+ *      get it in the same round trip — no second fetch.
+ *
+ * Pre-flighting at the call site keeps /applet/local a pure mount route.
+ */
+
+import {storage} from "./storage/storage"
+
+const REACHABILITY_TIMEOUT_MS = 1500
+
+export type DevManifest = {
+  packageName?: string
+  name?: string
+  /** First-found of `icon` / `iconUrl` / `logoUrl` (relative or absolute). */
+  icon?: string
+  permissions?: unknown
+  hardwareRequirements?: unknown
+  [key: string]: unknown
+}
+
+export type DevLaunchResult =
+  | {decision: "live"; manifest: DevManifest}
+  | {decision: "offline"; manifest: null}
+
+/**
+ * GET <devUrl>/miniapp.json with a hard timeout. Returns the parsed
+ * manifest on success ("live") or null ("offline").
+ *
+ * Side effect: on success, writes <packageName>_dev_last_reachable so
+ * the dev-offline screen can show "Last reached: N min ago" the next
+ * time the user lands there.
+ *
+ * The fetch doubles as the reachability probe AND the manifest source —
+ * one request per launch attempt instead of two.
+ */
+export async function decideDevLaunchRoute(
+  packageName: string,
+  devUrl: string,
+): Promise<DevLaunchResult> {
+  const controller = new AbortController()
+  const timer = setTimeout(() => controller.abort(), REACHABILITY_TIMEOUT_MS)
+  try {
+    const res = await fetch(`${devUrl.replace(/\/$/, "")}/miniapp.json`, {
+      method: "GET",
+      signal: controller.signal,
+    })
+    if (!res.ok) return {decision: "offline", manifest: null}
+    const manifest = (await res.json()) as DevManifest
+    if (packageName) {
+      storage.save(`${packageName}_dev_last_reachable`, Date.now())
+    }
+    return {decision: "live", manifest}
+  } catch {
+    return {decision: "offline", manifest: null}
+  } finally {
+    clearTimeout(timer)
+  }
+}