@mentra/engine 0.1.0-beta.0

package/src/types/applet.ts

@@ -0,0 +1,112 @@
+/**
+ * @mentra/types - App/Applet types for client interfaces
+ */
+
+import type {ReactNode} from "react"
+
+import {HardwareRequirement} from "./hardware"
+
+import type {CompatibilityResult} from "../utils/hardware/hardware"
+
+/**
+ * App execution model types
+ */
+export type AppletType = "standard" | "background" | "system_dashboard"
+
+/**
+ * Permission types that apps can request
+ */
+export type AppPermissionType =
+  | "ALL"
+  | "MICROPHONE"
+  | "CAMERA"
+  | "CALENDAR"
+  | "LOCATION"
+  | "BACKGROUND_LOCATION"
+  | "READ_NOTIFICATIONS"
+  | "POST_NOTIFICATIONS"
+
+/**
+ * Permission object with type and description
+ */
+export interface AppletPermission {
+  type: AppPermissionType
+  description?: string
+  required?: boolean
+}
+
+/**
+ * Minimal app interface for client home screen display
+ * Optimized for fast rendering - only essential fields
+ *
+ * This is the client-facing interface used by mobile apps.
+ * Internal cloud services use AppI from models (more fields).
+ */
+export interface AppletInterface {
+  packageName: string
+  name: string
+  webviewUrl: string
+  logoUrl: string
+  iconComponent?: ReactNode
+  type: AppletType
+  permissions: AppletPermission[]
+  running: boolean
+  /**
+   * True while this app's UI WebView is mounted in the Compositor overlay.
+   * At most one app is foregrounded at a time. Distinct from `running`:
+   * a foregrounded app is always running, but a running app may be
+   * backgrounded (JSContext alive, WebView torn down). Normalized to a
+   * concrete boolean by the apps store's projectApps; literal call sites
+   * may omit it.
+   */
+  foregrounded?: boolean
+  healthy: boolean
+  hardwareRequirements: HardwareRequirement[]
+  /** ISO date string when the app was installed */
+  installedDate?: string
+  /** ISO date string when the app was last run. Undefined if never run (app is "new") */
+  lastActiveAt?: string
+}
+
+/**
+ * Runtime applet shape used by the engine apps store and OEM hosts.
+ *
+ * Extends AppletInterface with the host-side fields the manager / OEM tracks
+ * outside the manifest: offline routing, hardware compatibility result,
+ * loading/hidden/local flags, lifecycle hooks, screenshot, and the
+ * dev-miniapp escape hatches (`devUrl` / `isMiniappDev`) that surface a
+ * `mentra-miniapp dev` snapshot.
+ */
+/**
+ * A manifest-declared action a miniapp exposes for other (system) miniapps to
+ * invoke via `session.actions`. Populated from `miniapp.json`'s `actions` field.
+ * Shape maps 1:1 onto an MCP tool (id→name, description, parameters→inputSchema).
+ */
+export interface DeclaredAction {
+  id: string
+  description: string
+  /** JSON-Schema input descriptor (MCP-compatible subset). */
+  parameters?: Record<string, unknown>
+}
+
+export interface ClientApp extends AppletInterface {
+  offline: boolean
+  offlineRoute: string
+  compatibility?: CompatibilityResult
+  loading: boolean
+  local: boolean
+  hidden: boolean
+  onStart?: () => void
+  onStop?: () => void
+  screenshot?: string
+  runtimePermissions?: string[]
+  declaredPermissions?: string[]
+  version?: string
+  needsPcm?: boolean
+  needsTranscript?: boolean
+  devUrl?: string
+  devPort?: number
+  isMiniappDev?: boolean
+  /** Manifest-declared actions (for session.miniapps.list + invoke gating). */
+  actions?: DeclaredAction[]
+}

package/src/types/asg.ts

@@ -0,0 +1,58 @@
+/**
+ * Gallery / ASG HTTP-API types (relocated from the host `@/types/asg`). `PhotoInfo`
+ * lives with the gallerySync store that owns it; the rest describe the glasses'
+ * on-device camera HTTP API that asgCameraApi talks to. The host `@/types/asg`
+ * re-exports these from `@mentra/engine` so existing importers stay unchanged.
+ */
+import type {PhotoInfo} from "../stores/gallerySync"
+export type {PhotoInfo}
+
+export interface CaptureFile {
+  name: string // "IMG_xxx/base.jpg" or "IMG_xxx.jpg" (legacy)
+  size: number
+  role: "primary" | "bracket" | "sidecar"
+  modified?: number
+  mime_type?: string
+  etag?: string
+  sha256?: string
+  download_url?: string
+  hash_url?: string
+}
+
+export interface CaptureGroup {
+  capture_id: string // folder name: "IMG_20250302_143022_456_123" or "IMG_..._<requestId>"
+  request_id?: string // originating SDK requestId embedded in the folder name (absent for button captures / legacy files)
+  type: "photo" | "video"
+  timestamp: number
+  total_size: number
+  files: CaptureFile[]
+  thumbnail_data?: string
+  thumbnail_url?: string
+  duration?: number // video only
+}
+
+export interface GalleryResponse {
+  status: "success" | "error"
+  data: {
+    photos: PhotoInfo[]
+  }
+}
+
+export interface ServerStatus {
+  status: string
+  uptime: number
+  version: string
+  timestamp: string
+}
+
+export interface HealthResponse {
+  status: "healthy" | "unhealthy"
+  timestamp: string
+  version: string
+}
+
+export interface GalleryEvent {
+  type: "photo_added" | "photo_deleted" | "gallery_updated"
+  photo?: PhotoInfo
+  timestamp: string
+}

package/src/types/capabilities/even-realities-g1.ts

@@ -0,0 +1,67 @@
+/**
+ * @fileoverview Even Realities G1 Hardware Capabilities
+ *
+ * Capability profile for the Even Realities G1 smart glasses model.
+ * Defines the hardware and software features available on this device.
+ */
+
+import type { Capabilities } from "../hardware";
+
+/**
+ * Even Realities G1 capability profile
+ */
+export const evenRealitiesG1: Capabilities = {
+  modelName: "Even Realities G1",
+
+  // Camera capabilities - G1 does not have a camera
+  hasCamera: false,
+  camera: null,
+
+  // Display capabilities - G1 has a green monochrome display
+  hasDisplay: true,
+  display: {
+    count: 2,
+    isColor: false,
+    color: "green",
+    canDisplayBitmap: false, // broken in practice — capability stops lying (design doc §3.4.8)
+    resolution: { width: 640, height: 200 },
+    fieldOfView: { horizontal: 25 },
+    maxTextLines: 5,
+    adjustBrightness: true,
+
+    // Scene display API — no positioned elements; scenes degrade to text walls.
+    canPosition: false,
+    maxImageElements: 0,
+  },
+
+  // Microphone capabilities - G1 has one microphone without VAD
+  hasMicrophone: true,
+  microphone: {
+    count: 1,
+    hasVAD: false,
+  },
+
+  // Speaker capabilities - G1 does not have a speaker
+  hasSpeaker: false,
+  speaker: null,
+
+  // IMU capabilities - G1 has IMU for head-up/down detection but raw data not exposed to apps
+  hasIMU: true,
+  imu: null,
+
+  // Button capabilities - G1 does not have buttons
+  hasButton: false,
+  button: null,
+
+  // Light capabilities - G1 does not have lights
+  hasLight: false,
+  light: null,
+
+  // Power capabilities - G1 does not have external battery
+  power: {
+    hasExternalBattery: false,
+  },
+
+  // WiFi capabilities - G1 does not support WiFi
+  hasWifi: false,
+};

package/src/types/capabilities/even-realities-g2.ts

@@ -0,0 +1,88 @@
+/**
+ * @fileoverview Even Realities G2 Hardware Capabilities
+ *
+ * Capability profile for the Even Realities G2 smart glasses model.
+ * G2 uses the EvenHub protocol with protobuf-based commands.
+ */
+
+import type { Capabilities } from "../hardware";
+
+/**
+ * Even Realities G2 capability profile
+ */
+export const evenRealitiesG2: Capabilities = {
+  modelName: "Even Realities G2",
+
+  // Camera capabilities - G2 does not have a camera
+  hasCamera: false,
+  camera: null,
+
+  // Display capabilities - G2 has a green monochrome display (similar to G1)
+  hasDisplay: true,
+  display: {
+    count: 2,
+    isColor: false,
+    color: "green",
+    canDisplayBitmap: true,
+    resolution: { width: 640, height: 200 },
+    fieldOfView: { horizontal: 25 },
+    maxTextLines: 8,
+    adjustBrightness: true,
+
+    // Scene display API — EvenHub retained containers.
+    width: 576,
+    height: 288,
+    canPosition: true,
+    maxTextElements: 6, // firmware text-container pool (rects share it)
+    maxImageElements: 4, // firmware image-container pool
+    // No maxImagePx: the SGC tiles larger images across multiple firmware
+    // containers (each ≤200x100 — the hardware-verified per-container transfer
+    // envelope). Pathological sizes needing more than the 4-container pool are
+    // dropped SGC-side with a log.
+    shapes: ["rect"], // bordered empty container ≈ rect
+    intensityLevels: 2,
+    partialUpdate: true,
+  },
+
+  // Microphone capabilities - G2 has one microphone (right side), LC3 codec
+  hasMicrophone: true,
+  microphone: {
+    count: 1,
+    hasVAD: false,
+  },
+
+  // Speaker capabilities - G2 does not have a speaker
+  hasSpeaker: false,
+  speaker: null,
+
+  // IMU capabilities - G2 has IMU
+  hasIMU: true,
+  imu: null,
+
+  // Button capabilities - G2 has a capacitive touchbar
+  hasButton: true,
+  button: {
+    count: 1,
+    buttons: [{
+      type: "swipe1d",
+      events: ["TAP", "DOUBLE_TAP", "TRIPLE_TAP", "PRESS_HOLD", "SWIPE_UP", "SWIPE_DOWN"],
+      isCapacitive: true,
+    }],
+  },
+
+  // Light capabilities - G2 does not have lights
+  hasLight: false,
+  light: null,
+
+  // Power capabilities
+  power: {
+    hasExternalBattery: false,
+  },
+
+  // WiFi capabilities - G2 does not support WiFi
+  hasWifi: false,
+
+  // Dashboard - G2 renders Even Realities' native dashboard in firmware, so
+  // MentraOS does not manage the dashboard or expose dashboard settings for it
+  hasNativeDashboard: true,
+};

package/src/types/capabilities/mentra-display.ts

@@ -0,0 +1,75 @@
+/**
+ * @fileoverview Mentra Display Hardware Capabilities
+ *
+ * Capability profile for the Mentra Display smart glasses model.
+ * Defines the hardware and software features available on this device.
+ */
+
+import type { Capabilities } from "../hardware";
+
+/**
+ * Mentra Display capability profile
+ */
+export const mentraDisplay: Capabilities = {
+  modelName: "Mentra Display",
+
+  // Camera capabilities - Mentra Display does not have a camera
+  hasCamera: false,
+  camera: null,
+
+  // Display capabilities - Mentra Display has a green monochrome display
+  hasDisplay: true,
+  display: {
+    count: 2,
+    isColor: false,
+    color: "green",
+    canDisplayBitmap: true,
+    resolution: { width: 640, height: 480 },
+    fieldOfView: { horizontal: 25 },
+    maxTextLines: 5,
+    adjustBrightness: true,
+
+    // Scene display API — firmware canvas components on the 500x220 virtual
+    // screen centered in the 640x480 A6N panel (mos_display_config.c).
+    width: 500,
+    height: 220,
+    canPosition: true,
+    maxTextElements: 6, // firmware TEXTBOX pool ids 1-6 (rects share it)
+    maxImageElements: 4, // firmware BITMAP pool ids 10-13
+    maxImagePx: { width: 200, height: 200 }, // CANVAS_IMAGE_MAX_I1_BYTES / NAV_IMAGE_MAX_W/H
+    shapes: ["rect"], // bordered empty TEXTBOX ≈ rect (no shape primitive)
+    intensityLevels: 2, // 1-bpp panel today
+    partialUpdate: true,
+  },
+
+  // Microphone capabilities - Mentra Display has one microphone without VAD
+  hasMicrophone: true,
+  microphone: {
+    count: 1,
+    hasVAD: false,
+  },
+
+  // Speaker capabilities - Mentra Display does not have a speaker
+  hasSpeaker: false,
+  speaker: null,
+
+  // IMU capabilities - Mentra Display has IMU for head-up/down detection but raw data not exposed to apps
+  hasIMU: true,
+  imu: null,
+
+  // Button capabilities - Mentra Display does not have buttons
+  hasButton: false,
+  button: null,
+
+  // Light capabilities - Mentra Display does not have lights
+  hasLight: false,
+  light: null,
+
+  // Power capabilities - Mentra Display does not have external battery
+  power: {
+    hasExternalBattery: false,
+  },
+
+  // WiFi capabilities - Mentra Display does not support WiFi
+  hasWifi: false,
+};

package/src/types/capabilities/mentra-live.ts

@@ -0,0 +1,103 @@
+/**
+ * @fileoverview Mentra Live Hardware Capabilities
+ *
+ * Capability profile for the Mentra Live smart glasses model.
+ * Defines the hardware and software features available on this device.
+ */
+
+import type { Capabilities } from "../hardware";
+
+/**
+ * Mentra Live capability profile
+ */
+export const mentraLive: Capabilities = {
+  modelName: "Mentra Live",
+
+  // Camera capabilities - Mentra Live has camera with streaming
+  hasCamera: true,
+  camera: {
+    resolution: { width: 1920, height: 1080 },
+    hasHDR: false,
+    hasFocus: true,
+    video: {
+      canRecord: true,
+      canStream: true,
+      supportedStreamTypes: ["rtmp"],
+      supportedResolutions: [
+        { width: 1920, height: 1080 },
+        { width: 1280, height: 720 },
+        { width: 640, height: 480 },
+      ],
+    },
+  },
+
+  // Display capabilities - Mentra Live does not have a display
+  hasDisplay: false,
+  display: null,
+
+  // Microphone capabilities - Mentra Live has one microphone with VAD
+  hasMicrophone: true,
+  microphone: {
+    count: 1,
+    hasVAD: true,
+  },
+
+  // Speaker capabilities - Mentra Live has one standard speaker
+  hasSpeaker: true,
+  speaker: {
+    count: 1,
+    isPrivate: false,
+  },
+
+  // IMU capabilities - Mentra Live has 6-axis IMU
+  hasIMU: true,
+  imu: {
+    axisCount: 6,
+    hasAccelerometer: true,
+    hasCompass: false,
+    hasGyroscope: true,
+  },
+
+  // Button capabilities - Mentra Live has one physical button
+  hasButton: true,
+  button: {
+    count: 1,
+    buttons: [
+      {
+        type: "press",
+        events: ["press", "double_press", "long_press"],
+        isCapacitive: false,
+      },
+    ],
+  },
+
+  // Light capabilities - Mentra Live has two lights: privacy light + RGB LED
+  hasLight: true,
+  light: {
+    count: 2,
+    lights: [
+      {
+        id: "privacy",
+        purpose: "privacy",
+        isFullColor: false,
+        color: "white",
+        position: "front_facing",
+      },
+      {
+        id: "user_feedback",
+        purpose: "user_feedback",
+        isFullColor: true,
+        color: "rgb",
+        position: "user_facing",
+      },
+    ],
+  },
+
+  // Power capabilities - Mentra Live does not have external battery
+  power: {
+    hasExternalBattery: false,
+  },
+
+  // WiFi capabilities - Mentra Live supports WiFi
+  hasWifi: true,
+};

package/src/types/capabilities/none.ts

@@ -0,0 +1,76 @@
+/**
+ * @fileoverview Simulated Glasses Hardware Capabilities
+ *
+ * Capability profile for the Simulated Glasses model.
+ * Defines the hardware and software features available on this device.
+ * This profile matches the Vuzix Z100 capabilities for testing purposes.
+ */
+
+import type { Capabilities } from "../hardware";
+
+/**
+ * Simulated Glasses capability profile
+ */
+export const none: Capabilities = {
+  modelName: "None",
+
+  // Camera capabilities - does not have a camera
+  hasCamera: false,
+  camera: null,
+
+  // Display capabilities - has a green monochrome display
+  hasDisplay: false,
+  display: {
+    count: 1,
+    isColor: false,
+    color: "green",
+    canDisplayBitmap: false,
+    resolution: { width: 640, height: 480 },
+    fieldOfView: { horizontal: 30 },
+    maxTextLines: 7,
+    adjustBrightness: true,
+  },
+
+  // Microphone capabilities - has a microphone (phone mic)
+  hasMicrophone: false,
+  microphone: {
+    count: 1,
+    hasVAD: false,
+  },
+
+  // Speaker capabilities - has a speaker (phone speaker)
+  hasSpeaker: false,
+  speaker: {
+    count: 1,
+    isPrivate: false,
+  },
+
+  // IMU capabilities - does not have an IMU
+  hasIMU: false,
+  imu: null,
+
+  // Button capabilities - Has one simulated button
+  hasButton: false,
+  button: {
+    count: 1,
+    buttons: [
+      {
+        type: "press",
+        events: ["press"],
+        isCapacitive: false,
+      },
+    ],
+  },
+
+  // Light capabilities - does not have lights
+  hasLight: false,
+  light: null,
+
+  // Power capabilities - does not have external battery
+  power: {
+    hasExternalBattery: false,
+  },
+
+  // WiFi capabilities - does not support WiFi
+  hasWifi: false,
+};

package/src/types/capabilities/simulated-glasses.ts

@@ -0,0 +1,76 @@
+/**
+ * @fileoverview Simulated Glasses Hardware Capabilities
+ *
+ * Capability profile for the Simulated Glasses model.
+ * Defines the hardware and software features available on this device.
+ * This profile matches the Vuzix Z100 capabilities for testing purposes.
+ */
+
+import type { Capabilities } from "../hardware";
+
+/**
+ * Simulated Glasses capability profile
+ */
+export const simulatedGlasses: Capabilities = {
+  modelName: "Simulated Glasses",
+
+  // Camera capabilities - does not have a camera
+  hasCamera: false,
+  camera: null,
+
+  // Display capabilities - has a green monochrome display
+  hasDisplay: true,
+  display: {
+    count: 1,
+    isColor: false,
+    color: "green",
+    canDisplayBitmap: false,
+    resolution: { width: 640, height: 480 },
+    fieldOfView: { horizontal: 30 },
+    maxTextLines: 7,
+    adjustBrightness: true,
+  },
+
+  // Microphone capabilities - has a microphone (phone mic)
+  hasMicrophone: true,
+  microphone: {
+    count: 1,
+    hasVAD: false,
+  },
+
+  // Speaker capabilities - has a speaker (phone speaker)
+  hasSpeaker: true,
+  speaker: {
+    count: 1,
+    isPrivate: false,
+  },
+
+  // IMU capabilities - does not have an IMU
+  hasIMU: false,
+  imu: null,
+
+  // Button capabilities - Has one simulated button
+  hasButton: true,
+  button: {
+    count: 1,
+    buttons: [
+      {
+        type: "press",
+        events: ["press"],
+        isCapacitive: false,
+      },
+    ],
+  },
+
+  // Light capabilities - does not have lights
+  hasLight: false,
+  light: null,
+
+  // Power capabilities - does not have external battery
+  power: {
+    hasExternalBattery: false,
+  },
+
+  // WiFi capabilities - does not support WiFi
+  hasWifi: false,
+};