@mentra/miniapp 0.3.0-dev.0

package/src/background/index.ts

@@ -0,0 +1,152 @@
+/**
+ * @mentra/miniapp/background — background-side SDK entry point.
+ *
+ * Imported from a miniapp's `src/background/index.ts` to access the
+ * per-miniapp `MiniappSession` and its typed `session.*` module
+ * wrappers. This is the **always-running JSContext side** of a two-layer
+ * miniapp.
+ *
+ * What's NOT in this entry point:
+ *   - `mentra` WebView global (UI-only — import from `@mentra/miniapp/ui`).
+ *   - `MentraProvider` / React hooks (UI-only).
+ *   - Any DOM-bound API. The JSContext has no DOM.
+ *
+ * Importing the wrong sub-path is caught at compile time by the
+ * separate type-roots on each `exports` entry.
+ */
+
+export {MiniappSession, type MiniappSessionOptions} from "../session"
+export {registerMiniapp, type MiniappInitHandler} from "./register"
+
+// Event / data type re-exports — these are payload shapes a miniapp's
+// background-side handlers consume from session.transcription.on, etc.
+export type {
+  TranscriptionData,
+  TranslationData,
+  ButtonPressData,
+  AudioChunkData,
+  VadData,
+  BatteryData,
+  ConnectionData,
+  WifiData,
+  HeadPositionData,
+  AccelData,
+  LocationData,
+  PhoneNotificationData,
+  NotificationDismissedData,
+  CalendarEventData,
+  HeadingData,
+  TouchData,
+  UnsubscribeFn,
+} from "../modules/events"
+
+// Public envelope + protocol types so authors can write strongly-typed
+// glue when they need to fall back to session.sendOneShot / sendRequest.
+export {MiniappRequestType, MiniappResponseType, MiniappStreamType, MiniappErrorCode} from "../protocol"
+
+// Action handler types — for typing `session.actions.handle(id, fn)` handlers
+// (the registering side lives in the background JSContext). `InvokeOptions` is
+// for system miniapps that call other apps' actions via `session.actions.invoke`.
+export type {ActionContext, ActionHandler, InvokeOptions} from "../modules/actions"
+
+// Session module types — useful for typing controller classes or
+// utility helpers that take a session-like dependency.
+export type {DisplayManager} from "../modules/display"
+export type {
+  CameraFovPreset,
+  CameraFovRequest,
+  CameraFovResult,
+  CameraModule,
+  CameraRoiPosition,
+  SetCameraFovOptions,
+} from "../modules/camera"
+export type {
+  CloudClientAudioTransport,
+  CloudClientConnectionStatus,
+  CloudClientStatus,
+  CloudModule,
+} from "../modules/cloud"
+export type {DashboardAPI} from "../modules/dashboard"
+export type {GlassesModule} from "../modules/glasses"
+export type {HeadingModule} from "../modules/heading"
+export type {ImuModule} from "../modules/imu"
+export type {InputModule} from "../modules/input"
+export type {LedModule} from "../modules/led"
+export type {LocationModule} from "../modules/location"
+export type {MicModule} from "../modules/mic"
+export type {NavigationModule} from "../modules/navigation"
+export type {PermissionsModule} from "../modules/permissions"
+export type {PhoneModule} from "../modules/phone"
+export type {SimpleStorage} from "../modules/storage"
+export {BlobModule, BlobWriter, BlobReader, BLOB_WRITE_CHUNK_BYTES, BLOB_READ_ALL_MAX_BYTES} from "../modules/blob"
+export type {BlobMeta, BlobSetOptions, BlobSetFromUrlOptions, BlobImportOptions} from "../modules/blob"
+export {bytesToBase64, base64ToBytes} from "../modules/base64"
+export type {SpeakerModule} from "../modules/speaker"
+export type {StreamModule} from "../modules/stream"
+export type {SystemModule} from "../modules/system"
+export type {TranscriptionModule} from "../modules/transcription"
+export type {TranslationModule} from "../modules/translation"
+export type {UIModule, UIChannelHandler, UIUnsubscribe} from "../modules/ui"
+export type {Rpc, IsRpc, RpcReq, RpcRes, RpcRequestOptions, RpcHandlerContext} from "../modules/ui"
+export {MentraRpcError, MentraRpcTimeoutError} from "../modules/ui"
+
+// Argument enums + option types — for authors writing controllers that pass
+// these into session.* method calls (or that build typed helpers around them).
+// Intentionally NOT re-exported from `@mentra/miniapp/ui` — the WebView half
+// never touches these directly. UI talks to background via mentra.send with
+// the author's own `Channels` registry; background is where these enums show
+// up as method args.
+export type {LedColor, LedControlOptions} from "../modules/led"
+export type {
+  ViewType,
+  DisplayBreakMode,
+  RenderBox,
+  RenderElement,
+  RenderOptions,
+  RenderRectStyle,
+  RenderResult,
+  RenderTextStyle,
+} from "../modules/display"
+export type {DashboardMode} from "../modules/dashboard"
+export type {PlayAudioOptions, SpeakOptions, SpeakResult, SpeakerState, SpeakerStateEvent} from "../modules/speaker"
+export type {ShareOptions, ShareResult, DownloadOptions, DownloadResult} from "../modules/system"
+export type {TranscriptionConfig} from "../modules/transcription"
+export type {PermissionErrorEvent} from "../modules/permissions"
+export type {
+  AuthUpdatePayload,
+  MiniappVisibility,
+  PermissionType,
+  PermissionRecord,
+  GlassesCapabilities,
+  ConnectAckPayload,
+  MiniappAuthState,
+  MiniappRequestError,
+} from "../session"
+export type {AuthFetchOptions, AuthModule} from "../modules/auth"
+export type {MiniappColorScheme} from "../globals"
+// Navigation — exported as a single block since the types reference each other.
+export type {
+  LatLng,
+  TravelMode,
+  ManeuverKind,
+  RouteAvoidances,
+  NavManeuver,
+  NavOffRoute,
+  NavRerouting,
+  NavArrived,
+  NavError,
+  NavUpdate,
+  NavRoute,
+  PivotOptions,
+  Pivot,
+  PivotEvent,
+  NavStep,
+  NavigationDev,
+  StartNavigationOptions,
+  NavState,
+  NavPermissionResult,
+  ComputeRouteOptions,
+  ComputedRouteStep,
+  ComputedRoute,
+  ComputeRouteResult,
+} from "../modules/navigation"

package/src/background/register.ts

@@ -0,0 +1,85 @@
+/**
+ * registerMiniapp — entry hook for the background JSContext.
+ *
+ * The MentraJS host loads the polyfill (which installs __dispatch /
+ * __deliver / timers / fetch / etc.), then evaluates the miniapp's
+ * background bundle. After both scripts have evaluated successfully,
+ * the host delivers a single `{kind: "init", sessionId}` envelope.
+ * The polyfill turns that into a call to `__mentraInitCallback(sessionId)`.
+ *
+ * This module wires that callback to:
+ *   1. Construct a `MiniappSession` (DispatchTransport picks itself
+ *      automatically because `__dispatch` is on globalThis).
+ *   2. Call the user-supplied handler synchronously so subscriptions
+ *      are registered before any host events fan out.
+ *   3. Call `session.connect()` so the host receives CONNECT and the
+ *      first CONNECT_ACK populates `userId`, `capabilities`, etc.
+ *
+ * The handler runs once per spawn. If the host kills + respawns the
+ * JSContext (crash recovery, dev reload), the polyfill + bundle are
+ * re-evaluated and the handler fires again with a fresh session.
+ */
+
+import {MiniappSession, type MiniappSessionOptions} from "../session"
+
+export type MiniappInitHandler = (session: MiniappSession) => void | Promise<void>
+
+interface InitGlobals {
+  __mentraInitCallback?: (sessionId: string) => void
+  /**
+   * Polyfill-injected host bridge for surfacing structured errors to the
+   * host's MentraJSRouter (which feeds them into MentraJSCrashController).
+   * Optional because tests / pure-Node environments don't have it.
+   */
+  __hostError?: (payloadJson: string) => void
+}
+
+/**
+ * Register the miniapp's startup handler. Call once at the top level of
+ * `src/background/index.ts`. Top-level side effects are fine, but the
+ * vast majority of setup should live inside the handler so it can run
+ * with a connected session.
+ *
+ * @example
+ *   registerMiniapp((session) => {
+ *     session.transcription.on((tx) => {
+ *       session.display.render([{type: "text", id: "tx", box, text: tx.text}])
+ *     })
+ *   })
+ */
+export function registerMiniapp(handler: MiniappInitHandler, options: MiniappSessionOptions = {}): void {
+  const g = globalThis as unknown as InitGlobals
+  g.__mentraInitCallback = (_sessionId: string) => {
+    const session = new MiniappSession(options)
+    // Fire the user handler first so any session.* subscriptions get
+    // registered before the CONNECT_ACK fan-out lands.
+    try {
+      const result = handler(session)
+      if (result && typeof (result as Promise<void>).then === "function") {
+        ;(result as Promise<void>).catch((err: unknown) => {
+          // eslint-disable-next-line no-console
+          console.error("[mentra-miniapp] registerMiniapp handler rejected:", err)
+        })
+      }
+    } catch (err) {
+      // eslint-disable-next-line no-console
+      console.error("[mentra-miniapp] registerMiniapp handler threw:", err)
+    }
+    session.connect().catch((err: unknown) => {
+      // eslint-disable-next-line no-console
+      console.error("[mentra-miniapp] session.connect() rejected:", err)
+      // Surface to the host's crash controller as a structured uncaught
+      // error so the existing backoff + crashloop machinery handles it.
+      // Repeated connect failures eventually flip the miniapp to
+      // CRASHLOOP_DISABLED, which the host turns into a user-facing
+      // alert + automatic incident report.
+      try {
+        const message = err instanceof Error ? err.message : String(err)
+        const stack = err instanceof Error && err.stack ? err.stack : ""
+        g.__hostError?.(JSON.stringify({message: `session.connect() failed: ${message}`, stack}))
+      } catch {
+        /* host bridge missing in tests — ignore */
+      }
+    })
+  }
+}

package/src/dev-reload.ts

@@ -0,0 +1,60 @@
+/**
+ * @fileoverview SDK-side dev-reload listener — auto-installed when the host
+ * indicates we're running in a dev miniapp.
+ *
+ * Companion to the phone-side console-tap shim (which is injected by the
+ * MentraOS app, not by the SDK). The SDK installs this listener on import so
+ * authors get live reload without any opt-in code.
+ *
+ * Mechanism:
+ *   - Phone-side `DevServerBridge` receives `{type: "reload"}` from the dev
+ *     server's WebSocket.
+ *   - Phone-side `MiniappHost` then injects a `MessageEvent` into the WebView
+ *     with payload `{type: "miniapp_dev_reload"}`.
+ *   - This listener catches that MessageEvent and calls `location.reload()`.
+ *
+ * Gated on `window.MentraOS.miniappDeveloperMode === true` so production
+ * miniapps never set up the listener. In production WebViews the host won't
+ * inject the message anyway, but belt-and-suspenders.
+ */
+
+const RELOAD_MESSAGE_TYPE = "miniapp_dev_reload"
+
+export function installDevReloadListenerIfDevMode(): void {
+  if (typeof window === "undefined") return
+  const mentra = (window as {MentraOS?: {miniappDeveloperMode?: boolean}}).MentraOS
+  if (!mentra?.miniappDeveloperMode) return
+
+  // Avoid double-install if the SDK is hot-reloaded mid-session.
+  const flagKey = "__mentraDevReloadInstalled"
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  if ((window as any)[flagKey]) return
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  ;(window as any)[flagKey] = true
+
+  const handler = (ev: MessageEvent): void => {
+    const data = typeof ev.data === "string" ? ev.data : null
+    if (!data) return
+    let parsed: unknown
+    try {
+      parsed = JSON.parse(data)
+    } catch {
+      return
+    }
+    const env = parsed as {payload?: {type?: string}}
+    const type = env?.payload?.type
+    if (type !== RELOAD_MESSAGE_TYPE) return
+    // eslint-disable-next-line no-console
+    console.log("[mentra-miniapp] dev reload signal received — reloading")
+    try {
+      location.reload()
+    } catch {
+      // Some embedded contexts forbid reload; ignore.
+    }
+  }
+
+  window.addEventListener("message", handler)
+  if (typeof document !== "undefined") {
+    document.addEventListener("message", handler as unknown as EventListener)
+  }
+}

package/src/envelope.ts

@@ -0,0 +1,54 @@
+/**
+ * @fileoverview Bridge envelope format.
+ *
+ * Every message between @mentra/miniapp and LocalMiniappRuntime is wrapped in
+ * this envelope for request/response correlation.
+ *
+ *   { payload: {...}, requestId?: string }
+ */
+
+export interface MiniappEnvelope<P = unknown> {
+  payload: P
+  /** Correlates request/response pairs. Set by the sender when it wants a reply. */
+  requestId?: string
+}
+
+/** Serialize an envelope for postMessage / WebSocket transport. */
+export function serializeEnvelope(envelope: MiniappEnvelope): string {
+  return JSON.stringify(envelope)
+}
+
+/**
+ * Parse a serialized envelope. Returns null for any malformed input — never throws.
+ * Transports should pass raw strings in and silently drop nulls.
+ */
+export function parseEnvelope(raw: unknown): MiniappEnvelope | null {
+  if (typeof raw !== "string") return null
+
+  let parsed: unknown
+  try {
+    parsed = JSON.parse(raw)
+  } catch {
+    return null
+  }
+
+  if (typeof parsed !== "object" || parsed === null) return null
+  const obj = parsed as Record<string, unknown>
+
+  if (typeof obj.payload !== "object" || obj.payload === null) return null
+  if (obj.requestId !== undefined && typeof obj.requestId !== "string") return null
+
+  return {
+    payload: obj.payload as unknown,
+    requestId: obj.requestId as string | undefined,
+  }
+}
+
+/** Generate a short random requestId. Browser-native (crypto.randomUUID). */
+export function makeRequestId(): string {
+  if (typeof crypto !== "undefined" && typeof crypto.randomUUID === "function") {
+    return crypto.randomUUID()
+  }
+  // Fallback for older engines: timestamp + random suffix. Not cryptographically secure.
+  return `req_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 10)}`
+}

package/src/globals.ts

@@ -0,0 +1,62 @@
+/**
+ * @fileoverview Shape of window.MentraOS, injected by the host MentraOS app
+ * before the miniapp's content loads. Miniapp authors generally won't read
+ * this directly — use the typed React hooks (useSafeArea, etc.) instead.
+ */
+
+export interface MiniappSafeAreaInsets {
+  top: number
+  bottom: number
+  left: number
+  right: number
+}
+
+export interface MiniappCapsuleMenuRect {
+  top: number
+  right: number
+  bottom: number
+  left: number
+  width: number
+  height: number
+}
+
+export type MiniappColorScheme = "light" | "dark"
+
+export interface MentraOSGlobals {
+  /** Package id the host assigned to this miniapp (e.g. "com.mentra.example"). */
+  packageName?: string
+  /** "ios" | "android" | other. */
+  platform?: string
+  /** Capabilities the host exposes to this miniapp (e.g. ["share", "open_url"]). */
+  capabilities?: string[]
+  /** True when this miniapp is running locally on-device (vs. hosted in the cloud). */
+  miniappLocal?: boolean
+  /** True when the miniapp is running via the dev workflow (QR sideload). */
+  miniappDeveloperMode?: boolean
+  /** Safe-area insets around the WebView content. Miniapps should pad accordingly. */
+  safeAreaInsets?: MiniappSafeAreaInsets
+  /**
+   * Bounding rect (in CSS pixels) of the floating capsule menu the host draws
+   * over the WebView. The menu lives in the top-right and miniapps should
+   * avoid placing interactive elements beneath it.
+   */
+  capsuleMenu?: MiniappCapsuleMenuRect
+  /**
+   * The host's current color scheme. Miniapps can follow this to match the
+   * phone's appearance. Updates at runtime arrive via the session's
+   * `colorScheme` event — see `useColorScheme()` in @mentra/miniapp/react.
+   */
+  colorScheme?: MiniappColorScheme
+}
+
+declare global {
+  interface Window {
+    MentraOS?: MentraOSGlobals
+  }
+}
+
+/** Reads window.MentraOS safely — returns an empty object if not set. */
+export function getMentraOSGlobals(): MentraOSGlobals {
+  if (typeof window === "undefined") return {}
+  return window.MentraOS ?? {}
+}

package/src/index.ts

@@ -0,0 +1,171 @@
+/**
+ * @mentra/miniapp — SDK for building MentraOS local miniapps.
+ *
+ * Public entry point. Consumers do:
+ *
+ *   import {MiniappSession} from "@mentra/miniapp"            // background JSContext
+ *   import {useColorScheme} from "@mentra/miniapp/react"      // UI WebView hooks
+ *   import {MiniappRequestType} from "@mentra/miniapp/protocol"
+ */
+
+import {installDevReloadListenerIfDevMode} from "./dev-reload"
+
+// Auto-install the dev-reload listener on module import so authors get live
+// reload for free in dev builds. No-op in production (gated on
+// window.MentraOS.miniappDeveloperMode).
+installDevReloadListenerIfDevMode()
+
+export {MiniappSession, NotConnectedError} from "./session"
+export type {
+  AuthUpdatePayload,
+  ConnectAckPayload,
+  DisplayCapabilities,
+  GlassesCapabilities,
+  MiniappAuthState,
+  MiniappRequestError,
+  MiniappSessionOptions,
+  MiniappVisibility,
+} from "./session"
+export type {AuthFetchOptions, AuthModule} from "./modules/auth"
+
+export {makeRequestId, parseEnvelope, serializeEnvelope} from "./envelope"
+export type {MiniappEnvelope} from "./envelope"
+
+export {getMentraOSGlobals} from "./globals"
+export type {MentraOSGlobals, MiniappCapsuleMenuRect, MiniappColorScheme, MiniappSafeAreaInsets} from "./globals"
+
+export {MiniappErrorCode, MiniappRequestType, MiniappResponseType, MiniappStreamType} from "./protocol"
+export {CLOUD_STATUS_STREAM} from "./modules/cloud"
+export type {CloudClientAudioTransport, CloudClientConnectionStatus, CloudClientStatus} from "./modules/cloud"
+
+// Hardware requirement types — re-exported from @mentra/types so miniapp
+// authors can type their miniapp.json manifest without pulling in the types
+// package directly. Keep explicit exports (enums as value, interfaces as
+// type) per @mentra/types' Bun-compat convention.
+export {HardwareType, HardwareRequirementLevel} from "@mentra/types"
+export type {HardwareRequirement} from "@mentra/types"
+
+// Transports — exported for advanced uses (forced transport injection, tests)
+export {createTransport} from "./transport/auto"
+export type {CreateTransportOptions} from "./transport/auto"
+export {PostMessageTransport} from "./transport/postmessage"
+export {LocalSocketTransport} from "./transport/local-socket"
+export type {LocalSocketTransportOptions} from "./transport/local-socket"
+export {MockTransport, isMockExplicitlyRequested} from "./transport/mock"
+export type {MockTransportOptions} from "./transport/mock"
+export type {Transport, TransportDisconnectHandler, TransportMessageHandler} from "./transport/types"
+
+// Module types — useful for typing handlers in consumer code
+export type {
+  DisplayBreakMode,
+  RenderBox,
+  RenderElement,
+  RenderOptions,
+  RenderRectStyle,
+  RenderResult,
+  RenderTextStyle,
+  ViewType,
+} from "./modules/display"
+export type {
+  AccelData,
+  AudioChunkData,
+  BatteryData,
+  ButtonPressData,
+  CalendarEventData,
+  ConnectionData,
+  HeadingData,
+  HeadPositionData,
+  LocationData,
+  NotificationDismissedData,
+  PhoneNotificationData,
+  TouchData,
+  TranscriptionData,
+  TranslationData,
+  UnsubscribeFn,
+  VadData,
+  WifiData,
+} from "./modules/events"
+export type {
+  PlayAudioOptions,
+  SpeakOptions,
+  SpeakResult,
+  SpeakerState,
+  SpeakerStateEvent,
+  SpeakerStreamOptions,
+  SpeakerStreamSampleRate,
+  SpeakerStreamWriteResult,
+} from "./modules/speaker"
+export {SpeakerStreamWriter, SPEAKER_STREAM_MAX_BUFFERED_MS, SPEAKER_WRITE_CHUNK_BYTES} from "./modules/speaker"
+export type {
+  CameraFovPreset,
+  CameraFovRequest,
+  CameraFovResult,
+  CameraRoiPosition,
+  PhotoTaken,
+  SetCameraFovOptions,
+  StartVideoRecordingOptions,
+  StopVideoRecordingOptions,
+  TakePhotoOptions,
+  VideoRecordingStarted,
+} from "./modules/camera"
+export type {DashboardMode} from "./modules/dashboard"
+export type {CloudModule} from "./modules/cloud"
+export type {LedColor, LedControlOptions} from "./modules/led"
+export type {
+  RestreamDestination,
+  StartStreamOptions,
+  StreamAudioConfig,
+  StreamResolvedConfig,
+  StreamResult,
+  StreamStatus,
+  StreamVideoConfig,
+} from "./modules/stream"
+export type {ShareOptions, ShareResult, DownloadOptions, DownloadResult} from "./modules/system"
+export type {MiniappInfo, MiniappActionInfo, MiniappCompatibility, ListMiniappsOptions} from "./modules/miniapps"
+export type {ActionContext, ActionHandler, InvokeOptions} from "./modules/actions"
+
+// Domain module types — exported so consumers can type module references
+// (rare; most authors interact via session.<module>.<method> directly).
+export type {DisplayManager} from "./modules/display"
+export type {MiniappsModule} from "./modules/miniapps"
+export type {ActionsModule} from "./modules/actions"
+export type {GlassesModule} from "./modules/glasses"
+export type {HeadingModule} from "./modules/heading"
+export type {ImuModule} from "./modules/imu"
+export type {InputModule} from "./modules/input"
+export type {LocationModule} from "./modules/location"
+export type {MicModule} from "./modules/mic"
+export type {
+  ComputedRoute,
+  ComputedRouteStep,
+  ComputeRouteOptions,
+  ComputeRouteResult,
+  LatLng,
+  NavArrived,
+  NavError,
+  NavigationModule,
+  NavManeuver,
+  NavOffRoute,
+  NavPermissionResult,
+  NavPlaceDetails,
+  NavPlaceSuggestion,
+  NavRerouting,
+  NavRoute,
+  NavState,
+  NavStep,
+  NavUpdate,
+  Pivot,
+  PivotEvent,
+  PivotOptions,
+  RouteAvoidances,
+  StartNavigationOptions,
+  TravelMode,
+} from "./modules/navigation"
+export type {PermissionsModule, PermissionErrorEvent} from "./modules/permissions"
+export type {PhoneModule, PhoneNotificationsModule, PhoneCalendarModule} from "./modules/phone"
+export type {TranscriptionModule, TranscriptionConfig} from "./modules/transcription"
+export type {TranslationModule} from "./modules/translation"
+export type {SpeakerModule} from "./modules/speaker"
+
+// Permission types
+export type {PermissionType, PermissionRecord} from "./session"