@mentra/miniapp 0.3.0-dev.0

package/src/modules/camera.ts

@@ -0,0 +1,196 @@
+/**
+ * @fileoverview CameraModule — glasses camera control and photo capture.
+ */
+
+import {MiniappRequestType} from "../protocol"
+import {MiniappSession} from "../session"
+
+export type CameraRoiPosition = "center" | "bottom" | "top"
+export type CameraFovPreset = "narrow" | "standard" | "wide"
+
+export type CameraFovRequest =
+  | {
+      /** Horizontal FOV, degrees. */
+      fov: number
+      /** Crop/region position: center, bottom, or top. Defaults to center. */
+      roiPosition?: CameraRoiPosition
+    }
+  | {
+      /** Named FOV preset. Presets use center ROI. */
+      preset: CameraFovPreset
+    }
+
+export type SetCameraFovOptions = CameraFovRequest
+
+export interface CameraFovResult {
+  requestId: string
+  fov: number
+  roiPosition: CameraRoiPosition
+  timestamp: number
+}
+
+export interface TakePhotoOptions {
+  size?: "low" | "medium" | "high" | "max"
+  /** Capture at maximum source quality and optimize BLE delivery for readable text. */
+  mode?: "photo" | "text"
+  compress?: "none" | "low" | "medium" | "high"
+  sound?: boolean
+  saveToGallery?: boolean
+  /**
+   * Manual shutter / exposure time in nanoseconds. Omit (or pass undefined)
+   * to let the glasses auto-expose. Honored only on cameras that support
+   * manual exposure; ignored otherwise.
+   */
+  exposureTimeNs?: number
+}
+
+export interface PhotoTaken {
+  /** Request id that correlates ASG status, upload, and phone/cloud logs. */
+  requestId: string
+  photoUrl: string
+  mimeType: string
+  size: number
+}
+
+export interface WarmUpCameraOptions {
+  size?: "low" | "medium" | "high" | "max"
+  exposureTimeNs?: number
+  durationMs?: number
+}
+
+export interface StartVideoRecordingOptions {
+  /** Video width in pixels. Omit to use the device's saved button-video default. */
+  width?: number
+  /** Video height in pixels. Omit to use the device's saved button-video default. */
+  height?: number
+  /**
+   * Frames per second. Lower frame rates keep the glasses cooler and produce
+   * smaller files — useful for long recordings where smooth motion isn't needed.
+   * Omit to use the device default.
+   */
+  fps?: number
+  /** Play start/stop capture sounds. Defaults to true. */
+  sound?: boolean
+  /** Keep the recording on the glasses after it finishes. Defaults to false. */
+  save?: boolean
+}
+
+export interface VideoRecordingStarted {
+  /** Identifier for this recording session; pass to {@link stopVideoRecording}. */
+  recordingId: string
+}
+
+export interface StopVideoRecordingOptions {
+  /**
+   * Upload the finished recording to this URL with a multipart POST (the file is
+   * the `video` part). The glasses upload it directly. Omit to keep the recording
+   * on the glasses instead.
+   */
+  uploadUrl?: string
+  /** Bearer token sent as the `Authorization` header on the upload, if your endpoint needs one. */
+  uploadAuthToken?: string
+}
+
+export class CameraModule {
+  constructor(private readonly session: MiniappSession) {}
+
+  /** True iff `CAMERA` is declared in the miniapp's manifest. */
+  get hasPermission(): boolean {
+    return this.session._hasManifestPermission("CAMERA")
+  }
+
+  /**
+   * Apply camera FOV/ROI settings on the glasses.
+   *
+   * Resolves after the ASG client reports that the setting was applied to camera
+   * hardware after the restart cooldown. Requires CAMERA permission declared in miniapp.json.
+   */
+  async setFov(request: CameraFovRequest): Promise<CameraFovResult> {
+    return this.session.sendRequest<CameraFovResult>({
+      type: MiniappRequestType.CAMERA_FOV,
+      ...request,
+    })
+  }
+
+  /**
+   * Take a photo via the glasses camera. Returns a URL to the captured image.
+   * Requires CAMERA permission declared in miniapp.json.
+   *
+   * The photo is uploaded to cloud storage; the returned URL is a short-TTL
+   * (~30 minute) signed download URL. If the glasses don't have a camera,
+   * the phone-side handler rejects with an error. Check
+   * `session.capabilities.hasCamera` before calling.
+   */
+  async takePhoto(options: TakePhotoOptions = {}): Promise<PhotoTaken> {
+    return this.session.sendRequest<PhotoTaken>({
+      type: MiniappRequestType.PHOTO,
+      size: options.size ?? "medium",
+      mode: options.mode ?? "photo",
+      compress: options.compress ?? "none",
+      sound: options.sound ?? true,
+      saveToGallery: options.saveToGallery ?? false,
+      exposureTimeNs: options.exposureTimeNs,
+    })
+  }
+
+  /**
+   * Pre-warm the glasses camera so the next takePhoto() is near-instant.
+   * The camera stays warm for ~durationMs (default 15s); call warmUp() again to
+   * extend it. Warm with the same `size` you'll capture with — a mismatched size
+   * forces the camera to reconfigure and loses the speedup. Requires CAMERA
+   * permission in miniapp.json. Resolves once the camera reports ready.
+   *
+   * Warm-ups are serialized: only one runs at a time and none may start while a
+   * photo is being captured. If the camera is busy (a capture is in flight, or
+   * another warm-up is still opening) this rejects with `camera_busy` — retry
+   * shortly, or just call takePhoto() (it works regardless, only slower). The
+   * normal warmUp() → takePhoto() sequence never hits this.
+   */
+  async warmUp(options: WarmUpCameraOptions = {}): Promise<void> {
+    await this.session.sendRequest<void>({
+      type: MiniappRequestType.CAMERA_WARM_UP,
+      size: options.size ?? "medium",
+      exposureTimeNs: options.exposureTimeNs,
+      durationMs: options.durationMs ?? 15000,
+    })
+  }
+
+  /**
+   * Start recording video on the glasses camera. Returns a `recordingId` to pass
+   * to {@link stopVideoRecording} after the glasses report that recording
+   * started. Requires CAMERA permission declared in miniapp.json. Check
+   * `session.capabilities.hasCamera` before calling.
+   *
+   * Resolution and frame rate are optional — omit them to use the device's saved
+   * button-video settings. Lowering `fps` (e.g. to 5) keeps the glasses cooler
+   * during long recordings.
+   */
+  async startVideoRecording(options: StartVideoRecordingOptions = {}): Promise<VideoRecordingStarted> {
+    return this.session.sendRequest<VideoRecordingStarted>({
+      type: MiniappRequestType.VIDEO_RECORDING_START,
+      width: options.width,
+      height: options.height,
+      fps: options.fps,
+      sound: options.sound ?? true,
+      save: options.save ?? false,
+    })
+  }
+
+  /**
+   * Stop an in-progress video recording started with {@link startVideoRecording}.
+   *
+   * By default the recording stays on the glasses. Pass `uploadUrl` to have the
+   * glasses upload the finished clip to your own endpoint.
+   *
+   * @param recordingId The id returned from `startVideoRecording`.
+   * @param options     Optional upload destination for the finished recording.
+   */
+  async stopVideoRecording(recordingId: string, options: StopVideoRecordingOptions = {}): Promise<void> {
+    await this.session.sendRequest<void>({
+      type: MiniappRequestType.VIDEO_RECORDING_STOP,
+      recordingId,
+      uploadUrl: options.uploadUrl,
+      uploadAuthToken: options.uploadAuthToken,
+    })
+  }
+}

package/src/modules/cloud.ts

@@ -0,0 +1,94 @@
+/**
+ * @fileoverview CloudModule — local miniapp view of the phone-owned cloud client.
+ *
+ * Miniapps do not own or configure the cloud connection. The phone owns one
+ * cloud-client instance and pushes status snapshots here so miniapps can render
+ * honest UX such as online/offline captions indicators.
+ */
+
+import {EventEmitter} from "eventemitter3"
+
+import {MiniappSession} from "../session"
+import type {UnsubscribeFn} from "./events"
+
+export type CloudClientConnectionStatus = "connected" | "connecting" | "reconnecting" | "disconnected"
+export type CloudClientAudioTransport = "udp" | "ws" | "offline" | "none"
+
+export interface CloudClientStatus {
+  status: CloudClientConnectionStatus
+  audioTransport: CloudClientAudioTransport
+}
+
+type CloudEmitterEvents = {
+  status: (status: CloudClientStatus) => void
+}
+
+export const CLOUD_STATUS_STREAM = "_cloud_status"
+
+const DEFAULT_STATUS: CloudClientStatus = {
+  status: "disconnected",
+  audioTransport: "none",
+}
+
+function normalizeCloudStatus(data: unknown): CloudClientStatus | null {
+  if (!data || typeof data !== "object") return null
+  const raw = data as Record<string, unknown>
+  const status = raw.status
+  const audioTransport = raw.audioTransport
+
+  if (status !== "connected" && status !== "connecting" && status !== "reconnecting" && status !== "disconnected") {
+    return null
+  }
+  if (
+    audioTransport !== "udp" &&
+    audioTransport !== "ws" &&
+    audioTransport !== "offline" &&
+    audioTransport !== "none"
+  ) {
+    return null
+  }
+
+  return {status, audioTransport}
+}
+
+export class CloudModule {
+  private readonly emitter = new EventEmitter<CloudEmitterEvents>()
+  private current: CloudClientStatus = {...DEFAULT_STATUS}
+
+  constructor(private readonly session: MiniappSession) {
+    this.session._subscribe(CLOUD_STATUS_STREAM, (data) => {
+      const next = normalizeCloudStatus(data)
+      if (!next) return
+      if (next.status === this.current.status && next.audioTransport === this.current.audioTransport) return
+      this.current = next
+      this.emitter.emit("status", {...this.current})
+    })
+  }
+
+  /** Latest status snapshot pushed by the phone runtime. */
+  get status(): CloudClientStatus {
+    return {...this.current}
+  }
+
+  /** True when the phone's cloud-client runtime handshake is complete. */
+  get connected(): boolean {
+    return this.current.status === "connected"
+  }
+
+  /** Method alias for callers who prefer function-style state reads. */
+  isConnected(): boolean {
+    return this.connected
+  }
+
+  /**
+   * Subscribe to cloud-client status updates. The current value is delivered
+   * immediately so UI code can hydrate without waiting for the next transition.
+   */
+  onStatusChanged(handler: (status: CloudClientStatus) => void): UnsubscribeFn {
+    handler({...this.current})
+    this.emitter.on("status", handler)
+    return () => {
+      this.emitter.off("status", handler)
+    }
+  }
+}

package/src/modules/dashboard.ts

@@ -0,0 +1,31 @@
+/**
+ * @fileoverview DashboardAPI — noop surface in v1.
+ *
+ * The cloud DashboardManager owns widget rendering in OS-ranked
+ * rotation. Keeping the API shape so miniapps compile, but calls are
+ * noop + console.warn.
+ */
+
+import {MiniappRequestType} from "../protocol"
+import {MiniappSession} from "../session"
+
+export type DashboardMode = "main" | "expanded" | "always_on"
+
+export class DashboardAPI {
+  private warned = false
+
+  constructor(private readonly session: MiniappSession) {}
+
+  setContent(mode: DashboardMode, content: string): void {
+    if (!this.warned) {
+      console.warn("[@mentra/miniapp] dashboard.setContent() is deferred in v1.")
+      this.warned = true
+    }
+    // Still forward so the phone can log/ignore consistently.
+    this.session.sendOneShot({
+      type: MiniappRequestType.DASHBOARD_CONTENT_UPDATE,
+      mode,
+      content,
+    })
+  }
+}

package/src/modules/display.ts

@@ -0,0 +1,122 @@
+/**
+ * @fileoverview DisplayManager — the glasses display, via `render()`.
+ *
+ * render(elements) is replace-the-frame scene rendering: each call describes
+ * everything that should be on screen, the host diffs it against the previous
+ * frame per device, and `render([])` clears. The legacy one-shot `show*`
+ * layout methods were removed once nothing first-party or known-external
+ * called them; hosts still accept the historical `miniapp_display` wire shape
+ * from bundles packed with older SDKs.
+ */
+
+import {MiniappRequestType} from "../protocol"
+import {MiniappSession} from "../session"
+
+export type ViewType = "main" | "dashboard"
+
+export type DisplayBreakMode = "character" | "character-no-hyphen" | "word" | "strict-word"
+
+// ============================================================================
+// render() — the scene API
+// ============================================================================
+
+/** Pixel-space bounding box on the device's drawable canvas (see `capabilities.display`). */
+export interface RenderBox {
+  x: number
+  y: number
+  w: number
+  h: number
+}
+
+export interface RenderTextStyle {
+  /** Border width in px (0/absent = none). */
+  border?: number
+  /** Border corner radius in px. */
+  radius?: number
+  /** What happens to text that doesn't fit the box after wrapping. Default "clip". */
+  overflow?: "clip" | "ellipsis"
+  /** Line-break policy for wrapping (host wraps; the box carries pre-wrapped text). */
+  breakMode?: DisplayBreakMode
+}
+
+export interface RenderRectStyle {
+  border?: number
+  radius?: number
+}
+
+/**
+ * One element of a rendered scene. `id` is optional but recommended for
+ * anything that updates over time: elements with a stable id update in place on
+ * the glasses (no flicker); the host matches unnamed elements by geometry.
+ */
+export type RenderElement =
+  | {type: "text"; id?: string; box: RenderBox; text: string; style?: RenderTextStyle}
+  | {type: "image"; id?: string; box: RenderBox; data: string}
+  | {type: "rect"; id?: string; box: RenderBox; style?: RenderRectStyle}
+
+export interface RenderOptions {
+  view?: ViewType
+  /** Auto-clear after this many ms (same semantics as legacy display options). */
+  durationMs?: number
+}
+
+/**
+ * Outcome of a render request, resolved when the host's display arbitration
+ * settles it. "displayed" = accepted and sent to the device — NOT a
+ * render confirmation. "blocked" = another app owns the display (or the
+ * request failed); `reason` says why.
+ */
+export interface RenderResult {
+  status: "displayed" | "blocked"
+  /** True when the host adjusted the scene (clamped boxes, dropped elements, degraded for the device). */
+  degraded?: boolean
+  /** Ids of elements the host dropped (budget/bounds/device limits) — dropped is never silent. */
+  dropped?: string[]
+  reason?: string
+}
+
+export class DisplayManager {
+  constructor(private readonly session: MiniappSession) {}
+
+  /**
+   * Render a whole scene of positioned elements — replace-the-frame.
+   *
+   * Each call describes everything that should be on screen; the host diffs it
+   * against the previous frame per device (elements with a stable `id` update
+   * in place; elements you stop sending are removed). There is no lifecycle to
+   * manage and no remove calls — `render([])` clears.
+   *
+   * Coordinates are raw pixels on the device's drawable canvas — read
+   * `session.capabilities.display` (populated on the "ready" event) for the
+   * real width/height and element budgets. Out-of-bounds boxes are clamped and
+   * over-budget elements are dropped tail-first; both are reported via the
+   * result, never silently.
+   *
+   * Awaiting is OPT-IN — a plain fire-and-forget call is fine. The returned
+   * promise never rejects.
+   *
+   * @example
+   * const result = await session.display.render([
+   *   {type: "text",  id: "stats", box: {x: 12, y: 9, w: 200, h: 28}, text: "863 m  11 min"},
+   *   {type: "image", id: "map",   box: {x: 335, y: 14, w: 150, h: 150}, data: mapPng},
+   *   {type: "rect",  box: {x: 0, y: 0, w: 500, h: 220}, style: {radius: 4}},
+   * ])
+   * if (result.degraded) console.warn("dropped:", result.dropped)
+   */
+  render(elements: RenderElement[], options: RenderOptions = {}): Promise<RenderResult> {
+    return this.session
+      .sendRequest<RenderResult>({
+        type: MiniappRequestType.RENDER,
+        view: options.view ?? "main",
+        elements,
+        durationMs: options.durationMs,
+      })
+      .catch((err) => ({
+        status: "blocked" as const,
+        reason:
+          typeof err === "object" && err !== null && "message" in err
+            ? String((err as {message: unknown}).message)
+            : String(err),
+      }))
+  }
+}