@mentra/miniapp 0.3.0-dev.0

package/src/modules/events.ts

@@ -0,0 +1,264 @@
+/**
+ * @fileoverview EventManager — internal subscription registry + escape hatch.
+ *
+ * Most miniapp authors should NOT touch this directly. Use the typed methods
+ * on domain modules instead:
+ *   - session.transcription.on(...)
+ *   - session.mic.onAudioChunk(...) / onVoiceActivity(...)
+ *   - session.input.onButtonPress(...)
+ *   - session.imu.onHeadPosition(...)
+ *   - session.location.onUpdate(...)
+ *   - session.glasses.onBattery(...) / onConnection(...)
+ *   - session.phone.onNotification(...) / onCalendarEvent(...) / onBattery(...)
+ *
+ * This module's only public method is `subscribe(rawStreamType, handler)` —
+ * a forward-compat escape hatch for new event types not yet wrapped on a
+ * domain module. Officially undocumented; the typed methods on domain
+ * modules are the canonical surface.
+ *
+ * Internally, EventManager owns:
+ *   1. The ref-count map. Outbound SUBSCRIBE is only sent when a stream's
+ *      ref count transitions 0↔1, so multiple components listening for the
+ *      same stream issue one wire-level subscribe.
+ *   2. Inbound event fan-out via `_forwardEvent(streamType, data)`, called
+ *      by MiniappSession.handleIncoming when an EVENT envelope arrives.
+ *
+ * Domain modules call back into the session via session._subscribe(...) which
+ * in turn delegates to this class — the session is the integration point;
+ * domain modules don't see this class directly.
+ */
+
+import {EventEmitter} from "eventemitter3"
+
+import {MiniappRequestType, MiniappStreamType} from "../protocol"
+import {MiniappSession} from "../session"
+
+export type UnsubscribeFn = () => void
+
+// ---------------------------------------------------------------------------
+// Shared event data shapes — re-exported by index.ts so consumers can type
+// their handlers without importing this internal module.
+// ---------------------------------------------------------------------------
+
+export interface TranscriptionData {
+  text: string
+  isFinal: boolean
+  language?: string
+}
+
+export interface TranslationData {
+  text: string
+  isFinal: boolean
+  sourceLanguage: string
+  targetLanguage: string
+  /** Source-language text of the same utterance, when the provider supplies it. */
+  originalText?: string
+  /** Stable id correlating interim + final results of one utterance. */
+  utteranceId?: string
+  /** Speaker id when the provider reports diarization. */
+  speakerId?: string
+}
+
+export interface ButtonPressData {
+  buttonId: string
+  pressType: "short" | "long"
+}
+
+export interface HeadPositionData {
+  position: "up" | "down"
+}
+
+export interface AccelData {
+  /** Accelerometer axes in g (gravity-normalized). */
+  x: number
+  y: number
+  z: number
+  /** Unix ms timestamp of the reading. */
+  timestamp: number
+}
+
+export interface LocationData {
+  lat: number
+  lng: number
+  /** Accuracy in meters, if the platform reported it. */
+  accuracy?: number
+  /** Unix ms timestamp of the fix. */
+  timestamp?: number
+  /** Set when this event is a response to a single-location request. */
+  correlationId?: string
+}
+
+export interface HeadingData {
+  /** Compass heading in degrees, 0 = north, 90 = east. */
+  degrees: number
+}
+
+export interface BatteryData {
+  level: number
+  charging: boolean
+}
+
+export interface ConnectionData {
+  connected: boolean
+  modelName?: string
+}
+
+/** Glasses Wi-Fi state. `connected` is false on glasses without Wi-Fi or when offline. */
+export interface WifiData {
+  connected: boolean
+  ssid?: string
+  localIp?: string
+}
+
+export interface PhoneNotificationData {
+  /** Stable id from the phone's notification listener. */
+  notificationId: string
+  /** Human app name (e.g. "Messages"). */
+  app: string
+  title: string
+  content: string
+  /** Android priority string; empty on iOS. */
+  priority: string
+  timestamp: number
+  /** Reverse-DNS package/bundle id of the originating app. */
+  packageName: string
+}
+
+export interface NotificationDismissedData {
+  /** Same id as the matching post event from `notifications.on(...)`. */
+  notificationId: string
+  /** Android NotificationKey. More stable than notificationId across reposts. */
+  notificationKey?: string
+  /** Reverse-DNS package/bundle id of the originating app. */
+  packageName?: string
+  /** Unix ms timestamp of the dismissal. */
+  timestamp: number
+}
+
+export interface CalendarEventData {
+  eventId: string
+  title: string
+  /** ISO 8601 start time. */
+  dtStart: string
+  /** ISO 8601 end time. */
+  dtEnd: string
+  timezone: string
+  allDay: boolean
+  location: string
+  notes: string
+  calendarId: string
+}
+
+export interface VadData {
+  /** True while the user is speaking (voice detected), false when silent. */
+  status: boolean
+}
+
+export interface TouchData {
+  /** The gesture: single_tap, double_tap, triple_tap, long_press, swipe_up, or swipe_down. */
+  kind: "single_tap" | "double_tap" | "triple_tap" | "long_press" | "swipe_up" | "swipe_down" | string
+}
+
+export interface AudioChunkData {
+  /** PCM or LC3, base64-encoded. Format depends on phone's mic mode. */
+  data: string
+  sampleRate?: number
+  format?: string
+}
+
+// ---------------------------------------------------------------------------
+// Registry
+// ---------------------------------------------------------------------------
+
+export class EventManager {
+  private readonly emitter = new EventEmitter()
+  /** Stream -> ref count. Outbound SUBSCRIBE is sent when refs transition 0↔1. */
+  private readonly refCounts = new Map<string, number>()
+
+  constructor(private readonly session: MiniappSession) {}
+
+  /**
+   * Generic subscribe. `stream` is the raw wire value, including any language
+   * suffix like `"transcription:en-US"`. Forward-compat escape hatch for new
+   * event types not yet wrapped on a domain module. Most authors should use
+   * typed methods on domain modules instead.
+   *
+   * Stream names beginning with `_` (e.g. `_ui` for the session.ui bus) are
+   * INTERNAL — they ride the local _forwardEvent fan-out and never appear in
+   * the outbound SUBSCRIBE list. Native already knows to deliver UI envelopes
+   * via a separate envelope `type`; the EventManager only routes them
+   * locally.
+   */
+  subscribe(stream: string, handler: (data: unknown) => void): UnsubscribeFn {
+    this.emitter.on(stream, handler)
+    const isInternal = stream.startsWith("_")
+    if (!isInternal) {
+      const before = this.refCounts.get(stream) ?? 0
+      this.refCounts.set(stream, before + 1)
+      if (before === 0) {
+        this.sendSubscriptionUpdate()
+      }
+    }
+    return () => {
+      this.emitter.off(stream, handler)
+      if (isInternal) return
+      const current = this.refCounts.get(stream) ?? 0
+      if (current <= 1) {
+        this.refCounts.delete(stream)
+        this.sendSubscriptionUpdate()
+      } else {
+        this.refCounts.set(stream, current - 1)
+      }
+    }
+  }
+
+  /** Unsubscribe every handler on every stream this EventManager owns. */
+  unsubscribeAll(): void {
+    this.emitter.removeAllListeners()
+    this.refCounts.clear()
+    this.sendSubscriptionUpdate()
+  }
+
+  // -------------------------------------------------------------------------
+  // Internal — called by MiniappSession when EVENT arrives from phone
+  // -------------------------------------------------------------------------
+
+  /** @internal */
+  _forwardEvent(stream: string, data: unknown): void {
+    this.emitter.emit(stream, data)
+
+    // Wildcard fan-out: handlers register under wildcard patterns
+    // ("transcription:auto", "translation:*:<target>", …) but the host
+    // delivers events under the CONCRETE stream key
+    // ("transcription:en-US", "translation:en:es"). Re-emit on every
+    // pattern the concrete key satisfies — the same pattern set
+    // LocalMiniappRuntime.forwardEvent matches on the host side —
+    // otherwise a wildcard subscriber never fires.
+    if (stream.startsWith("transcription:") && stream !== "transcription:auto") {
+      this.emitter.emit("transcription:auto", data)
+    } else if (stream.startsWith("translation:")) {
+      const parts = stream.split(":")
+      const patterns = new Set<string>(["translation:auto"])
+      if (parts.length === 3) {
+        const [, source, target] = parts
+        patterns.add("translation:*:*")
+        patterns.add(`translation:*:${target}`)
+        patterns.add(`translation:${source}:*`)
+      }
+      patterns.delete(stream) // exact key already emitted above
+      for (const pattern of patterns) {
+        this.emitter.emit(pattern, data)
+      }
+    }
+  }
+
+  private sendSubscriptionUpdate(): void {
+    const subscriptions = Array.from(this.refCounts.keys()).map((stream) =>
+      stream === MiniappStreamType.LOCATION_UPDATE ? {stream: "location_stream", rate: "realtime"} : stream,
+    )
+    this.session.sendOneShot({
+      type: MiniappRequestType.SUBSCRIBE,
+      subscriptions,
+    })
+  }
+}

package/src/modules/glasses.ts

@@ -0,0 +1,44 @@
+/**
+ * @fileoverview GlassesModule — device-state events for the glasses themselves.
+ *
+ * Reports on the glasses hardware: battery level + charging state, connection
+ * status (connected/disconnected, model name). The phone has its own battery
+ * + connection events on `session.phone`.
+ */
+
+import {MiniappRequestType, MiniappStreamType} from "../protocol"
+import {MiniappSession} from "../session"
+import type {BatteryData, ConnectionData, UnsubscribeFn, WifiData} from "./events"
+
+export class GlassesModule {
+  constructor(private readonly session: MiniappSession) {}
+
+  onBattery(handler: (data: BatteryData) => void): UnsubscribeFn {
+    return this.session._subscribe(MiniappStreamType.GLASSES_BATTERY, handler as (data: unknown) => void)
+  }
+
+  onConnection(handler: (data: ConnectionData) => void): UnsubscribeFn {
+    return this.session._subscribe(MiniappStreamType.GLASSES_CONNECTION, handler as (data: unknown) => void)
+  }
+
+  /**
+   * Subscribe to glasses Wi-Fi state. Fires the current state on subscribe and
+   * on every change. `connected` is false on glasses without Wi-Fi. Streaming
+   * requires Wi-Fi — pair this with {@link requestWifiSetup} to prompt the user.
+   */
+  onWifi(handler: (data: WifiData) => void): UnsubscribeFn {
+    return this.session._subscribe(MiniappStreamType.GLASSES_WIFI, handler as (data: unknown) => void)
+  }
+
+  /**
+   * Ask the host to open the glasses Wi-Fi setup flow on the phone. Mirrors the
+   * cloud SDK's `session.requestWifiSetup(reason)`. `reason` is a user-facing
+   * line shown in the setup prompt (e.g. "Streaming needs your glasses on Wi-Fi").
+   */
+  async requestWifiSetup(reason?: string): Promise<void> {
+    await this.session.sendRequest<void>({
+      type: MiniappRequestType.REQUEST_WIFI_SETUP,
+      reason,
+    })
+  }
+}

package/src/modules/heading.ts

@@ -0,0 +1,23 @@
+/**
+ * @fileoverview HeadingModule — phone compass heading events.
+ *
+ * Works on both Android and iOS. LOCATION permission must be declared in miniapp.json.
+ */
+
+import {MiniappStreamType} from "../protocol"
+import {MiniappSession} from "../session"
+import type {HeadingData, UnsubscribeFn} from "./events"
+
+export class HeadingModule {
+  constructor(private readonly session: MiniappSession) {}
+
+  /** True iff `LOCATION` is declared in the miniapp's manifest. */
+  get hasPermission(): boolean {
+    return this.session._hasManifestPermission("LOCATION")
+  }
+
+  /** Subscribe to continuous compass heading updates. */
+  onUpdate(handler: (data: HeadingData) => void): UnsubscribeFn {
+    return this.session._subscribe(MiniappStreamType.HEADING_UPDATE, handler as (data: unknown) => void)
+  }
+}

package/src/modules/imu.ts

@@ -0,0 +1,44 @@
+/**
+ * @fileoverview ImuModule — head position + motion events.
+ *
+ * Glasses' inertial measurement unit. Exposes head-up/down position and a
+ * raw accelerometer stream (`onAccel`). Gyroscope / magnetometer / fused
+ * orientation are future work — when added they'll surface as a combined IMU
+ * event rather than extending this single-sensor accel stream.
+ */
+
+import {MiniappRequestType, MiniappStreamType} from "../protocol"
+import {MiniappSession} from "../session"
+import type {AccelData, HeadPositionData, UnsubscribeFn} from "./events"
+
+export class ImuModule {
+  constructor(private readonly session: MiniappSession) {}
+
+  onHeadPosition(handler: (data: HeadPositionData) => void): UnsubscribeFn {
+    return this.session._subscribe(MiniappStreamType.HEAD_POSITION, handler as (data: unknown) => void)
+  }
+
+  /**
+   * Subscribe to raw accelerometer readings (`{x, y, z}` in g) from the
+   * glasses IMU. G2 only today; on glasses without an exposed IMU stream the
+   * subscription succeeds but no events fire.
+   */
+  onAccel(handler: (data: AccelData) => void): UnsubscribeFn {
+    return this.session._subscribe(MiniappStreamType.ACCEL_DATA, handler as (data: unknown) => void)
+  }
+
+  /**
+   * Explicitly enable or disable raw accelerometer streaming on the glasses.
+   *
+   * Subscribing via `onAccel` already auto-enables the sensor and unsubscribing
+   * disables it, so most callers don't need this. Use it when you want to drive
+   * the IMU directly — e.g. a diagnostic toggle. G2 only today; a no-op on
+   * glasses without an exposed IMU stream.
+   */
+  setEnabled(enabled: boolean): Promise<void> {
+    return this.session.sendRequest<void>({
+      type: MiniappRequestType.IMU_SET_ENABLED,
+      enabled,
+    })
+  }
+}

package/src/modules/input.ts

@@ -0,0 +1,71 @@
+/**
+ * @fileoverview InputModule — physical control events on the glasses.
+ *
+ * Combines button + touch surfaces. Future input modes (gesture, voice
+ * command, eye tracking) will extend this module rather than spawning new
+ * top-level modules.
+ *
+ * Touch overloads:
+ *
+ *   session.input.onTouch(handler)
+ *   session.input.onTouch("single_tap", handler)
+ *   session.input.onTouch(["swipe_up", "swipe_down"], handler)
+ *
+ * Per-gesture filtering rides on `touch_event:<gesture>` stream variants
+ * the phone runtime fans out alongside the bare `touch_event` stream.
+ */
+
+import {MiniappStreamType} from "../protocol"
+import {MiniappSession} from "../session"
+import type {ButtonPressData, TouchData, UnsubscribeFn} from "./events"
+
+export class InputModule {
+  constructor(private readonly session: MiniappSession) {}
+
+  onButtonPress(handler: (data: ButtonPressData) => void): UnsubscribeFn {
+    return this.session._subscribe(MiniappStreamType.BUTTON_PRESS, handler as (data: unknown) => void)
+  }
+
+  /**
+   * Subscribe to touch events.
+   *
+   *   onTouch(handler)                — all touch events
+   *   onTouch("single_tap", handler)  — only single taps
+   *   onTouch(["a","b"], handler)     — multiple gestures, single subscription
+   */
+  onTouch(handler: (data: TouchData) => void): UnsubscribeFn
+  onTouch(gesture: string, handler: (data: TouchData) => void): UnsubscribeFn
+  onTouch(gestures: string[], handler: (data: TouchData) => void): UnsubscribeFn
+  onTouch(
+    gestureOrHandler: string | string[] | ((data: TouchData) => void),
+    maybeHandler?: (data: TouchData) => void,
+  ): UnsubscribeFn {
+    // Plain handler — subscribe to all touches.
+    if (typeof gestureOrHandler === "function") {
+      return this.session._subscribe(
+        MiniappStreamType.TOUCH_EVENT,
+        gestureOrHandler as (data: unknown) => void,
+      )
+    }
+
+    const handler = maybeHandler!
+    const gestures = Array.isArray(gestureOrHandler) ? gestureOrHandler : [gestureOrHandler]
+    if (gestures.length === 0) return () => {}
+
+    const unsubs: UnsubscribeFn[] = []
+    for (const g of gestures) {
+      unsubs.push(
+        this.session._subscribe(`${MiniappStreamType.TOUCH_EVENT}:${g}`, handler as (data: unknown) => void),
+      )
+    }
+    return () => {
+      for (const u of unsubs) {
+        try {
+          u()
+        } catch {
+          /* ignore */
+        }
+      }
+    }
+  }
+}

package/src/modules/led.ts

@@ -0,0 +1,61 @@
+/**
+ * @fileoverview LedModule — glasses RGB LED control.
+ *
+ * API mirrors the cloud SDK's LED module. Colors are named strings
+ * (the phone maps them to per-device LED indices). Actions are "on" / "off".
+ */
+
+import {MiniappRequestType} from "../protocol"
+import {MiniappSession} from "../session"
+
+export type LedColor = "red" | "green" | "blue" | "orange" | "white"
+
+export type LedControlResult = {
+  type?: "rgb_led_control_response"
+  state: "success"
+  requestId: string
+}
+
+export interface LedControlOptions {
+  color?: LedColor
+  /** LED on duration in ms. */
+  ontime?: number
+  /** LED off duration in ms. */
+  offtime?: number
+  /** Number of on/off cycles. */
+  count?: number
+}
+
+export class LedModule {
+  constructor(private readonly session: MiniappSession) {}
+
+  /** Turn an LED on with the given pattern. Resolves after the glasses acknowledge the command. */
+  async turnOn(options: LedControlOptions = {}): Promise<LedControlResult> {
+    return this.session.sendRequest<LedControlResult>({
+      type: MiniappRequestType.RGB_LED,
+      action: "on",
+      color: options.color ?? "red",
+      ontime: options.ontime ?? 1000,
+      offtime: options.offtime ?? 0,
+      count: options.count ?? 1,
+    })
+  }
+
+  /** Turn all LEDs off. Resolves after the glasses acknowledge the command. */
+  async turnOff(): Promise<LedControlResult> {
+    return this.session.sendRequest<LedControlResult>({
+      type: MiniappRequestType.RGB_LED,
+      action: "off",
+    })
+  }
+
+  /** Blink pattern — repeats `count` times with ontime/offtime. */
+  async blink(color: LedColor, ontime: number, offtime: number, count: number): Promise<LedControlResult> {
+    return this.turnOn({color, ontime, offtime, count})
+  }
+
+  /** Solid LED for a fixed duration. */
+  async solid(color: LedColor, duration: number): Promise<LedControlResult> {
+    return this.turnOn({color, ontime: duration, offtime: 0, count: 1})
+  }
+}

package/src/modules/location.ts

@@ -0,0 +1,39 @@
+/**
+ * @fileoverview LocationModule — phone location events.
+ *
+ * Exposes both a continuous `onUpdate` subscription and a one-shot
+ * `getOnce()` poll over `MiniappRequestType.LOCATION_POLL`.
+ *
+ * LOCATION permission must be declared in miniapp.json for either path to
+ * succeed; the phone runtime rejects with PERMISSION_NOT_DECLARED
+ * otherwise.
+ */
+
+import {MiniappRequestType, MiniappStreamType} from "../protocol"
+import {MiniappSession} from "../session"
+import type {LocationData, UnsubscribeFn} from "./events"
+
+export class LocationModule {
+  constructor(private readonly session: MiniappSession) {}
+
+  /** True iff `LOCATION` is declared in the miniapp's manifest. */
+  get hasPermission(): boolean {
+    return this.session._hasManifestPermission("LOCATION")
+  }
+
+  /** Subscribe to continuous location updates. */
+  onUpdate(handler: (data: LocationData) => void): UnsubscribeFn {
+    return this.session._subscribe(MiniappStreamType.LOCATION_UPDATE, handler as (data: unknown) => void)
+  }
+
+  /**
+   * Request a single location fix. Resolves with the next available
+   * reading from the phone. Useful at app load to seed UI before a
+   * continuous stream of updates begins.
+   */
+  getOnce(): Promise<LocationData> {
+    return this.session.sendRequest<LocationData>({
+      type: MiniappRequestType.LOCATION_POLL,
+    })
+  }
+}

package/src/modules/mic.ts

@@ -0,0 +1,74 @@
+/**
+ * @fileoverview MicModule — low-level audio input subscriptions.
+ *
+ * Mirrors cloud SDK v3's MicManager naming. Houses raw audio chunks + VAD;
+ * transcription and translation hoisted to top-level (`session.transcription`
+ * / `session.translation`) in the v3-alignment round so authors don't have to
+ * mentally model "transcription is a microphone thing" — it's just a
+ * top-level domain. This module was called `MicrophoneModule` /
+ * `session.microphone` before that round.
+ *
+ * Audio *output* (TTS, file playback) lives on `session.speaker`.
+ *
+ * MICROPHONE permission must be declared in miniapp.json for any of these
+ * subscriptions to succeed; the phone runtime rejects with
+ * PERMISSION_NOT_DECLARED otherwise.
+ */
+
+import {MiniappStreamType} from "../protocol"
+import {MiniappSession} from "../session"
+import type {AudioChunkData, UnsubscribeFn, VadData} from "./events"
+
+export class MicModule {
+  /** All active unsubscribe functions for stop() to tear down at once. */
+  private readonly unsubs = new Set<UnsubscribeFn>()
+
+  constructor(private readonly session: MiniappSession) {}
+
+  /**
+   * Subscribe to voice activity detection (VAD) events. `data.status` is
+   * `true` while the user is speaking, `false` when silent.
+   */
+  onVoiceActivity(handler: (data: VadData) => void): UnsubscribeFn {
+    return this.track(this.session._subscribe(MiniappStreamType.VAD, handler as (data: unknown) => void))
+  }
+
+  /**
+   * Subscribe to raw audio chunks. Format depends on the phone's mic mode
+   * (PCM or LC3, base64-encoded).
+   */
+  onAudioChunk(handler: (data: AudioChunkData) => void): UnsubscribeFn {
+    return this.track(this.session._subscribe(MiniappStreamType.AUDIO_CHUNK, handler as (data: unknown) => void))
+  }
+
+  /**
+   * Tear down every subscription this module owns. Useful when a component
+   * is unmounting and wants to free everything at once without tracking
+   * individual unsubscribe functions.
+   */
+  stop(): void {
+    for (const u of this.unsubs) {
+      try {
+        u()
+      } catch {
+        /* ignore */
+      }
+    }
+    this.unsubs.clear()
+  }
+
+  /** True iff `MICROPHONE` is declared in the miniapp's manifest. */
+  get hasPermission(): boolean {
+    return this.session._hasManifestPermission("MICROPHONE")
+  }
+
+  // ------------------------------------------------------------------------
+
+  private track(unsub: UnsubscribeFn): UnsubscribeFn {
+    this.unsubs.add(unsub)
+    return () => {
+      this.unsubs.delete(unsub)
+      unsub()
+    }
+  }
+}