@mentra/miniapp 0.3.0-dev.0

package/src/modules/storage.ts

@@ -0,0 +1,109 @@
+/**
+ * @fileoverview SimpleStorage — phone-local key-value store scoped to
+ * (userId, packageName). Mirrors cloud SDK v3's StorageManager surface.
+ *
+ * All operations round-trip to LocalMiniappRuntime, which reads/writes
+ * the phone's storage with a namespaced key format:
+ *   mentraos_localstorage_{userId}_{packageName}_{key}
+ *
+ * Values are plain strings. Callers serialize structured data with
+ * JSON.stringify themselves (matching cloud's SimpleStorage shape).
+ */
+
+import {MiniappRequestType} from "../protocol"
+import {MiniappSession} from "../session"
+
+export class SimpleStorage {
+  constructor(private readonly session: MiniappSession) {}
+
+  /** Get a value by key. Resolves to null when the key is unset. */
+  async get(key: string): Promise<string | null> {
+    const result = await this.session.sendRequest<{value: string | null}>({
+      type: MiniappRequestType.STORAGE_GET,
+      key,
+    })
+    return result?.value ?? null
+  }
+
+  /** Set a single key/value pair. Overwrites silently. */
+  async set(key: string, value: string): Promise<void> {
+    await this.session.sendRequest<void>({
+      type: MiniappRequestType.STORAGE_SET,
+      key,
+      value,
+    })
+  }
+
+  /** Delete one key. No-op if the key is unset. */
+  async delete(key: string): Promise<void> {
+    await this.session.sendRequest<void>({
+      type: MiniappRequestType.STORAGE_DELETE,
+      key,
+    })
+  }
+
+  /** List every key in this miniapp's namespace. */
+  async keys(): Promise<string[]> {
+    const result = await this.session.sendRequest<{keys: string[]}>({
+      type: MiniappRequestType.STORAGE_LIST,
+    })
+    return result?.keys ?? []
+  }
+
+  /**
+   * @deprecated Renamed to `keys()` for cloud SDK parity. This alias will be
+   * removed in a future release.
+   */
+  async list(): Promise<string[]> {
+    return this.keys()
+  }
+
+  /** Drop every key in this miniapp's namespace. */
+  async clear(): Promise<void> {
+    await this.session.sendRequest<void>({
+      type: MiniappRequestType.STORAGE_CLEAR,
+    })
+  }
+
+  /** True iff `key` is set in this miniapp's namespace. */
+  async has(key: string): Promise<boolean> {
+    const result = await this.session.sendRequest<{has: boolean}>({
+      type: MiniappRequestType.STORAGE_HAS,
+      key,
+    })
+    return !!result?.has
+  }
+
+  /**
+   * Read every key/value pair in this miniapp's namespace. Useful for
+   * hydrating state on bootstrap. Bounded by the underlying namespace
+   * size — keep total stored payload small (typical < 1 MB) to avoid
+   * stalling the JSContext when this resolves.
+   */
+  async getAll(): Promise<Record<string, string>> {
+    const result = await this.session.sendRequest<{values: Record<string, string>}>({
+      type: MiniappRequestType.STORAGE_GET_ALL,
+    })
+    return result?.values ?? {}
+  }
+
+  /** Write many key/value pairs in one round-trip. */
+  async setMultiple(values: Record<string, string>): Promise<void> {
+    await this.session.sendRequest<void>({
+      type: MiniappRequestType.STORAGE_SET_MULTIPLE,
+      values,
+    })
+  }
+
+  /**
+   * Force any pending in-flight writes to disk. No-op today because the
+   * host writes through immediately, but reserved so a future debounced
+   * backend can honor "flush before I quit" calls without breaking the
+   * API.
+   */
+  async flush(): Promise<void> {
+    await this.session.sendRequest<void>({
+      type: MiniappRequestType.STORAGE_FLUSH,
+    })
+  }
+}

package/src/modules/stream.ts

@@ -0,0 +1,163 @@
+/**
+ * @fileoverview StreamModule -- video streaming from glasses.
+ *
+ * Wires to cloud streaming extensions via the __phone__ subscriber path.
+ *
+ * One entry point: `startStream`. By default it provisions a Mentra-hosted
+ * stream and returns playback URLs (managed). Pass `direct` to publish straight
+ * to your own ingest URL with no relay.
+ */
+
+import {MiniappRequestType} from "../protocol"
+import {MiniappSession} from "../session"
+
+export interface StreamVideoConfig {
+  width?: number
+  height?: number
+  bitrate?: number
+  fps?: number
+}
+
+export interface StreamAudioConfig {
+  bitrate?: number
+  sampleRate?: number
+  echoCancellation?: boolean
+  noiseSuppression?: boolean
+}
+
+export interface StreamResolvedConfig {
+  transport?: "rtmp" | "srt" | "whip"
+  video?: {
+    width: number
+    height: number
+    captureWidth?: number
+    captureHeight?: number
+    bitrate: number
+    fps: number
+  }
+  audio?: {
+    bitrate?: number
+    sampleRate?: number
+    echoCancellation?: boolean
+    noiseSuppression?: boolean
+  }
+}
+
+/**
+ * Restream destination. The full stream key is part of the URL (e.g.
+ * `rtmp://yt.com/live/STREAM-KEY`). `name` is a human label that surfaces
+ * in dashboards but doesn't affect routing.
+ */
+export interface RestreamDestination {
+  url: string
+  name?: string
+}
+
+export interface StartStreamOptions {
+  /**
+   * Publish straight to your own ingest URL (rtmp/rtmps/srt/whip). The glasses
+   * connect to it directly with no Mentra relay, so no playback URLs come back.
+   * Omit this to use the managed relay (the default, best for most apps).
+   */
+  direct?: string
+  /**
+   * Restream destinations the managed relay fans out to (YouTube, Twitch, …).
+   * Bare URL strings or `{url, name?}` objects; mix freely. Managed streams
+   * only — ignored when `direct` is set.
+   */
+  destinations?: Array<string | RestreamDestination>
+  video?: StreamVideoConfig
+  audio?: StreamAudioConfig
+  /** Play glasses-side stream start/stop sounds. Defaults to true. */
+  sound?: boolean
+  /**
+   * Managed ingest protocol, a latency/durability trade. Ignored when `direct`
+   * is set.
+   *   - "srt" (default): ~10-20s latency via LL-HLS, shareable HLS/DASH URLs,
+   *     and automatic recording.
+   *   - "whip": sub-second WebRTC playback (use `webrtcUrl`/WHEP), but no
+   *     HLS/DASH and no recording.
+   */
+  ingest?: "srt" | "whip"
+}
+
+export interface StreamResult {
+  streamId: string
+  status: string
+  resolvedConfig?: StreamResolvedConfig
+  /** Managed streams only: the live-input UID for building hosted-player URLs. */
+  liveInputId?: string
+  /** Managed streams only: which playback this stream serves. In "webrtc" mode
+   *  `hlsUrl` never serves content. */
+  mode?: "hls" | "webrtc"
+  /** Managed streams only: HLS playback URL. */
+  hlsUrl?: string
+  /** Managed streams only: DASH playback URL. */
+  dashUrl?: string
+  /** Managed streams only: WHEP playback URL (present in "webrtc" mode). */
+  webrtcUrl?: string
+}
+
+/** Live encoder telemetry, when the glasses emit it (no firmware does yet). */
+export interface StreamLiveStats {
+  /** Current encoded video bitrate in bits per second. */
+  bitrate?: number
+  /** Current encode frame rate. */
+  fps?: number
+  droppedFrames?: number
+  /** Seconds since the stream started. */
+  duration?: number
+  /** Device temperature in °C, if the hardware reports it. */
+  temperatureC?: number
+}
+
+export interface StreamStatus {
+  streamId: string
+  status: string
+  errorDetails?: string
+  /** Negotiated encode config — forwarded once, on the first status ack. */
+  resolvedConfig?: StreamResolvedConfig
+  stats?: StreamLiveStats
+}
+
+export class StreamModule {
+  constructor(private readonly session: MiniappSession) {}
+
+  /**
+   * Start a live video stream from the glasses camera.
+   *
+   * Managed (default) -- Mentra hosts the stream and the result carries
+   * `hlsUrl`/`dashUrl`/`webrtcUrl` for playback. Optionally fan out to your own
+   * `destinations`.
+   *
+   * Direct -- pass `direct` with your own ingest URL; the glasses publish to it
+   * and no playback URLs are returned.
+   */
+  async startStream(options: StartStreamOptions = {}): Promise<StreamResult> {
+    if (options.direct !== undefined) {
+      return this.session.sendRequest<StreamResult>({
+        type: MiniappRequestType.STREAM_START,
+        streamUrl: options.direct,
+        video: options.video,
+        audio: options.audio,
+        sound: options.sound ?? true,
+      })
+    }
+    return this.session.sendRequest<StreamResult>({
+      type: MiniappRequestType.MANAGED_STREAM_START,
+      restreamDestinations: options.destinations,
+      video: options.video,
+      audio: options.audio,
+      sound: options.sound ?? true,
+      ingest: options.ingest,
+    })
+  }
+
+  /** Stop the active stream, or a specific one by `streamId`. */
+  async stop(streamId?: string): Promise<void> {
+    await this.session.sendRequest<void>({
+      type: MiniappRequestType.STREAM_STOP,
+      streamId,
+    })
+  }
+}

package/src/modules/system.ts

@@ -0,0 +1,76 @@
+/**
+ * @fileoverview SystemModule — OS-level utilities (share, open URL, clipboard, download).
+ *
+ * These bridge to native phone capabilities via LocalMiniappRuntime.
+ */
+
+import { MiniappRequestType } from "../protocol"
+import { MiniappSession } from "../session"
+
+export interface ShareOptions {
+  text?: string
+  url?: string
+  title?: string
+  /** Base64-encoded file data for file sharing. */
+  base64?: string
+  /** MIME type when sharing base64 data. */
+  mimeType?: string
+  /** Filename when sharing base64 data. */
+  filename?: string
+}
+
+export interface ShareResult {
+  success: boolean
+  cancelled?: boolean
+}
+
+export interface DownloadOptions {
+  /** URL to download from, OR base64 data. */
+  url?: string
+  base64?: string
+  filename?: string
+  mimeType?: string
+}
+
+export interface DownloadResult {
+  success: boolean
+  cancelled?: boolean
+}
+
+export class SystemModule {
+  constructor(private readonly session: MiniappSession) {}
+
+  /** Open the OS share sheet with the given content. */
+  async share(options: ShareOptions): Promise<ShareResult> {
+    const result = await this.session.sendRequest<ShareResult>({
+      type: MiniappRequestType.SHARE,
+      ...options,
+    })
+    return result ?? { success: false }
+  }
+
+  /** Open a URL in the system browser. Blocks dangerous schemes (javascript:, file:). */
+  openUrl(url: string): void {
+    this.session.sendOneShot({
+      type: MiniappRequestType.OPEN_URL,
+      url,
+    })
+  }
+
+  /** Copy text to the system clipboard. */
+  async copyToClipboard(text: string): Promise<void> {
+    await this.session.sendRequest<void>({
+      type: MiniappRequestType.COPY_CLIPBOARD,
+      text,
+    })
+  }
+
+  /** Download a file. Opens the OS share sheet so user can choose save location. */
+  async download(options: DownloadOptions): Promise<DownloadResult> {
+    const result = await this.session.sendRequest<DownloadResult>({
+      type: MiniappRequestType.DOWNLOAD,
+      ...options,
+    })
+    return result ?? { success: false }
+  }
+}

package/src/modules/transcription.ts

@@ -0,0 +1,128 @@
+/**
+ * @fileoverview TranscriptionModule — top-level transcription API.
+ *
+ * Mirrors cloud SDK v3's TranscriptionManager:
+ *
+ *   session.transcription.on(handler)                                // auto-detect
+ *   session.transcription.forLanguage("en-US", handler)              // single language
+ *   session.transcription.forLanguage(["en-US", "es-ES"], handler)   // multi-language
+ *   session.transcription.configure({languageHints, vocabulary, diarization})
+ *   session.transcription.stop()                                     // tear down all
+ *
+ * Hoisted to top-level (vs. nested under `session.mic`) because transcription
+ * is a domain in its own right, not a microphone-input event. The `mic`
+ * module still exposes lower-level audio chunks + VAD; this module is for
+ * "give me text from speech."
+ *
+ * MICROPHONE permission must be declared in miniapp.json. The phone runtime
+ * rejects with PERMISSION_NOT_DECLARED otherwise.
+ */
+
+import {MiniappRequestType, MiniappStreamType} from "../protocol"
+import {MiniappSession} from "../session"
+import type {TranscriptionData, UnsubscribeFn} from "./events"
+
+/** Configuration for cloud-side transcription behavior. */
+export interface TranscriptionConfig {
+  /** ISO 639-1 language hints to improve detection accuracy (e.g. ["en", "ja"]). */
+  languageHints?: string[]
+  /** Custom vocabulary / boosted terms (e.g. ["MentraOS", "HIPAA"]). */
+  vocabulary?: string[]
+  /** Enable speaker diarisation. Defaults vary by provider. */
+  diarization?: boolean
+}
+
+export class TranscriptionModule {
+  private readonly unsubs = new Set<UnsubscribeFn>()
+  private currentConfig: TranscriptionConfig | null = null
+
+  constructor(private readonly session: MiniappSession) {}
+
+  /**
+   * Subscribe to all transcription events (auto-detect language). The
+   * detected language is in `data.language`.
+   *
+   * Wire-level: subscribes to `transcription:auto`. Today's wildcard fan-out
+   * (handlers on `transcription:auto` receive any `transcription:<lang>`
+   * event) preserves existing semantics.
+   */
+  on(handler: (data: TranscriptionData) => void): UnsubscribeFn {
+    return this.track(
+      this.session._subscribe(`${MiniappStreamType.TRANSCRIPTION}:auto`, handler as (data: unknown) => void),
+    )
+  }
+
+  /**
+   * Subscribe to transcription for one or more specific languages. Each call
+   * is independent; multiple can be active simultaneously.
+   *
+   * @param language - BCP-47 tag(s), e.g. `"en-US"` or `["en-US", "es-ES"]`.
+   * @param handler  - Called for every event in any of the listed languages.
+   */
+  forLanguage(language: string | string[], handler: (data: TranscriptionData) => void): UnsubscribeFn {
+    const langs = Array.isArray(language) ? language : [language]
+    if (langs.length === 0) return () => {}
+
+    const unsubs: UnsubscribeFn[] = []
+    for (const lang of langs) {
+      unsubs.push(
+        this.session._subscribe(`${MiniappStreamType.TRANSCRIPTION}:${lang}`, handler as (data: unknown) => void),
+      )
+    }
+    const combined: UnsubscribeFn = () => {
+      for (const u of unsubs) {
+        try {
+          u()
+        } catch {
+          /* ignore */
+        }
+      }
+    }
+    return this.track(combined)
+  }
+
+  /**
+   * Apply transcription configuration (language hints, custom vocabulary,
+   * diarisation toggle). Sent to the cloud immediately. Cached locally so we
+   * could re-send on reconnect (future).
+   */
+  configure(config: TranscriptionConfig): void {
+    this.currentConfig = {...config}
+    this.session.sendOneShot({
+      type: MiniappRequestType.TRANSCRIPTION_CONFIG,
+      config: {...config},
+    })
+  }
+
+  /** Tear down every transcription subscription this module owns. */
+  stop(): void {
+    for (const u of this.unsubs) {
+      try {
+        u()
+      } catch {
+        /* ignore */
+      }
+    }
+    this.unsubs.clear()
+  }
+
+  /** True iff `MICROPHONE` is declared in the miniapp's manifest. */
+  get hasPermission(): boolean {
+    return this.session._hasManifestPermission("MICROPHONE")
+  }
+
+  /** @internal — current config, read by tests / future reconnect logic. */
+  _getConfig(): TranscriptionConfig | null {
+    return this.currentConfig ? {...this.currentConfig} : null
+  }
+
+  // ------------------------------------------------------------------------
+
+  private track(unsub: UnsubscribeFn): UnsubscribeFn {
+    this.unsubs.add(unsub)
+    return () => {
+      this.unsubs.delete(unsub)
+      unsub()
+    }
+  }
+}

package/src/modules/translation.ts

@@ -0,0 +1,149 @@
+/**
+ * @fileoverview TranslationModule — top-level translation API.
+ *
+ * Mirrors cloud SDK v3's TranslationManager surface:
+ *
+ *   session.translation.on(handler)                          // any pair
+ *   session.translation.to(target, handler)                  // any-source → target
+ *   session.translation.to([targets], handler)               // any-source → many
+ *   session.translation.fromTo(source, target, handler)      // specific pair
+ *   session.translation.fromTo(source, [targets], handler)   // source → many
+ *   session.translation.stop()                                // tear down all
+ *
+ * Internally each subscribe registers against a `translation:<source>:<target>`
+ * stream pattern; LocalMiniappRuntime's wildcard matcher routes incoming
+ * `translation:<src>:<dst>` events to any matching `*` patterns.
+ *
+ * MICROPHONE permission required.
+ */
+
+import {MiniappStreamType} from "../protocol"
+import {MiniappSession} from "../session"
+import type {TranslationData, UnsubscribeFn} from "./events"
+
+export type TranslationHandler = (data: TranslationData) => void
+
+export class TranslationModule {
+  private readonly unsubs = new Set<UnsubscribeFn>()
+
+  constructor(private readonly session: MiniappSession) {}
+
+  /**
+   * Subscribe to every translation event from the cloud, regardless of
+   * source or target language.
+   *
+   * Uses the wildcard stream `translation:*:*` — cheap to register on the
+   * client, expensive on the cloud side (cloud fans every active pair out
+   * to this subscriber). Prefer `to` or `fromTo` when you know the
+   * language(s) you care about.
+   */
+  on(handler: TranslationHandler): UnsubscribeFn {
+    return this.track(
+      this.session._subscribe(
+        `${MiniappStreamType.TRANSLATION}:*:*`,
+        handler as (data: unknown) => void,
+      ),
+    )
+  }
+
+  /**
+   * Subscribe to any translation that lands at the given target language(s),
+   * regardless of source. Useful for "I'm a Spanish speaker, translate
+   * whatever I hear to Spanish."
+   */
+  to(target: string | string[], handler: TranslationHandler): UnsubscribeFn {
+    const targets = Array.isArray(target) ? target : [target]
+    if (targets.length === 0) return () => {}
+    const unsubs: UnsubscribeFn[] = []
+    for (const t of targets) {
+      unsubs.push(
+        this.session._subscribe(
+          `${MiniappStreamType.TRANSLATION}:*:${t}`,
+          handler as (data: unknown) => void,
+        ),
+      )
+    }
+    const composite = () => {
+      for (const u of unsubs) {
+        try {
+          u()
+        } catch {
+          /* ignore */
+        }
+      }
+    }
+    return this.track(composite)
+  }
+
+  /**
+   * Subscribe to a specific source → target translation pair. Pass an
+   * array for `target` to fan a single handler across multiple targets
+   * from the same source.
+   */
+  fromTo(
+    source: string,
+    target: string | string[],
+    handler: TranslationHandler,
+  ): UnsubscribeFn {
+    const targets = Array.isArray(target) ? target : [target]
+    if (targets.length === 0) return () => {}
+    const unsubs: UnsubscribeFn[] = []
+    for (const t of targets) {
+      unsubs.push(
+        this.session._subscribe(
+          `${MiniappStreamType.TRANSLATION}:${source}:${t}`,
+          handler as (data: unknown) => void,
+        ),
+      )
+    }
+    const composite = () => {
+      for (const u of unsubs) {
+        try {
+          u()
+        } catch {
+          /* ignore */
+        }
+      }
+    }
+    return this.track(composite)
+  }
+
+  /**
+   * @deprecated Renamed to `fromTo(source, target, handler)` for cloud
+   * SDK v3 parity. This alias will be removed in a future release.
+   */
+  forLanguagePair(
+    fromLang: string,
+    toLang: string,
+    handler: TranslationHandler,
+  ): UnsubscribeFn {
+    return this.fromTo(fromLang, toLang, handler)
+  }
+
+  /** Tear down every translation subscription this module owns. */
+  stop(): void {
+    for (const u of this.unsubs) {
+      try {
+        u()
+      } catch {
+        /* ignore */
+      }
+    }
+    this.unsubs.clear()
+  }
+
+  /** True iff `MICROPHONE` is declared in the miniapp's manifest. */
+  get hasPermission(): boolean {
+    return this.session._hasManifestPermission("MICROPHONE")
+  }
+
+  // ------------------------------------------------------------------------
+
+  private track(unsub: UnsubscribeFn): UnsubscribeFn {
+    this.unsubs.add(unsub)
+    return () => {
+      this.unsubs.delete(unsub)
+      unsub()
+    }
+  }
+}