@mentra/miniapp 0.3.0-dev.0 → 3.1.0-dev.4

package/src/modules/speaker.ts

@@ -50,6 +50,17 @@ export interface SpeakOptions {
   voice_settings?: Record<string, unknown>
   volume?: number
   stopOtherAudio?: boolean
+  /**
+   * Normalize display-oriented markup and symbols into natural spoken words.
+   * Defaults to true. Set false when exact characters must reach the TTS engine.
+   */
+  enableSanitization?: boolean
+  /**
+   * Force this call to use on-device offline TTS, skipping cloud TTS
+   * entirely — even when cloud is connected. Rejects with `TTS_LOCAL_UNAVAILABLE`
+   * if the offline model isn't ready instead of falling back to cloud.
+   */
+  forceLocal?: boolean
 }
 
 export interface SpeakResult {
@@ -244,7 +255,13 @@ export class SpeakerModule {
    * Rejects with a MiniappRequestError containing a `code` field on cloud-side
    * TTS failures: `TTS_TEXT_TOO_LONG`, `TTS_INVALID_VOICE`, `TTS_UPSTREAM_ERROR`.
    */
-  async speak(text: string, options: SpeakOptions = {}): Promise<SpeakResult> {
+  async speak(text: string, options?: SpeakOptions): Promise<SpeakResult>
+  async speak(sentences: string[], options?: SpeakOptions): Promise<SpeakResult>
+  async speak(text: string | string[], options: SpeakOptions = {}): Promise<SpeakResult> {
+    const normalized = Array.isArray(text) ? text.map((sentence) => sentence.trim()).filter(Boolean) : text.trim()
+    if ((Array.isArray(normalized) && normalized.length === 0) || normalized === "") {
+      throw {code: MiniappErrorCode.INTERNAL, message: "speak requires at least one non-empty sentence"}
+    }
     try {
       // Like play(): resolves only when TTS playback completes, so opt out of the
       // default request timeout (long text can outlast it). Settled by the host
@@ -252,11 +269,13 @@ export class SpeakerModule {
       const result = await this.session.sendRequest<SpeakResult | null>(
         {
           type: MiniappRequestType.SPEAK,
-          text,
+          text: normalized,
           voice_id: options.voice_id,
           voice_settings: options.voice_settings,
           volume: options.volume,
           stopOtherAudio: options.stopOtherAudio ?? false,
+          enableSanitization: options.enableSanitization ?? true,
+          forceLocal: options.forceLocal ?? false,
         },
         {timeoutMs: 0},
       )

package/src/modules/stream.ts

@@ -77,8 +77,12 @@ export interface StartStreamOptions {
    *     and automatic recording.
    *   - "whip": sub-second WebRTC playback (use `webrtcUrl`/WHEP), but no
    *     HLS/DASH and no recording.
+   *   - "rtmp": RTMPS ingest over TCP 443 → HLS playback. Use when WHIP/SRT
+   *     UDP is blocked (then play `hlsUrl`, not WHEP).
    */
-  ingest?: "srt" | "whip"
+  ingest?: "srt" | "whip" | "rtmp"
+  /** Optional Bearer token for direct WHIP Authorization (custom authenticated endpoints). */
+  authToken?: string
 }
 
 export interface StreamResult {
@@ -98,7 +102,7 @@ export interface StreamResult {
   webrtcUrl?: string
 }
 
-/** Live encoder telemetry, when the glasses emit it (no firmware does yet). */
+/** Live encoder and device telemetry emitted periodically by supported glasses firmware. */
 export interface StreamLiveStats {
   /** Current encoded video bitrate in bits per second. */
   bitrate?: number
@@ -141,6 +145,7 @@ export class StreamModule {
         video: options.video,
         audio: options.audio,
         sound: options.sound ?? true,
+        ...(options.authToken ? {authToken: options.authToken} : {}),
       })
     }
     return this.session.sendRequest<StreamResult>({

package/src/modules/system.ts

@@ -1,5 +1,5 @@
 /**
- * @fileoverview SystemModule — OS-level utilities (share, open URL, clipboard, download).
+ * @fileoverview SystemModule — OS-level utilities (share, open URL, clipboard, download, QR scan).
  *
  * These bridge to native phone capabilities via LocalMiniappRuntime.
  */
@@ -37,6 +37,15 @@ export interface DownloadResult {
   cancelled?: boolean
 }
 
+export interface ScanQrOptions {
+  /** Header title on the phone scanner overlay. */
+  title?: string
+  /** Hint shown over the camera viewfinder. */
+  hint?: string
+}
+
+export type ScanQrResult = {data: string; cancelled?: false} | {cancelled: true; data?: undefined}
+
 export class SystemModule {
   constructor(private readonly session: MiniappSession) {}
 
@@ -73,4 +82,20 @@ export class SystemModule {
     })
     return result ?? { success: false }
   }
+
+  /**
+   * Open the phone camera QR scanner as a host overlay. Does not unmount the
+   * miniapp (so live sessions survive the scan). Resolves with the raw QR
+   * payload, or `{cancelled: true}` if the user dismisses the scanner.
+   */
+  async scanQr(options: ScanQrOptions = {}): Promise<ScanQrResult> {
+    const result = await this.session.sendRequest<ScanQrResult>(
+      {
+        type: MiniappRequestType.SCAN_QR,
+        ...options,
+      },
+      {timeoutMs: 0},
+    )
+    return result ?? { cancelled: true }
+  }
 }

package/src/modules/transcription.ts

@@ -21,17 +21,33 @@
 import {MiniappRequestType, MiniappStreamType} from "../protocol"
 import {MiniappSession} from "../session"
 import type {TranscriptionData, UnsubscribeFn} from "./events"
+import {requireLanguageHint, requireTranscriptionLanguage} from "./languages"
+import type {LanguageHint, TranscriptionLanguage} from "./languages"
 
 /** Configuration for cloud-side transcription behavior. */
 export interface TranscriptionConfig {
-  /** ISO 639-1 language hints to improve detection accuracy (e.g. ["en", "ja"]). */
-  languageHints?: string[]
+  /**
+   * Language hints to improve auto-detection accuracy. Bare ISO 639-1 codes
+   * from the registry (e.g. ["en", "ja"]); registry BCP-47 tags are accepted
+   * and normalized ("en-US" -> "en"). Anything else throws
+   * `MiniappValidationError`.
+   */
+  languageHints?: LanguageHint[]
   /** Custom vocabulary / boosted terms (e.g. ["MentraOS", "HIPAA"]). */
   vocabulary?: string[]
   /** Enable speaker diarisation. Defaults vary by provider. */
   diarization?: boolean
 }
 
+export interface TranscriptionOptions {
+  /**
+   * Use only on-device STT for this listener; never deliver cloud transcripts.
+   * Other listeners on the same stream keep their own routing choice. Requires
+   * a downloaded local STT model.
+   */
+  forceLocal?: boolean
+}
+
 export class TranscriptionModule {
   private readonly unsubs = new Set<UnsubscribeFn>()
   private currentConfig: TranscriptionConfig | null = null
@@ -46,9 +62,9 @@ export class TranscriptionModule {
    * (handlers on `transcription:auto` receive any `transcription:<lang>`
    * event) preserves existing semantics.
    */
-  on(handler: (data: TranscriptionData) => void): UnsubscribeFn {
+  on(handler: (data: TranscriptionData) => void, options: TranscriptionOptions = {}): UnsubscribeFn {
     return this.track(
-      this.session._subscribe(`${MiniappStreamType.TRANSCRIPTION}:auto`, handler as (data: unknown) => void),
+      this.session._subscribe(`${MiniappStreamType.TRANSCRIPTION}:auto`, handler as (data: unknown) => void, options),
     )
   }
 
@@ -56,17 +72,34 @@ export class TranscriptionModule {
    * Subscribe to transcription for one or more specific languages. Each call
    * is independent; multiple can be active simultaneously.
    *
-   * @param language - BCP-47 tag(s), e.g. `"en-US"` or `["en-US", "es-ES"]`.
+   * Languages are validated against the platform registry
+   * (`SUPPORTED_TRANSCRIPTION_LANGUAGES`): registry BCP-47 tags pass through,
+   * bare registry codes canonicalize ("fr" -> "fr-FR"), anything else throws
+   * `MiniappValidationError` immediately — an invalid language must never
+   * become a subscription that silently transcribes the wrong thing
+   * (OS-1746 / issue 021).
+   *
+   * @param language - Registry tag(s), e.g. `"en-US"` or `["en-US", "es-ES"]`.
    * @param handler  - Called for every event in any of the listed languages.
    */
-  forLanguage(language: string | string[], handler: (data: TranscriptionData) => void): UnsubscribeFn {
-    const langs = Array.isArray(language) ? language : [language]
+  forLanguage(
+    language: TranscriptionLanguage | TranscriptionLanguage[],
+    handler: (data: TranscriptionData) => void,
+    options: TranscriptionOptions = {},
+  ): UnsubscribeFn {
+    const langs = (Array.isArray(language) ? language : [language]).map((lang) =>
+      requireTranscriptionLanguage(lang, "transcription.forLanguage(language)"),
+    )
     if (langs.length === 0) return () => {}
 
     const unsubs: UnsubscribeFn[] = []
     for (const lang of langs) {
       unsubs.push(
-        this.session._subscribe(`${MiniappStreamType.TRANSCRIPTION}:${lang}`, handler as (data: unknown) => void),
+        this.session._subscribe(
+          `${MiniappStreamType.TRANSCRIPTION}:${lang}`,
+          handler as (data: unknown) => void,
+          options,
+        ),
       )
     }
     const combined: UnsubscribeFn = () => {
@@ -83,15 +116,29 @@ export class TranscriptionModule {
 
   /**
    * Apply transcription configuration (language hints, custom vocabulary,
-   * diarisation toggle). Sent to the cloud immediately. Cached locally so we
-   * could re-send on reconnect (future).
+   * diarisation toggle). Hints are validated against the registry and
+   * normalized to bare codes ("en-US" -> "en"); unknown values throw
+   * `MiniappValidationError` synchronously.
+   *
+   * Sent to the phone runtime as a REQUEST (not a one-shot) so the returned
+   * promise rejects if the runtime cannot apply it — a config that the
+   * platform ignores must be loud, not silent (issue 021 S3). Cached locally
+   * so the runtime can re-apply on reconnect.
    */
-  configure(config: TranscriptionConfig): void {
-    this.currentConfig = {...config}
-    this.session.sendOneShot({
-      type: MiniappRequestType.TRANSCRIPTION_CONFIG,
-      config: {...config},
-    })
+  configure(config: TranscriptionConfig): Promise<void> {
+    const normalized: TranscriptionConfig = {
+      ...config,
+      languageHints: config.languageHints?.map((hint) =>
+        requireLanguageHint(hint, "transcription.configure(languageHints)"),
+      ),
+    }
+    this.currentConfig = {...normalized}
+    return this.session
+      .sendRequest<void>({
+        type: MiniappRequestType.TRANSCRIPTION_CONFIG,
+        config: {...normalized},
+      })
+      .then(() => undefined)
   }
 
   /** Tear down every transcription subscription this module owns. */

package/src/modules/translation.ts

@@ -20,9 +20,17 @@
 import {MiniappStreamType} from "../protocol"
 import {MiniappSession} from "../session"
 import type {TranslationData, UnsubscribeFn} from "./events"
+import {requireTranscriptionLanguage} from "./languages"
+import type {TranscriptionLanguage} from "./languages"
 
 export type TranslationHandler = (data: TranslationData) => void
 
+/**
+ * Translation source: a registry language, or "auto" to let the cloud detect
+ * the spoken language.
+ */
+export type TranslationSource = TranscriptionLanguage | "auto"
+
 export class TranslationModule {
   private readonly unsubs = new Set<UnsubscribeFn>()
 
@@ -51,8 +59,10 @@ export class TranslationModule {
    * regardless of source. Useful for "I'm a Spanish speaker, translate
    * whatever I hear to Spanish."
    */
-  to(target: string | string[], handler: TranslationHandler): UnsubscribeFn {
-    const targets = Array.isArray(target) ? target : [target]
+  to(target: TranscriptionLanguage | TranscriptionLanguage[], handler: TranslationHandler): UnsubscribeFn {
+    const targets = (Array.isArray(target) ? target : [target]).map((t) =>
+      requireTranscriptionLanguage(t, "translation.to(target)"),
+    )
     if (targets.length === 0) return () => {}
     const unsubs: UnsubscribeFn[] = []
     for (const t of targets) {
@@ -81,17 +91,21 @@ export class TranslationModule {
    * from the same source.
    */
   fromTo(
-    source: string,
-    target: string | string[],
+    source: TranslationSource,
+    target: TranscriptionLanguage | TranscriptionLanguage[],
     handler: TranslationHandler,
   ): UnsubscribeFn {
-    const targets = Array.isArray(target) ? target : [target]
+    const validSource =
+      source === "auto" ? "auto" : requireTranscriptionLanguage(source, "translation.fromTo(source)")
+    const targets = (Array.isArray(target) ? target : [target]).map((t) =>
+      requireTranscriptionLanguage(t, "translation.fromTo(target)"),
+    )
     if (targets.length === 0) return () => {}
     const unsubs: UnsubscribeFn[] = []
     for (const t of targets) {
       unsubs.push(
         this.session._subscribe(
-          `${MiniappStreamType.TRANSLATION}:${source}:${t}`,
+          `${MiniappStreamType.TRANSLATION}:${validSource}:${t}`,
           handler as (data: unknown) => void,
         ),
       )
@@ -113,8 +127,8 @@ export class TranslationModule {
    * SDK v3 parity. This alias will be removed in a future release.
    */
   forLanguagePair(
-    fromLang: string,
-    toLang: string,
+    fromLang: TranslationSource,
+    toLang: TranscriptionLanguage,
     handler: TranslationHandler,
   ): UnsubscribeFn {
     return this.fromTo(fromLang, toLang, handler)

package/src/protocol.ts

@@ -60,6 +60,9 @@ export enum MiniappRequestType {
   /** One-shot location poll. */
   LOCATION_POLL = "miniapp_location_poll",
 
+  /** Read a bounded snapshot of phone calendar events. */
+  CALENDAR_LIST_EVENTS = "miniapp_calendar_list_events",
+
   /** Start a turn-by-turn navigation trip. Android only. */
   NAVIGATION_START = "miniapp_navigation_start",
   /** Stop the active navigation trip (if any). */
@@ -114,6 +117,17 @@ export enum MiniappRequestType {
    */
   IMU_SET_ENABLED = "miniapp_imu_set_enabled",
 
+  /** Explicitly enable/disable glasses-side VAD (GX8002). Mirrors the app's mic settings toggle. */
+  MIC_SET_VAD_ENABLED = "miniapp_mic_set_vad_enabled",
+  /** Explicitly enable/disable the center-mic loudness gate ("Barrier"). */
+  MIC_SET_LOUDNESS_GATE_ENABLED = "miniapp_mic_set_loudness_gate_enabled",
+
+  /**
+   * Enable or disable Wi-Fi ADB (wireless debugging) on Mentra Live.
+   * SYSTEM-only — host rejects with NOT_PERMITTED for non-system callers.
+   */
+  SET_WIFI_ADB_STATE = "miniapp_set_wifi_adb_state",
+
   /** Share content via the OS share sheet. */
   SHARE = "miniapp_share",
   /** Open a URL in the system browser. */
@@ -122,6 +136,11 @@ export enum MiniappRequestType {
   COPY_CLIPBOARD = "miniapp_copy_clipboard",
   /** Download a file (triggers OS share sheet for save location). */
   DOWNLOAD = "miniapp_download",
+  /**
+   * Open a phone-camera QR scanner overlay. Host must not unmount the miniapp
+   * (clearing foreground fires UI_CLOSE and can hang up live sessions).
+   */
+  SCAN_QR = "miniapp_scan_qr",
 
   /**
    * Phone-local persistent binary blob storage, scoped to (userId, packageName).
@@ -300,8 +319,6 @@ export enum MiniappStreamType {
    * fire. See agents/miniapp-speaker-state-and-notif-dismissed-plan.md.
    */
   PHONE_NOTIFICATION_DISMISSED = "phone_notification_dismissed",
-  CALENDAR_EVENT = "calendar_event",
-
   // Photos, streaming
   PHOTO_TAKEN = "photo_taken",
   STREAM_STATUS = "stream_status",
@@ -318,6 +335,9 @@ export enum MiniappErrorCode {
   /** The miniapp subscribed to a stream whose required permission wasn't in its manifest. */
   PERMISSION_NOT_DECLARED = "PERMISSION_NOT_DECLARED",
 
+  /** The required phone OS permission is not currently granted. */
+  PERMISSION_DENIED = "PERMISSION_DENIED",
+
   /** Request routed to a method that isn't supported yet. */
   NOT_IMPLEMENTED = "NOT_IMPLEMENTED",
 
@@ -332,11 +352,17 @@ export enum MiniappErrorCode {
   TTS_INVALID_VOICE = "TTS_INVALID_VOICE",
   TTS_UPSTREAM_ERROR = "TTS_UPSTREAM_ERROR",
 
+  /** `speak({forceLocal: true})` but the on-device offline TTS model isn't downloaded/ready. */
+  TTS_LOCAL_UNAVAILABLE = "TTS_LOCAL_UNAVAILABLE",
+
   /** Not connected / pre-ACK and transport closed. */
   NOT_CONNECTED = "NOT_CONNECTED",
 
   // ----- Inter-miniapp interop -----
-  /** Caller is not a system app — interop APIs (list/start/stop/invoke) are SYSTEM-only. */
+  /**
+   * Caller is not a system app — SYSTEM-only APIs (interop list/start/stop/invoke,
+   * setWifiAdbState) reject with this code.
+   */
   NOT_PERMITTED = "NOT_PERMITTED",
   /** Target miniapp is not installed. */
   APP_NOT_FOUND = "APP_NOT_FOUND",

package/src/react/capsuleHeaderLayout.ts

@@ -0,0 +1,11 @@
+import type {MiniappCapsuleMenuRect} from "../globals"
+
+export function getCapsuleHeaderPaddingRight(
+  capsuleMenu: MiniappCapsuleMenuRect,
+  viewportWidth: number,
+  safeAreaRight: number,
+  rightGap: number,
+): number {
+  const capsuleOffsetFromContentRight = viewportWidth - capsuleMenu.left - safeAreaRight
+  return Math.max(rightGap, capsuleOffsetFromContentRight + rightGap)
+}

package/src/react/useCapsuleHeaderStyle.ts

@@ -15,6 +15,7 @@
 
 import type {CSSProperties} from "react"
 
+import {getCapsuleHeaderPaddingRight} from "./capsuleHeaderLayout"
 import {useSafeArea} from "./useSafeArea"
 
 export interface UseCapsuleHeaderStyleOptions {
@@ -39,7 +40,11 @@ export function useCapsuleHeaderStyle(
     ? capsuleMenu.top + capsuleMenu.height / 2 - insets.top
     : fallbackMarginTop + height / 2
   const marginTop = Math.max(0, center - height / 2)
-  const paddingRight = capsuleMenu ? capsuleMenu.width + rightGap : leftPadding
+  const paddingRight = capsuleMenu
+    ? typeof window === "undefined"
+      ? capsuleMenu.width + rightGap
+      : getCapsuleHeaderPaddingRight(capsuleMenu, window.innerWidth, insets.right, rightGap)
+    : leftPadding
 
   return {
     display: "flex",

package/src/session.ts

@@ -24,7 +24,7 @@ import {AuthModule} from "./modules/auth"
 import {CloudModule} from "./modules/cloud"
 import {DashboardAPI} from "./modules/dashboard"
 import {DisplayManager} from "./modules/display"
-import {EventManager, type UnsubscribeFn} from "./modules/events"
+import {EventManager, type TranscriptionEventRoute, type UnsubscribeFn} from "./modules/events"
 import {GlassesModule} from "./modules/glasses"
 import {HeadingModule} from "./modules/heading"
 import {ImuModule} from "./modules/imu"
@@ -194,7 +194,11 @@ type SessionEmitterEvents = {
   auth: (auth: MiniappAuthState) => void
 }
 
-export class MiniappSession {
+// The default preserves the pre-channel-registry behavior for code that creates
+// or accepts a bare MiniappSession. `registerMiniapp<Channels>` supplies the
+// concrete mapping for scaffolded miniapps.
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export class MiniappSession<TChannels extends object = any> {
   public readonly auth: AuthModule
   public readonly display: DisplayManager
   /**
@@ -238,7 +242,7 @@ export class MiniappSession {
    * with inverted buffering policy (background drops when no WebView is
    * bound; the WebView buffers until ready).
    */
-  public readonly ui: UIModule
+  public readonly ui: UIModule<TChannels>
   /**
    * Inter-miniapp lifecycle + discovery (list / start / stop). SYSTEM-only —
    * calls reject with NOT_PERMITTED unless this miniapp is a system app.
@@ -324,7 +328,7 @@ export class MiniappSession {
     this.system = new SystemModule(this)
     this.transcription = new TranscriptionModule(this)
     this.translation = new TranslationModule(this)
-    this.ui = new UIModuleImpl(this)
+    this.ui = new UIModuleImpl<TChannels>(this)
     this.miniapps = new MiniappsModule(this)
     this.actions = new ActionsModule(this)
   }
@@ -392,8 +396,12 @@ export class MiniappSession {
    * session.transcription.on(...)
    * etc. instead."
    */
-  _subscribe(streamType: string, handler: (data: unknown) => void): UnsubscribeFn {
-    return this.events.subscribe(streamType, handler)
+  _subscribe(
+    streamType: string,
+    handler: (data: unknown) => void,
+    options: {forceLocal?: boolean} = {},
+  ): UnsubscribeFn {
+    return this.events.subscribe(streamType, handler, options)
   }
 
   // -------------------------------------------------------------------------
@@ -677,7 +685,11 @@ export class MiniappSession {
       case MiniappResponseType.EVENT: {
         const streamType = payload.streamType as string | undefined
         if (!streamType) return
-        this.events._forwardEvent(streamType, payload.data)
+        this.events._forwardEvent(
+          streamType,
+          payload.data,
+          payload.transcriptionRoute as TranscriptionEventRoute | undefined,
+        )
         return
       }
 

package/src/sub-path-types.test-d.ts

@@ -23,12 +23,23 @@
  */
 
 // ----- @mentra/miniapp/background ---------------------------------------
-import type {MiniappSession} from "./background/index"
+import {registerMiniapp} from "./background/index"
+import type {MiniappSession, UIModule} from "./background/index"
 
 // ✅ MiniappSession is reachable from the background entry.
 const _bg: MiniappSession | undefined = undefined
 void _bg
 
+// ✅ registerMiniapp<Channels> carries the shared channel registry into
+//    session.ui on the background side.
+interface _TestChannels {
+  ping: {at: number}
+}
+type _RegisteredHandler = Parameters<typeof registerMiniapp<_TestChannels>>[0]
+type _RegisteredSession = Parameters<_RegisteredHandler>[0]
+type _Assert<T extends true> = T
+export type _TypedBackground = _Assert<_RegisteredSession["ui"] extends UIModule<_TestChannels> ? true : false>
+
 // ----- @mentra/miniapp/ui -----------------------------------------------
 import type {MentraUiGlobal, MentraTyped} from "./ui/index"
 

package/src/transport/mock.ts

@@ -226,6 +226,9 @@ function syntheticDataFor(requestId: string, requestType: string, requestPayload
       // Atlantic, which makes maps/navigation unusable in the WebView.
       return {lat: 37.7956, lng: -122.3933, accuracy: 0, timestamp: Date.now()}
 
+    case MiniappRequestType.CALENDAR_LIST_EVENTS:
+      return {events: [], truncated: false}
+
     case MiniappRequestType.STORAGE_GET:
       return {value: null}
 
@@ -240,8 +243,12 @@ function syntheticDataFor(requestId: string, requestType: string, requestPayload
     case MiniappRequestType.COPY_CLIPBOARD:
     case MiniappRequestType.DOWNLOAD:
     case MiniappRequestType.REQUEST_WIFI_SETUP:
+    case MiniappRequestType.SET_WIFI_ADB_STATE:
       return {ok: true}
 
+    case MiniappRequestType.SCAN_QR:
+      return {cancelled: true}
+
     default:
       return null
   }