@mentra/miniapp 3.2.0-dev.115 → 3.2.0-dev.120

package/src/modules/meeting.ts

@@ -0,0 +1,237 @@
+/**
+ * @fileoverview MeetingModule — phone-native meeting (ACS Teams).
+ *
+ * V1 token pass-through is deliberate technical debt (identity ticket):
+ * later, join(meetingUrl, whepUrl) and the host fetches the credential from
+ * Porter. Miniapps must not persist the token.
+ */
+
+import {MiniappErrorCode, MiniappRequestType} from "../protocol"
+import type {MiniappRequestError} from "../session"
+import {MiniappSession} from "../session"
+import type {UnsubscribeFn} from "./events"
+
+export const MEETING_HOST_UPDATE_MESSAGE = "Update the Mentra App to use Teams calling"
+
+export type MeetingProvider = "acs-teams"
+
+export type MeetingPhase = "idle" | "connecting" | "lobby" | "connected" | "disconnected" | "error"
+
+export interface MeetingVideoSource {
+  type: "whep"
+  url: string
+}
+
+/** Advertised ACS outgoing format. Omitted hosts keep 1280×720@15. */
+export interface MeetingOutgoingVideo {
+  width: number
+  height: number
+  fps: number
+  maxBitrateBps: number
+}
+
+export interface MeetingJoinOptions {
+  provider: MeetingProvider
+  meetingUrl: string
+  videoSource: MeetingVideoSource
+  /** V1-only: Porter-minted ACS guest token. Do not persist. */
+  token: string
+  displayName?: string
+  video?: MeetingOutgoingVideo
+}
+
+export type MeetingParticipantState = "idle" | "connecting" | "connected" | "lobby" | "hold" | "disconnected"
+
+/** A remote participant as reported by the phone-native meeting client. */
+export interface MeetingParticipant {
+  /** Stable provider identifier (ACS raw id). */
+  id: string
+  displayName: string | null
+  state: MeetingParticipantState
+  isMuted: boolean
+  isSpeaking: boolean
+}
+
+export interface MeetingState {
+  state: MeetingPhase
+  muted: boolean
+  error?: string
+  meetingUrl?: string
+  provider?: MeetingProvider
+  audioSource?: "glasses" | "phone"
+  audioSourceReason?:
+    | "explicit"
+    | "current-mic"
+    | "ranking"
+    | "fallback-glasses-connected"
+    | "fallback-no-glasses"
+  activeStream?: "none" | "virtual" | "local"
+  audioSafety?: "safe" | "degraded" | "unsafe"
+  /**
+   * Health of the glasses video the phone is forwarding into the meeting.
+   * `live` means a frame reached the meeting client, so it is the only honest
+   * "remote participants can see the camera" signal — the WHEP subscription
+   * answers seconds earlier. Omitted by hosts that predate the field, which
+   * must be read as "unknown", never as "not live".
+   */
+  mediaSource?: MeetingMediaSource
+  /** Remote roster. Omitted by hosts that predate participant reporting. */
+  participants?: MeetingParticipant[]
+}
+
+export type MeetingMediaSource = "idle" | "connecting" | "live" | "failed"
+
+const PARTICIPANT_STATES: ReadonlySet<string> = new Set(["idle", "connecting", "connected", "lobby", "hold", "disconnected"])
+
+const MEDIA_SOURCES: ReadonlySet<string> = new Set(["idle", "connecting", "live", "failed"])
+
+/** Tolerant parse of a host `mediaSource`. Unknown values read as unknown. */
+export function parseMeetingMediaSource(raw: unknown): MeetingMediaSource | undefined {
+  return MEDIA_SOURCES.has(String(raw)) ? (raw as MeetingMediaSource) : undefined
+}
+
+/** Tolerant parse of a host `participants` payload. Unknown shapes are skipped. */
+export function parseMeetingParticipants(raw: unknown): MeetingParticipant[] | undefined {
+  if (!Array.isArray(raw)) return undefined
+  const result: MeetingParticipant[] = []
+  for (const entry of raw) {
+    if (!entry || typeof entry !== "object") continue
+    const value = entry as Record<string, unknown>
+    if (typeof value.id !== "string" || !value.id) continue
+    result.push({
+      id: value.id,
+      displayName: typeof value.displayName === "string" && value.displayName ? value.displayName : null,
+      state: PARTICIPANT_STATES.has(String(value.state)) ? (value.state as MeetingParticipantState) : "idle",
+      isMuted: Boolean(value.isMuted),
+      isSpeaking: Boolean(value.isSpeaking),
+    })
+  }
+  return result
+}
+
+export type MeetingStateHandler = (state: MeetingState) => void
+
+function isMiniappRequestError(error: unknown): error is MiniappRequestError {
+  return Boolean(error && typeof error === "object" && "code" in error)
+}
+
+function mapHostError(error: unknown): never {
+  if (isMiniappRequestError(error) && error.code === MiniappErrorCode.NOT_IMPLEMENTED) {
+    throw {code: MiniappErrorCode.NOT_IMPLEMENTED, message: MEETING_HOST_UPDATE_MESSAGE}
+  }
+  throw error
+}
+
+export class MeetingModule {
+  private _state: MeetingState = {state: "idle", muted: false}
+
+  constructor(private readonly session: MiniappSession) {}
+
+  get state(): MeetingState {
+    return {...this._state}
+  }
+
+  /**
+   * Join a Teams meeting via the phone ACS client.
+   * Resolves once the host has accepted the join (state may still be connecting/lobby).
+   */
+  async join(options: MeetingJoinOptions): Promise<MeetingState> {
+    if (options.provider !== "acs-teams") {
+      throw {code: MiniappErrorCode.INVALID_ARGUMENT, message: `Unsupported meeting provider: ${options.provider}`}
+    }
+    if (!options.meetingUrl?.trim()) {
+      throw {code: MiniappErrorCode.INVALID_ARGUMENT, message: "meetingUrl is required"}
+    }
+    if (options.videoSource?.type !== "whep" || !options.videoSource.url?.trim()) {
+      throw {code: MiniappErrorCode.INVALID_ARGUMENT, message: "videoSource must be a WHEP URL"}
+    }
+    if (!options.token?.trim()) {
+      throw {code: MiniappErrorCode.INVALID_ARGUMENT, message: "token is required"}
+    }
+    try {
+      const result = await this.session.sendRequest<MeetingState | null>(
+        {
+          type: MiniappRequestType.MEETING_JOIN,
+          provider: options.provider,
+          meetingUrl: options.meetingUrl,
+          videoSource: options.videoSource,
+          token: options.token,
+          displayName: options.displayName,
+          ...(options.video ? {video: options.video} : {}),
+        },
+        {timeoutMs: 0},
+      )
+      if (result) this._applyState(result)
+      return this.state
+    } catch (error) {
+      mapHostError(error)
+    }
+  }
+
+  async leave(): Promise<void> {
+    try {
+      await this.session.sendRequest<void>({type: MiniappRequestType.MEETING_LEAVE})
+    } catch (error) {
+      mapHostError(error)
+    }
+  }
+
+  async setMuted(muted: boolean): Promise<void> {
+    try {
+      const result = await this.session.sendRequest<MeetingState | null>({
+        type: MiniappRequestType.MEETING_SET_MUTED,
+        muted,
+      })
+      if (result) this._applyState(result)
+    } catch (error) {
+      mapHostError(error)
+    }
+  }
+
+  async updateVideoSource(source: MeetingVideoSource): Promise<void> {
+    if (source?.type !== "whep" || !source.url?.trim()) {
+      throw {code: MiniappErrorCode.INVALID_ARGUMENT, message: "videoSource must be a WHEP URL"}
+    }
+    try {
+      await this.session.sendRequest<void>({
+        type: MiniappRequestType.MEETING_UPDATE_VIDEO_SOURCE,
+        videoSource: source,
+      })
+    } catch (error) {
+      mapHostError(error)
+    }
+  }
+
+  async getState(): Promise<MeetingState> {
+    try {
+      const result = await this.session.sendRequest<MeetingState | null>({
+        type: MiniappRequestType.MEETING_GET_STATE,
+      })
+      if (result) this._applyState(result)
+      return this.state
+    } catch (error) {
+      mapHostError(error)
+    }
+  }
+
+  onState(handler: MeetingStateHandler): UnsubscribeFn {
+    return this.session.on("meetingState", handler)
+  }
+
+  /** @internal — applied by MiniappSession on inbound MEETING_STATE. */
+  _applyState(event: MeetingState): void {
+    this._state = {
+      state: event.state,
+      muted: Boolean(event.muted),
+      error: event.error,
+      meetingUrl: event.meetingUrl,
+      provider: event.provider,
+      audioSource: event.audioSource,
+      audioSourceReason: event.audioSourceReason,
+      activeStream: event.activeStream,
+      audioSafety: event.audioSafety,
+      mediaSource: parseMeetingMediaSource(event.mediaSource),
+      participants: parseMeetingParticipants(event.participants),
+    }
+  }
+}

package/src/modules/stream.ts

@@ -83,6 +83,11 @@ export interface StartStreamOptions {
   ingest?: "srt" | "whip" | "rtmp"
   /** Optional Bearer token for direct WHIP Authorization (custom authenticated endpoints). */
   authToken?: string
+  /**
+   * When false, glasses skip mic capture for this WHIP session. Fixed for the
+   * lifetime of the stream. Defaults to true. Older Mentra Apps ignore this.
+   */
+  captureAudio?: boolean
 }
 
 export interface StreamResult {
@@ -146,6 +151,7 @@ export class StreamModule {
         audio: options.audio,
         sound: options.sound ?? true,
         ...(options.authToken ? {authToken: options.authToken} : {}),
+        ...(typeof options.captureAudio === "boolean" ? {captureAudio: options.captureAudio} : {}),
       })
     }
     return this.session.sendRequest<StreamResult>({
@@ -155,6 +161,7 @@ export class StreamModule {
       audio: options.audio,
       sound: options.sound ?? true,
       ingest: options.ingest,
+      ...(typeof options.captureAudio === "boolean" ? {captureAudio: options.captureAudio} : {}),
     })
   }
 

package/src/protocol.ts

@@ -205,6 +205,16 @@ export enum MiniappRequestType {
   ACTION_INVOKE = "miniapp_action_invoke",
   /** Target → host: the result of a delivered ACTION_CALL, correlated by callId. */
   ACTION_RESULT = "miniapp_action_result",
+
+  /**
+   * Phone-native meeting (ACS Teams). Join/leave/mute live in the MentraOS
+   * host so the miniapp never holds the ACS Calling SDK.
+   */
+  MEETING_JOIN = "miniapp_meeting_join",
+  MEETING_LEAVE = "miniapp_meeting_leave",
+  MEETING_SET_MUTED = "miniapp_meeting_set_muted",
+  MEETING_UPDATE_VIDEO_SOURCE = "miniapp_meeting_update_video_source",
+  MEETING_GET_STATE = "miniapp_meeting_get_state",
 }
 
 // ============================================================================
@@ -261,6 +271,12 @@ export enum MiniappResponseType {
    */
   ACTION_CALL = "miniapp_action_call",
 
+  /**
+   * Push: native meeting state changed. Carries {state, muted?, error?}.
+   * See MeetingModule.onState().
+   */
+  MEETING_STATE = "miniapp_meeting_state",
+
   /**
    * Push: phone is about to tear down the miniapp's session. Gives the SDK
    * a brief window (~50ms grace on the phone side) to fire one last

package/src/session.ts

@@ -45,6 +45,7 @@ import {SystemModule} from "./modules/system"
 import {MiniappsModule} from "./modules/miniapps"
 import {ActionsModule} from "./modules/actions"
 import {BlobModule} from "./modules/blob"
+import {MeetingModule, parseMeetingMediaSource, parseMeetingParticipants} from "./modules/meeting"
 
 // ---------------------------------------------------------------------------
 // Public types
@@ -105,6 +106,13 @@ export interface ConnectAckPayload {
   permissions?: PermissionRecord
   /** Miniapp-scoped backend auth. Never a Core or runtime token. */
   auth?: MiniappAuthState
+  /** Host-advertised optional features. Absent on older Mentra Apps. */
+  hostFeatures?: HostFeatures
+}
+
+export interface HostFeatures {
+  /** Host honors `startStream({captureAudio})` for the lifetime of a WHIP session. */
+  captureAudio?: boolean
 }
 
 export interface MiniappAuthState {
@@ -191,6 +199,7 @@ type SessionEmitterEvents = {
   colorScheme: (scheme: MiniappColorScheme) => void
   permissions: (perms: PermissionRecord) => void
   speakerState: (event: import("./modules/speaker").SpeakerStateEvent) => void
+  meetingState: (event: import("./modules/meeting").MeetingState) => void
   auth: (auth: MiniappAuthState) => void
 }
 
@@ -211,6 +220,7 @@ export class MiniappSession<TChannels extends object = any> {
    */
   public readonly events: EventManager
   public readonly speaker: SpeakerModule
+  public readonly meeting: MeetingModule
   public readonly camera: CameraModule
   public readonly cloud: CloudModule
   public readonly dashboard: DashboardAPI
@@ -256,6 +266,11 @@ export class MiniappSession<TChannels extends object = any> {
 
   /** Phone-declared glasses capabilities. Null until CONNECT_ACK arrives. */
   public capabilities: GlassesCapabilities | null = null
+  /**
+   * Host-advertised optional features. Null until CONNECT_ACK. Older Mentra Apps
+   * omit this, so miniapps must not assume `captureAudio` is honored.
+   */
+  public hostFeatures: HostFeatures | null = null
   public userId = ""
   public packageName = ""
   public visibility: MiniappVisibility = "foreground"
@@ -308,6 +323,7 @@ export class MiniappSession<TChannels extends object = any> {
     this.auth = new AuthModule(this)
     this.events = new EventManager(this)
     this.speaker = new SpeakerModule(this)
+    this.meeting = new MeetingModule(this)
     this.camera = new CameraModule(this)
     this.cloud = new CloudModule(this)
     this.dashboard = new DashboardAPI(this)
@@ -619,6 +635,7 @@ export class MiniappSession<TChannels extends object = any> {
         this.userId = ack.userId ?? ""
         if (ack.packageName) this.packageName = ack.packageName
         this.capabilities = ack.capabilities ?? null
+        this.hostFeatures = ack.hostFeatures ?? null
         if (ack.visibility) this.visibility = ack.visibility
         if (ack.colorScheme === "light" || ack.colorScheme === "dark") {
           this.colorScheme = ack.colorScheme
@@ -667,6 +684,27 @@ export class MiniappSession<TChannels extends object = any> {
         return
       }
 
+      case MiniappResponseType.MEETING_STATE: {
+        const state = payload.state as import("./modules/meeting").MeetingPhase | undefined
+        if (!state) return
+        const event: import("./modules/meeting").MeetingState = {
+          state,
+          muted: Boolean(payload.muted),
+          error: payload.error as string | undefined,
+          meetingUrl: payload.meetingUrl as string | undefined,
+          provider: payload.provider as import("./modules/meeting").MeetingProvider | undefined,
+          audioSource: payload.audioSource as import("./modules/meeting").MeetingState["audioSource"],
+          audioSourceReason: payload.audioSourceReason as import("./modules/meeting").MeetingState["audioSourceReason"],
+          activeStream: payload.activeStream as import("./modules/meeting").MeetingState["activeStream"],
+          audioSafety: payload.audioSafety as import("./modules/meeting").MeetingState["audioSafety"],
+          mediaSource: parseMeetingMediaSource(payload.mediaSource),
+          participants: parseMeetingParticipants(payload.participants),
+        }
+        this.meeting._applyState(event)
+        this.emitter.emit("meetingState", event)
+        return
+      }
+
       case MiniappRequestType.PING: {
         // Phone → miniapp keepalive ping. Auto-reply with PONG.
         const pong: object = {type: MiniappResponseType.PONG}