@mentra/miniapp 3.2.1-dev.314 → 3.3.0-beta.942

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/README.md +16 -0
  2. package/dist/background/index.d.ts +6 -3
  3. package/dist/background/index.d.ts.map +1 -1
  4. package/dist/background/index.js +1 -0
  5. package/dist/background/index.js.map +1 -1
  6. package/dist/background/register.d.ts +6 -0
  7. package/dist/background/register.d.ts.map +1 -1
  8. package/dist/background/register.js +19 -4
  9. package/dist/background/register.js.map +1 -1
  10. package/dist/index.d.ts +7 -4
  11. package/dist/index.d.ts.map +1 -1
  12. package/dist/index.js +2 -0
  13. package/dist/index.js.map +1 -1
  14. package/dist/modules/display.d.ts +26 -0
  15. package/dist/modules/display.d.ts.map +1 -1
  16. package/dist/modules/display.js +1 -0
  17. package/dist/modules/display.js.map +1 -1
  18. package/dist/modules/events.d.ts +8 -4
  19. package/dist/modules/events.d.ts.map +1 -1
  20. package/dist/modules/events.js +46 -6
  21. package/dist/modules/events.js.map +1 -1
  22. package/dist/modules/meeting.d.ts +129 -0
  23. package/dist/modules/meeting.d.ts.map +1 -1
  24. package/dist/modules/meeting.js +200 -14
  25. package/dist/modules/meeting.js.map +1 -1
  26. package/dist/modules/mic.d.ts +5 -4
  27. package/dist/modules/mic.d.ts.map +1 -1
  28. package/dist/modules/mic.js +5 -4
  29. package/dist/modules/mic.js.map +1 -1
  30. package/dist/modules/miniapps.d.ts +101 -4
  31. package/dist/modules/miniapps.d.ts.map +1 -1
  32. package/dist/modules/miniapps.js +59 -4
  33. package/dist/modules/miniapps.js.map +1 -1
  34. package/dist/modules/speaker.d.ts +2 -0
  35. package/dist/modules/speaker.d.ts.map +1 -1
  36. package/dist/modules/speaker.js +5 -0
  37. package/dist/modules/speaker.js.map +1 -1
  38. package/dist/modules/stream.d.ts +46 -0
  39. package/dist/modules/stream.d.ts.map +1 -1
  40. package/dist/modules/stream.js +107 -0
  41. package/dist/modules/stream.js.map +1 -1
  42. package/dist/protocol.d.ts +86 -1
  43. package/dist/protocol.d.ts.map +1 -1
  44. package/dist/protocol.js +82 -0
  45. package/dist/protocol.js.map +1 -1
  46. package/dist/react/StreamPreview.d.ts +41 -0
  47. package/dist/react/StreamPreview.d.ts.map +1 -0
  48. package/dist/react/StreamPreview.js +144 -0
  49. package/dist/react/StreamPreview.js.map +1 -0
  50. package/dist/react/index.d.ts +3 -0
  51. package/dist/react/index.d.ts.map +1 -1
  52. package/dist/react/index.js +2 -0
  53. package/dist/react/index.js.map +1 -1
  54. package/dist/react/stream-preview/PreviewConnection.d.ts +146 -0
  55. package/dist/react/stream-preview/PreviewConnection.d.ts.map +1 -0
  56. package/dist/react/stream-preview/PreviewConnection.js +493 -0
  57. package/dist/react/stream-preview/PreviewConnection.js.map +1 -0
  58. package/dist/react/stream-preview/controlChannel.d.ts +75 -0
  59. package/dist/react/stream-preview/controlChannel.d.ts.map +1 -0
  60. package/dist/react/stream-preview/controlChannel.js +55 -0
  61. package/dist/react/stream-preview/controlChannel.js.map +1 -0
  62. package/dist/react/stream-preview/creditLoop.d.ts +81 -0
  63. package/dist/react/stream-preview/creditLoop.d.ts.map +1 -0
  64. package/dist/react/stream-preview/creditLoop.js +146 -0
  65. package/dist/react/stream-preview/creditLoop.js.map +1 -0
  66. package/dist/react/stream-preview/protocol.d.ts +113 -0
  67. package/dist/react/stream-preview/protocol.d.ts.map +1 -0
  68. package/dist/react/stream-preview/protocol.js +148 -0
  69. package/dist/react/stream-preview/protocol.js.map +1 -0
  70. package/dist/react/stream-preview/resizeCoalescer.d.ts +38 -0
  71. package/dist/react/stream-preview/resizeCoalescer.d.ts.map +1 -0
  72. package/dist/react/stream-preview/resizeCoalescer.js +71 -0
  73. package/dist/react/stream-preview/resizeCoalescer.js.map +1 -0
  74. package/dist/react/stream-preview/testFrames.d.ts +38 -0
  75. package/dist/react/stream-preview/testFrames.d.ts.map +1 -0
  76. package/dist/react/stream-preview/testFrames.js +50 -0
  77. package/dist/react/stream-preview/testFrames.js.map +1 -0
  78. package/dist/react/stream-preview/tiers.d.ts +21 -0
  79. package/dist/react/stream-preview/tiers.d.ts.map +1 -0
  80. package/dist/react/stream-preview/tiers.js +33 -0
  81. package/dist/react/stream-preview/tiers.js.map +1 -0
  82. package/dist/react/stream-preview/trace.d.ts +23 -0
  83. package/dist/react/stream-preview/trace.d.ts.map +1 -0
  84. package/dist/react/stream-preview/trace.js +71 -0
  85. package/dist/react/stream-preview/trace.js.map +1 -0
  86. package/dist/react/stream-preview/transport.d.ts +79 -0
  87. package/dist/react/stream-preview/transport.d.ts.map +1 -0
  88. package/dist/react/stream-preview/transport.js +209 -0
  89. package/dist/react/stream-preview/transport.js.map +1 -0
  90. package/dist/react/stream-preview/yuvRenderer.d.ts +31 -0
  91. package/dist/react/stream-preview/yuvRenderer.d.ts.map +1 -0
  92. package/dist/react/stream-preview/yuvRenderer.js +398 -0
  93. package/dist/react/stream-preview/yuvRenderer.js.map +1 -0
  94. package/dist/react/useHistoryState.d.ts +35 -0
  95. package/dist/react/useHistoryState.d.ts.map +1 -0
  96. package/dist/react/useHistoryState.js +117 -0
  97. package/dist/react/useHistoryState.js.map +1 -0
  98. package/dist/session.d.ts +25 -0
  99. package/dist/session.d.ts.map +1 -1
  100. package/dist/session.js +61 -1
  101. package/dist/session.js.map +1 -1
  102. package/dist/ui/index.d.ts +3 -0
  103. package/dist/ui/index.d.ts.map +1 -1
  104. package/dist/ui/index.js +2 -0
  105. package/dist/ui/index.js.map +1 -1
  106. package/package.json +8 -3
  107. package/src/background/index.ts +28 -1
  108. package/src/background/register.ts +28 -6
  109. package/src/index.ts +29 -1
  110. package/src/modules/display.ts +29 -0
  111. package/src/modules/events.ts +51 -6
  112. package/src/modules/meeting.ts +292 -15
  113. package/src/modules/mic.ts +5 -4
  114. package/src/modules/miniapps.ts +145 -4
  115. package/src/modules/speaker.ts +10 -4
  116. package/src/modules/stream.ts +153 -1
  117. package/src/protocol.ts +125 -0
  118. package/src/react/StreamPreview.tsx +187 -0
  119. package/src/react/index.ts +3 -0
  120. package/src/react/stream-preview/PreviewConnection.ts +587 -0
  121. package/src/react/stream-preview/controlChannel.ts +119 -0
  122. package/src/react/stream-preview/creditLoop.ts +189 -0
  123. package/src/react/stream-preview/protocol.ts +227 -0
  124. package/src/react/stream-preview/resizeCoalescer.ts +88 -0
  125. package/src/react/stream-preview/testFrames.ts +86 -0
  126. package/src/react/stream-preview/tiers.ts +37 -0
  127. package/src/react/stream-preview/trace.ts +81 -0
  128. package/src/react/stream-preview/transport.ts +242 -0
  129. package/src/react/stream-preview/yuvRenderer.ts +453 -0
  130. package/src/react/useHistoryState.ts +135 -0
  131. package/src/session.ts +76 -0
  132. package/src/ui/index.ts +3 -0
package/src/session.ts CHANGED
@@ -66,6 +66,8 @@ import {
66
66
  * not zero. Populated on the "ready" event (null in `start()`).
67
67
  */
68
68
  export interface DisplayCapabilities {
69
+ /** Legacy panel dimensions. Prefer drawable width/height when present. */
70
+ resolution?: {width: number; height: number}
69
71
  /** Public drawable canvas in px — raw coordinate space for `display.render()` boxes. */
70
72
  width?: number
71
73
  height?: number
@@ -122,8 +124,12 @@ export interface ConnectAckPayload {
122
124
  }
123
125
 
124
126
  export interface HostFeatures {
127
+ /** @internal Runtime projects transcription independently for each listener. */
128
+ transcriptionListeners?: boolean
125
129
  /** Host honors `startStream({captureAudio})` for the lifetime of a WHIP session. */
126
130
  captureAudio?: boolean
131
+ /** Host keeps the UI closed until a background session that announced it sends READY. */
132
+ initReady?: boolean
127
133
  }
128
134
 
129
135
  export interface MiniappAuthState {
@@ -211,6 +217,8 @@ type SessionEmitterEvents = {
211
217
  permissions: (perms: PermissionRecord) => void
212
218
  speakerState: (event: import("./modules/speaker").SpeakerStateEvent) => void
213
219
  meetingState: (event: import("./modules/meeting").MeetingState) => void
220
+ meetingVideoPublisher: (event: import("./modules/meeting").MeetingVideoPublisherEvent) => void
221
+ meetingStill: (event: import("./modules/meeting").MeetingStillProgressEvent) => void
214
222
  auth: (auth: MiniappAuthState) => void
215
223
  }
216
224
 
@@ -313,6 +321,12 @@ export class MiniappSession<TChannels extends object = any> {
313
321
  private readonly pendingRequests = new Map<string, PendingRequest>()
314
322
  private connectPromise: Promise<void> | null = null
315
323
  private disposed = false
324
+ /**
325
+ * Set by `registerMiniapp` to the host's id for this spawn: CONNECT tells the
326
+ * host to wait for READY, and both carry the id so the host can ignore
327
+ * frames from a previous context.
328
+ */
329
+ private initSessionId: string | null = null
316
330
 
317
331
  /** Manifest-declared permission cache. Updated on CONNECT_ACK / PERMISSIONS_UPDATE. */
318
332
  private _permissions: PermissionRecord = {
@@ -482,6 +496,7 @@ export class MiniappSession<TChannels extends object = any> {
482
496
  const connectPayload = {
483
497
  type: MiniappRequestType.CONNECT,
484
498
  packageName: this.packageName,
499
+ ...(this.initSessionId !== null ? {initReady: true, sessionId: this.initSessionId} : {}),
485
500
  }
486
501
  this.transport.send(serializeEnvelope({payload: connectPayload, requestId}))
487
502
 
@@ -491,6 +506,21 @@ export class MiniappSession<TChannels extends object = any> {
491
506
  return this.connectPromise
492
507
  }
493
508
 
509
+ /**
510
+ * @internal — called by `registerMiniapp` before `connect()`. The host then
511
+ * keeps the UI closed until {@link reportInitReady}, so UI requests cannot
512
+ * reach the background before its `session.ui.handle` handlers exist.
513
+ */
514
+ announceInitReady(sessionId: string): void {
515
+ this.initSessionId = sessionId
516
+ }
517
+
518
+ /** @internal — the `registerMiniapp` handler settled; let the host open the UI. */
519
+ reportInitReady(): void {
520
+ if (this.initSessionId === null || this.disposed || !this.hostFeatures?.initReady) return
521
+ this.sendOneShot({type: MiniappRequestType.READY, sessionId: this.initSessionId})
522
+ }
523
+
494
524
  /** Resolves when `ready` becomes true, or rejects if connect failed. */
495
525
  waitForReady(): Promise<void> {
496
526
  if (this.ready) return Promise.resolve()
@@ -707,9 +737,17 @@ export class MiniappSession<TChannels extends object = any> {
707
737
  case MiniappResponseType.MEETING_STATE: {
708
738
  const state = payload.state as import("./modules/meeting").MeetingPhase | undefined
709
739
  if (!state) return
740
+ const identityMode =
741
+ payload.identityMode === "guest" || payload.identityMode === "teams-user" ? payload.identityMode : undefined
710
742
  const event: import("./modules/meeting").MeetingState = {
711
743
  state,
744
+ identityMode,
745
+ guestReason:
746
+ identityMode === "guest"
747
+ ? (payload.guestReason as import("./modules/meeting").MeetingState["guestReason"])
748
+ : undefined,
712
749
  muted: Boolean(payload.muted),
750
+ videoEnabled: typeof payload.videoEnabled === "boolean" ? payload.videoEnabled : undefined,
713
751
  error: payload.error as string | undefined,
714
752
  meetingUrl: payload.meetingUrl as string | undefined,
715
753
  provider: payload.provider as import("./modules/meeting").MeetingProvider | undefined,
@@ -729,6 +767,27 @@ export class MiniappSession<TChannels extends object = any> {
729
767
  return
730
768
  }
731
769
 
770
+ case MiniappResponseType.MEETING_VIDEO_PUBLISHER: {
771
+ const pauseId = typeof payload.pauseId === "string" ? payload.pauseId : ""
772
+ if (!pauseId || payload.status !== "expired") return
773
+ const event = {pauseId, status: "expired" as const}
774
+ this.emitter.emit("meetingVideoPublisher", event)
775
+ return
776
+ }
777
+
778
+ case MiniappResponseType.MEETING_STILL: {
779
+ const stillId = typeof payload.stillId === "string" ? payload.stillId : ""
780
+ const phase = payload.phase
781
+ if (!stillId || (phase !== "card" && phase !== "uploading" && phase !== "shown")) return
782
+ this.emitter.emit("meetingStill", {stillId, phase})
783
+ return
784
+ }
785
+
786
+ case MiniappResponseType.STREAM_PREVIEW_STATUS: {
787
+ this.stream._applyPreviewStatus(payload)
788
+ return
789
+ }
790
+
732
791
  case MiniappRequestType.PING: {
733
792
  // Phone → miniapp keepalive ping. Auto-reply with PONG.
734
793
  const pong: object = {type: MiniappResponseType.PONG}
@@ -751,6 +810,7 @@ export class MiniappSession<TChannels extends object = any> {
751
810
  streamType,
752
811
  payload.data,
753
812
  payload.transcriptionRoute as TranscriptionEventRoute | undefined,
813
+ payload.listenerId as string | undefined,
754
814
  )
755
815
  return
756
816
  }
@@ -793,6 +853,22 @@ export class MiniappSession<TChannels extends object = any> {
793
853
  return
794
854
  }
795
855
 
856
+ case MiniappResponseType.MINIAPPS_INSTALL_PROGRESS: {
857
+ const {packageName, version, phase} = payload as Record<string, unknown>
858
+ if (
859
+ typeof packageName === "string" &&
860
+ typeof version === "string" &&
861
+ (phase === "downloading" ||
862
+ phase === "verifying" ||
863
+ phase === "extracting" ||
864
+ phase === "activating" ||
865
+ phase === "complete")
866
+ ) {
867
+ this.miniapps._deliverInstallProgress({packageName, version, phase})
868
+ }
869
+ return
870
+ }
871
+
796
872
  case MiniappResponseType.REQUEST_RESULT: {
797
873
  const requestId = envelope.requestId
798
874
  if (!requestId) return
package/src/ui/index.ts CHANGED
@@ -93,6 +93,9 @@ export {useConnected} from "../react/useConnected"
93
93
  export {useSafeArea} from "../react/useSafeArea"
94
94
  export {useCapsuleHeaderStyle} from "../react/useCapsuleHeaderStyle"
95
95
  export {useRpc, type RpcCallable} from "../react/useRpc"
96
+ export {useHistoryState, type SetHistoryState} from "../react/useHistoryState"
97
+ export {StreamPreview} from "../react/StreamPreview"
98
+ export type {StreamPreviewProps, StreamPreviewStatus, PreviewFit, PreviewConnectionState} from "../react/StreamPreview"
96
99
 
97
100
  /**
98
101
  * Type helper for declaring a typed `mentra` global inside a miniapp.