@mentra/miniapp 3.2.1-dev.314 → 3.3.0-beta.1000

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/README.md +16 -0
  2. package/dist/background/index.d.ts +6 -3
  3. package/dist/background/index.d.ts.map +1 -1
  4. package/dist/background/index.js +1 -0
  5. package/dist/background/index.js.map +1 -1
  6. package/dist/background/register.d.ts +6 -0
  7. package/dist/background/register.d.ts.map +1 -1
  8. package/dist/background/register.js +19 -4
  9. package/dist/background/register.js.map +1 -1
  10. package/dist/index.d.ts +7 -4
  11. package/dist/index.d.ts.map +1 -1
  12. package/dist/index.js +2 -0
  13. package/dist/index.js.map +1 -1
  14. package/dist/modules/display.d.ts +26 -0
  15. package/dist/modules/display.d.ts.map +1 -1
  16. package/dist/modules/display.js +1 -0
  17. package/dist/modules/display.js.map +1 -1
  18. package/dist/modules/events.d.ts +8 -4
  19. package/dist/modules/events.d.ts.map +1 -1
  20. package/dist/modules/events.js +46 -6
  21. package/dist/modules/events.js.map +1 -1
  22. package/dist/modules/meeting.d.ts +129 -0
  23. package/dist/modules/meeting.d.ts.map +1 -1
  24. package/dist/modules/meeting.js +200 -14
  25. package/dist/modules/meeting.js.map +1 -1
  26. package/dist/modules/mic.d.ts +5 -4
  27. package/dist/modules/mic.d.ts.map +1 -1
  28. package/dist/modules/mic.js +5 -4
  29. package/dist/modules/mic.js.map +1 -1
  30. package/dist/modules/miniapps.d.ts +101 -4
  31. package/dist/modules/miniapps.d.ts.map +1 -1
  32. package/dist/modules/miniapps.js +59 -4
  33. package/dist/modules/miniapps.js.map +1 -1
  34. package/dist/modules/speaker.d.ts +2 -0
  35. package/dist/modules/speaker.d.ts.map +1 -1
  36. package/dist/modules/speaker.js +5 -0
  37. package/dist/modules/speaker.js.map +1 -1
  38. package/dist/modules/stream.d.ts +46 -0
  39. package/dist/modules/stream.d.ts.map +1 -1
  40. package/dist/modules/stream.js +107 -0
  41. package/dist/modules/stream.js.map +1 -1
  42. package/dist/protocol.d.ts +86 -1
  43. package/dist/protocol.d.ts.map +1 -1
  44. package/dist/protocol.js +82 -0
  45. package/dist/protocol.js.map +1 -1
  46. package/dist/react/StreamPreview.d.ts +41 -0
  47. package/dist/react/StreamPreview.d.ts.map +1 -0
  48. package/dist/react/StreamPreview.js +144 -0
  49. package/dist/react/StreamPreview.js.map +1 -0
  50. package/dist/react/index.d.ts +3 -0
  51. package/dist/react/index.d.ts.map +1 -1
  52. package/dist/react/index.js +2 -0
  53. package/dist/react/index.js.map +1 -1
  54. package/dist/react/stream-preview/PreviewConnection.d.ts +146 -0
  55. package/dist/react/stream-preview/PreviewConnection.d.ts.map +1 -0
  56. package/dist/react/stream-preview/PreviewConnection.js +493 -0
  57. package/dist/react/stream-preview/PreviewConnection.js.map +1 -0
  58. package/dist/react/stream-preview/controlChannel.d.ts +75 -0
  59. package/dist/react/stream-preview/controlChannel.d.ts.map +1 -0
  60. package/dist/react/stream-preview/controlChannel.js +55 -0
  61. package/dist/react/stream-preview/controlChannel.js.map +1 -0
  62. package/dist/react/stream-preview/creditLoop.d.ts +81 -0
  63. package/dist/react/stream-preview/creditLoop.d.ts.map +1 -0
  64. package/dist/react/stream-preview/creditLoop.js +146 -0
  65. package/dist/react/stream-preview/creditLoop.js.map +1 -0
  66. package/dist/react/stream-preview/protocol.d.ts +113 -0
  67. package/dist/react/stream-preview/protocol.d.ts.map +1 -0
  68. package/dist/react/stream-preview/protocol.js +148 -0
  69. package/dist/react/stream-preview/protocol.js.map +1 -0
  70. package/dist/react/stream-preview/resizeCoalescer.d.ts +38 -0
  71. package/dist/react/stream-preview/resizeCoalescer.d.ts.map +1 -0
  72. package/dist/react/stream-preview/resizeCoalescer.js +71 -0
  73. package/dist/react/stream-preview/resizeCoalescer.js.map +1 -0
  74. package/dist/react/stream-preview/testFrames.d.ts +38 -0
  75. package/dist/react/stream-preview/testFrames.d.ts.map +1 -0
  76. package/dist/react/stream-preview/testFrames.js +50 -0
  77. package/dist/react/stream-preview/testFrames.js.map +1 -0
  78. package/dist/react/stream-preview/tiers.d.ts +21 -0
  79. package/dist/react/stream-preview/tiers.d.ts.map +1 -0
  80. package/dist/react/stream-preview/tiers.js +33 -0
  81. package/dist/react/stream-preview/tiers.js.map +1 -0
  82. package/dist/react/stream-preview/trace.d.ts +23 -0
  83. package/dist/react/stream-preview/trace.d.ts.map +1 -0
  84. package/dist/react/stream-preview/trace.js +71 -0
  85. package/dist/react/stream-preview/trace.js.map +1 -0
  86. package/dist/react/stream-preview/transport.d.ts +79 -0
  87. package/dist/react/stream-preview/transport.d.ts.map +1 -0
  88. package/dist/react/stream-preview/transport.js +209 -0
  89. package/dist/react/stream-preview/transport.js.map +1 -0
  90. package/dist/react/stream-preview/yuvRenderer.d.ts +31 -0
  91. package/dist/react/stream-preview/yuvRenderer.d.ts.map +1 -0
  92. package/dist/react/stream-preview/yuvRenderer.js +398 -0
  93. package/dist/react/stream-preview/yuvRenderer.js.map +1 -0
  94. package/dist/react/useHistoryState.d.ts +35 -0
  95. package/dist/react/useHistoryState.d.ts.map +1 -0
  96. package/dist/react/useHistoryState.js +117 -0
  97. package/dist/react/useHistoryState.js.map +1 -0
  98. package/dist/session.d.ts +25 -0
  99. package/dist/session.d.ts.map +1 -1
  100. package/dist/session.js +61 -1
  101. package/dist/session.js.map +1 -1
  102. package/dist/ui/index.d.ts +3 -0
  103. package/dist/ui/index.d.ts.map +1 -1
  104. package/dist/ui/index.js +2 -0
  105. package/dist/ui/index.js.map +1 -1
  106. package/package.json +8 -3
  107. package/src/background/index.ts +28 -1
  108. package/src/background/register.ts +28 -6
  109. package/src/index.ts +29 -1
  110. package/src/modules/display.ts +29 -0
  111. package/src/modules/events.ts +51 -6
  112. package/src/modules/meeting.ts +292 -15
  113. package/src/modules/mic.ts +5 -4
  114. package/src/modules/miniapps.ts +145 -4
  115. package/src/modules/speaker.ts +10 -4
  116. package/src/modules/stream.ts +153 -1
  117. package/src/protocol.ts +125 -0
  118. package/src/react/StreamPreview.tsx +187 -0
  119. package/src/react/index.ts +3 -0
  120. package/src/react/stream-preview/PreviewConnection.ts +587 -0
  121. package/src/react/stream-preview/controlChannel.ts +119 -0
  122. package/src/react/stream-preview/creditLoop.ts +189 -0
  123. package/src/react/stream-preview/protocol.ts +227 -0
  124. package/src/react/stream-preview/resizeCoalescer.ts +88 -0
  125. package/src/react/stream-preview/testFrames.ts +86 -0
  126. package/src/react/stream-preview/tiers.ts +37 -0
  127. package/src/react/stream-preview/trace.ts +81 -0
  128. package/src/react/stream-preview/transport.ts +242 -0
  129. package/src/react/stream-preview/yuvRenderer.ts +453 -0
  130. package/src/react/useHistoryState.ts +135 -0
  131. package/src/session.ts +76 -0
  132. package/src/ui/index.ts +3 -0
@@ -8,7 +8,7 @@
8
8
  * to your own ingest URL with no relay.
9
9
  */
10
10
 
11
- import {MiniappRequestType} from "../protocol"
11
+ import {MiniappRequestType, type PreviewErrorCode, type PreviewSource} from "../protocol"
12
12
  import {MiniappSession} from "../session"
13
13
 
14
14
  export interface StreamVideoConfig {
@@ -133,9 +133,161 @@ export interface StreamStatus {
133
133
  stats?: StreamLiveStats
134
134
  }
135
135
 
136
+ export interface StreamPreviewOptions {
137
+ /** Defaults to `"call"`. `"glasses"` is reserved and rejects with `unsupported`. */
138
+ source?: PreviewSource
139
+ }
140
+
141
+ export type PreviewHandleState = "held" | "ended"
142
+
143
+ export interface PreviewHandleStatus {
144
+ state: PreviewHandleState
145
+ /** Set when `state` is `"ended"`: `stopped`, `source_ended`, `runtime_stopped`, ... */
146
+ reason?: string
147
+ }
148
+
149
+ /** A preview request or lease failed. `code` is one of {@link PreviewErrorCode} when the host sent one. */
150
+ export class PreviewError extends Error {
151
+ readonly code: PreviewErrorCode | string
152
+
153
+ constructor(code: PreviewErrorCode | string, message?: string) {
154
+ super(message ?? code)
155
+ this.name = "PreviewError"
156
+ this.code = code
157
+ }
158
+ }
159
+
160
+ /**
161
+ * The background-owned lease on a preview source.
162
+ *
163
+ * The lease is what lets a `<StreamPreview>` in the UI show frames. It survives the UI closing and
164
+ * the WebView being destroyed, and ends on `stop()`, when the source ends (the meeting is over), or
165
+ * when the background runtime stops. Hiding the component does not release it.
166
+ */
167
+ export interface PreviewHandle {
168
+ readonly source: PreviewSource
169
+ readonly handleId: string
170
+ /** Correlation id stamped on every `PREVIEW_TRACE` log line for this lease. */
171
+ readonly previewTraceId: string
172
+ readonly state: PreviewHandleState
173
+ /** Release the lease. Idempotent. */
174
+ stop(): Promise<void>
175
+ /** Lease transitions. Fires `{state: "ended", reason}` exactly once. */
176
+ onStatus(cb: (status: PreviewHandleStatus) => void): () => void
177
+ }
178
+
179
+ interface PreviewStartResult {
180
+ handleId: string
181
+ previewTraceId: string
182
+ source: PreviewSource
183
+ }
184
+
185
+ function toPreviewError(error: unknown): PreviewError {
186
+ if (error instanceof PreviewError) return error
187
+ const value = (error ?? {}) as {code?: unknown; message?: unknown}
188
+ const code = typeof value.code === "string" ? value.code : "unsupported"
189
+ const message = typeof value.message === "string" ? value.message : undefined
190
+ return new PreviewError(code, message)
191
+ }
192
+
193
+ class PreviewHandleImpl implements PreviewHandle {
194
+ state: PreviewHandleState = "held"
195
+ private readonly listeners = new Set<(status: PreviewHandleStatus) => void>()
196
+
197
+ constructor(
198
+ readonly source: PreviewSource,
199
+ readonly handleId: string,
200
+ readonly previewTraceId: string,
201
+ private readonly release: (handle: PreviewHandleImpl) => Promise<void>,
202
+ ) {}
203
+
204
+ async stop(): Promise<void> {
205
+ if (this.state === "ended") return
206
+ this.end("stopped")
207
+ await this.release(this)
208
+ }
209
+
210
+ onStatus(cb: (status: PreviewHandleStatus) => void): () => void {
211
+ this.listeners.add(cb)
212
+ return () => this.listeners.delete(cb)
213
+ }
214
+
215
+ /** @internal */
216
+ end(reason: string): void {
217
+ if (this.state === "ended") return
218
+ this.state = "ended"
219
+ this.emit({state: "ended", reason})
220
+ }
221
+
222
+ private emit(status: PreviewHandleStatus): void {
223
+ for (const listener of [...this.listeners]) {
224
+ try {
225
+ listener(status)
226
+ } catch {
227
+ // A throwing listener must not stop the others from hearing that the lease ended.
228
+ }
229
+ }
230
+ }
231
+ }
232
+
136
233
  export class StreamModule {
234
+ private readonly previewHandles = new Map<string, PreviewHandleImpl>()
235
+
137
236
  constructor(private readonly session: MiniappSession) {}
138
237
 
238
+ /**
239
+ * Take the preview lease on a source so a `<StreamPreview>` in the UI can show its frames.
240
+ *
241
+ * For `"call"` the caller must declare `CAMERA` and own the active meeting (`session.meeting`).
242
+ * Only one lease exists at a time; a second request fails with `preview_busy`. Rejects with a
243
+ * {@link PreviewError}. Never await this on a call's connect path: preview is additive.
244
+ */
245
+ async preview(options: StreamPreviewOptions = {}): Promise<PreviewHandle> {
246
+ const source = options.source ?? "call"
247
+ if (source !== "call") {
248
+ throw new PreviewError("unsupported", `Preview source "${String(source)}" is not supported yet`)
249
+ }
250
+ let result: PreviewStartResult
251
+ try {
252
+ result = await this.session.sendRequest<PreviewStartResult>({
253
+ type: MiniappRequestType.STREAM_PREVIEW_START,
254
+ source,
255
+ })
256
+ } catch (error) {
257
+ throw toPreviewError(error)
258
+ }
259
+ const handle = new PreviewHandleImpl(result.source ?? source, result.handleId, result.previewTraceId, (h) =>
260
+ this.releasePreview(h),
261
+ )
262
+ this.previewHandles.set(handle.handleId, handle)
263
+ handle.onStatus((status) => {
264
+ if (status.state === "ended") this.previewHandles.delete(handle.handleId)
265
+ })
266
+ return handle
267
+ }
268
+
269
+ private async releasePreview(handle: PreviewHandleImpl): Promise<void> {
270
+ this.previewHandles.delete(handle.handleId)
271
+ try {
272
+ await this.session.sendRequest<void>({
273
+ type: MiniappRequestType.STREAM_PREVIEW_STOP,
274
+ handleId: handle.handleId,
275
+ })
276
+ } catch (error) {
277
+ throw toPreviewError(error)
278
+ }
279
+ }
280
+
281
+ /** @internal — applied by MiniappSession on inbound STREAM_PREVIEW_STATUS. */
282
+ _applyPreviewStatus(payload: Record<string, unknown>): void {
283
+ if (typeof payload.handleId !== "string") return
284
+ const handle = this.previewHandles.get(payload.handleId)
285
+ if (!handle) return
286
+ if (payload.state === "ended") {
287
+ handle.end(typeof payload.reason === "string" ? payload.reason : "ended")
288
+ }
289
+ }
290
+
139
291
  /**
140
292
  * Start a live video stream from the glasses camera.
141
293
  *
package/src/protocol.ts CHANGED
@@ -16,6 +16,13 @@ export enum MiniappRequestType {
16
16
  /** Handshake: miniapp announces itself and asks phone to bind the session. */
17
17
  CONNECT = "miniapp_connect",
18
18
 
19
+ /**
20
+ * One-shot: the background's `registerMiniapp` handler has settled, so its
21
+ * `session.ui.handle` handlers exist. Sent only to hosts advertising
22
+ * `hostFeatures.initReady`; the host keeps the UI closed until it arrives.
23
+ */
24
+ READY = "miniapp_ready",
25
+
19
26
  /** Request a fresh miniapp-scoped backend auth token. */
20
27
  AUTH_REFRESH = "miniapp_auth_refresh",
21
28
 
@@ -196,6 +203,13 @@ export enum MiniappRequestType {
196
203
  STREAM_STOP = "miniapp_stream_stop",
197
204
  MANAGED_STREAM_START = "miniapp_managed_stream_start",
198
205
  MANAGED_STREAM_STOP = "miniapp_managed_stream_stop",
206
+ /**
207
+ * Take the single raw decoded-frame preview lease on a source. See
208
+ * `session.stream.preview()`. Replies `{handleId, previewTraceId, source}`.
209
+ */
210
+ STREAM_PREVIEW_START = "miniapp_stream_preview_start",
211
+ /** Release a preview lease by `handleId`. */
212
+ STREAM_PREVIEW_STOP = "miniapp_stream_preview_stop",
199
213
 
200
214
  /** Ask the host to open the glasses Wi-Fi setup flow (mirrors the cloud SDK's requestWifiSetup). */
201
215
  REQUEST_WIFI_SETUP = "miniapp_request_wifi_setup",
@@ -211,6 +225,12 @@ export enum MiniappRequestType {
211
225
  MINIAPPS_START = "miniapp_apps_start",
212
226
  /** Stop another miniapp. */
213
227
  MINIAPPS_STOP = "miniapp_apps_stop",
228
+ /** Install/update an approved bundle. Bundled host-trusted Stores only. */
229
+ MINIAPPS_INSTALL = "miniapp_apps_install",
230
+ /** Preflight a release's host/SDK/hardware requirements without downloading it. */
231
+ MINIAPPS_INSTALL_CHECK = "miniapp_apps_install_check",
232
+ /** Uninstall a package owned by the calling Store. Bundled host-trusted Stores only. */
233
+ MINIAPPS_UNINSTALL = "miniapp_apps_uninstall",
214
234
  /** Invoke a declared action on another miniapp; headless-wakes it if stopped. */
215
235
  ACTION_INVOKE = "miniapp_action_invoke",
216
236
  /** Target → host: the result of a delivered ACTION_CALL, correlated by callId. */
@@ -220,7 +240,10 @@ export enum MiniappRequestType {
220
240
  * Phone-native meeting (ACS Teams). Join/leave/mute live in the MentraOS
221
241
  * host so the miniapp never holds the ACS Calling SDK.
222
242
  */
243
+ MEETING_GET_IDENTITY = "miniapp_meeting_get_identity",
223
244
  MEETING_GET_CONFIGURATION = "miniapp_meeting_get_configuration",
245
+ MEETING_CREATE = "miniapp_meeting_create",
246
+ MEETING_RETIRE = "miniapp_meeting_retire",
224
247
  MEETING_JOIN = "miniapp_meeting_join",
225
248
  MEETING_LEAVE = "miniapp_meeting_leave",
226
249
  /** Terminate the meeting for everyone, not just this device. See `meeting.end()`. */
@@ -228,8 +251,24 @@ export enum MiniappRequestType {
228
251
  /** Admit one waiting participant, when the host has Teams lobby permission. */
229
252
  MEETING_ADMIT = "miniapp_meeting_admit",
230
253
  MEETING_SET_MUTED = "miniapp_meeting_set_muted",
254
+ /** Stop or resume the glasses camera the meeting receives, without leaving. */
255
+ MEETING_SET_VIDEO_ENABLED = "miniapp_meeting_set_video_enabled",
231
256
  MEETING_UPDATE_VIDEO_SOURCE = "miniapp_meeting_update_video_source",
232
257
  MEETING_GET_STATE = "miniapp_meeting_get_state",
258
+ /** Stop the glasses publisher for a photo. The meeting, hotspot, and audio stay up. */
259
+ MEETING_PAUSE_VIDEO_PUBLISHER = "miniapp_meeting_pause_video_publisher",
260
+ MEETING_RESUME_VIDEO_PUBLISHER = "miniapp_meeting_resume_video_publisher",
261
+ /** Full-frame "Taking a photo..." card on the outgoing video. Resolves after the first card frame. */
262
+ MEETING_SHOW_CARD = "miniapp_meeting_show_card",
263
+ /** Replace the card with a still. `durationMs` is counted by the caller from `shownAt`. */
264
+ MEETING_SHOW_IMAGE = "miniapp_meeting_show_image",
265
+ /** Stop holding a card or still so glasses frames reach Teams again. */
266
+ MEETING_SHOW_LIVE = "miniapp_meeting_show_live",
267
+ /**
268
+ * Direct link only: card, full-size glasses photo, still, and live again, all run by the host
269
+ * without stopping the glasses publisher. Resolves after the still has been held `durationMs`.
270
+ */
271
+ MEETING_CAPTURE_STILL = "miniapp_meeting_capture_still",
233
272
  }
234
273
 
235
274
  // ============================================================================
@@ -255,6 +294,9 @@ export enum MiniappResponseType {
255
294
  /** Push: host color scheme (light/dark) changed. */
256
295
  COLOR_SCHEME_CHANGE = "miniapp_color_scheme_change",
257
296
 
297
+ /** Push: a Store installation moved to another host-side phase. */
298
+ MINIAPPS_INSTALL_PROGRESS = "miniapp_apps_install_progress",
299
+
258
300
  /**
259
301
  * Push: speaker playback state transitioned. Per-miniapp; the runtime sends
260
302
  * to the owning miniapp only. Carries {state, errorCode?, errorMessage?,
@@ -292,6 +334,25 @@ export enum MiniappResponseType {
292
334
  */
293
335
  MEETING_STATE = "miniapp_meeting_state",
294
336
 
337
+ /**
338
+ * Push: a photo pause hit its ceiling and the publisher was started again.
339
+ * Carries {pauseId, status: "expired"}. A resume the miniapp sends itself is not this event.
340
+ */
341
+ MEETING_VIDEO_PUBLISHER = "miniapp_meeting_video_publisher",
342
+
343
+ /**
344
+ * Push: progress of one `meeting.captureStill`. Carries {stillId, phase} with phase `card`
345
+ * (card on the tile), `uploading` (the glasses captured and are uploading), or `shown` (the
346
+ * still is on the tile). The request result still settles the call.
347
+ */
348
+ MEETING_STILL = "miniapp_meeting_still",
349
+
350
+ /**
351
+ * Push: a preview lease changed state. Carries {handleId, state: "held" | "ended", reason?}.
352
+ * See PreviewHandle.onStatus().
353
+ */
354
+ STREAM_PREVIEW_STATUS = "miniapp_stream_preview_status",
355
+
295
356
  /**
296
357
  * Push: phone is about to tear down the miniapp's session. Gives the SDK
297
358
  * a brief window (~50ms grace on the phone side) to fire one last
@@ -405,6 +466,14 @@ export enum MiniappErrorCode {
405
466
  APP_NOT_FOUND = "APP_NOT_FOUND",
406
467
  /** Target miniapp is incompatible with the connected glasses. */
407
468
  APP_NOT_COMPATIBLE = "APP_NOT_COMPATIBLE",
469
+ /** Bundle download, validation, or activation failed. */
470
+ INSTALL_FAILED = "INSTALL_FAILED",
471
+ /** Another Store installation is already in progress for the same package. */
472
+ INSTALL_IN_PROGRESS = "INSTALL_IN_PROGRESS",
473
+ /** An automatic update was deferred because the target is running or starting. */
474
+ APP_RUNNING = "APP_RUNNING",
475
+ /** Another Store owns the installed package's update provenance. */
476
+ STORE_OWNERSHIP_CONFLICT = "STORE_OWNERSHIP_CONFLICT",
408
477
  /** Target miniapp does not declare the requested action. */
409
478
  ACTION_NOT_FOUND = "ACTION_NOT_FOUND",
410
479
  /** Target connected but never registered a handler for the action (within the buffer window). */
@@ -432,3 +501,59 @@ export enum MiniappErrorCode {
432
501
  /** `blob.importFile` failed (not the user cancelling — that resolves to null). */
433
502
  BLOB_IMPORT_FAILED = "BLOB_IMPORT_FAILED",
434
503
  }
504
+
505
+ // ============================================================================
506
+ // Stream preview
507
+ // ============================================================================
508
+
509
+ /**
510
+ * Where a raw preview comes from. `"call"` is the decoded video of the phone-native meeting the
511
+ * caller owns. `"glasses"` is reserved and currently rejects with `unsupported`.
512
+ */
513
+ export type PreviewSource = "call" | "glasses"
514
+
515
+ /**
516
+ * Typed preview failures and non-terminal statuses, shared by the background handle, the UI
517
+ * component and the host. `waiting_for_lease` and `paused_background` are statuses, not errors.
518
+ */
519
+ export type PreviewErrorCode =
520
+ | "permission_denied"
521
+ | "not_meeting_owner"
522
+ | "preview_busy"
523
+ | "waiting_for_lease"
524
+ | "source_ended"
525
+ | "unsupported"
526
+ | "ack_timeout"
527
+ | "transport_failed"
528
+ | "pack_failed"
529
+ | "paused_background"
530
+ | "diagnostics_disabled"
531
+
532
+ export const PREVIEW_ERROR_CODES: readonly PreviewErrorCode[] = [
533
+ "permission_denied",
534
+ "not_meeting_owner",
535
+ "preview_busy",
536
+ "waiting_for_lease",
537
+ "source_ended",
538
+ "unsupported",
539
+ "ack_timeout",
540
+ "transport_failed",
541
+ "pack_failed",
542
+ "paused_background",
543
+ "diagnostics_disabled",
544
+ ]
545
+
546
+ export function isPreviewErrorCode(value: unknown): value is PreviewErrorCode {
547
+ return typeof value === "string" && (PREVIEW_ERROR_CODES as readonly string[]).includes(value)
548
+ }
549
+
550
+ /** Lease state pushed to the background with `STREAM_PREVIEW_STATUS`. */
551
+ export interface PreviewStatus {
552
+ handleId: string
553
+ state: "held" | "ended"
554
+ /** Why the lease ended: `stopped`, `source_ended`, `runtime_stopped`, ... */
555
+ reason?: string
556
+ }
557
+
558
+ /** The single reserved UI channel the host answers itself for preview control. */
559
+ export const PREVIEW_UI_CHANNEL = "_preview"
@@ -0,0 +1,187 @@
1
+ /**
2
+ * `<StreamPreview>` — draws the raw decoded frames of the source the miniapp's background has
3
+ * leased with `session.stream.preview()`.
4
+ *
5
+ * The component owns only the renderer (canvas, WebGL context, textures). Mounting it shows the
6
+ * preview; unmounting it hides the preview and stops production, but keeps the document's
7
+ * connection open so the next mount starts without a handshake. Releasing the lease is the
8
+ * background's job (`handle.stop()`), not this component's.
9
+ *
10
+ * Frame bytes never enter React state: frames go from the transport to the renderer directly.
11
+ */
12
+
13
+ import {useEffect, useRef, type CSSProperties} from "react"
14
+
15
+ import type {PreviewConnectionState, PreviewSink} from "./stream-preview/PreviewConnection"
16
+ import {getPreviewConnection} from "./stream-preview/PreviewConnection"
17
+ import {ResizeCoalescer, type PreviewBox} from "./stream-preview/resizeCoalescer"
18
+ import {quantizePreviewTier, sameTier, type PreviewTier} from "./stream-preview/tiers"
19
+ import {previewTrace, previewTraceWarn} from "./stream-preview/trace"
20
+ import {
21
+ createYuvRenderer,
22
+ type PreviewFit,
23
+ type YuvRenderer,
24
+ type YuvRendererOptions,
25
+ } from "./stream-preview/yuvRenderer"
26
+
27
+ export type {PreviewFit} from "./stream-preview/yuvRenderer"
28
+ export type {PreviewConnectionState} from "./stream-preview/PreviewConnection"
29
+
30
+ export interface StreamPreviewStatus {
31
+ state: PreviewConnectionState
32
+ /** The Mentra App is in the background; production is paused and resumes on return. */
33
+ paused: boolean
34
+ /** Why the state changed, when the host said (`source_ended`, `stopped`, ...). */
35
+ reason?: string
36
+ /** Error code for `error` / `unsupported`. */
37
+ code?: string
38
+ /** Latest once-a-second host counters plus page-side counters. */
39
+ stats?: Record<string, unknown>
40
+ }
41
+
42
+ export interface StreamPreviewProps {
43
+ /** `contain` letterboxes (default); `cover` fills and crops. The picture is never stretched. */
44
+ fit?: PreviewFit
45
+ onStatus?: (status: StreamPreviewStatus) => void
46
+ /** Typed preview error codes: `source_ended`, `unsupported`, `transport_failed`, ... */
47
+ onError?: (code: string) => void
48
+ className?: string
49
+ style?: CSSProperties
50
+ }
51
+
52
+ type RendererFactory = (canvas: HTMLCanvasElement, options: YuvRendererOptions) => YuvRenderer | null
53
+
54
+ let rendererFactory: RendererFactory = createYuvRenderer
55
+
56
+ /** Test seam: WebGL is not available in headless DOMs. */
57
+ export function setStreamPreviewRendererFactoryForTests(factory: RendererFactory | null): void {
58
+ rendererFactory = factory ?? createYuvRenderer
59
+ }
60
+
61
+ const DEFAULT_STYLE: CSSProperties = {display: "block", width: "100%", height: "100%"}
62
+
63
+ function measure(canvas: HTMLCanvasElement): PreviewBox {
64
+ const rect = canvas.getBoundingClientRect()
65
+ const ratio = typeof window === "undefined" ? 1 : window.devicePixelRatio || 1
66
+ return {width: Math.round(rect.width * ratio), height: Math.round(rect.height * ratio)}
67
+ }
68
+
69
+ function isDocumentVisible(): boolean {
70
+ return typeof document === "undefined" || document.visibilityState !== "hidden"
71
+ }
72
+
73
+ export function StreamPreview({fit = "contain", onStatus, onError, className, style}: StreamPreviewProps) {
74
+ const canvasRef = useRef<HTMLCanvasElement | null>(null)
75
+ const rendererRef = useRef<YuvRenderer | null>(null)
76
+ const fitRef = useRef(fit)
77
+ const onStatusRef = useRef(onStatus)
78
+ const onErrorRef = useRef(onError)
79
+ fitRef.current = fit
80
+ onStatusRef.current = onStatus
81
+ onErrorRef.current = onError
82
+
83
+ useEffect(() => {
84
+ rendererRef.current?.setFit(fit)
85
+ }, [fit])
86
+
87
+ useEffect(() => {
88
+ const canvas = canvasRef.current
89
+ if (!canvas) return
90
+ const connection = getPreviewConnection()
91
+ const mountEpoch = connection.nextMountEpoch()
92
+ let contextLosses = 0
93
+ let status: StreamPreviewStatus = {state: connection.currentState, paused: false}
94
+ let box: PreviewBox = {width: 0, height: 0}
95
+ let tier: PreviewTier | null = null
96
+ let running = isDocumentVisible()
97
+ let unmounted = false
98
+
99
+ const emit = (next: Partial<StreamPreviewStatus>) => {
100
+ status = {...status, ...next}
101
+ onStatusRef.current?.(status)
102
+ }
103
+
104
+ const buildRenderer = () => {
105
+ rendererRef.current?.dispose()
106
+ rendererRef.current = rendererFactory(canvas, {
107
+ fit: fitRef.current,
108
+ onError: (reason) => {
109
+ previewTraceWarn("renderer_error", {reason, mountEpoch, docGen: connection.documentGeneration})
110
+ connection.reportRendererError(mountEpoch, reason)
111
+ if (reason === "webgl_context_lost") contextLosses += 1
112
+ },
113
+ })
114
+ if (!rendererRef.current) connection.reportRendererError(mountEpoch, "webgl_unavailable")
115
+ }
116
+ buildRenderer()
117
+
118
+ // Context loss rebuilds only the renderer. The connection and the lease are not involved,
119
+ // and frames that arrive meanwhile are still acked so the sender does not stall.
120
+ const onContextRestored = () => {
121
+ if (unmounted) return
122
+ previewTrace("renderer_rebuilt", {mountEpoch, contextLosses})
123
+ buildRenderer()
124
+ }
125
+ canvas.addEventListener("webglcontextrestored", onContextRestored)
126
+
127
+ const sink: PreviewSink = {
128
+ mountEpoch,
129
+ draw: (frame) => rendererRef.current?.drawFrame(frame) ?? null,
130
+ onState: (state, detail) =>
131
+ emit({state, paused: detail.paused ?? false, reason: detail.reason, code: detail.code}),
132
+ onStatus: (stats) => emit({stats: {...stats, contextLosses}}),
133
+ onError: (code) => onErrorRef.current?.(code),
134
+ }
135
+ const detach = connection.attach(sink)
136
+ previewTrace("mount", {mountEpoch, docGen: connection.documentGeneration})
137
+
138
+ const push = () => connection.update(mountEpoch, {boxWidth: box.width, boxHeight: box.height, tier, running})
139
+
140
+ const coalescer = new ResizeCoalescer({
141
+ apply: (next) => {
142
+ box = next
143
+ const nextTier = quantizePreviewTier(next.width, next.height)
144
+ if (sameTier(nextTier, tier) && tier !== null) return
145
+ tier = nextTier
146
+ if (!tier) previewTrace("hidden_zero_size", {mountEpoch})
147
+ push()
148
+ },
149
+ })
150
+
151
+ let resizeObserver: ResizeObserver | null = null
152
+ const onWindowResize = () => coalescer.push(measure(canvas))
153
+ if (typeof ResizeObserver === "function") {
154
+ resizeObserver = new ResizeObserver(() => coalescer.push(measure(canvas)))
155
+ resizeObserver.observe(canvas)
156
+ } else if (typeof window !== "undefined") {
157
+ window.addEventListener("resize", onWindowResize)
158
+ }
159
+ coalescer.push(measure(canvas))
160
+
161
+ const onVisibility = () => {
162
+ const visible = isDocumentVisible()
163
+ if (visible === running) return
164
+ running = visible
165
+ previewTrace("visibility_change", {mountEpoch, visible})
166
+ push()
167
+ }
168
+ document.addEventListener("visibilitychange", onVisibility)
169
+
170
+ return () => {
171
+ unmounted = true
172
+ previewTrace("unmount", {mountEpoch, docGen: connection.documentGeneration})
173
+ document.removeEventListener("visibilitychange", onVisibility)
174
+ resizeObserver?.disconnect()
175
+ if (typeof window !== "undefined") window.removeEventListener("resize", onWindowResize)
176
+ coalescer.cancel()
177
+ canvas.removeEventListener("webglcontextrestored", onContextRestored)
178
+ running = false
179
+ push()
180
+ detach()
181
+ rendererRef.current?.dispose()
182
+ rendererRef.current = null
183
+ }
184
+ }, [])
185
+
186
+ return <canvas ref={canvasRef} className={className} style={{...DEFAULT_STYLE, ...style}} />
187
+ }
@@ -10,3 +10,6 @@ export type {MiniappHeaderProps} from "./MiniappHeader"
10
10
  export {MentraProvider} from "./MentraProvider"
11
11
  export type {MentraProviderProps} from "./MentraProvider"
12
12
  export {useRpc, type RpcCallable} from "./useRpc"
13
+ export {useHistoryState, type SetHistoryState} from "./useHistoryState"
14
+ export {StreamPreview} from "./StreamPreview"
15
+ export type {StreamPreviewProps, StreamPreviewStatus, PreviewFit, PreviewConnectionState} from "./StreamPreview"