@mentra/miniapp 3.3.0-dev.328 → 3.3.0-dev.329

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. package/dist/background/index.d.ts +3 -1
  2. package/dist/background/index.d.ts.map +1 -1
  3. package/dist/background/index.js +1 -0
  4. package/dist/background/index.js.map +1 -1
  5. package/dist/index.d.ts +4 -1
  6. package/dist/index.d.ts.map +1 -1
  7. package/dist/index.js +2 -0
  8. package/dist/index.js.map +1 -1
  9. package/dist/modules/meeting.d.ts +11 -0
  10. package/dist/modules/meeting.d.ts.map +1 -1
  11. package/dist/modules/meeting.js +19 -0
  12. package/dist/modules/meeting.js.map +1 -1
  13. package/dist/modules/stream.d.ts +46 -0
  14. package/dist/modules/stream.d.ts.map +1 -1
  15. package/dist/modules/stream.js +107 -0
  16. package/dist/modules/stream.js.map +1 -1
  17. package/dist/protocol.d.ts +35 -0
  18. package/dist/protocol.d.ts.map +1 -1
  19. package/dist/protocol.js +32 -0
  20. package/dist/protocol.js.map +1 -1
  21. package/dist/react/StreamPreview.d.ts +41 -0
  22. package/dist/react/StreamPreview.d.ts.map +1 -0
  23. package/dist/react/StreamPreview.js +144 -0
  24. package/dist/react/StreamPreview.js.map +1 -0
  25. package/dist/react/index.d.ts +2 -0
  26. package/dist/react/index.d.ts.map +1 -1
  27. package/dist/react/index.js +1 -0
  28. package/dist/react/index.js.map +1 -1
  29. package/dist/react/stream-preview/PreviewConnection.d.ts +139 -0
  30. package/dist/react/stream-preview/PreviewConnection.d.ts.map +1 -0
  31. package/dist/react/stream-preview/PreviewConnection.js +454 -0
  32. package/dist/react/stream-preview/PreviewConnection.js.map +1 -0
  33. package/dist/react/stream-preview/controlChannel.d.ts +75 -0
  34. package/dist/react/stream-preview/controlChannel.d.ts.map +1 -0
  35. package/dist/react/stream-preview/controlChannel.js +55 -0
  36. package/dist/react/stream-preview/controlChannel.js.map +1 -0
  37. package/dist/react/stream-preview/creditLoop.d.ts +81 -0
  38. package/dist/react/stream-preview/creditLoop.d.ts.map +1 -0
  39. package/dist/react/stream-preview/creditLoop.js +146 -0
  40. package/dist/react/stream-preview/creditLoop.js.map +1 -0
  41. package/dist/react/stream-preview/protocol.d.ts +113 -0
  42. package/dist/react/stream-preview/protocol.d.ts.map +1 -0
  43. package/dist/react/stream-preview/protocol.js +148 -0
  44. package/dist/react/stream-preview/protocol.js.map +1 -0
  45. package/dist/react/stream-preview/resizeCoalescer.d.ts +38 -0
  46. package/dist/react/stream-preview/resizeCoalescer.d.ts.map +1 -0
  47. package/dist/react/stream-preview/resizeCoalescer.js +71 -0
  48. package/dist/react/stream-preview/resizeCoalescer.js.map +1 -0
  49. package/dist/react/stream-preview/testFrames.d.ts +38 -0
  50. package/dist/react/stream-preview/testFrames.d.ts.map +1 -0
  51. package/dist/react/stream-preview/testFrames.js +50 -0
  52. package/dist/react/stream-preview/testFrames.js.map +1 -0
  53. package/dist/react/stream-preview/tiers.d.ts +21 -0
  54. package/dist/react/stream-preview/tiers.d.ts.map +1 -0
  55. package/dist/react/stream-preview/tiers.js +33 -0
  56. package/dist/react/stream-preview/tiers.js.map +1 -0
  57. package/dist/react/stream-preview/trace.d.ts +23 -0
  58. package/dist/react/stream-preview/trace.d.ts.map +1 -0
  59. package/dist/react/stream-preview/trace.js +71 -0
  60. package/dist/react/stream-preview/trace.js.map +1 -0
  61. package/dist/react/stream-preview/transport.d.ts +79 -0
  62. package/dist/react/stream-preview/transport.d.ts.map +1 -0
  63. package/dist/react/stream-preview/transport.js +209 -0
  64. package/dist/react/stream-preview/transport.js.map +1 -0
  65. package/dist/react/stream-preview/yuvRenderer.d.ts +31 -0
  66. package/dist/react/stream-preview/yuvRenderer.d.ts.map +1 -0
  67. package/dist/react/stream-preview/yuvRenderer.js +398 -0
  68. package/dist/react/stream-preview/yuvRenderer.js.map +1 -0
  69. package/dist/session.d.ts.map +1 -1
  70. package/dist/session.js +5 -0
  71. package/dist/session.js.map +1 -1
  72. package/dist/ui/index.d.ts +2 -0
  73. package/dist/ui/index.d.ts.map +1 -1
  74. package/dist/ui/index.js +1 -0
  75. package/dist/ui/index.js.map +1 -1
  76. package/package.json +8 -3
  77. package/src/background/index.ts +9 -1
  78. package/src/index.ts +7 -0
  79. package/src/modules/meeting.ts +23 -0
  80. package/src/modules/stream.ts +153 -1
  81. package/src/protocol.ts +71 -0
  82. package/src/react/StreamPreview.tsx +187 -0
  83. package/src/react/index.ts +2 -0
  84. package/src/react/stream-preview/PreviewConnection.ts +543 -0
  85. package/src/react/stream-preview/controlChannel.ts +119 -0
  86. package/src/react/stream-preview/creditLoop.ts +189 -0
  87. package/src/react/stream-preview/protocol.ts +227 -0
  88. package/src/react/stream-preview/resizeCoalescer.ts +88 -0
  89. package/src/react/stream-preview/testFrames.ts +86 -0
  90. package/src/react/stream-preview/tiers.ts +37 -0
  91. package/src/react/stream-preview/trace.ts +81 -0
  92. package/src/react/stream-preview/transport.ts +242 -0
  93. package/src/react/stream-preview/yuvRenderer.ts +453 -0
  94. package/src/session.ts +6 -0
  95. package/src/ui/index.ts +2 -0
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@mentra/miniapp",
3
- "version": "3.3.0-dev.328",
3
+ "version": "3.3.0-dev.329",
4
4
  "description": "SDK for building MentraOS local miniapps (browser/WebView)",
5
5
  "author": "Mentra Labs",
6
6
  "license": "Apache-2.0",
@@ -57,18 +57,23 @@
57
57
  }
58
58
  },
59
59
  "dependencies": {
60
- "@mentra/cloud-protocol": "3.3.0-dev.328",
60
+ "@mentra/cloud-protocol": "3.3.0-dev.329",
61
61
  "eventemitter3": "^5.0.1"
62
62
  },
63
63
  "devDependencies": {
64
+ "@happy-dom/global-registrator": "^17.6.3",
64
65
  "@types/react": "^19.1.6",
66
+ "@types/react-dom": "^19.1.0",
67
+ "happy-dom": "^17.6.3",
65
68
  "react": "^19.1.0",
69
+ "react-dom": "^19.2.0",
66
70
  "typescript": "^5.0.0"
67
71
  },
68
72
  "files": [
69
73
  "dist",
70
74
  "src",
71
- "!src/**/*.test.ts"
75
+ "!src/**/*.test.ts",
76
+ "!src/**/*.test.tsx"
72
77
  ],
73
78
  "keywords": [
74
79
  "mentra",
@@ -111,7 +111,15 @@ export {
111
111
  parseMeetingParticipants,
112
112
  validateMeetingVideoSource,
113
113
  } from "../modules/meeting"
114
- export type {StreamModule} from "../modules/stream"
114
+ export type {
115
+ StreamModule,
116
+ StreamPreviewOptions,
117
+ PreviewHandle,
118
+ PreviewHandleState,
119
+ PreviewHandleStatus,
120
+ } from "../modules/stream"
121
+ export {PreviewError} from "../modules/stream"
122
+ export type {PreviewErrorCode, PreviewSource, PreviewStatus} from "../protocol"
115
123
  export type {SystemModule} from "../modules/system"
116
124
  export type {TranscriptionModule} from "../modules/transcription"
117
125
  export type {TranslationModule} from "../modules/translation"
package/src/index.ts CHANGED
@@ -131,7 +131,14 @@ export type {
131
131
  StreamResult,
132
132
  StreamStatus,
133
133
  StreamVideoConfig,
134
+ StreamPreviewOptions,
135
+ PreviewHandle,
136
+ PreviewHandleState,
137
+ PreviewHandleStatus,
134
138
  } from "./modules/stream"
139
+ export {PreviewError} from "./modules/stream"
140
+ export {PREVIEW_ERROR_CODES, PREVIEW_UI_CHANNEL, isPreviewErrorCode} from "./protocol"
141
+ export type {PreviewErrorCode, PreviewSource, PreviewStatus} from "./protocol"
135
142
  export type {
136
143
  ShareOptions,
137
144
  ShareResult,
@@ -154,6 +154,11 @@ export interface MeetingState {
154
154
  guestReason?: MeetingGuestReason
155
155
  state: MeetingPhase
156
156
  muted: boolean
157
+ /**
158
+ * Whether remote participants receive the glasses camera. False after `setVideoEnabled(false)`;
159
+ * every new join starts true. Omitted by hosts that cannot toggle outgoing video.
160
+ */
161
+ videoEnabled?: boolean
157
162
  error?: string
158
163
  /** Provider termination details, including Teams' invalid meeting-link codes. */
159
164
  endReason?: MeetingEndReason
@@ -518,6 +523,23 @@ export class MeetingModule {
518
523
  }
519
524
  }
520
525
 
526
+ /**
527
+ * Stop or resume the glasses camera the meeting receives. The call and the glasses stream stay
528
+ * up, so a local `<StreamPreview>` keeps drawing while remote participants see the camera off.
529
+ * Check `state.videoEnabled !== undefined` before offering it; older hosts reject.
530
+ */
531
+ async setVideoEnabled(enabled: boolean): Promise<void> {
532
+ try {
533
+ const result = await this.session.sendRequest<MeetingState | null>({
534
+ type: MiniappRequestType.MEETING_SET_VIDEO_ENABLED,
535
+ enabled,
536
+ })
537
+ if (result) this._applyState(result)
538
+ } catch (error) {
539
+ mapHostError(error)
540
+ }
541
+ }
542
+
521
543
  /**
522
544
  * Repoint the host at a new WHEP URL mid-call, the recovery path for a re-published stream.
523
545
  *
@@ -566,6 +588,7 @@ export class MeetingModule {
566
588
  guestReason: event.identityMode === "guest" ? event.guestReason : undefined,
567
589
  state: event.state,
568
590
  muted: Boolean(event.muted),
591
+ videoEnabled: typeof event.videoEnabled === "boolean" ? event.videoEnabled : undefined,
569
592
  error: event.error,
570
593
  endReason: parseMeetingEndReason(event.endReason),
571
594
  meetingUrl: event.meetingUrl,
@@ -8,7 +8,7 @@
8
8
  * to your own ingest URL with no relay.
9
9
  */
10
10
 
11
- import {MiniappRequestType} from "../protocol"
11
+ import {MiniappRequestType, type PreviewErrorCode, type PreviewSource} from "../protocol"
12
12
  import {MiniappSession} from "../session"
13
13
 
14
14
  export interface StreamVideoConfig {
@@ -133,9 +133,161 @@ export interface StreamStatus {
133
133
  stats?: StreamLiveStats
134
134
  }
135
135
 
136
+ export interface StreamPreviewOptions {
137
+ /** Defaults to `"call"`. `"glasses"` is reserved and rejects with `unsupported`. */
138
+ source?: PreviewSource
139
+ }
140
+
141
+ export type PreviewHandleState = "held" | "ended"
142
+
143
+ export interface PreviewHandleStatus {
144
+ state: PreviewHandleState
145
+ /** Set when `state` is `"ended"`: `stopped`, `source_ended`, `runtime_stopped`, ... */
146
+ reason?: string
147
+ }
148
+
149
+ /** A preview request or lease failed. `code` is one of {@link PreviewErrorCode} when the host sent one. */
150
+ export class PreviewError extends Error {
151
+ readonly code: PreviewErrorCode | string
152
+
153
+ constructor(code: PreviewErrorCode | string, message?: string) {
154
+ super(message ?? code)
155
+ this.name = "PreviewError"
156
+ this.code = code
157
+ }
158
+ }
159
+
160
+ /**
161
+ * The background-owned lease on a preview source.
162
+ *
163
+ * The lease is what lets a `<StreamPreview>` in the UI show frames. It survives the UI closing and
164
+ * the WebView being destroyed, and ends on `stop()`, when the source ends (the meeting is over), or
165
+ * when the background runtime stops. Hiding the component does not release it.
166
+ */
167
+ export interface PreviewHandle {
168
+ readonly source: PreviewSource
169
+ readonly handleId: string
170
+ /** Correlation id stamped on every `PREVIEW_TRACE` log line for this lease. */
171
+ readonly previewTraceId: string
172
+ readonly state: PreviewHandleState
173
+ /** Release the lease. Idempotent. */
174
+ stop(): Promise<void>
175
+ /** Lease transitions. Fires `{state: "ended", reason}` exactly once. */
176
+ onStatus(cb: (status: PreviewHandleStatus) => void): () => void
177
+ }
178
+
179
+ interface PreviewStartResult {
180
+ handleId: string
181
+ previewTraceId: string
182
+ source: PreviewSource
183
+ }
184
+
185
+ function toPreviewError(error: unknown): PreviewError {
186
+ if (error instanceof PreviewError) return error
187
+ const value = (error ?? {}) as {code?: unknown; message?: unknown}
188
+ const code = typeof value.code === "string" ? value.code : "unsupported"
189
+ const message = typeof value.message === "string" ? value.message : undefined
190
+ return new PreviewError(code, message)
191
+ }
192
+
193
+ class PreviewHandleImpl implements PreviewHandle {
194
+ state: PreviewHandleState = "held"
195
+ private readonly listeners = new Set<(status: PreviewHandleStatus) => void>()
196
+
197
+ constructor(
198
+ readonly source: PreviewSource,
199
+ readonly handleId: string,
200
+ readonly previewTraceId: string,
201
+ private readonly release: (handle: PreviewHandleImpl) => Promise<void>,
202
+ ) {}
203
+
204
+ async stop(): Promise<void> {
205
+ if (this.state === "ended") return
206
+ this.end("stopped")
207
+ await this.release(this)
208
+ }
209
+
210
+ onStatus(cb: (status: PreviewHandleStatus) => void): () => void {
211
+ this.listeners.add(cb)
212
+ return () => this.listeners.delete(cb)
213
+ }
214
+
215
+ /** @internal */
216
+ end(reason: string): void {
217
+ if (this.state === "ended") return
218
+ this.state = "ended"
219
+ this.emit({state: "ended", reason})
220
+ }
221
+
222
+ private emit(status: PreviewHandleStatus): void {
223
+ for (const listener of [...this.listeners]) {
224
+ try {
225
+ listener(status)
226
+ } catch {
227
+ // A throwing listener must not stop the others from hearing that the lease ended.
228
+ }
229
+ }
230
+ }
231
+ }
232
+
136
233
  export class StreamModule {
234
+ private readonly previewHandles = new Map<string, PreviewHandleImpl>()
235
+
137
236
  constructor(private readonly session: MiniappSession) {}
138
237
 
238
+ /**
239
+ * Take the preview lease on a source so a `<StreamPreview>` in the UI can show its frames.
240
+ *
241
+ * For `"call"` the caller must declare `CAMERA` and own the active meeting (`session.meeting`).
242
+ * Only one lease exists at a time; a second request fails with `preview_busy`. Rejects with a
243
+ * {@link PreviewError}. Never await this on a call's connect path: preview is additive.
244
+ */
245
+ async preview(options: StreamPreviewOptions = {}): Promise<PreviewHandle> {
246
+ const source = options.source ?? "call"
247
+ if (source !== "call") {
248
+ throw new PreviewError("unsupported", `Preview source "${String(source)}" is not supported yet`)
249
+ }
250
+ let result: PreviewStartResult
251
+ try {
252
+ result = await this.session.sendRequest<PreviewStartResult>({
253
+ type: MiniappRequestType.STREAM_PREVIEW_START,
254
+ source,
255
+ })
256
+ } catch (error) {
257
+ throw toPreviewError(error)
258
+ }
259
+ const handle = new PreviewHandleImpl(result.source ?? source, result.handleId, result.previewTraceId, (h) =>
260
+ this.releasePreview(h),
261
+ )
262
+ this.previewHandles.set(handle.handleId, handle)
263
+ handle.onStatus((status) => {
264
+ if (status.state === "ended") this.previewHandles.delete(handle.handleId)
265
+ })
266
+ return handle
267
+ }
268
+
269
+ private async releasePreview(handle: PreviewHandleImpl): Promise<void> {
270
+ this.previewHandles.delete(handle.handleId)
271
+ try {
272
+ await this.session.sendRequest<void>({
273
+ type: MiniappRequestType.STREAM_PREVIEW_STOP,
274
+ handleId: handle.handleId,
275
+ })
276
+ } catch (error) {
277
+ throw toPreviewError(error)
278
+ }
279
+ }
280
+
281
+ /** @internal — applied by MiniappSession on inbound STREAM_PREVIEW_STATUS. */
282
+ _applyPreviewStatus(payload: Record<string, unknown>): void {
283
+ if (typeof payload.handleId !== "string") return
284
+ const handle = this.previewHandles.get(payload.handleId)
285
+ if (!handle) return
286
+ if (payload.state === "ended") {
287
+ handle.end(typeof payload.reason === "string" ? payload.reason : "ended")
288
+ }
289
+ }
290
+
139
291
  /**
140
292
  * Start a live video stream from the glasses camera.
141
293
  *
package/src/protocol.ts CHANGED
@@ -196,6 +196,13 @@ export enum MiniappRequestType {
196
196
  STREAM_STOP = "miniapp_stream_stop",
197
197
  MANAGED_STREAM_START = "miniapp_managed_stream_start",
198
198
  MANAGED_STREAM_STOP = "miniapp_managed_stream_stop",
199
+ /**
200
+ * Take the single raw decoded-frame preview lease on a source. See
201
+ * `session.stream.preview()`. Replies `{handleId, previewTraceId, source}`.
202
+ */
203
+ STREAM_PREVIEW_START = "miniapp_stream_preview_start",
204
+ /** Release a preview lease by `handleId`. */
205
+ STREAM_PREVIEW_STOP = "miniapp_stream_preview_stop",
199
206
 
200
207
  /** Ask the host to open the glasses Wi-Fi setup flow (mirrors the cloud SDK's requestWifiSetup). */
201
208
  REQUEST_WIFI_SETUP = "miniapp_request_wifi_setup",
@@ -230,6 +237,8 @@ export enum MiniappRequestType {
230
237
  /** Admit one waiting participant, when the host has Teams lobby permission. */
231
238
  MEETING_ADMIT = "miniapp_meeting_admit",
232
239
  MEETING_SET_MUTED = "miniapp_meeting_set_muted",
240
+ /** Stop or resume the glasses camera the meeting receives, without leaving. */
241
+ MEETING_SET_VIDEO_ENABLED = "miniapp_meeting_set_video_enabled",
233
242
  MEETING_UPDATE_VIDEO_SOURCE = "miniapp_meeting_update_video_source",
234
243
  MEETING_GET_STATE = "miniapp_meeting_get_state",
235
244
  }
@@ -294,6 +303,12 @@ export enum MiniappResponseType {
294
303
  */
295
304
  MEETING_STATE = "miniapp_meeting_state",
296
305
 
306
+ /**
307
+ * Push: a preview lease changed state. Carries {handleId, state: "held" | "ended", reason?}.
308
+ * See PreviewHandle.onStatus().
309
+ */
310
+ STREAM_PREVIEW_STATUS = "miniapp_stream_preview_status",
311
+
297
312
  /**
298
313
  * Push: phone is about to tear down the miniapp's session. Gives the SDK
299
314
  * a brief window (~50ms grace on the phone side) to fire one last
@@ -434,3 +449,59 @@ export enum MiniappErrorCode {
434
449
  /** `blob.importFile` failed (not the user cancelling — that resolves to null). */
435
450
  BLOB_IMPORT_FAILED = "BLOB_IMPORT_FAILED",
436
451
  }
452
+
453
+ // ============================================================================
454
+ // Stream preview
455
+ // ============================================================================
456
+
457
+ /**
458
+ * Where a raw preview comes from. `"call"` is the decoded video of the phone-native meeting the
459
+ * caller owns. `"glasses"` is reserved and currently rejects with `unsupported`.
460
+ */
461
+ export type PreviewSource = "call" | "glasses"
462
+
463
+ /**
464
+ * Typed preview failures and non-terminal statuses, shared by the background handle, the UI
465
+ * component and the host. `waiting_for_lease` and `paused_background` are statuses, not errors.
466
+ */
467
+ export type PreviewErrorCode =
468
+ | "permission_denied"
469
+ | "not_meeting_owner"
470
+ | "preview_busy"
471
+ | "waiting_for_lease"
472
+ | "source_ended"
473
+ | "unsupported"
474
+ | "ack_timeout"
475
+ | "transport_failed"
476
+ | "pack_failed"
477
+ | "paused_background"
478
+ | "diagnostics_disabled"
479
+
480
+ export const PREVIEW_ERROR_CODES: readonly PreviewErrorCode[] = [
481
+ "permission_denied",
482
+ "not_meeting_owner",
483
+ "preview_busy",
484
+ "waiting_for_lease",
485
+ "source_ended",
486
+ "unsupported",
487
+ "ack_timeout",
488
+ "transport_failed",
489
+ "pack_failed",
490
+ "paused_background",
491
+ "diagnostics_disabled",
492
+ ]
493
+
494
+ export function isPreviewErrorCode(value: unknown): value is PreviewErrorCode {
495
+ return typeof value === "string" && (PREVIEW_ERROR_CODES as readonly string[]).includes(value)
496
+ }
497
+
498
+ /** Lease state pushed to the background with `STREAM_PREVIEW_STATUS`. */
499
+ export interface PreviewStatus {
500
+ handleId: string
501
+ state: "held" | "ended"
502
+ /** Why the lease ended: `stopped`, `source_ended`, `runtime_stopped`, ... */
503
+ reason?: string
504
+ }
505
+
506
+ /** The single reserved UI channel the host answers itself for preview control. */
507
+ export const PREVIEW_UI_CHANNEL = "_preview"
@@ -0,0 +1,187 @@
1
+ /**
2
+ * `<StreamPreview>` — draws the raw decoded frames of the source the miniapp's background has
3
+ * leased with `session.stream.preview()`.
4
+ *
5
+ * The component owns only the renderer (canvas, WebGL context, textures). Mounting it shows the
6
+ * preview; unmounting it hides the preview and stops production, but keeps the document's
7
+ * connection open so the next mount starts without a handshake. Releasing the lease is the
8
+ * background's job (`handle.stop()`), not this component's.
9
+ *
10
+ * Frame bytes never enter React state: frames go from the transport to the renderer directly.
11
+ */
12
+
13
+ import {useEffect, useRef, type CSSProperties} from "react"
14
+
15
+ import type {PreviewConnectionState, PreviewSink} from "./stream-preview/PreviewConnection"
16
+ import {getPreviewConnection} from "./stream-preview/PreviewConnection"
17
+ import {ResizeCoalescer, type PreviewBox} from "./stream-preview/resizeCoalescer"
18
+ import {quantizePreviewTier, sameTier, type PreviewTier} from "./stream-preview/tiers"
19
+ import {previewTrace, previewTraceWarn} from "./stream-preview/trace"
20
+ import {
21
+ createYuvRenderer,
22
+ type PreviewFit,
23
+ type YuvRenderer,
24
+ type YuvRendererOptions,
25
+ } from "./stream-preview/yuvRenderer"
26
+
27
+ export type {PreviewFit} from "./stream-preview/yuvRenderer"
28
+ export type {PreviewConnectionState} from "./stream-preview/PreviewConnection"
29
+
30
+ export interface StreamPreviewStatus {
31
+ state: PreviewConnectionState
32
+ /** The Mentra App is in the background; production is paused and resumes on return. */
33
+ paused: boolean
34
+ /** Why the state changed, when the host said (`source_ended`, `stopped`, ...). */
35
+ reason?: string
36
+ /** Error code for `error` / `unsupported`. */
37
+ code?: string
38
+ /** Latest once-a-second host counters plus page-side counters. */
39
+ stats?: Record<string, unknown>
40
+ }
41
+
42
+ export interface StreamPreviewProps {
43
+ /** `contain` letterboxes (default); `cover` fills and crops. The picture is never stretched. */
44
+ fit?: PreviewFit
45
+ onStatus?: (status: StreamPreviewStatus) => void
46
+ /** Typed preview error codes: `source_ended`, `unsupported`, `transport_failed`, ... */
47
+ onError?: (code: string) => void
48
+ className?: string
49
+ style?: CSSProperties
50
+ }
51
+
52
+ type RendererFactory = (canvas: HTMLCanvasElement, options: YuvRendererOptions) => YuvRenderer | null
53
+
54
+ let rendererFactory: RendererFactory = createYuvRenderer
55
+
56
+ /** Test seam: WebGL is not available in headless DOMs. */
57
+ export function setStreamPreviewRendererFactoryForTests(factory: RendererFactory | null): void {
58
+ rendererFactory = factory ?? createYuvRenderer
59
+ }
60
+
61
+ const DEFAULT_STYLE: CSSProperties = {display: "block", width: "100%", height: "100%"}
62
+
63
+ function measure(canvas: HTMLCanvasElement): PreviewBox {
64
+ const rect = canvas.getBoundingClientRect()
65
+ const ratio = typeof window === "undefined" ? 1 : window.devicePixelRatio || 1
66
+ return {width: Math.round(rect.width * ratio), height: Math.round(rect.height * ratio)}
67
+ }
68
+
69
+ function isDocumentVisible(): boolean {
70
+ return typeof document === "undefined" || document.visibilityState !== "hidden"
71
+ }
72
+
73
+ export function StreamPreview({fit = "contain", onStatus, onError, className, style}: StreamPreviewProps) {
74
+ const canvasRef = useRef<HTMLCanvasElement | null>(null)
75
+ const rendererRef = useRef<YuvRenderer | null>(null)
76
+ const fitRef = useRef(fit)
77
+ const onStatusRef = useRef(onStatus)
78
+ const onErrorRef = useRef(onError)
79
+ fitRef.current = fit
80
+ onStatusRef.current = onStatus
81
+ onErrorRef.current = onError
82
+
83
+ useEffect(() => {
84
+ rendererRef.current?.setFit(fit)
85
+ }, [fit])
86
+
87
+ useEffect(() => {
88
+ const canvas = canvasRef.current
89
+ if (!canvas) return
90
+ const connection = getPreviewConnection()
91
+ const mountEpoch = connection.nextMountEpoch()
92
+ let contextLosses = 0
93
+ let status: StreamPreviewStatus = {state: connection.currentState, paused: false}
94
+ let box: PreviewBox = {width: 0, height: 0}
95
+ let tier: PreviewTier | null = null
96
+ let running = isDocumentVisible()
97
+ let unmounted = false
98
+
99
+ const emit = (next: Partial<StreamPreviewStatus>) => {
100
+ status = {...status, ...next}
101
+ onStatusRef.current?.(status)
102
+ }
103
+
104
+ const buildRenderer = () => {
105
+ rendererRef.current?.dispose()
106
+ rendererRef.current = rendererFactory(canvas, {
107
+ fit: fitRef.current,
108
+ onError: (reason) => {
109
+ previewTraceWarn("renderer_error", {reason, mountEpoch, docGen: connection.documentGeneration})
110
+ connection.reportRendererError(mountEpoch, reason)
111
+ if (reason === "webgl_context_lost") contextLosses += 1
112
+ },
113
+ })
114
+ if (!rendererRef.current) connection.reportRendererError(mountEpoch, "webgl_unavailable")
115
+ }
116
+ buildRenderer()
117
+
118
+ // Context loss rebuilds only the renderer. The connection and the lease are not involved,
119
+ // and frames that arrive meanwhile are still acked so the sender does not stall.
120
+ const onContextRestored = () => {
121
+ if (unmounted) return
122
+ previewTrace("renderer_rebuilt", {mountEpoch, contextLosses})
123
+ buildRenderer()
124
+ }
125
+ canvas.addEventListener("webglcontextrestored", onContextRestored)
126
+
127
+ const sink: PreviewSink = {
128
+ mountEpoch,
129
+ draw: (frame) => rendererRef.current?.drawFrame(frame) ?? null,
130
+ onState: (state, detail) =>
131
+ emit({state, paused: detail.paused ?? false, reason: detail.reason, code: detail.code}),
132
+ onStatus: (stats) => emit({stats: {...stats, contextLosses}}),
133
+ onError: (code) => onErrorRef.current?.(code),
134
+ }
135
+ const detach = connection.attach(sink)
136
+ previewTrace("mount", {mountEpoch, docGen: connection.documentGeneration})
137
+
138
+ const push = () => connection.update(mountEpoch, {boxWidth: box.width, boxHeight: box.height, tier, running})
139
+
140
+ const coalescer = new ResizeCoalescer({
141
+ apply: (next) => {
142
+ box = next
143
+ const nextTier = quantizePreviewTier(next.width, next.height)
144
+ if (sameTier(nextTier, tier) && tier !== null) return
145
+ tier = nextTier
146
+ if (!tier) previewTrace("hidden_zero_size", {mountEpoch})
147
+ push()
148
+ },
149
+ })
150
+
151
+ let resizeObserver: ResizeObserver | null = null
152
+ const onWindowResize = () => coalescer.push(measure(canvas))
153
+ if (typeof ResizeObserver === "function") {
154
+ resizeObserver = new ResizeObserver(() => coalescer.push(measure(canvas)))
155
+ resizeObserver.observe(canvas)
156
+ } else if (typeof window !== "undefined") {
157
+ window.addEventListener("resize", onWindowResize)
158
+ }
159
+ coalescer.push(measure(canvas))
160
+
161
+ const onVisibility = () => {
162
+ const visible = isDocumentVisible()
163
+ if (visible === running) return
164
+ running = visible
165
+ previewTrace("visibility_change", {mountEpoch, visible})
166
+ push()
167
+ }
168
+ document.addEventListener("visibilitychange", onVisibility)
169
+
170
+ return () => {
171
+ unmounted = true
172
+ previewTrace("unmount", {mountEpoch, docGen: connection.documentGeneration})
173
+ document.removeEventListener("visibilitychange", onVisibility)
174
+ resizeObserver?.disconnect()
175
+ if (typeof window !== "undefined") window.removeEventListener("resize", onWindowResize)
176
+ coalescer.cancel()
177
+ canvas.removeEventListener("webglcontextrestored", onContextRestored)
178
+ running = false
179
+ push()
180
+ detach()
181
+ rendererRef.current?.dispose()
182
+ rendererRef.current = null
183
+ }
184
+ }, [])
185
+
186
+ return <canvas ref={canvasRef} className={className} style={{...DEFAULT_STYLE, ...style}} />
187
+ }
@@ -10,3 +10,5 @@ export type {MiniappHeaderProps} from "./MiniappHeader"
10
10
  export {MentraProvider} from "./MentraProvider"
11
11
  export type {MentraProviderProps} from "./MentraProvider"
12
12
  export {useRpc, type RpcCallable} from "./useRpc"
13
+ export {StreamPreview} from "./StreamPreview"
14
+ export type {StreamPreviewProps, StreamPreviewStatus, PreviewFit, PreviewConnectionState} from "./StreamPreview"