@mentra/miniapp 3.2.1-dev.314 → 3.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -0
- package/dist/background/index.d.ts +4 -2
- package/dist/background/index.d.ts.map +1 -1
- package/dist/background/index.js +1 -0
- package/dist/background/index.js.map +1 -1
- package/dist/index.d.ts +6 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/modules/display.d.ts +26 -0
- package/dist/modules/display.d.ts.map +1 -1
- package/dist/modules/display.js +1 -0
- package/dist/modules/display.js.map +1 -1
- package/dist/modules/meeting.d.ts +47 -0
- package/dist/modules/meeting.d.ts.map +1 -1
- package/dist/modules/meeting.js +63 -12
- package/dist/modules/meeting.js.map +1 -1
- package/dist/modules/mic.d.ts +5 -4
- package/dist/modules/mic.d.ts.map +1 -1
- package/dist/modules/mic.js +5 -4
- package/dist/modules/mic.js.map +1 -1
- package/dist/modules/speaker.d.ts +2 -0
- package/dist/modules/speaker.d.ts.map +1 -1
- package/dist/modules/speaker.js +5 -0
- package/dist/modules/speaker.js.map +1 -1
- package/dist/modules/stream.d.ts +46 -0
- package/dist/modules/stream.d.ts.map +1 -1
- package/dist/modules/stream.js +107 -0
- package/dist/modules/stream.js.map +1 -1
- package/dist/protocol.d.ts +38 -0
- package/dist/protocol.d.ts.map +1 -1
- package/dist/protocol.js +35 -0
- package/dist/protocol.js.map +1 -1
- package/dist/react/StreamPreview.d.ts +41 -0
- package/dist/react/StreamPreview.d.ts.map +1 -0
- package/dist/react/StreamPreview.js +144 -0
- package/dist/react/StreamPreview.js.map +1 -0
- package/dist/react/index.d.ts +3 -0
- package/dist/react/index.d.ts.map +1 -1
- package/dist/react/index.js +2 -0
- package/dist/react/index.js.map +1 -1
- package/dist/react/stream-preview/PreviewConnection.d.ts +139 -0
- package/dist/react/stream-preview/PreviewConnection.d.ts.map +1 -0
- package/dist/react/stream-preview/PreviewConnection.js +462 -0
- package/dist/react/stream-preview/PreviewConnection.js.map +1 -0
- package/dist/react/stream-preview/controlChannel.d.ts +75 -0
- package/dist/react/stream-preview/controlChannel.d.ts.map +1 -0
- package/dist/react/stream-preview/controlChannel.js +55 -0
- package/dist/react/stream-preview/controlChannel.js.map +1 -0
- package/dist/react/stream-preview/creditLoop.d.ts +81 -0
- package/dist/react/stream-preview/creditLoop.d.ts.map +1 -0
- package/dist/react/stream-preview/creditLoop.js +146 -0
- package/dist/react/stream-preview/creditLoop.js.map +1 -0
- package/dist/react/stream-preview/protocol.d.ts +113 -0
- package/dist/react/stream-preview/protocol.d.ts.map +1 -0
- package/dist/react/stream-preview/protocol.js +148 -0
- package/dist/react/stream-preview/protocol.js.map +1 -0
- package/dist/react/stream-preview/resizeCoalescer.d.ts +38 -0
- package/dist/react/stream-preview/resizeCoalescer.d.ts.map +1 -0
- package/dist/react/stream-preview/resizeCoalescer.js +71 -0
- package/dist/react/stream-preview/resizeCoalescer.js.map +1 -0
- package/dist/react/stream-preview/testFrames.d.ts +38 -0
- package/dist/react/stream-preview/testFrames.d.ts.map +1 -0
- package/dist/react/stream-preview/testFrames.js +50 -0
- package/dist/react/stream-preview/testFrames.js.map +1 -0
- package/dist/react/stream-preview/tiers.d.ts +21 -0
- package/dist/react/stream-preview/tiers.d.ts.map +1 -0
- package/dist/react/stream-preview/tiers.js +33 -0
- package/dist/react/stream-preview/tiers.js.map +1 -0
- package/dist/react/stream-preview/trace.d.ts +23 -0
- package/dist/react/stream-preview/trace.d.ts.map +1 -0
- package/dist/react/stream-preview/trace.js +71 -0
- package/dist/react/stream-preview/trace.js.map +1 -0
- package/dist/react/stream-preview/transport.d.ts +79 -0
- package/dist/react/stream-preview/transport.d.ts.map +1 -0
- package/dist/react/stream-preview/transport.js +209 -0
- package/dist/react/stream-preview/transport.js.map +1 -0
- package/dist/react/stream-preview/yuvRenderer.d.ts +31 -0
- package/dist/react/stream-preview/yuvRenderer.d.ts.map +1 -0
- package/dist/react/stream-preview/yuvRenderer.js +398 -0
- package/dist/react/stream-preview/yuvRenderer.js.map +1 -0
- package/dist/react/useHistoryState.d.ts +35 -0
- package/dist/react/useHistoryState.d.ts.map +1 -0
- package/dist/react/useHistoryState.js +117 -0
- package/dist/react/useHistoryState.js.map +1 -0
- package/dist/session.d.ts +5 -0
- package/dist/session.d.ts.map +1 -1
- package/dist/session.js +5 -0
- package/dist/session.js.map +1 -1
- package/dist/ui/index.d.ts +3 -0
- package/dist/ui/index.d.ts.map +1 -1
- package/dist/ui/index.js +2 -0
- package/dist/ui/index.js.map +1 -1
- package/package.json +8 -3
- package/src/background/index.ts +11 -1
- package/src/index.ts +12 -0
- package/src/modules/display.ts +29 -0
- package/src/modules/meeting.ts +104 -13
- package/src/modules/mic.ts +5 -4
- package/src/modules/speaker.ts +10 -4
- package/src/modules/stream.ts +153 -1
- package/src/protocol.ts +74 -0
- package/src/react/StreamPreview.tsx +187 -0
- package/src/react/index.ts +3 -0
- package/src/react/stream-preview/PreviewConnection.ts +552 -0
- package/src/react/stream-preview/controlChannel.ts +119 -0
- package/src/react/stream-preview/creditLoop.ts +189 -0
- package/src/react/stream-preview/protocol.ts +227 -0
- package/src/react/stream-preview/resizeCoalescer.ts +88 -0
- package/src/react/stream-preview/testFrames.ts +86 -0
- package/src/react/stream-preview/tiers.ts +37 -0
- package/src/react/stream-preview/trace.ts +81 -0
- package/src/react/stream-preview/transport.ts +242 -0
- package/src/react/stream-preview/yuvRenderer.ts +453 -0
- package/src/react/useHistoryState.ts +135 -0
- package/src/session.ts +8 -0
- package/src/ui/index.ts +3 -0
package/src/modules/stream.ts
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* to your own ingest URL with no relay.
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
|
-
import {MiniappRequestType} from "../protocol"
|
|
11
|
+
import {MiniappRequestType, type PreviewErrorCode, type PreviewSource} from "../protocol"
|
|
12
12
|
import {MiniappSession} from "../session"
|
|
13
13
|
|
|
14
14
|
export interface StreamVideoConfig {
|
|
@@ -133,9 +133,161 @@ export interface StreamStatus {
|
|
|
133
133
|
stats?: StreamLiveStats
|
|
134
134
|
}
|
|
135
135
|
|
|
136
|
+
export interface StreamPreviewOptions {
|
|
137
|
+
/** Defaults to `"call"`. `"glasses"` is reserved and rejects with `unsupported`. */
|
|
138
|
+
source?: PreviewSource
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
export type PreviewHandleState = "held" | "ended"
|
|
142
|
+
|
|
143
|
+
export interface PreviewHandleStatus {
|
|
144
|
+
state: PreviewHandleState
|
|
145
|
+
/** Set when `state` is `"ended"`: `stopped`, `source_ended`, `runtime_stopped`, ... */
|
|
146
|
+
reason?: string
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/** A preview request or lease failed. `code` is one of {@link PreviewErrorCode} when the host sent one. */
|
|
150
|
+
export class PreviewError extends Error {
|
|
151
|
+
readonly code: PreviewErrorCode | string
|
|
152
|
+
|
|
153
|
+
constructor(code: PreviewErrorCode | string, message?: string) {
|
|
154
|
+
super(message ?? code)
|
|
155
|
+
this.name = "PreviewError"
|
|
156
|
+
this.code = code
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* The background-owned lease on a preview source.
|
|
162
|
+
*
|
|
163
|
+
* The lease is what lets a `<StreamPreview>` in the UI show frames. It survives the UI closing and
|
|
164
|
+
* the WebView being destroyed, and ends on `stop()`, when the source ends (the meeting is over), or
|
|
165
|
+
* when the background runtime stops. Hiding the component does not release it.
|
|
166
|
+
*/
|
|
167
|
+
export interface PreviewHandle {
|
|
168
|
+
readonly source: PreviewSource
|
|
169
|
+
readonly handleId: string
|
|
170
|
+
/** Correlation id stamped on every `PREVIEW_TRACE` log line for this lease. */
|
|
171
|
+
readonly previewTraceId: string
|
|
172
|
+
readonly state: PreviewHandleState
|
|
173
|
+
/** Release the lease. Idempotent. */
|
|
174
|
+
stop(): Promise<void>
|
|
175
|
+
/** Lease transitions. Fires `{state: "ended", reason}` exactly once. */
|
|
176
|
+
onStatus(cb: (status: PreviewHandleStatus) => void): () => void
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
interface PreviewStartResult {
|
|
180
|
+
handleId: string
|
|
181
|
+
previewTraceId: string
|
|
182
|
+
source: PreviewSource
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
function toPreviewError(error: unknown): PreviewError {
|
|
186
|
+
if (error instanceof PreviewError) return error
|
|
187
|
+
const value = (error ?? {}) as {code?: unknown; message?: unknown}
|
|
188
|
+
const code = typeof value.code === "string" ? value.code : "unsupported"
|
|
189
|
+
const message = typeof value.message === "string" ? value.message : undefined
|
|
190
|
+
return new PreviewError(code, message)
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
class PreviewHandleImpl implements PreviewHandle {
|
|
194
|
+
state: PreviewHandleState = "held"
|
|
195
|
+
private readonly listeners = new Set<(status: PreviewHandleStatus) => void>()
|
|
196
|
+
|
|
197
|
+
constructor(
|
|
198
|
+
readonly source: PreviewSource,
|
|
199
|
+
readonly handleId: string,
|
|
200
|
+
readonly previewTraceId: string,
|
|
201
|
+
private readonly release: (handle: PreviewHandleImpl) => Promise<void>,
|
|
202
|
+
) {}
|
|
203
|
+
|
|
204
|
+
async stop(): Promise<void> {
|
|
205
|
+
if (this.state === "ended") return
|
|
206
|
+
this.end("stopped")
|
|
207
|
+
await this.release(this)
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
onStatus(cb: (status: PreviewHandleStatus) => void): () => void {
|
|
211
|
+
this.listeners.add(cb)
|
|
212
|
+
return () => this.listeners.delete(cb)
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/** @internal */
|
|
216
|
+
end(reason: string): void {
|
|
217
|
+
if (this.state === "ended") return
|
|
218
|
+
this.state = "ended"
|
|
219
|
+
this.emit({state: "ended", reason})
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
private emit(status: PreviewHandleStatus): void {
|
|
223
|
+
for (const listener of [...this.listeners]) {
|
|
224
|
+
try {
|
|
225
|
+
listener(status)
|
|
226
|
+
} catch {
|
|
227
|
+
// A throwing listener must not stop the others from hearing that the lease ended.
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
|
|
136
233
|
export class StreamModule {
|
|
234
|
+
private readonly previewHandles = new Map<string, PreviewHandleImpl>()
|
|
235
|
+
|
|
137
236
|
constructor(private readonly session: MiniappSession) {}
|
|
138
237
|
|
|
238
|
+
/**
|
|
239
|
+
* Take the preview lease on a source so a `<StreamPreview>` in the UI can show its frames.
|
|
240
|
+
*
|
|
241
|
+
* For `"call"` the caller must declare `CAMERA` and own the active meeting (`session.meeting`).
|
|
242
|
+
* Only one lease exists at a time; a second request fails with `preview_busy`. Rejects with a
|
|
243
|
+
* {@link PreviewError}. Never await this on a call's connect path: preview is additive.
|
|
244
|
+
*/
|
|
245
|
+
async preview(options: StreamPreviewOptions = {}): Promise<PreviewHandle> {
|
|
246
|
+
const source = options.source ?? "call"
|
|
247
|
+
if (source !== "call") {
|
|
248
|
+
throw new PreviewError("unsupported", `Preview source "${String(source)}" is not supported yet`)
|
|
249
|
+
}
|
|
250
|
+
let result: PreviewStartResult
|
|
251
|
+
try {
|
|
252
|
+
result = await this.session.sendRequest<PreviewStartResult>({
|
|
253
|
+
type: MiniappRequestType.STREAM_PREVIEW_START,
|
|
254
|
+
source,
|
|
255
|
+
})
|
|
256
|
+
} catch (error) {
|
|
257
|
+
throw toPreviewError(error)
|
|
258
|
+
}
|
|
259
|
+
const handle = new PreviewHandleImpl(result.source ?? source, result.handleId, result.previewTraceId, (h) =>
|
|
260
|
+
this.releasePreview(h),
|
|
261
|
+
)
|
|
262
|
+
this.previewHandles.set(handle.handleId, handle)
|
|
263
|
+
handle.onStatus((status) => {
|
|
264
|
+
if (status.state === "ended") this.previewHandles.delete(handle.handleId)
|
|
265
|
+
})
|
|
266
|
+
return handle
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
private async releasePreview(handle: PreviewHandleImpl): Promise<void> {
|
|
270
|
+
this.previewHandles.delete(handle.handleId)
|
|
271
|
+
try {
|
|
272
|
+
await this.session.sendRequest<void>({
|
|
273
|
+
type: MiniappRequestType.STREAM_PREVIEW_STOP,
|
|
274
|
+
handleId: handle.handleId,
|
|
275
|
+
})
|
|
276
|
+
} catch (error) {
|
|
277
|
+
throw toPreviewError(error)
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/** @internal — applied by MiniappSession on inbound STREAM_PREVIEW_STATUS. */
|
|
282
|
+
_applyPreviewStatus(payload: Record<string, unknown>): void {
|
|
283
|
+
if (typeof payload.handleId !== "string") return
|
|
284
|
+
const handle = this.previewHandles.get(payload.handleId)
|
|
285
|
+
if (!handle) return
|
|
286
|
+
if (payload.state === "ended") {
|
|
287
|
+
handle.end(typeof payload.reason === "string" ? payload.reason : "ended")
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
|
|
139
291
|
/**
|
|
140
292
|
* Start a live video stream from the glasses camera.
|
|
141
293
|
*
|
package/src/protocol.ts
CHANGED
|
@@ -196,6 +196,13 @@ export enum MiniappRequestType {
|
|
|
196
196
|
STREAM_STOP = "miniapp_stream_stop",
|
|
197
197
|
MANAGED_STREAM_START = "miniapp_managed_stream_start",
|
|
198
198
|
MANAGED_STREAM_STOP = "miniapp_managed_stream_stop",
|
|
199
|
+
/**
|
|
200
|
+
* Take the single raw decoded-frame preview lease on a source. See
|
|
201
|
+
* `session.stream.preview()`. Replies `{handleId, previewTraceId, source}`.
|
|
202
|
+
*/
|
|
203
|
+
STREAM_PREVIEW_START = "miniapp_stream_preview_start",
|
|
204
|
+
/** Release a preview lease by `handleId`. */
|
|
205
|
+
STREAM_PREVIEW_STOP = "miniapp_stream_preview_stop",
|
|
199
206
|
|
|
200
207
|
/** Ask the host to open the glasses Wi-Fi setup flow (mirrors the cloud SDK's requestWifiSetup). */
|
|
201
208
|
REQUEST_WIFI_SETUP = "miniapp_request_wifi_setup",
|
|
@@ -220,7 +227,10 @@ export enum MiniappRequestType {
|
|
|
220
227
|
* Phone-native meeting (ACS Teams). Join/leave/mute live in the MentraOS
|
|
221
228
|
* host so the miniapp never holds the ACS Calling SDK.
|
|
222
229
|
*/
|
|
230
|
+
MEETING_GET_IDENTITY = "miniapp_meeting_get_identity",
|
|
223
231
|
MEETING_GET_CONFIGURATION = "miniapp_meeting_get_configuration",
|
|
232
|
+
MEETING_CREATE = "miniapp_meeting_create",
|
|
233
|
+
MEETING_RETIRE = "miniapp_meeting_retire",
|
|
224
234
|
MEETING_JOIN = "miniapp_meeting_join",
|
|
225
235
|
MEETING_LEAVE = "miniapp_meeting_leave",
|
|
226
236
|
/** Terminate the meeting for everyone, not just this device. See `meeting.end()`. */
|
|
@@ -228,6 +238,8 @@ export enum MiniappRequestType {
|
|
|
228
238
|
/** Admit one waiting participant, when the host has Teams lobby permission. */
|
|
229
239
|
MEETING_ADMIT = "miniapp_meeting_admit",
|
|
230
240
|
MEETING_SET_MUTED = "miniapp_meeting_set_muted",
|
|
241
|
+
/** Stop or resume the glasses camera the meeting receives, without leaving. */
|
|
242
|
+
MEETING_SET_VIDEO_ENABLED = "miniapp_meeting_set_video_enabled",
|
|
231
243
|
MEETING_UPDATE_VIDEO_SOURCE = "miniapp_meeting_update_video_source",
|
|
232
244
|
MEETING_GET_STATE = "miniapp_meeting_get_state",
|
|
233
245
|
}
|
|
@@ -292,6 +304,12 @@ export enum MiniappResponseType {
|
|
|
292
304
|
*/
|
|
293
305
|
MEETING_STATE = "miniapp_meeting_state",
|
|
294
306
|
|
|
307
|
+
/**
|
|
308
|
+
* Push: a preview lease changed state. Carries {handleId, state: "held" | "ended", reason?}.
|
|
309
|
+
* See PreviewHandle.onStatus().
|
|
310
|
+
*/
|
|
311
|
+
STREAM_PREVIEW_STATUS = "miniapp_stream_preview_status",
|
|
312
|
+
|
|
295
313
|
/**
|
|
296
314
|
* Push: phone is about to tear down the miniapp's session. Gives the SDK
|
|
297
315
|
* a brief window (~50ms grace on the phone side) to fire one last
|
|
@@ -432,3 +450,59 @@ export enum MiniappErrorCode {
|
|
|
432
450
|
/** `blob.importFile` failed (not the user cancelling — that resolves to null). */
|
|
433
451
|
BLOB_IMPORT_FAILED = "BLOB_IMPORT_FAILED",
|
|
434
452
|
}
|
|
453
|
+
|
|
454
|
+
// ============================================================================
|
|
455
|
+
// Stream preview
|
|
456
|
+
// ============================================================================
|
|
457
|
+
|
|
458
|
+
/**
|
|
459
|
+
* Where a raw preview comes from. `"call"` is the decoded video of the phone-native meeting the
|
|
460
|
+
* caller owns. `"glasses"` is reserved and currently rejects with `unsupported`.
|
|
461
|
+
*/
|
|
462
|
+
export type PreviewSource = "call" | "glasses"
|
|
463
|
+
|
|
464
|
+
/**
|
|
465
|
+
* Typed preview failures and non-terminal statuses, shared by the background handle, the UI
|
|
466
|
+
* component and the host. `waiting_for_lease` and `paused_background` are statuses, not errors.
|
|
467
|
+
*/
|
|
468
|
+
export type PreviewErrorCode =
|
|
469
|
+
| "permission_denied"
|
|
470
|
+
| "not_meeting_owner"
|
|
471
|
+
| "preview_busy"
|
|
472
|
+
| "waiting_for_lease"
|
|
473
|
+
| "source_ended"
|
|
474
|
+
| "unsupported"
|
|
475
|
+
| "ack_timeout"
|
|
476
|
+
| "transport_failed"
|
|
477
|
+
| "pack_failed"
|
|
478
|
+
| "paused_background"
|
|
479
|
+
| "diagnostics_disabled"
|
|
480
|
+
|
|
481
|
+
export const PREVIEW_ERROR_CODES: readonly PreviewErrorCode[] = [
|
|
482
|
+
"permission_denied",
|
|
483
|
+
"not_meeting_owner",
|
|
484
|
+
"preview_busy",
|
|
485
|
+
"waiting_for_lease",
|
|
486
|
+
"source_ended",
|
|
487
|
+
"unsupported",
|
|
488
|
+
"ack_timeout",
|
|
489
|
+
"transport_failed",
|
|
490
|
+
"pack_failed",
|
|
491
|
+
"paused_background",
|
|
492
|
+
"diagnostics_disabled",
|
|
493
|
+
]
|
|
494
|
+
|
|
495
|
+
export function isPreviewErrorCode(value: unknown): value is PreviewErrorCode {
|
|
496
|
+
return typeof value === "string" && (PREVIEW_ERROR_CODES as readonly string[]).includes(value)
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
/** Lease state pushed to the background with `STREAM_PREVIEW_STATUS`. */
|
|
500
|
+
export interface PreviewStatus {
|
|
501
|
+
handleId: string
|
|
502
|
+
state: "held" | "ended"
|
|
503
|
+
/** Why the lease ended: `stopped`, `source_ended`, `runtime_stopped`, ... */
|
|
504
|
+
reason?: string
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
/** The single reserved UI channel the host answers itself for preview control. */
|
|
508
|
+
export const PREVIEW_UI_CHANNEL = "_preview"
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `<StreamPreview>` — draws the raw decoded frames of the source the miniapp's background has
|
|
3
|
+
* leased with `session.stream.preview()`.
|
|
4
|
+
*
|
|
5
|
+
* The component owns only the renderer (canvas, WebGL context, textures). Mounting it shows the
|
|
6
|
+
* preview; unmounting it hides the preview and stops production, but keeps the document's
|
|
7
|
+
* connection open so the next mount starts without a handshake. Releasing the lease is the
|
|
8
|
+
* background's job (`handle.stop()`), not this component's.
|
|
9
|
+
*
|
|
10
|
+
* Frame bytes never enter React state: frames go from the transport to the renderer directly.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import {useEffect, useRef, type CSSProperties} from "react"
|
|
14
|
+
|
|
15
|
+
import type {PreviewConnectionState, PreviewSink} from "./stream-preview/PreviewConnection"
|
|
16
|
+
import {getPreviewConnection} from "./stream-preview/PreviewConnection"
|
|
17
|
+
import {ResizeCoalescer, type PreviewBox} from "./stream-preview/resizeCoalescer"
|
|
18
|
+
import {quantizePreviewTier, sameTier, type PreviewTier} from "./stream-preview/tiers"
|
|
19
|
+
import {previewTrace, previewTraceWarn} from "./stream-preview/trace"
|
|
20
|
+
import {
|
|
21
|
+
createYuvRenderer,
|
|
22
|
+
type PreviewFit,
|
|
23
|
+
type YuvRenderer,
|
|
24
|
+
type YuvRendererOptions,
|
|
25
|
+
} from "./stream-preview/yuvRenderer"
|
|
26
|
+
|
|
27
|
+
export type {PreviewFit} from "./stream-preview/yuvRenderer"
|
|
28
|
+
export type {PreviewConnectionState} from "./stream-preview/PreviewConnection"
|
|
29
|
+
|
|
30
|
+
export interface StreamPreviewStatus {
|
|
31
|
+
state: PreviewConnectionState
|
|
32
|
+
/** The Mentra App is in the background; production is paused and resumes on return. */
|
|
33
|
+
paused: boolean
|
|
34
|
+
/** Why the state changed, when the host said (`source_ended`, `stopped`, ...). */
|
|
35
|
+
reason?: string
|
|
36
|
+
/** Error code for `error` / `unsupported`. */
|
|
37
|
+
code?: string
|
|
38
|
+
/** Latest once-a-second host counters plus page-side counters. */
|
|
39
|
+
stats?: Record<string, unknown>
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export interface StreamPreviewProps {
|
|
43
|
+
/** `contain` letterboxes (default); `cover` fills and crops. The picture is never stretched. */
|
|
44
|
+
fit?: PreviewFit
|
|
45
|
+
onStatus?: (status: StreamPreviewStatus) => void
|
|
46
|
+
/** Typed preview error codes: `source_ended`, `unsupported`, `transport_failed`, ... */
|
|
47
|
+
onError?: (code: string) => void
|
|
48
|
+
className?: string
|
|
49
|
+
style?: CSSProperties
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
type RendererFactory = (canvas: HTMLCanvasElement, options: YuvRendererOptions) => YuvRenderer | null
|
|
53
|
+
|
|
54
|
+
let rendererFactory: RendererFactory = createYuvRenderer
|
|
55
|
+
|
|
56
|
+
/** Test seam: WebGL is not available in headless DOMs. */
|
|
57
|
+
export function setStreamPreviewRendererFactoryForTests(factory: RendererFactory | null): void {
|
|
58
|
+
rendererFactory = factory ?? createYuvRenderer
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
const DEFAULT_STYLE: CSSProperties = {display: "block", width: "100%", height: "100%"}
|
|
62
|
+
|
|
63
|
+
function measure(canvas: HTMLCanvasElement): PreviewBox {
|
|
64
|
+
const rect = canvas.getBoundingClientRect()
|
|
65
|
+
const ratio = typeof window === "undefined" ? 1 : window.devicePixelRatio || 1
|
|
66
|
+
return {width: Math.round(rect.width * ratio), height: Math.round(rect.height * ratio)}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function isDocumentVisible(): boolean {
|
|
70
|
+
return typeof document === "undefined" || document.visibilityState !== "hidden"
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export function StreamPreview({fit = "contain", onStatus, onError, className, style}: StreamPreviewProps) {
|
|
74
|
+
const canvasRef = useRef<HTMLCanvasElement | null>(null)
|
|
75
|
+
const rendererRef = useRef<YuvRenderer | null>(null)
|
|
76
|
+
const fitRef = useRef(fit)
|
|
77
|
+
const onStatusRef = useRef(onStatus)
|
|
78
|
+
const onErrorRef = useRef(onError)
|
|
79
|
+
fitRef.current = fit
|
|
80
|
+
onStatusRef.current = onStatus
|
|
81
|
+
onErrorRef.current = onError
|
|
82
|
+
|
|
83
|
+
useEffect(() => {
|
|
84
|
+
rendererRef.current?.setFit(fit)
|
|
85
|
+
}, [fit])
|
|
86
|
+
|
|
87
|
+
useEffect(() => {
|
|
88
|
+
const canvas = canvasRef.current
|
|
89
|
+
if (!canvas) return
|
|
90
|
+
const connection = getPreviewConnection()
|
|
91
|
+
const mountEpoch = connection.nextMountEpoch()
|
|
92
|
+
let contextLosses = 0
|
|
93
|
+
let status: StreamPreviewStatus = {state: connection.currentState, paused: false}
|
|
94
|
+
let box: PreviewBox = {width: 0, height: 0}
|
|
95
|
+
let tier: PreviewTier | null = null
|
|
96
|
+
let running = isDocumentVisible()
|
|
97
|
+
let unmounted = false
|
|
98
|
+
|
|
99
|
+
const emit = (next: Partial<StreamPreviewStatus>) => {
|
|
100
|
+
status = {...status, ...next}
|
|
101
|
+
onStatusRef.current?.(status)
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
const buildRenderer = () => {
|
|
105
|
+
rendererRef.current?.dispose()
|
|
106
|
+
rendererRef.current = rendererFactory(canvas, {
|
|
107
|
+
fit: fitRef.current,
|
|
108
|
+
onError: (reason) => {
|
|
109
|
+
previewTraceWarn("renderer_error", {reason, mountEpoch, docGen: connection.documentGeneration})
|
|
110
|
+
connection.reportRendererError(mountEpoch, reason)
|
|
111
|
+
if (reason === "webgl_context_lost") contextLosses += 1
|
|
112
|
+
},
|
|
113
|
+
})
|
|
114
|
+
if (!rendererRef.current) connection.reportRendererError(mountEpoch, "webgl_unavailable")
|
|
115
|
+
}
|
|
116
|
+
buildRenderer()
|
|
117
|
+
|
|
118
|
+
// Context loss rebuilds only the renderer. The connection and the lease are not involved,
|
|
119
|
+
// and frames that arrive meanwhile are still acked so the sender does not stall.
|
|
120
|
+
const onContextRestored = () => {
|
|
121
|
+
if (unmounted) return
|
|
122
|
+
previewTrace("renderer_rebuilt", {mountEpoch, contextLosses})
|
|
123
|
+
buildRenderer()
|
|
124
|
+
}
|
|
125
|
+
canvas.addEventListener("webglcontextrestored", onContextRestored)
|
|
126
|
+
|
|
127
|
+
const sink: PreviewSink = {
|
|
128
|
+
mountEpoch,
|
|
129
|
+
draw: (frame) => rendererRef.current?.drawFrame(frame) ?? null,
|
|
130
|
+
onState: (state, detail) =>
|
|
131
|
+
emit({state, paused: detail.paused ?? false, reason: detail.reason, code: detail.code}),
|
|
132
|
+
onStatus: (stats) => emit({stats: {...stats, contextLosses}}),
|
|
133
|
+
onError: (code) => onErrorRef.current?.(code),
|
|
134
|
+
}
|
|
135
|
+
const detach = connection.attach(sink)
|
|
136
|
+
previewTrace("mount", {mountEpoch, docGen: connection.documentGeneration})
|
|
137
|
+
|
|
138
|
+
const push = () => connection.update(mountEpoch, {boxWidth: box.width, boxHeight: box.height, tier, running})
|
|
139
|
+
|
|
140
|
+
const coalescer = new ResizeCoalescer({
|
|
141
|
+
apply: (next) => {
|
|
142
|
+
box = next
|
|
143
|
+
const nextTier = quantizePreviewTier(next.width, next.height)
|
|
144
|
+
if (sameTier(nextTier, tier) && tier !== null) return
|
|
145
|
+
tier = nextTier
|
|
146
|
+
if (!tier) previewTrace("hidden_zero_size", {mountEpoch})
|
|
147
|
+
push()
|
|
148
|
+
},
|
|
149
|
+
})
|
|
150
|
+
|
|
151
|
+
let resizeObserver: ResizeObserver | null = null
|
|
152
|
+
const onWindowResize = () => coalescer.push(measure(canvas))
|
|
153
|
+
if (typeof ResizeObserver === "function") {
|
|
154
|
+
resizeObserver = new ResizeObserver(() => coalescer.push(measure(canvas)))
|
|
155
|
+
resizeObserver.observe(canvas)
|
|
156
|
+
} else if (typeof window !== "undefined") {
|
|
157
|
+
window.addEventListener("resize", onWindowResize)
|
|
158
|
+
}
|
|
159
|
+
coalescer.push(measure(canvas))
|
|
160
|
+
|
|
161
|
+
const onVisibility = () => {
|
|
162
|
+
const visible = isDocumentVisible()
|
|
163
|
+
if (visible === running) return
|
|
164
|
+
running = visible
|
|
165
|
+
previewTrace("visibility_change", {mountEpoch, visible})
|
|
166
|
+
push()
|
|
167
|
+
}
|
|
168
|
+
document.addEventListener("visibilitychange", onVisibility)
|
|
169
|
+
|
|
170
|
+
return () => {
|
|
171
|
+
unmounted = true
|
|
172
|
+
previewTrace("unmount", {mountEpoch, docGen: connection.documentGeneration})
|
|
173
|
+
document.removeEventListener("visibilitychange", onVisibility)
|
|
174
|
+
resizeObserver?.disconnect()
|
|
175
|
+
if (typeof window !== "undefined") window.removeEventListener("resize", onWindowResize)
|
|
176
|
+
coalescer.cancel()
|
|
177
|
+
canvas.removeEventListener("webglcontextrestored", onContextRestored)
|
|
178
|
+
running = false
|
|
179
|
+
push()
|
|
180
|
+
detach()
|
|
181
|
+
rendererRef.current?.dispose()
|
|
182
|
+
rendererRef.current = null
|
|
183
|
+
}
|
|
184
|
+
}, [])
|
|
185
|
+
|
|
186
|
+
return <canvas ref={canvasRef} className={className} style={{...DEFAULT_STYLE, ...style}} />
|
|
187
|
+
}
|
package/src/react/index.ts
CHANGED
|
@@ -10,3 +10,6 @@ export type {MiniappHeaderProps} from "./MiniappHeader"
|
|
|
10
10
|
export {MentraProvider} from "./MentraProvider"
|
|
11
11
|
export type {MentraProviderProps} from "./MentraProvider"
|
|
12
12
|
export {useRpc, type RpcCallable} from "./useRpc"
|
|
13
|
+
export {useHistoryState, type SetHistoryState} from "./useHistoryState"
|
|
14
|
+
export {StreamPreview} from "./StreamPreview"
|
|
15
|
+
export type {StreamPreviewProps, StreamPreviewStatus, PreviewFit, PreviewConnectionState} from "./StreamPreview"
|