@mentra/miniapp 3.2.1-dev.314 → 3.3.0-beta.1000

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/README.md +16 -0
  2. package/dist/background/index.d.ts +6 -3
  3. package/dist/background/index.d.ts.map +1 -1
  4. package/dist/background/index.js +1 -0
  5. package/dist/background/index.js.map +1 -1
  6. package/dist/background/register.d.ts +6 -0
  7. package/dist/background/register.d.ts.map +1 -1
  8. package/dist/background/register.js +19 -4
  9. package/dist/background/register.js.map +1 -1
  10. package/dist/index.d.ts +7 -4
  11. package/dist/index.d.ts.map +1 -1
  12. package/dist/index.js +2 -0
  13. package/dist/index.js.map +1 -1
  14. package/dist/modules/display.d.ts +26 -0
  15. package/dist/modules/display.d.ts.map +1 -1
  16. package/dist/modules/display.js +1 -0
  17. package/dist/modules/display.js.map +1 -1
  18. package/dist/modules/events.d.ts +8 -4
  19. package/dist/modules/events.d.ts.map +1 -1
  20. package/dist/modules/events.js +46 -6
  21. package/dist/modules/events.js.map +1 -1
  22. package/dist/modules/meeting.d.ts +129 -0
  23. package/dist/modules/meeting.d.ts.map +1 -1
  24. package/dist/modules/meeting.js +200 -14
  25. package/dist/modules/meeting.js.map +1 -1
  26. package/dist/modules/mic.d.ts +5 -4
  27. package/dist/modules/mic.d.ts.map +1 -1
  28. package/dist/modules/mic.js +5 -4
  29. package/dist/modules/mic.js.map +1 -1
  30. package/dist/modules/miniapps.d.ts +101 -4
  31. package/dist/modules/miniapps.d.ts.map +1 -1
  32. package/dist/modules/miniapps.js +59 -4
  33. package/dist/modules/miniapps.js.map +1 -1
  34. package/dist/modules/speaker.d.ts +2 -0
  35. package/dist/modules/speaker.d.ts.map +1 -1
  36. package/dist/modules/speaker.js +5 -0
  37. package/dist/modules/speaker.js.map +1 -1
  38. package/dist/modules/stream.d.ts +46 -0
  39. package/dist/modules/stream.d.ts.map +1 -1
  40. package/dist/modules/stream.js +107 -0
  41. package/dist/modules/stream.js.map +1 -1
  42. package/dist/protocol.d.ts +86 -1
  43. package/dist/protocol.d.ts.map +1 -1
  44. package/dist/protocol.js +82 -0
  45. package/dist/protocol.js.map +1 -1
  46. package/dist/react/StreamPreview.d.ts +41 -0
  47. package/dist/react/StreamPreview.d.ts.map +1 -0
  48. package/dist/react/StreamPreview.js +144 -0
  49. package/dist/react/StreamPreview.js.map +1 -0
  50. package/dist/react/index.d.ts +3 -0
  51. package/dist/react/index.d.ts.map +1 -1
  52. package/dist/react/index.js +2 -0
  53. package/dist/react/index.js.map +1 -1
  54. package/dist/react/stream-preview/PreviewConnection.d.ts +146 -0
  55. package/dist/react/stream-preview/PreviewConnection.d.ts.map +1 -0
  56. package/dist/react/stream-preview/PreviewConnection.js +493 -0
  57. package/dist/react/stream-preview/PreviewConnection.js.map +1 -0
  58. package/dist/react/stream-preview/controlChannel.d.ts +75 -0
  59. package/dist/react/stream-preview/controlChannel.d.ts.map +1 -0
  60. package/dist/react/stream-preview/controlChannel.js +55 -0
  61. package/dist/react/stream-preview/controlChannel.js.map +1 -0
  62. package/dist/react/stream-preview/creditLoop.d.ts +81 -0
  63. package/dist/react/stream-preview/creditLoop.d.ts.map +1 -0
  64. package/dist/react/stream-preview/creditLoop.js +146 -0
  65. package/dist/react/stream-preview/creditLoop.js.map +1 -0
  66. package/dist/react/stream-preview/protocol.d.ts +113 -0
  67. package/dist/react/stream-preview/protocol.d.ts.map +1 -0
  68. package/dist/react/stream-preview/protocol.js +148 -0
  69. package/dist/react/stream-preview/protocol.js.map +1 -0
  70. package/dist/react/stream-preview/resizeCoalescer.d.ts +38 -0
  71. package/dist/react/stream-preview/resizeCoalescer.d.ts.map +1 -0
  72. package/dist/react/stream-preview/resizeCoalescer.js +71 -0
  73. package/dist/react/stream-preview/resizeCoalescer.js.map +1 -0
  74. package/dist/react/stream-preview/testFrames.d.ts +38 -0
  75. package/dist/react/stream-preview/testFrames.d.ts.map +1 -0
  76. package/dist/react/stream-preview/testFrames.js +50 -0
  77. package/dist/react/stream-preview/testFrames.js.map +1 -0
  78. package/dist/react/stream-preview/tiers.d.ts +21 -0
  79. package/dist/react/stream-preview/tiers.d.ts.map +1 -0
  80. package/dist/react/stream-preview/tiers.js +33 -0
  81. package/dist/react/stream-preview/tiers.js.map +1 -0
  82. package/dist/react/stream-preview/trace.d.ts +23 -0
  83. package/dist/react/stream-preview/trace.d.ts.map +1 -0
  84. package/dist/react/stream-preview/trace.js +71 -0
  85. package/dist/react/stream-preview/trace.js.map +1 -0
  86. package/dist/react/stream-preview/transport.d.ts +79 -0
  87. package/dist/react/stream-preview/transport.d.ts.map +1 -0
  88. package/dist/react/stream-preview/transport.js +209 -0
  89. package/dist/react/stream-preview/transport.js.map +1 -0
  90. package/dist/react/stream-preview/yuvRenderer.d.ts +31 -0
  91. package/dist/react/stream-preview/yuvRenderer.d.ts.map +1 -0
  92. package/dist/react/stream-preview/yuvRenderer.js +398 -0
  93. package/dist/react/stream-preview/yuvRenderer.js.map +1 -0
  94. package/dist/react/useHistoryState.d.ts +35 -0
  95. package/dist/react/useHistoryState.d.ts.map +1 -0
  96. package/dist/react/useHistoryState.js +117 -0
  97. package/dist/react/useHistoryState.js.map +1 -0
  98. package/dist/session.d.ts +25 -0
  99. package/dist/session.d.ts.map +1 -1
  100. package/dist/session.js +61 -1
  101. package/dist/session.js.map +1 -1
  102. package/dist/ui/index.d.ts +3 -0
  103. package/dist/ui/index.d.ts.map +1 -1
  104. package/dist/ui/index.js +2 -0
  105. package/dist/ui/index.js.map +1 -1
  106. package/package.json +8 -3
  107. package/src/background/index.ts +28 -1
  108. package/src/background/register.ts +28 -6
  109. package/src/index.ts +29 -1
  110. package/src/modules/display.ts +29 -0
  111. package/src/modules/events.ts +51 -6
  112. package/src/modules/meeting.ts +292 -15
  113. package/src/modules/mic.ts +5 -4
  114. package/src/modules/miniapps.ts +145 -4
  115. package/src/modules/speaker.ts +10 -4
  116. package/src/modules/stream.ts +153 -1
  117. package/src/protocol.ts +125 -0
  118. package/src/react/StreamPreview.tsx +187 -0
  119. package/src/react/index.ts +3 -0
  120. package/src/react/stream-preview/PreviewConnection.ts +587 -0
  121. package/src/react/stream-preview/controlChannel.ts +119 -0
  122. package/src/react/stream-preview/creditLoop.ts +189 -0
  123. package/src/react/stream-preview/protocol.ts +227 -0
  124. package/src/react/stream-preview/resizeCoalescer.ts +88 -0
  125. package/src/react/stream-preview/testFrames.ts +86 -0
  126. package/src/react/stream-preview/tiers.ts +37 -0
  127. package/src/react/stream-preview/trace.ts +81 -0
  128. package/src/react/stream-preview/transport.ts +242 -0
  129. package/src/react/stream-preview/yuvRenderer.ts +453 -0
  130. package/src/react/useHistoryState.ts +135 -0
  131. package/src/session.ts +76 -0
  132. package/src/ui/index.ts +3 -0
@@ -0,0 +1,119 @@
1
+ /**
2
+ * The reserved `_preview` UI channel, as seen from the page.
3
+ *
4
+ * The Mentra App answers this channel itself; it never reaches the miniapp's background. Requests
5
+ * ride `mentra.request`, and host events arrive on `mentra.on("_preview")`. Every control request
6
+ * carries the page identity (`docGen`, `token`, `mountEpoch`) so the host can drop operations
7
+ * from a document or a component that has since been replaced.
8
+ */
9
+
10
+ import {PREVIEW_UI_CHANNEL} from "../../protocol"
11
+
12
+ /** `injectFault` is honoured only in debug builds or with the host's hidden developer setting. */
13
+ export type PreviewCommand = "handshake" | "configure" | "start" | "stop" | "rendererError" | "injectFault"
14
+
15
+ export interface PreviewControlRequest {
16
+ cmd: PreviewCommand
17
+ /** 0 until the first handshake has told the page which document it is. */
18
+ docGen: number
19
+ token?: string
20
+ mountEpoch: number
21
+ /** `configure` only: rendered box in device pixels, after rotation. */
22
+ boxWidth?: number
23
+ boxHeight?: number
24
+ /** `rendererError` only. */
25
+ message?: string
26
+ /** `injectFault` only: `ack_delay`, `ack_drop`, `transport_close`, `pack_throw`, `sink_throw`, `clear`. */
27
+ kind?: string
28
+ ms?: number
29
+ }
30
+
31
+ export interface PreviewDocumentConfig {
32
+ t: "config"
33
+ protocolVersion: number
34
+ transport: "webmessage" | "websocket"
35
+ url?: string
36
+ portName?: string
37
+ token: string
38
+ docGen: number
39
+ }
40
+
41
+ /** `stale`: the host minted a newer document while this handshake was in flight. */
42
+ export type PreviewHandshakeResult =
43
+ | PreviewDocumentConfig
44
+ | {t: "waiting_for_lease"; docGen: number}
45
+ | {t?: undefined; stale: true}
46
+
47
+ /** Reply to configure/start/stop. `stale` means the host dropped it; that is not an error. */
48
+ export interface PreviewControlReply {
49
+ applied?: boolean
50
+ stale?: boolean
51
+ }
52
+
53
+ export type PreviewHostEvent =
54
+ | PreviewDocumentConfig
55
+ | ({t: "status"} & Record<string, unknown>)
56
+ | {t: "lease_available"}
57
+ | {t: "lease_ended"; reason: string}
58
+ | {t: "error"; code: string; docGen?: number}
59
+ | {t: "paused_background"}
60
+ | {t: "resumed"}
61
+
62
+ /** A `_preview` request the host refused. `code` is a preview error code. */
63
+ export class PreviewControlError extends Error {
64
+ readonly code: string
65
+
66
+ constructor(code: string, message?: string) {
67
+ super(message ?? code)
68
+ this.name = "PreviewControlError"
69
+ this.code = code
70
+ }
71
+ }
72
+
73
+ export interface PreviewControlChannel {
74
+ request(request: PreviewControlRequest): Promise<unknown>
75
+ onEvent(cb: (event: PreviewHostEvent) => void): () => void
76
+ }
77
+
78
+ interface MentraLike {
79
+ request(channel: string, payload: unknown, options?: {timeout?: number}): Promise<unknown>
80
+ on(channel: string, cb: (payload: unknown) => void): () => void
81
+ }
82
+
83
+ /** Host replies are local and fast; this only bounds a host that never answers. */
84
+ const REQUEST_TIMEOUT_MS = 10_000
85
+
86
+ function mentraGlobal(): MentraLike | null {
87
+ const candidate = (globalThis as unknown as {mentra?: Partial<MentraLike>}).mentra
88
+ if (!candidate || typeof candidate.request !== "function" || typeof candidate.on !== "function") return null
89
+ return candidate as MentraLike
90
+ }
91
+
92
+ /** The channel over the injected `window.mentra` bridge. Null outside a Mentra App WebView. */
93
+ export function createMentraControlChannel(): PreviewControlChannel | null {
94
+ const mentra = mentraGlobal()
95
+ if (!mentra) return null
96
+ return {
97
+ async request(request) {
98
+ try {
99
+ return await mentra.request(PREVIEW_UI_CHANNEL, request, {timeout: REQUEST_TIMEOUT_MS})
100
+ } catch (error) {
101
+ const value = (error ?? {}) as {code?: unknown; name?: unknown; message?: unknown}
102
+ const code =
103
+ typeof value.code === "string"
104
+ ? value.code
105
+ : value.name === "MentraRpcTimeoutError"
106
+ ? "transport_failed"
107
+ : "unsupported"
108
+ throw new PreviewControlError(code, typeof value.message === "string" ? value.message : undefined)
109
+ }
110
+ },
111
+ onEvent(cb) {
112
+ return mentra.on(PREVIEW_UI_CHANNEL, (payload) => {
113
+ if (payload && typeof payload === "object" && typeof (payload as {t?: unknown}).t === "string") {
114
+ cb(payload as PreviewHostEvent)
115
+ }
116
+ })
117
+ },
118
+ }
119
+ }
@@ -0,0 +1,189 @@
1
+ /**
2
+ * Consumer side of the one-frame credit scheme.
3
+ *
4
+ * The native sender holds exactly one frame in flight and will not send the next until this side
5
+ * acknowledges the last, so there is no queue anywhere and a slow consumer shows up as a lower
6
+ * frame rate rather than as growing latency. This module is the half that makes that true: parse,
7
+ * draw, ack — and never hold a reference to a frame after its ack.
8
+ *
9
+ * Durations here would be `performance.now()` and describe JavaScript only; the header's
10
+ * `timestampNs` / `sentAtNs` are a native monotonic clock with a different epoch. End-to-end
11
+ * latency is measured natively (it sees both the send and the ack on one clock).
12
+ */
13
+
14
+ import {parsePreviewFrame, type ParsedFrame, type PreviewParseErrorReason} from "./protocol"
15
+
16
+ export interface CreditLoopFrameSummary {
17
+ gen: number
18
+ seq: number
19
+ width: number
20
+ height: number
21
+ pixelFormat: string
22
+ rotationQuarters: number
23
+ }
24
+
25
+ export interface CreditLoopStats {
26
+ framesReceived: number
27
+ drawsSubmitted: number
28
+ acksSent: number
29
+ parseErrors: number
30
+ parseErrorsByReason: Partial<Record<PreviewParseErrorReason, number>>
31
+ /**
32
+ * Frames that arrived while another was still being processed. Native guarantees one at a time,
33
+ * so any value above zero means the credit scheme is broken.
34
+ */
35
+ overlapped: number
36
+ /** Frames the sender skipped, counted from gaps in the header sequence. */
37
+ sequenceGaps: number
38
+ skippedBySender: number
39
+ lastError: string | null
40
+ lastFrame: CreditLoopFrameSummary | null
41
+ }
42
+
43
+ export interface CreditLoopOptions {
44
+ sendAck: (gen: number, seq: number) => void
45
+ /**
46
+ * Submit the frame. Returns the submit duration in ms, or null when there is no renderer (the
47
+ * component is unmounted or its GL context is lost). The frame is acked either way.
48
+ */
49
+ draw?: (frame: ParsedFrame) => number | null
50
+ /** Called once per rejected frame, before anything else happens to it. */
51
+ onParseError?: (reason: PreviewParseErrorReason, detail?: string) => void
52
+ }
53
+
54
+ export class CreditLoop {
55
+ private readonly errorsByReason = new Map<PreviewParseErrorReason, number>()
56
+ private idleWaiters: Array<() => void> = []
57
+ private inFlightCount = 0
58
+ private framesReceived = 0
59
+ private drawsSubmitted = 0
60
+ private acksSent = 0
61
+ private parseErrors = 0
62
+ private overlapped = 0
63
+ private sequenceGaps = 0
64
+ private skippedBySender = 0
65
+ private lastSeq: number | null = null
66
+ private lastGen: number | null = null
67
+ private lastError: string | null = null
68
+ private lastFrame: CreditLoopFrameSummary | null = null
69
+ private draw: CreditLoopOptions["draw"]
70
+
71
+ constructor(private readonly options: CreditLoopOptions) {
72
+ this.draw = options.draw
73
+ }
74
+
75
+ /** Swap the renderer without touching the credit state. Null means frames are acked undrawn. */
76
+ setDraw(draw: CreditLoopOptions["draw"] | null): void {
77
+ this.draw = draw ?? undefined
78
+ }
79
+
80
+ /** Frames currently being processed. The credit scheme means this is 0 or 1. */
81
+ get inFlight(): number {
82
+ return this.inFlightCount
83
+ }
84
+
85
+ /** Entry point for the transport. Deliberately not awaited: the transport must not block. */
86
+ handleBinary(buffer: ArrayBuffer): void {
87
+ if (this.inFlightCount > 0) this.overlapped += 1
88
+ this.inFlightCount += 1
89
+ void this.process(buffer).finally(() => {
90
+ this.inFlightCount -= 1
91
+ if (this.inFlightCount === 0) {
92
+ const waiters = this.idleWaiters
93
+ this.idleWaiters = []
94
+ for (const resolve of waiters) resolve()
95
+ }
96
+ })
97
+ }
98
+
99
+ /** Resolves once nothing is being processed. */
100
+ idle(): Promise<void> {
101
+ if (this.inFlightCount === 0) return Promise.resolve()
102
+ return new Promise((resolve) => this.idleWaiters.push(resolve))
103
+ }
104
+
105
+ recordError(reason: string): void {
106
+ this.lastError = reason
107
+ }
108
+
109
+ snapshot(): CreditLoopStats {
110
+ const parseErrorsByReason: Partial<Record<PreviewParseErrorReason, number>> = {}
111
+ for (const [reason, count] of this.errorsByReason) parseErrorsByReason[reason] = count
112
+ return {
113
+ framesReceived: this.framesReceived,
114
+ drawsSubmitted: this.drawsSubmitted,
115
+ acksSent: this.acksSent,
116
+ parseErrors: this.parseErrors,
117
+ parseErrorsByReason,
118
+ overlapped: this.overlapped,
119
+ sequenceGaps: this.sequenceGaps,
120
+ skippedBySender: this.skippedBySender,
121
+ lastError: this.lastError,
122
+ lastFrame: this.lastFrame,
123
+ }
124
+ }
125
+
126
+ reset(): void {
127
+ this.framesReceived = 0
128
+ this.drawsSubmitted = 0
129
+ this.acksSent = 0
130
+ this.parseErrors = 0
131
+ this.overlapped = 0
132
+ this.sequenceGaps = 0
133
+ this.skippedBySender = 0
134
+ this.lastSeq = null
135
+ this.lastGen = null
136
+ this.lastError = null
137
+ this.lastFrame = null
138
+ this.errorsByReason.clear()
139
+ }
140
+
141
+ private async process(buffer: ArrayBuffer): Promise<void> {
142
+ this.framesReceived += 1
143
+ const parsed = parsePreviewFrame(buffer)
144
+ if (!parsed.ok) {
145
+ // No ack: a frame this side could not read is not a frame it consumed, and acking it would
146
+ // hide the fault behind a healthy-looking rate.
147
+ this.parseErrors += 1
148
+ this.errorsByReason.set(parsed.reason, (this.errorsByReason.get(parsed.reason) ?? 0) + 1)
149
+ this.lastError = parsed.detail ? `${parsed.reason} (${parsed.detail})` : parsed.reason
150
+ this.options.onParseError?.(parsed.reason, parsed.detail)
151
+ return
152
+ }
153
+ this.trackSequence(parsed)
154
+ this.lastFrame = {
155
+ gen: parsed.sessionGen,
156
+ seq: parsed.frameSeq,
157
+ width: parsed.width,
158
+ height: parsed.height,
159
+ pixelFormat: parsed.pixelFormat,
160
+ rotationQuarters: parsed.rotationQuarters,
161
+ }
162
+ if (this.draw) {
163
+ const submitMs = this.draw(parsed)
164
+ if (submitMs === null) this.lastError = "draw_failed"
165
+ else this.drawsSubmitted += 1
166
+ }
167
+ // Ack last, after the draw has been submitted, so the credit the sender gets back means "this
168
+ // side is done with that frame" rather than "this side received some bytes". A dead canvas
169
+ // still acks: withholding credit would stall the sender on top of it.
170
+ this.options.sendAck(parsed.sessionGen, parsed.frameSeq)
171
+ this.acksSent += 1
172
+ }
173
+
174
+ private trackSequence(parsed: ParsedFrame): void {
175
+ if (this.lastGen !== parsed.sessionGen) {
176
+ this.lastGen = parsed.sessionGen
177
+ this.lastSeq = parsed.frameSeq
178
+ return
179
+ }
180
+ if (this.lastSeq !== null) {
181
+ const delta = parsed.frameSeq - this.lastSeq
182
+ if (delta > 1) {
183
+ this.sequenceGaps += 1
184
+ this.skippedBySender += delta - 1
185
+ }
186
+ }
187
+ this.lastSeq = parsed.frameSeq
188
+ }
189
+ }
@@ -0,0 +1,227 @@
1
+ /**
2
+ * Wire format for raw decoded-frame preview (`MFPV`): a 64-byte little-endian header followed by
3
+ * a tightly packed 8-bit YUV payload.
4
+ *
5
+ * Nothing in this module may copy the payload. `parsePreviewFrame` returns `Uint8Array` views onto
6
+ * the caller's ArrayBuffer and the renderer uploads those views straight to WebGL; a `.slice()`
7
+ * anywhere in this path turns a zero-copy preview into a per-frame memcpy.
8
+ *
9
+ * Bad input is expected — a truncated or stale frame is a measurement, not an exception — so the
10
+ * parser returns a discriminated union and never throws.
11
+ *
12
+ * offset size field
13
+ * 0 4 magic "MFPV"
14
+ * 4 2 version u16
15
+ * 6 2 headerLen u16
16
+ * 8 4 payloadLen u32
17
+ * 12 4 sessionGen u32
18
+ * 16 4 frameSeq u32
19
+ * 20 2 width u16
20
+ * 22 2 height u16
21
+ * 24 1 pixelFormat u8 (1 = I420, 2 = NV12)
22
+ * 25 1 rotation u8 (quarter turns clockwise)
23
+ * 26 1 colorMatrix u8 (0 unknown, 1 BT.601, 2 BT.709)
24
+ * 27 1 colorRange u8 (0 unknown, 1 limited, 2 full)
25
+ * 28 2 flags u16 (bit 0: the sender fell back on default colour metadata)
26
+ * 30 2 reserved u16
27
+ * 32 8 timestampNs i64 (native monotonic decode time, 0 when unknown)
28
+ * 40 8 sentAtNs i64 (native monotonic at the send call)
29
+ * 48 16 reserved
30
+ */
31
+
32
+ /** Fixed header size. A sender that disagrees is rejected rather than guessed at. */
33
+ export const PREVIEW_FRAME_HEADER_BYTES = 64
34
+ export const PREVIEW_FRAME_VERSION = 1
35
+ /** ASCII "MFPV". */
36
+ export const PREVIEW_FRAME_MAGIC = [0x4d, 0x46, 0x50, 0x56] as const
37
+ /** Neither the glasses nor a phone camera reaches this; anything above it is a corrupt header. */
38
+ export const PREVIEW_FRAME_MAX_DIMENSION = 4096
39
+
40
+ export const PIXEL_FORMAT_I420 = 1
41
+ export const PIXEL_FORMAT_NV12 = 2
42
+
43
+ export type PreviewPixelFormat = "i420" | "nv12"
44
+ export type PreviewColorMatrix = "bt601" | "bt709"
45
+ export type PreviewColorRange = "limited" | "full"
46
+
47
+ /**
48
+ * One code per rejected frame, because "the frame was bad" is not a finding. A run that drops
49
+ * frames on `truncated-payload` and one that drops them on `payload-size-mismatch` point at
50
+ * different halves of the native sender.
51
+ */
52
+ export type PreviewParseErrorReason =
53
+ | "short-buffer"
54
+ | "bad-magic"
55
+ | "unsupported-version"
56
+ | "bad-header-length"
57
+ | "bad-dimensions"
58
+ | "unknown-pixel-format"
59
+ | "payload-size-mismatch"
60
+ | "truncated-payload"
61
+
62
+ export const PREVIEW_PARSE_ERROR_REASONS: readonly PreviewParseErrorReason[] = [
63
+ "short-buffer",
64
+ "bad-magic",
65
+ "unsupported-version",
66
+ "bad-header-length",
67
+ "bad-dimensions",
68
+ "unknown-pixel-format",
69
+ "payload-size-mismatch",
70
+ "truncated-payload",
71
+ ]
72
+
73
+ interface ParsedFrameBase {
74
+ ok: true
75
+ sessionGen: number
76
+ frameSeq: number
77
+ width: number
78
+ height: number
79
+ /** Quarter turns clockwise, normalized to 0..3. */
80
+ rotationQuarters: number
81
+ colorMatrix: PreviewColorMatrix
82
+ colorRange: PreviewColorRange
83
+ /** The header said "unknown" and BT.601 was assumed. */
84
+ colorMatrixAssumed: boolean
85
+ /** The header said "unknown" and limited range was assumed. */
86
+ colorRangeAssumed: boolean
87
+ /** Header flag bit 0: the native sender itself had to fall back on default colour metadata. */
88
+ senderColorFallback: boolean
89
+ /**
90
+ * Native monotonic clock, nanoseconds. Never subtract a `performance.now()` value from these:
91
+ * the two clocks share no epoch. Only native-to-native differences mean anything.
92
+ */
93
+ timestampNs: bigint
94
+ sentAtNs: bigint
95
+ yStride: number
96
+ chromaWidth: number
97
+ chromaHeight: number
98
+ chromaStride: number
99
+ payloadLen: number
100
+ }
101
+
102
+ export interface ParsedI420Frame extends ParsedFrameBase {
103
+ pixelFormat: "i420"
104
+ y: Uint8Array
105
+ u: Uint8Array
106
+ v: Uint8Array
107
+ }
108
+
109
+ export interface ParsedNv12Frame extends ParsedFrameBase {
110
+ pixelFormat: "nv12"
111
+ y: Uint8Array
112
+ /** Interleaved U,V pairs: `chromaStride` is 2 × `chromaWidth` bytes. */
113
+ uv: Uint8Array
114
+ }
115
+
116
+ export type ParsedFrame = ParsedI420Frame | ParsedNv12Frame
117
+
118
+ export interface PreviewParseError {
119
+ ok: false
120
+ reason: PreviewParseErrorReason
121
+ /** Numbers the reason alone does not carry, for the log line. */
122
+ detail?: string
123
+ }
124
+
125
+ export type PreviewParseResult = ParsedFrame | PreviewParseError
126
+
127
+ /** Chroma plane dimensions for 4:2:0, which both supported formats are. */
128
+ export function chromaDimensions(width: number, height: number): {width: number; height: number} {
129
+ return {width: Math.ceil(width / 2), height: Math.ceil(height / 2)}
130
+ }
131
+
132
+ /** Bytes a tightly packed I420 or NV12 frame occupies. Both layouts are the same size. */
133
+ export function packedPayloadBytes(width: number, height: number): number {
134
+ const chroma = chromaDimensions(width, height)
135
+ return width * height + 2 * chroma.width * chroma.height
136
+ }
137
+
138
+ function fail(reason: PreviewParseErrorReason, detail?: string): PreviewParseError {
139
+ return detail === undefined ? {ok: false, reason} : {ok: false, reason, detail}
140
+ }
141
+
142
+ /**
143
+ * Validate a frame and expose its planes.
144
+ *
145
+ * The plane fields are views onto `buffer`, so they stay valid only as long as the caller holds
146
+ * that buffer and nothing else writes into it.
147
+ */
148
+ export function parsePreviewFrame(buffer: ArrayBuffer): PreviewParseResult {
149
+ if (buffer.byteLength < PREVIEW_FRAME_HEADER_BYTES) {
150
+ return fail("short-buffer", `${buffer.byteLength} bytes`)
151
+ }
152
+ const header = new DataView(buffer, 0, PREVIEW_FRAME_HEADER_BYTES)
153
+ const magic = new Uint8Array(buffer, 0, PREVIEW_FRAME_MAGIC.length)
154
+ if (PREVIEW_FRAME_MAGIC.some((byte, index) => magic[index] !== byte)) {
155
+ return fail("bad-magic", Array.from(magic, (byte) => byte.toString(16).padStart(2, "0")).join(" "))
156
+ }
157
+ const version = header.getUint16(4, true)
158
+ if (version !== PREVIEW_FRAME_VERSION) return fail("unsupported-version", `v${version}`)
159
+ const headerLen = header.getUint16(6, true)
160
+ if (headerLen !== PREVIEW_FRAME_HEADER_BYTES) return fail("bad-header-length", `${headerLen} bytes`)
161
+ const payloadLen = header.getUint32(8, true)
162
+ const sessionGen = header.getUint32(12, true)
163
+ const frameSeq = header.getUint32(16, true)
164
+ const width = header.getUint16(20, true)
165
+ const height = header.getUint16(22, true)
166
+ if (width === 0 || height === 0 || width > PREVIEW_FRAME_MAX_DIMENSION || height > PREVIEW_FRAME_MAX_DIMENSION) {
167
+ return fail("bad-dimensions", `${width}x${height}`)
168
+ }
169
+ const formatCode = header.getUint8(24)
170
+ if (formatCode !== PIXEL_FORMAT_I420 && formatCode !== PIXEL_FORMAT_NV12) {
171
+ return fail("unknown-pixel-format", `code ${formatCode}`)
172
+ }
173
+ const expectedPayload = packedPayloadBytes(width, height)
174
+ if (payloadLen !== expectedPayload) {
175
+ return fail("payload-size-mismatch", `${payloadLen} bytes, expected ${expectedPayload}`)
176
+ }
177
+ if (buffer.byteLength < headerLen + payloadLen) {
178
+ return fail("truncated-payload", `${buffer.byteLength - headerLen} of ${payloadLen} bytes`)
179
+ }
180
+
181
+ const matrixCode = header.getUint8(26)
182
+ const rangeCode = header.getUint8(27)
183
+ const flags = header.getUint16(28, true)
184
+ const chroma = chromaDimensions(width, height)
185
+ const luma = width * height
186
+ const base: ParsedFrameBase = {
187
+ ok: true,
188
+ sessionGen,
189
+ frameSeq,
190
+ width,
191
+ height,
192
+ rotationQuarters: header.getUint8(25) % 4,
193
+ // Unknown metadata is not a rejection: BT.601 limited is what an untagged 8-bit camera frame
194
+ // almost always is. The assumption is surfaced so a wrong-looking picture can be traced to it
195
+ // instead of to the shader.
196
+ colorMatrix: matrixCode === 2 ? "bt709" : "bt601",
197
+ colorRange: rangeCode === 2 ? "full" : "limited",
198
+ colorMatrixAssumed: matrixCode !== 1 && matrixCode !== 2,
199
+ colorRangeAssumed: rangeCode !== 1 && rangeCode !== 2,
200
+ senderColorFallback: (flags & 0x0001) !== 0,
201
+ timestampNs: header.getBigInt64(32, true),
202
+ sentAtNs: header.getBigInt64(40, true),
203
+ yStride: width,
204
+ chromaWidth: chroma.width,
205
+ chromaHeight: chroma.height,
206
+ chromaStride: formatCode === PIXEL_FORMAT_NV12 ? chroma.width * 2 : chroma.width,
207
+ payloadLen,
208
+ }
209
+
210
+ // Subviews, never copies.
211
+ if (formatCode === PIXEL_FORMAT_NV12) {
212
+ return {
213
+ ...base,
214
+ pixelFormat: "nv12",
215
+ y: new Uint8Array(buffer, headerLen, luma),
216
+ uv: new Uint8Array(buffer, headerLen + luma, 2 * chroma.width * chroma.height),
217
+ }
218
+ }
219
+ const chromaPlane = chroma.width * chroma.height
220
+ return {
221
+ ...base,
222
+ pixelFormat: "i420",
223
+ y: new Uint8Array(buffer, headerLen, luma),
224
+ u: new Uint8Array(buffer, headerLen + luma, chromaPlane),
225
+ v: new Uint8Array(buffer, headerLen + luma + chromaPlane, chromaPlane),
226
+ }
227
+ }
@@ -0,0 +1,88 @@
1
+ /**
2
+ * Collapse a stream of box measurements into the few that matter.
3
+ *
4
+ * A layout animation fires ResizeObserver every frame. Bursts inside one frame are merged with
5
+ * `requestAnimationFrame`, and a size is only applied once it has held still for the settle
6
+ * window. Hiding (a zero box) and the first measurement apply at once: the first so the preview
7
+ * starts without a delay, hiding so production stops as soon as nobody can see it.
8
+ */
9
+
10
+ export interface PreviewBox {
11
+ width: number
12
+ height: number
13
+ }
14
+
15
+ export interface ResizeCoalescerOptions {
16
+ apply: (box: PreviewBox) => void
17
+ settleMs?: number
18
+ requestFrame?: (cb: () => void) => unknown
19
+ setTimer?: (cb: () => void, ms: number) => unknown
20
+ clearTimer?: (handle: unknown) => void
21
+ }
22
+
23
+ export const RESIZE_SETTLE_MS = 250
24
+
25
+ function isZero(box: PreviewBox): boolean {
26
+ return !(box.width > 0) || !(box.height > 0)
27
+ }
28
+
29
+ export class ResizeCoalescer {
30
+ private latest: PreviewBox | null = null
31
+ private lastApplied: PreviewBox | null = null
32
+ private framePending = false
33
+ private timer: unknown = null
34
+ private cancelled = false
35
+ private readonly settleMs: number
36
+ private readonly requestFrame: (cb: () => void) => unknown
37
+ private readonly setTimer: (cb: () => void, ms: number) => unknown
38
+ private readonly clearTimer: (handle: unknown) => void
39
+
40
+ constructor(private readonly options: ResizeCoalescerOptions) {
41
+ this.settleMs = options.settleMs ?? RESIZE_SETTLE_MS
42
+ this.requestFrame =
43
+ options.requestFrame ??
44
+ ((cb) => (typeof requestAnimationFrame === "function" ? requestAnimationFrame(() => cb()) : setTimeout(cb, 16)))
45
+ this.setTimer = options.setTimer ?? ((cb, ms) => setTimeout(cb, ms))
46
+ this.clearTimer = options.clearTimer ?? ((handle) => clearTimeout(handle as ReturnType<typeof setTimeout>))
47
+ }
48
+
49
+ push(box: PreviewBox): void {
50
+ if (this.cancelled) return
51
+ this.latest = box
52
+ if (!this.lastApplied || isZero(box) || isZero(this.lastApplied)) {
53
+ this.clearSettle()
54
+ this.applyLatest()
55
+ return
56
+ }
57
+ if (this.framePending) return
58
+ this.framePending = true
59
+ this.requestFrame(() => {
60
+ this.framePending = false
61
+ if (this.cancelled) return
62
+ this.clearSettle()
63
+ this.timer = this.setTimer(() => {
64
+ this.timer = null
65
+ this.applyLatest()
66
+ }, this.settleMs)
67
+ })
68
+ }
69
+
70
+ cancel(): void {
71
+ this.cancelled = true
72
+ this.clearSettle()
73
+ }
74
+
75
+ private applyLatest(): void {
76
+ const box = this.latest
77
+ if (!box || this.cancelled) return
78
+ if (this.lastApplied && this.lastApplied.width === box.width && this.lastApplied.height === box.height) return
79
+ this.lastApplied = box
80
+ this.options.apply(box)
81
+ }
82
+
83
+ private clearSettle(): void {
84
+ if (this.timer === null) return
85
+ this.clearTimer(this.timer)
86
+ this.timer = null
87
+ }
88
+ }