@mentra/miniapp 3.2.1-beta.319 → 3.2.1-beta.354
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -0
- package/dist/background/index.d.ts +5 -3
- package/dist/background/index.d.ts.map +1 -1
- package/dist/background/index.js +1 -0
- package/dist/background/index.js.map +1 -1
- package/dist/index.d.ts +6 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/modules/display.d.ts +33 -0
- package/dist/modules/display.d.ts.map +1 -1
- package/dist/modules/display.js +2 -0
- package/dist/modules/display.js.map +1 -1
- package/dist/modules/meeting.d.ts +66 -6
- package/dist/modules/meeting.d.ts.map +1 -1
- package/dist/modules/meeting.js +72 -19
- package/dist/modules/meeting.js.map +1 -1
- package/dist/modules/stream.d.ts +46 -0
- package/dist/modules/stream.d.ts.map +1 -1
- package/dist/modules/stream.js +107 -0
- package/dist/modules/stream.js.map +1 -1
- package/dist/protocol.d.ts +39 -0
- package/dist/protocol.d.ts.map +1 -1
- package/dist/protocol.js +36 -0
- package/dist/protocol.js.map +1 -1
- package/dist/react/StreamPreview.d.ts +41 -0
- package/dist/react/StreamPreview.d.ts.map +1 -0
- package/dist/react/StreamPreview.js +144 -0
- package/dist/react/StreamPreview.js.map +1 -0
- package/dist/react/index.d.ts +2 -0
- package/dist/react/index.d.ts.map +1 -1
- package/dist/react/index.js +1 -0
- package/dist/react/index.js.map +1 -1
- package/dist/react/stream-preview/PreviewConnection.d.ts +139 -0
- package/dist/react/stream-preview/PreviewConnection.d.ts.map +1 -0
- package/dist/react/stream-preview/PreviewConnection.js +462 -0
- package/dist/react/stream-preview/PreviewConnection.js.map +1 -0
- package/dist/react/stream-preview/controlChannel.d.ts +75 -0
- package/dist/react/stream-preview/controlChannel.d.ts.map +1 -0
- package/dist/react/stream-preview/controlChannel.js +55 -0
- package/dist/react/stream-preview/controlChannel.js.map +1 -0
- package/dist/react/stream-preview/creditLoop.d.ts +81 -0
- package/dist/react/stream-preview/creditLoop.d.ts.map +1 -0
- package/dist/react/stream-preview/creditLoop.js +146 -0
- package/dist/react/stream-preview/creditLoop.js.map +1 -0
- package/dist/react/stream-preview/protocol.d.ts +113 -0
- package/dist/react/stream-preview/protocol.d.ts.map +1 -0
- package/dist/react/stream-preview/protocol.js +148 -0
- package/dist/react/stream-preview/protocol.js.map +1 -0
- package/dist/react/stream-preview/resizeCoalescer.d.ts +38 -0
- package/dist/react/stream-preview/resizeCoalescer.d.ts.map +1 -0
- package/dist/react/stream-preview/resizeCoalescer.js +71 -0
- package/dist/react/stream-preview/resizeCoalescer.js.map +1 -0
- package/dist/react/stream-preview/testFrames.d.ts +38 -0
- package/dist/react/stream-preview/testFrames.d.ts.map +1 -0
- package/dist/react/stream-preview/testFrames.js +50 -0
- package/dist/react/stream-preview/testFrames.js.map +1 -0
- package/dist/react/stream-preview/tiers.d.ts +21 -0
- package/dist/react/stream-preview/tiers.d.ts.map +1 -0
- package/dist/react/stream-preview/tiers.js +33 -0
- package/dist/react/stream-preview/tiers.js.map +1 -0
- package/dist/react/stream-preview/trace.d.ts +23 -0
- package/dist/react/stream-preview/trace.d.ts.map +1 -0
- package/dist/react/stream-preview/trace.js +71 -0
- package/dist/react/stream-preview/trace.js.map +1 -0
- package/dist/react/stream-preview/transport.d.ts +79 -0
- package/dist/react/stream-preview/transport.d.ts.map +1 -0
- package/dist/react/stream-preview/transport.js +209 -0
- package/dist/react/stream-preview/transport.js.map +1 -0
- package/dist/react/stream-preview/yuvRenderer.d.ts +31 -0
- package/dist/react/stream-preview/yuvRenderer.d.ts.map +1 -0
- package/dist/react/stream-preview/yuvRenderer.js +398 -0
- package/dist/react/stream-preview/yuvRenderer.js.map +1 -0
- package/dist/session.d.ts +5 -0
- package/dist/session.d.ts.map +1 -1
- package/dist/session.js +5 -0
- package/dist/session.js.map +1 -1
- package/dist/ui/index.d.ts +2 -0
- package/dist/ui/index.d.ts.map +1 -1
- package/dist/ui/index.js +1 -0
- package/dist/ui/index.js.map +1 -1
- package/package.json +8 -3
- package/src/background/index.ts +14 -1
- package/src/index.ts +15 -0
- package/src/modules/display.ts +37 -0
- package/src/modules/meeting.ts +133 -23
- package/src/modules/stream.ts +153 -1
- package/src/protocol.ts +75 -0
- package/src/react/StreamPreview.tsx +187 -0
- package/src/react/index.ts +2 -0
- package/src/react/stream-preview/PreviewConnection.ts +552 -0
- package/src/react/stream-preview/controlChannel.ts +119 -0
- package/src/react/stream-preview/creditLoop.ts +189 -0
- package/src/react/stream-preview/protocol.ts +227 -0
- package/src/react/stream-preview/resizeCoalescer.ts +88 -0
- package/src/react/stream-preview/testFrames.ts +86 -0
- package/src/react/stream-preview/tiers.ts +37 -0
- package/src/react/stream-preview/trace.ts +81 -0
- package/src/react/stream-preview/transport.ts +242 -0
- package/src/react/stream-preview/yuvRenderer.ts +453 -0
- package/src/session.ts +8 -0
- package/src/ui/index.ts +2 -0
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Consumer side of the one-frame credit scheme.
|
|
3
|
+
*
|
|
4
|
+
* The native sender holds exactly one frame in flight and will not send the next until this side
|
|
5
|
+
* acknowledges the last, so there is no queue anywhere and a slow consumer shows up as a lower
|
|
6
|
+
* frame rate rather than as growing latency. This module is the half that makes that true: parse,
|
|
7
|
+
* draw, ack — and never hold a reference to a frame after its ack.
|
|
8
|
+
*
|
|
9
|
+
* Durations here would be `performance.now()` and describe JavaScript only; the header's
|
|
10
|
+
* `timestampNs` / `sentAtNs` are a native monotonic clock with a different epoch. End-to-end
|
|
11
|
+
* latency is measured natively (it sees both the send and the ack on one clock).
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import {parsePreviewFrame, type ParsedFrame, type PreviewParseErrorReason} from "./protocol"
|
|
15
|
+
|
|
16
|
+
export interface CreditLoopFrameSummary {
|
|
17
|
+
gen: number
|
|
18
|
+
seq: number
|
|
19
|
+
width: number
|
|
20
|
+
height: number
|
|
21
|
+
pixelFormat: string
|
|
22
|
+
rotationQuarters: number
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export interface CreditLoopStats {
|
|
26
|
+
framesReceived: number
|
|
27
|
+
drawsSubmitted: number
|
|
28
|
+
acksSent: number
|
|
29
|
+
parseErrors: number
|
|
30
|
+
parseErrorsByReason: Partial<Record<PreviewParseErrorReason, number>>
|
|
31
|
+
/**
|
|
32
|
+
* Frames that arrived while another was still being processed. Native guarantees one at a time,
|
|
33
|
+
* so any value above zero means the credit scheme is broken.
|
|
34
|
+
*/
|
|
35
|
+
overlapped: number
|
|
36
|
+
/** Frames the sender skipped, counted from gaps in the header sequence. */
|
|
37
|
+
sequenceGaps: number
|
|
38
|
+
skippedBySender: number
|
|
39
|
+
lastError: string | null
|
|
40
|
+
lastFrame: CreditLoopFrameSummary | null
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export interface CreditLoopOptions {
|
|
44
|
+
sendAck: (gen: number, seq: number) => void
|
|
45
|
+
/**
|
|
46
|
+
* Submit the frame. Returns the submit duration in ms, or null when there is no renderer (the
|
|
47
|
+
* component is unmounted or its GL context is lost). The frame is acked either way.
|
|
48
|
+
*/
|
|
49
|
+
draw?: (frame: ParsedFrame) => number | null
|
|
50
|
+
/** Called once per rejected frame, before anything else happens to it. */
|
|
51
|
+
onParseError?: (reason: PreviewParseErrorReason, detail?: string) => void
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export class CreditLoop {
|
|
55
|
+
private readonly errorsByReason = new Map<PreviewParseErrorReason, number>()
|
|
56
|
+
private idleWaiters: Array<() => void> = []
|
|
57
|
+
private inFlightCount = 0
|
|
58
|
+
private framesReceived = 0
|
|
59
|
+
private drawsSubmitted = 0
|
|
60
|
+
private acksSent = 0
|
|
61
|
+
private parseErrors = 0
|
|
62
|
+
private overlapped = 0
|
|
63
|
+
private sequenceGaps = 0
|
|
64
|
+
private skippedBySender = 0
|
|
65
|
+
private lastSeq: number | null = null
|
|
66
|
+
private lastGen: number | null = null
|
|
67
|
+
private lastError: string | null = null
|
|
68
|
+
private lastFrame: CreditLoopFrameSummary | null = null
|
|
69
|
+
private draw: CreditLoopOptions["draw"]
|
|
70
|
+
|
|
71
|
+
constructor(private readonly options: CreditLoopOptions) {
|
|
72
|
+
this.draw = options.draw
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** Swap the renderer without touching the credit state. Null means frames are acked undrawn. */
|
|
76
|
+
setDraw(draw: CreditLoopOptions["draw"] | null): void {
|
|
77
|
+
this.draw = draw ?? undefined
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Frames currently being processed. The credit scheme means this is 0 or 1. */
|
|
81
|
+
get inFlight(): number {
|
|
82
|
+
return this.inFlightCount
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** Entry point for the transport. Deliberately not awaited: the transport must not block. */
|
|
86
|
+
handleBinary(buffer: ArrayBuffer): void {
|
|
87
|
+
if (this.inFlightCount > 0) this.overlapped += 1
|
|
88
|
+
this.inFlightCount += 1
|
|
89
|
+
void this.process(buffer).finally(() => {
|
|
90
|
+
this.inFlightCount -= 1
|
|
91
|
+
if (this.inFlightCount === 0) {
|
|
92
|
+
const waiters = this.idleWaiters
|
|
93
|
+
this.idleWaiters = []
|
|
94
|
+
for (const resolve of waiters) resolve()
|
|
95
|
+
}
|
|
96
|
+
})
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** Resolves once nothing is being processed. */
|
|
100
|
+
idle(): Promise<void> {
|
|
101
|
+
if (this.inFlightCount === 0) return Promise.resolve()
|
|
102
|
+
return new Promise((resolve) => this.idleWaiters.push(resolve))
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
recordError(reason: string): void {
|
|
106
|
+
this.lastError = reason
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
snapshot(): CreditLoopStats {
|
|
110
|
+
const parseErrorsByReason: Partial<Record<PreviewParseErrorReason, number>> = {}
|
|
111
|
+
for (const [reason, count] of this.errorsByReason) parseErrorsByReason[reason] = count
|
|
112
|
+
return {
|
|
113
|
+
framesReceived: this.framesReceived,
|
|
114
|
+
drawsSubmitted: this.drawsSubmitted,
|
|
115
|
+
acksSent: this.acksSent,
|
|
116
|
+
parseErrors: this.parseErrors,
|
|
117
|
+
parseErrorsByReason,
|
|
118
|
+
overlapped: this.overlapped,
|
|
119
|
+
sequenceGaps: this.sequenceGaps,
|
|
120
|
+
skippedBySender: this.skippedBySender,
|
|
121
|
+
lastError: this.lastError,
|
|
122
|
+
lastFrame: this.lastFrame,
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
reset(): void {
|
|
127
|
+
this.framesReceived = 0
|
|
128
|
+
this.drawsSubmitted = 0
|
|
129
|
+
this.acksSent = 0
|
|
130
|
+
this.parseErrors = 0
|
|
131
|
+
this.overlapped = 0
|
|
132
|
+
this.sequenceGaps = 0
|
|
133
|
+
this.skippedBySender = 0
|
|
134
|
+
this.lastSeq = null
|
|
135
|
+
this.lastGen = null
|
|
136
|
+
this.lastError = null
|
|
137
|
+
this.lastFrame = null
|
|
138
|
+
this.errorsByReason.clear()
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
private async process(buffer: ArrayBuffer): Promise<void> {
|
|
142
|
+
this.framesReceived += 1
|
|
143
|
+
const parsed = parsePreviewFrame(buffer)
|
|
144
|
+
if (!parsed.ok) {
|
|
145
|
+
// No ack: a frame this side could not read is not a frame it consumed, and acking it would
|
|
146
|
+
// hide the fault behind a healthy-looking rate.
|
|
147
|
+
this.parseErrors += 1
|
|
148
|
+
this.errorsByReason.set(parsed.reason, (this.errorsByReason.get(parsed.reason) ?? 0) + 1)
|
|
149
|
+
this.lastError = parsed.detail ? `${parsed.reason} (${parsed.detail})` : parsed.reason
|
|
150
|
+
this.options.onParseError?.(parsed.reason, parsed.detail)
|
|
151
|
+
return
|
|
152
|
+
}
|
|
153
|
+
this.trackSequence(parsed)
|
|
154
|
+
this.lastFrame = {
|
|
155
|
+
gen: parsed.sessionGen,
|
|
156
|
+
seq: parsed.frameSeq,
|
|
157
|
+
width: parsed.width,
|
|
158
|
+
height: parsed.height,
|
|
159
|
+
pixelFormat: parsed.pixelFormat,
|
|
160
|
+
rotationQuarters: parsed.rotationQuarters,
|
|
161
|
+
}
|
|
162
|
+
if (this.draw) {
|
|
163
|
+
const submitMs = this.draw(parsed)
|
|
164
|
+
if (submitMs === null) this.lastError = "draw_failed"
|
|
165
|
+
else this.drawsSubmitted += 1
|
|
166
|
+
}
|
|
167
|
+
// Ack last, after the draw has been submitted, so the credit the sender gets back means "this
|
|
168
|
+
// side is done with that frame" rather than "this side received some bytes". A dead canvas
|
|
169
|
+
// still acks: withholding credit would stall the sender on top of it.
|
|
170
|
+
this.options.sendAck(parsed.sessionGen, parsed.frameSeq)
|
|
171
|
+
this.acksSent += 1
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
private trackSequence(parsed: ParsedFrame): void {
|
|
175
|
+
if (this.lastGen !== parsed.sessionGen) {
|
|
176
|
+
this.lastGen = parsed.sessionGen
|
|
177
|
+
this.lastSeq = parsed.frameSeq
|
|
178
|
+
return
|
|
179
|
+
}
|
|
180
|
+
if (this.lastSeq !== null) {
|
|
181
|
+
const delta = parsed.frameSeq - this.lastSeq
|
|
182
|
+
if (delta > 1) {
|
|
183
|
+
this.sequenceGaps += 1
|
|
184
|
+
this.skippedBySender += delta - 1
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
this.lastSeq = parsed.frameSeq
|
|
188
|
+
}
|
|
189
|
+
}
|
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Wire format for raw decoded-frame preview (`MFPV`): a 64-byte little-endian header followed by
|
|
3
|
+
* a tightly packed 8-bit YUV payload.
|
|
4
|
+
*
|
|
5
|
+
* Nothing in this module may copy the payload. `parsePreviewFrame` returns `Uint8Array` views onto
|
|
6
|
+
* the caller's ArrayBuffer and the renderer uploads those views straight to WebGL; a `.slice()`
|
|
7
|
+
* anywhere in this path turns a zero-copy preview into a per-frame memcpy.
|
|
8
|
+
*
|
|
9
|
+
* Bad input is expected — a truncated or stale frame is a measurement, not an exception — so the
|
|
10
|
+
* parser returns a discriminated union and never throws.
|
|
11
|
+
*
|
|
12
|
+
* offset size field
|
|
13
|
+
* 0 4 magic "MFPV"
|
|
14
|
+
* 4 2 version u16
|
|
15
|
+
* 6 2 headerLen u16
|
|
16
|
+
* 8 4 payloadLen u32
|
|
17
|
+
* 12 4 sessionGen u32
|
|
18
|
+
* 16 4 frameSeq u32
|
|
19
|
+
* 20 2 width u16
|
|
20
|
+
* 22 2 height u16
|
|
21
|
+
* 24 1 pixelFormat u8 (1 = I420, 2 = NV12)
|
|
22
|
+
* 25 1 rotation u8 (quarter turns clockwise)
|
|
23
|
+
* 26 1 colorMatrix u8 (0 unknown, 1 BT.601, 2 BT.709)
|
|
24
|
+
* 27 1 colorRange u8 (0 unknown, 1 limited, 2 full)
|
|
25
|
+
* 28 2 flags u16 (bit 0: the sender fell back on default colour metadata)
|
|
26
|
+
* 30 2 reserved u16
|
|
27
|
+
* 32 8 timestampNs i64 (native monotonic decode time, 0 when unknown)
|
|
28
|
+
* 40 8 sentAtNs i64 (native monotonic at the send call)
|
|
29
|
+
* 48 16 reserved
|
|
30
|
+
*/
|
|
31
|
+
|
|
32
|
+
/** Fixed header size. A sender that disagrees is rejected rather than guessed at. */
|
|
33
|
+
export const PREVIEW_FRAME_HEADER_BYTES = 64
|
|
34
|
+
export const PREVIEW_FRAME_VERSION = 1
|
|
35
|
+
/** ASCII "MFPV". */
|
|
36
|
+
export const PREVIEW_FRAME_MAGIC = [0x4d, 0x46, 0x50, 0x56] as const
|
|
37
|
+
/** Neither the glasses nor a phone camera reaches this; anything above it is a corrupt header. */
|
|
38
|
+
export const PREVIEW_FRAME_MAX_DIMENSION = 4096
|
|
39
|
+
|
|
40
|
+
export const PIXEL_FORMAT_I420 = 1
|
|
41
|
+
export const PIXEL_FORMAT_NV12 = 2
|
|
42
|
+
|
|
43
|
+
export type PreviewPixelFormat = "i420" | "nv12"
|
|
44
|
+
export type PreviewColorMatrix = "bt601" | "bt709"
|
|
45
|
+
export type PreviewColorRange = "limited" | "full"
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* One code per rejected frame, because "the frame was bad" is not a finding. A run that drops
|
|
49
|
+
* frames on `truncated-payload` and one that drops them on `payload-size-mismatch` point at
|
|
50
|
+
* different halves of the native sender.
|
|
51
|
+
*/
|
|
52
|
+
export type PreviewParseErrorReason =
|
|
53
|
+
| "short-buffer"
|
|
54
|
+
| "bad-magic"
|
|
55
|
+
| "unsupported-version"
|
|
56
|
+
| "bad-header-length"
|
|
57
|
+
| "bad-dimensions"
|
|
58
|
+
| "unknown-pixel-format"
|
|
59
|
+
| "payload-size-mismatch"
|
|
60
|
+
| "truncated-payload"
|
|
61
|
+
|
|
62
|
+
export const PREVIEW_PARSE_ERROR_REASONS: readonly PreviewParseErrorReason[] = [
|
|
63
|
+
"short-buffer",
|
|
64
|
+
"bad-magic",
|
|
65
|
+
"unsupported-version",
|
|
66
|
+
"bad-header-length",
|
|
67
|
+
"bad-dimensions",
|
|
68
|
+
"unknown-pixel-format",
|
|
69
|
+
"payload-size-mismatch",
|
|
70
|
+
"truncated-payload",
|
|
71
|
+
]
|
|
72
|
+
|
|
73
|
+
interface ParsedFrameBase {
|
|
74
|
+
ok: true
|
|
75
|
+
sessionGen: number
|
|
76
|
+
frameSeq: number
|
|
77
|
+
width: number
|
|
78
|
+
height: number
|
|
79
|
+
/** Quarter turns clockwise, normalized to 0..3. */
|
|
80
|
+
rotationQuarters: number
|
|
81
|
+
colorMatrix: PreviewColorMatrix
|
|
82
|
+
colorRange: PreviewColorRange
|
|
83
|
+
/** The header said "unknown" and BT.601 was assumed. */
|
|
84
|
+
colorMatrixAssumed: boolean
|
|
85
|
+
/** The header said "unknown" and limited range was assumed. */
|
|
86
|
+
colorRangeAssumed: boolean
|
|
87
|
+
/** Header flag bit 0: the native sender itself had to fall back on default colour metadata. */
|
|
88
|
+
senderColorFallback: boolean
|
|
89
|
+
/**
|
|
90
|
+
* Native monotonic clock, nanoseconds. Never subtract a `performance.now()` value from these:
|
|
91
|
+
* the two clocks share no epoch. Only native-to-native differences mean anything.
|
|
92
|
+
*/
|
|
93
|
+
timestampNs: bigint
|
|
94
|
+
sentAtNs: bigint
|
|
95
|
+
yStride: number
|
|
96
|
+
chromaWidth: number
|
|
97
|
+
chromaHeight: number
|
|
98
|
+
chromaStride: number
|
|
99
|
+
payloadLen: number
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export interface ParsedI420Frame extends ParsedFrameBase {
|
|
103
|
+
pixelFormat: "i420"
|
|
104
|
+
y: Uint8Array
|
|
105
|
+
u: Uint8Array
|
|
106
|
+
v: Uint8Array
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
export interface ParsedNv12Frame extends ParsedFrameBase {
|
|
110
|
+
pixelFormat: "nv12"
|
|
111
|
+
y: Uint8Array
|
|
112
|
+
/** Interleaved U,V pairs: `chromaStride` is 2 × `chromaWidth` bytes. */
|
|
113
|
+
uv: Uint8Array
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
export type ParsedFrame = ParsedI420Frame | ParsedNv12Frame
|
|
117
|
+
|
|
118
|
+
export interface PreviewParseError {
|
|
119
|
+
ok: false
|
|
120
|
+
reason: PreviewParseErrorReason
|
|
121
|
+
/** Numbers the reason alone does not carry, for the log line. */
|
|
122
|
+
detail?: string
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
export type PreviewParseResult = ParsedFrame | PreviewParseError
|
|
126
|
+
|
|
127
|
+
/** Chroma plane dimensions for 4:2:0, which both supported formats are. */
|
|
128
|
+
export function chromaDimensions(width: number, height: number): {width: number; height: number} {
|
|
129
|
+
return {width: Math.ceil(width / 2), height: Math.ceil(height / 2)}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** Bytes a tightly packed I420 or NV12 frame occupies. Both layouts are the same size. */
|
|
133
|
+
export function packedPayloadBytes(width: number, height: number): number {
|
|
134
|
+
const chroma = chromaDimensions(width, height)
|
|
135
|
+
return width * height + 2 * chroma.width * chroma.height
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function fail(reason: PreviewParseErrorReason, detail?: string): PreviewParseError {
|
|
139
|
+
return detail === undefined ? {ok: false, reason} : {ok: false, reason, detail}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Validate a frame and expose its planes.
|
|
144
|
+
*
|
|
145
|
+
* The plane fields are views onto `buffer`, so they stay valid only as long as the caller holds
|
|
146
|
+
* that buffer and nothing else writes into it.
|
|
147
|
+
*/
|
|
148
|
+
export function parsePreviewFrame(buffer: ArrayBuffer): PreviewParseResult {
|
|
149
|
+
if (buffer.byteLength < PREVIEW_FRAME_HEADER_BYTES) {
|
|
150
|
+
return fail("short-buffer", `${buffer.byteLength} bytes`)
|
|
151
|
+
}
|
|
152
|
+
const header = new DataView(buffer, 0, PREVIEW_FRAME_HEADER_BYTES)
|
|
153
|
+
const magic = new Uint8Array(buffer, 0, PREVIEW_FRAME_MAGIC.length)
|
|
154
|
+
if (PREVIEW_FRAME_MAGIC.some((byte, index) => magic[index] !== byte)) {
|
|
155
|
+
return fail("bad-magic", Array.from(magic, (byte) => byte.toString(16).padStart(2, "0")).join(" "))
|
|
156
|
+
}
|
|
157
|
+
const version = header.getUint16(4, true)
|
|
158
|
+
if (version !== PREVIEW_FRAME_VERSION) return fail("unsupported-version", `v${version}`)
|
|
159
|
+
const headerLen = header.getUint16(6, true)
|
|
160
|
+
if (headerLen !== PREVIEW_FRAME_HEADER_BYTES) return fail("bad-header-length", `${headerLen} bytes`)
|
|
161
|
+
const payloadLen = header.getUint32(8, true)
|
|
162
|
+
const sessionGen = header.getUint32(12, true)
|
|
163
|
+
const frameSeq = header.getUint32(16, true)
|
|
164
|
+
const width = header.getUint16(20, true)
|
|
165
|
+
const height = header.getUint16(22, true)
|
|
166
|
+
if (width === 0 || height === 0 || width > PREVIEW_FRAME_MAX_DIMENSION || height > PREVIEW_FRAME_MAX_DIMENSION) {
|
|
167
|
+
return fail("bad-dimensions", `${width}x${height}`)
|
|
168
|
+
}
|
|
169
|
+
const formatCode = header.getUint8(24)
|
|
170
|
+
if (formatCode !== PIXEL_FORMAT_I420 && formatCode !== PIXEL_FORMAT_NV12) {
|
|
171
|
+
return fail("unknown-pixel-format", `code ${formatCode}`)
|
|
172
|
+
}
|
|
173
|
+
const expectedPayload = packedPayloadBytes(width, height)
|
|
174
|
+
if (payloadLen !== expectedPayload) {
|
|
175
|
+
return fail("payload-size-mismatch", `${payloadLen} bytes, expected ${expectedPayload}`)
|
|
176
|
+
}
|
|
177
|
+
if (buffer.byteLength < headerLen + payloadLen) {
|
|
178
|
+
return fail("truncated-payload", `${buffer.byteLength - headerLen} of ${payloadLen} bytes`)
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
const matrixCode = header.getUint8(26)
|
|
182
|
+
const rangeCode = header.getUint8(27)
|
|
183
|
+
const flags = header.getUint16(28, true)
|
|
184
|
+
const chroma = chromaDimensions(width, height)
|
|
185
|
+
const luma = width * height
|
|
186
|
+
const base: ParsedFrameBase = {
|
|
187
|
+
ok: true,
|
|
188
|
+
sessionGen,
|
|
189
|
+
frameSeq,
|
|
190
|
+
width,
|
|
191
|
+
height,
|
|
192
|
+
rotationQuarters: header.getUint8(25) % 4,
|
|
193
|
+
// Unknown metadata is not a rejection: BT.601 limited is what an untagged 8-bit camera frame
|
|
194
|
+
// almost always is. The assumption is surfaced so a wrong-looking picture can be traced to it
|
|
195
|
+
// instead of to the shader.
|
|
196
|
+
colorMatrix: matrixCode === 2 ? "bt709" : "bt601",
|
|
197
|
+
colorRange: rangeCode === 2 ? "full" : "limited",
|
|
198
|
+
colorMatrixAssumed: matrixCode !== 1 && matrixCode !== 2,
|
|
199
|
+
colorRangeAssumed: rangeCode !== 1 && rangeCode !== 2,
|
|
200
|
+
senderColorFallback: (flags & 0x0001) !== 0,
|
|
201
|
+
timestampNs: header.getBigInt64(32, true),
|
|
202
|
+
sentAtNs: header.getBigInt64(40, true),
|
|
203
|
+
yStride: width,
|
|
204
|
+
chromaWidth: chroma.width,
|
|
205
|
+
chromaHeight: chroma.height,
|
|
206
|
+
chromaStride: formatCode === PIXEL_FORMAT_NV12 ? chroma.width * 2 : chroma.width,
|
|
207
|
+
payloadLen,
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
// Subviews, never copies.
|
|
211
|
+
if (formatCode === PIXEL_FORMAT_NV12) {
|
|
212
|
+
return {
|
|
213
|
+
...base,
|
|
214
|
+
pixelFormat: "nv12",
|
|
215
|
+
y: new Uint8Array(buffer, headerLen, luma),
|
|
216
|
+
uv: new Uint8Array(buffer, headerLen + luma, 2 * chroma.width * chroma.height),
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
const chromaPlane = chroma.width * chroma.height
|
|
220
|
+
return {
|
|
221
|
+
...base,
|
|
222
|
+
pixelFormat: "i420",
|
|
223
|
+
y: new Uint8Array(buffer, headerLen, luma),
|
|
224
|
+
u: new Uint8Array(buffer, headerLen + luma, chromaPlane),
|
|
225
|
+
v: new Uint8Array(buffer, headerLen + luma + chromaPlane, chromaPlane),
|
|
226
|
+
}
|
|
227
|
+
}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Collapse a stream of box measurements into the few that matter.
|
|
3
|
+
*
|
|
4
|
+
* A layout animation fires ResizeObserver every frame. Bursts inside one frame are merged with
|
|
5
|
+
* `requestAnimationFrame`, and a size is only applied once it has held still for the settle
|
|
6
|
+
* window. Hiding (a zero box) and the first measurement apply at once: the first so the preview
|
|
7
|
+
* starts without a delay, hiding so production stops as soon as nobody can see it.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
export interface PreviewBox {
|
|
11
|
+
width: number
|
|
12
|
+
height: number
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export interface ResizeCoalescerOptions {
|
|
16
|
+
apply: (box: PreviewBox) => void
|
|
17
|
+
settleMs?: number
|
|
18
|
+
requestFrame?: (cb: () => void) => unknown
|
|
19
|
+
setTimer?: (cb: () => void, ms: number) => unknown
|
|
20
|
+
clearTimer?: (handle: unknown) => void
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export const RESIZE_SETTLE_MS = 250
|
|
24
|
+
|
|
25
|
+
function isZero(box: PreviewBox): boolean {
|
|
26
|
+
return !(box.width > 0) || !(box.height > 0)
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export class ResizeCoalescer {
|
|
30
|
+
private latest: PreviewBox | null = null
|
|
31
|
+
private lastApplied: PreviewBox | null = null
|
|
32
|
+
private framePending = false
|
|
33
|
+
private timer: unknown = null
|
|
34
|
+
private cancelled = false
|
|
35
|
+
private readonly settleMs: number
|
|
36
|
+
private readonly requestFrame: (cb: () => void) => unknown
|
|
37
|
+
private readonly setTimer: (cb: () => void, ms: number) => unknown
|
|
38
|
+
private readonly clearTimer: (handle: unknown) => void
|
|
39
|
+
|
|
40
|
+
constructor(private readonly options: ResizeCoalescerOptions) {
|
|
41
|
+
this.settleMs = options.settleMs ?? RESIZE_SETTLE_MS
|
|
42
|
+
this.requestFrame =
|
|
43
|
+
options.requestFrame ??
|
|
44
|
+
((cb) => (typeof requestAnimationFrame === "function" ? requestAnimationFrame(() => cb()) : setTimeout(cb, 16)))
|
|
45
|
+
this.setTimer = options.setTimer ?? ((cb, ms) => setTimeout(cb, ms))
|
|
46
|
+
this.clearTimer = options.clearTimer ?? ((handle) => clearTimeout(handle as ReturnType<typeof setTimeout>))
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
push(box: PreviewBox): void {
|
|
50
|
+
if (this.cancelled) return
|
|
51
|
+
this.latest = box
|
|
52
|
+
if (!this.lastApplied || isZero(box) || isZero(this.lastApplied)) {
|
|
53
|
+
this.clearSettle()
|
|
54
|
+
this.applyLatest()
|
|
55
|
+
return
|
|
56
|
+
}
|
|
57
|
+
if (this.framePending) return
|
|
58
|
+
this.framePending = true
|
|
59
|
+
this.requestFrame(() => {
|
|
60
|
+
this.framePending = false
|
|
61
|
+
if (this.cancelled) return
|
|
62
|
+
this.clearSettle()
|
|
63
|
+
this.timer = this.setTimer(() => {
|
|
64
|
+
this.timer = null
|
|
65
|
+
this.applyLatest()
|
|
66
|
+
}, this.settleMs)
|
|
67
|
+
})
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
cancel(): void {
|
|
71
|
+
this.cancelled = true
|
|
72
|
+
this.clearSettle()
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
private applyLatest(): void {
|
|
76
|
+
const box = this.latest
|
|
77
|
+
if (!box || this.cancelled) return
|
|
78
|
+
if (this.lastApplied && this.lastApplied.width === box.width && this.lastApplied.height === box.height) return
|
|
79
|
+
this.lastApplied = box
|
|
80
|
+
this.options.apply(box)
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
private clearSettle(): void {
|
|
84
|
+
if (this.timer === null) return
|
|
85
|
+
this.clearTimer(this.timer)
|
|
86
|
+
this.timer = null
|
|
87
|
+
}
|
|
88
|
+
}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Build `MFPV` frames for tests. Lives beside the parser so the golden-bytes test and the parser
|
|
3
|
+
* cannot drift apart silently. Not exported from the package entry.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import {
|
|
7
|
+
chromaDimensions,
|
|
8
|
+
packedPayloadBytes,
|
|
9
|
+
PIXEL_FORMAT_I420,
|
|
10
|
+
PIXEL_FORMAT_NV12,
|
|
11
|
+
PREVIEW_FRAME_HEADER_BYTES,
|
|
12
|
+
PREVIEW_FRAME_MAGIC,
|
|
13
|
+
PREVIEW_FRAME_VERSION,
|
|
14
|
+
type PreviewPixelFormat,
|
|
15
|
+
} from "./protocol"
|
|
16
|
+
|
|
17
|
+
export interface BuildPreviewFrameOptions {
|
|
18
|
+
width?: number
|
|
19
|
+
height?: number
|
|
20
|
+
pixelFormat?: PreviewPixelFormat
|
|
21
|
+
sessionGen?: number
|
|
22
|
+
frameSeq?: number
|
|
23
|
+
rotationQuarters?: number
|
|
24
|
+
/** Raw header codes so a test can send "unknown" (0) and check the fallback. */
|
|
25
|
+
colorMatrixCode?: number
|
|
26
|
+
colorRangeCode?: number
|
|
27
|
+
flags?: number
|
|
28
|
+
timestampNs?: bigint
|
|
29
|
+
sentAtNs?: bigint
|
|
30
|
+
/** Byte written into every sample of each plane, so plane views can be told apart. */
|
|
31
|
+
fill?: {y?: number; u?: number; v?: number}
|
|
32
|
+
/** Deliberate corruption, for the rejection tests. Every field overrides what would be written. */
|
|
33
|
+
corrupt?: {
|
|
34
|
+
magic?: readonly number[]
|
|
35
|
+
version?: number
|
|
36
|
+
headerLen?: number
|
|
37
|
+
payloadLen?: number
|
|
38
|
+
/** Payload bytes actually appended, independent of the declared `payloadLen`. */
|
|
39
|
+
payloadBytes?: number
|
|
40
|
+
pixelFormatCode?: number
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** Build a wire-format frame. */
|
|
45
|
+
export function buildPreviewFrame(options: BuildPreviewFrameOptions = {}): ArrayBuffer {
|
|
46
|
+
const width = options.width ?? 1280
|
|
47
|
+
const height = options.height ?? 720
|
|
48
|
+
const pixelFormat = options.pixelFormat ?? "i420"
|
|
49
|
+
const corrupt = options.corrupt ?? {}
|
|
50
|
+
const chroma = chromaDimensions(width, height)
|
|
51
|
+
const declaredPayload = corrupt.payloadLen ?? packedPayloadBytes(width, height)
|
|
52
|
+
const actualPayload = corrupt.payloadBytes ?? packedPayloadBytes(width, height)
|
|
53
|
+
const buffer = new ArrayBuffer(PREVIEW_FRAME_HEADER_BYTES + actualPayload)
|
|
54
|
+
const header = new DataView(buffer, 0, PREVIEW_FRAME_HEADER_BYTES)
|
|
55
|
+
const magic = corrupt.magic ?? PREVIEW_FRAME_MAGIC
|
|
56
|
+
for (let index = 0; index < 4; index += 1) header.setUint8(index, magic[index] ?? 0)
|
|
57
|
+
header.setUint16(4, corrupt.version ?? PREVIEW_FRAME_VERSION, true)
|
|
58
|
+
header.setUint16(6, corrupt.headerLen ?? PREVIEW_FRAME_HEADER_BYTES, true)
|
|
59
|
+
header.setUint32(8, declaredPayload, true)
|
|
60
|
+
header.setUint32(12, options.sessionGen ?? 1, true)
|
|
61
|
+
header.setUint32(16, options.frameSeq ?? 1, true)
|
|
62
|
+
header.setUint16(20, width, true)
|
|
63
|
+
header.setUint16(22, height, true)
|
|
64
|
+
header.setUint8(24, corrupt.pixelFormatCode ?? (pixelFormat === "nv12" ? PIXEL_FORMAT_NV12 : PIXEL_FORMAT_I420))
|
|
65
|
+
header.setUint8(25, options.rotationQuarters ?? 0)
|
|
66
|
+
header.setUint8(26, options.colorMatrixCode ?? 1)
|
|
67
|
+
header.setUint8(27, options.colorRangeCode ?? 1)
|
|
68
|
+
header.setUint16(28, options.flags ?? 0, true)
|
|
69
|
+
header.setBigInt64(32, options.timestampNs ?? 0n, true)
|
|
70
|
+
header.setBigInt64(40, options.sentAtNs ?? 0n, true)
|
|
71
|
+
|
|
72
|
+
const payload = new Uint8Array(buffer, PREVIEW_FRAME_HEADER_BYTES)
|
|
73
|
+
const luma = Math.min(width * height, payload.length)
|
|
74
|
+
const chromaPlane = chroma.width * chroma.height
|
|
75
|
+
payload.fill(options.fill?.y ?? 0x80, 0, luma)
|
|
76
|
+
if (pixelFormat === "nv12") {
|
|
77
|
+
for (let index = luma; index + 1 < payload.length; index += 2) {
|
|
78
|
+
payload[index] = options.fill?.u ?? 0x40
|
|
79
|
+
payload[index + 1] = options.fill?.v ?? 0xc0
|
|
80
|
+
}
|
|
81
|
+
} else {
|
|
82
|
+
payload.fill(options.fill?.u ?? 0x40, luma, Math.min(luma + chromaPlane, payload.length))
|
|
83
|
+
payload.fill(options.fill?.v ?? 0xc0, Math.min(luma + chromaPlane, payload.length))
|
|
84
|
+
}
|
|
85
|
+
return buffer
|
|
86
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Preview sizing tiers.
|
|
3
|
+
*
|
|
4
|
+
* The page measures its box (CSS pixels × devicePixelRatio, after rotation) and only tells the
|
|
5
|
+
* host when the quantized tier changes, so a resize storm costs at most one reconfigure per tier.
|
|
6
|
+
* The host quantizes again with the same table and is the authority; native never upscales.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
export interface PreviewTier {
|
|
10
|
+
width: number
|
|
11
|
+
height: number
|
|
12
|
+
maxFps: number
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/** Smallest first. The last entry is the ceiling until measurements justify raising it. */
|
|
16
|
+
export const PREVIEW_TIERS: readonly PreviewTier[] = [
|
|
17
|
+
{width: 320, height: 180, maxFps: 15},
|
|
18
|
+
{width: 640, height: 360, maxFps: 15},
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* The smallest tier that covers the box, capped at the ceiling. A zero-area box means the preview
|
|
23
|
+
* is hidden and returns null.
|
|
24
|
+
*/
|
|
25
|
+
export function quantizePreviewTier(boxWidth: number, boxHeight: number): PreviewTier | null {
|
|
26
|
+
if (!(boxWidth > 0) || !(boxHeight > 0)) return null
|
|
27
|
+
for (const tier of PREVIEW_TIERS) {
|
|
28
|
+
if (tier.width >= boxWidth && tier.height >= boxHeight) return tier
|
|
29
|
+
}
|
|
30
|
+
return PREVIEW_TIERS[PREVIEW_TIERS.length - 1]!
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function sameTier(a: PreviewTier | null, b: PreviewTier | null): boolean {
|
|
34
|
+
if (a === b) return true
|
|
35
|
+
if (!a || !b) return false
|
|
36
|
+
return a.width === b.width && a.height === b.height && a.maxFps === b.maxFps
|
|
37
|
+
}
|