@mentra/miniapp 3.2.1-dev.311 → 3.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -0
- package/dist/background/index.d.ts +5 -3
- package/dist/background/index.d.ts.map +1 -1
- package/dist/background/index.js +1 -0
- package/dist/background/index.js.map +1 -1
- package/dist/index.d.ts +6 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/modules/display.d.ts +33 -0
- package/dist/modules/display.d.ts.map +1 -1
- package/dist/modules/display.js +2 -0
- package/dist/modules/display.js.map +1 -1
- package/dist/modules/meeting.d.ts +66 -6
- package/dist/modules/meeting.d.ts.map +1 -1
- package/dist/modules/meeting.js +72 -19
- package/dist/modules/meeting.js.map +1 -1
- package/dist/modules/mic.d.ts +5 -4
- package/dist/modules/mic.d.ts.map +1 -1
- package/dist/modules/mic.js +5 -4
- package/dist/modules/mic.js.map +1 -1
- package/dist/modules/speaker.d.ts +2 -0
- package/dist/modules/speaker.d.ts.map +1 -1
- package/dist/modules/speaker.js +5 -0
- package/dist/modules/speaker.js.map +1 -1
- package/dist/modules/stream.d.ts +46 -0
- package/dist/modules/stream.d.ts.map +1 -1
- package/dist/modules/stream.js +107 -0
- package/dist/modules/stream.js.map +1 -1
- package/dist/protocol.d.ts +39 -0
- package/dist/protocol.d.ts.map +1 -1
- package/dist/protocol.js +36 -0
- package/dist/protocol.js.map +1 -1
- package/dist/react/StreamPreview.d.ts +41 -0
- package/dist/react/StreamPreview.d.ts.map +1 -0
- package/dist/react/StreamPreview.js +144 -0
- package/dist/react/StreamPreview.js.map +1 -0
- package/dist/react/index.d.ts +3 -0
- package/dist/react/index.d.ts.map +1 -1
- package/dist/react/index.js +2 -0
- package/dist/react/index.js.map +1 -1
- package/dist/react/stream-preview/PreviewConnection.d.ts +139 -0
- package/dist/react/stream-preview/PreviewConnection.d.ts.map +1 -0
- package/dist/react/stream-preview/PreviewConnection.js +462 -0
- package/dist/react/stream-preview/PreviewConnection.js.map +1 -0
- package/dist/react/stream-preview/controlChannel.d.ts +75 -0
- package/dist/react/stream-preview/controlChannel.d.ts.map +1 -0
- package/dist/react/stream-preview/controlChannel.js +55 -0
- package/dist/react/stream-preview/controlChannel.js.map +1 -0
- package/dist/react/stream-preview/creditLoop.d.ts +81 -0
- package/dist/react/stream-preview/creditLoop.d.ts.map +1 -0
- package/dist/react/stream-preview/creditLoop.js +146 -0
- package/dist/react/stream-preview/creditLoop.js.map +1 -0
- package/dist/react/stream-preview/protocol.d.ts +113 -0
- package/dist/react/stream-preview/protocol.d.ts.map +1 -0
- package/dist/react/stream-preview/protocol.js +148 -0
- package/dist/react/stream-preview/protocol.js.map +1 -0
- package/dist/react/stream-preview/resizeCoalescer.d.ts +38 -0
- package/dist/react/stream-preview/resizeCoalescer.d.ts.map +1 -0
- package/dist/react/stream-preview/resizeCoalescer.js +71 -0
- package/dist/react/stream-preview/resizeCoalescer.js.map +1 -0
- package/dist/react/stream-preview/testFrames.d.ts +38 -0
- package/dist/react/stream-preview/testFrames.d.ts.map +1 -0
- package/dist/react/stream-preview/testFrames.js +50 -0
- package/dist/react/stream-preview/testFrames.js.map +1 -0
- package/dist/react/stream-preview/tiers.d.ts +21 -0
- package/dist/react/stream-preview/tiers.d.ts.map +1 -0
- package/dist/react/stream-preview/tiers.js +33 -0
- package/dist/react/stream-preview/tiers.js.map +1 -0
- package/dist/react/stream-preview/trace.d.ts +23 -0
- package/dist/react/stream-preview/trace.d.ts.map +1 -0
- package/dist/react/stream-preview/trace.js +71 -0
- package/dist/react/stream-preview/trace.js.map +1 -0
- package/dist/react/stream-preview/transport.d.ts +79 -0
- package/dist/react/stream-preview/transport.d.ts.map +1 -0
- package/dist/react/stream-preview/transport.js +209 -0
- package/dist/react/stream-preview/transport.js.map +1 -0
- package/dist/react/stream-preview/yuvRenderer.d.ts +31 -0
- package/dist/react/stream-preview/yuvRenderer.d.ts.map +1 -0
- package/dist/react/stream-preview/yuvRenderer.js +398 -0
- package/dist/react/stream-preview/yuvRenderer.js.map +1 -0
- package/dist/react/useHistoryState.d.ts +35 -0
- package/dist/react/useHistoryState.d.ts.map +1 -0
- package/dist/react/useHistoryState.js +117 -0
- package/dist/react/useHistoryState.js.map +1 -0
- package/dist/session.d.ts +5 -0
- package/dist/session.d.ts.map +1 -1
- package/dist/session.js +5 -0
- package/dist/session.js.map +1 -1
- package/dist/ui/index.d.ts +3 -0
- package/dist/ui/index.d.ts.map +1 -1
- package/dist/ui/index.js +2 -0
- package/dist/ui/index.js.map +1 -1
- package/package.json +8 -3
- package/src/background/index.ts +14 -1
- package/src/index.ts +15 -0
- package/src/modules/display.ts +37 -0
- package/src/modules/meeting.ts +133 -23
- package/src/modules/mic.ts +5 -4
- package/src/modules/speaker.ts +10 -4
- package/src/modules/stream.ts +153 -1
- package/src/protocol.ts +75 -0
- package/src/react/StreamPreview.tsx +187 -0
- package/src/react/index.ts +3 -0
- package/src/react/stream-preview/PreviewConnection.ts +552 -0
- package/src/react/stream-preview/controlChannel.ts +119 -0
- package/src/react/stream-preview/creditLoop.ts +189 -0
- package/src/react/stream-preview/protocol.ts +227 -0
- package/src/react/stream-preview/resizeCoalescer.ts +88 -0
- package/src/react/stream-preview/testFrames.ts +86 -0
- package/src/react/stream-preview/tiers.ts +37 -0
- package/src/react/stream-preview/trace.ts +81 -0
- package/src/react/stream-preview/transport.ts +242 -0
- package/src/react/stream-preview/yuvRenderer.ts +453 -0
- package/src/react/useHistoryState.ts +135 -0
- package/src/session.ts +8 -0
- package/src/ui/index.ts +3 -0
|
@@ -0,0 +1,453 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* WebGL1 YUV → RGB renderer for raw preview frames.
|
|
3
|
+
*
|
|
4
|
+
* WebGL1 only, on purpose: the Mentra App supports iOS 15.5 / Safari 15, where WebGL2, WebCodecs,
|
|
5
|
+
* `VideoFrame` and `OffscreenCanvas` are either absent or unreliable. Everything here is the
|
|
6
|
+
* 2013 feature set — `LUMINANCE` textures, one draw call, a fragment shader doing the colour
|
|
7
|
+
* conversion.
|
|
8
|
+
*
|
|
9
|
+
* Rules:
|
|
10
|
+
* - JavaScript never touches a pixel. Planes go to `texImage2D`/`texSubImage2D` as views.
|
|
11
|
+
* - `readPixels` is never called; it would stall the pipeline.
|
|
12
|
+
* - `drawFrame` times the SUBMIT, not the paint. See the comment on that method.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import type {ParsedFrame, PreviewColorMatrix, PreviewColorRange, PreviewPixelFormat} from "./protocol"
|
|
16
|
+
import {previewTrace, previewTraceWarn} from "./trace"
|
|
17
|
+
|
|
18
|
+
const VERTEX_SHADER = `
|
|
19
|
+
attribute vec2 aPosition;
|
|
20
|
+
attribute vec2 aTexCoord;
|
|
21
|
+
uniform mat2 uTransform;
|
|
22
|
+
varying vec2 vTexCoord;
|
|
23
|
+
void main() {
|
|
24
|
+
gl_Position = vec4(uTransform * aPosition, 0.0, 1.0);
|
|
25
|
+
vTexCoord = aTexCoord;
|
|
26
|
+
}
|
|
27
|
+
`
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* `uCoeff` is (Cr→R, Cb→G, Cr→G, Cb→B) for the frame's matrix; `uYOffset`/`uYScale`/`uCScale`
|
|
31
|
+
* carry the range. Both are uniforms rather than shader variants so a stream that switches
|
|
32
|
+
* colour metadata mid-run does not force a recompile.
|
|
33
|
+
*/
|
|
34
|
+
function fragmentShader(format: PreviewPixelFormat): string {
|
|
35
|
+
const chroma =
|
|
36
|
+
format === "nv12"
|
|
37
|
+
? `uniform sampler2D uUv;
|
|
38
|
+
vec2 chromaSample() {
|
|
39
|
+
vec4 texel = texture2D(uUv, vTexCoord);
|
|
40
|
+
return vec2(texel.r, texel.a);
|
|
41
|
+
}`
|
|
42
|
+
: `uniform sampler2D uU;
|
|
43
|
+
uniform sampler2D uV;
|
|
44
|
+
vec2 chromaSample() {
|
|
45
|
+
return vec2(texture2D(uU, vTexCoord).r, texture2D(uV, vTexCoord).r);
|
|
46
|
+
}`
|
|
47
|
+
return `
|
|
48
|
+
#ifdef GL_FRAGMENT_PRECISION_HIGH
|
|
49
|
+
precision highp float;
|
|
50
|
+
#else
|
|
51
|
+
precision mediump float;
|
|
52
|
+
#endif
|
|
53
|
+
varying vec2 vTexCoord;
|
|
54
|
+
uniform sampler2D uY;
|
|
55
|
+
uniform float uYOffset;
|
|
56
|
+
uniform float uYScale;
|
|
57
|
+
uniform float uCScale;
|
|
58
|
+
uniform vec4 uCoeff;
|
|
59
|
+
${chroma}
|
|
60
|
+
void main() {
|
|
61
|
+
float y = (texture2D(uY, vTexCoord).r - uYOffset) * uYScale;
|
|
62
|
+
vec2 c = (chromaSample() - 0.5019608) * uCScale;
|
|
63
|
+
vec3 rgb = vec3(
|
|
64
|
+
y + uCoeff.x * c.y,
|
|
65
|
+
y + uCoeff.y * c.x + uCoeff.z * c.y,
|
|
66
|
+
y + uCoeff.w * c.x);
|
|
67
|
+
gl_FragColor = vec4(clamp(rgb, 0.0, 1.0), 1.0);
|
|
68
|
+
}
|
|
69
|
+
`
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** Standard BT.601 and BT.709 conversion coefficients, as (Cr→R, Cb→G, Cr→G, Cb→B). */
|
|
73
|
+
const MATRIX_COEFFICIENTS: Record<PreviewColorMatrix, readonly [number, number, number, number]> = {
|
|
74
|
+
bt601: [1.402, -0.344136, -0.714136, 1.772],
|
|
75
|
+
bt709: [1.5748, -0.187324, -0.468124, 1.8556],
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** Limited range is Y 16..235 and chroma 16..240; full range uses the whole byte. */
|
|
79
|
+
const RANGE_SCALES: Record<PreviewColorRange, {yOffset: number; yScale: number; cScale: number}> = {
|
|
80
|
+
limited: {yOffset: 16 / 255, yScale: 255 / 219, cScale: 255 / 224},
|
|
81
|
+
full: {yOffset: 0, yScale: 1, cScale: 1},
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** Clockwise quarter turns, as exact cos/sin so 90° stays 90°. */
|
|
85
|
+
const ROTATIONS: ReadonlyArray<{cos: number; sin: number}> = [
|
|
86
|
+
{cos: 1, sin: 0},
|
|
87
|
+
{cos: 0, sin: 1},
|
|
88
|
+
{cos: -1, sin: 0},
|
|
89
|
+
{cos: 0, sin: -1},
|
|
90
|
+
]
|
|
91
|
+
|
|
92
|
+
/** `contain` letterboxes; `cover` fills the canvas and crops the overflow. Never stretches. */
|
|
93
|
+
export type PreviewFit = "contain" | "cover"
|
|
94
|
+
|
|
95
|
+
export interface YuvRendererOptions {
|
|
96
|
+
/** Context loss, a failed compile, a failed upload: anything that makes the canvas stop. */
|
|
97
|
+
onError?: (reason: string) => void
|
|
98
|
+
fit?: PreviewFit
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
export interface YuvRenderer {
|
|
102
|
+
/** False once the GL context is lost, until it is restored. */
|
|
103
|
+
readonly alive: boolean
|
|
104
|
+
setFit(fit: PreviewFit): void
|
|
105
|
+
drawFrame(frame: ParsedFrame): number | null
|
|
106
|
+
dispose(): void
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
interface ProgramBundle {
|
|
110
|
+
program: WebGLProgram
|
|
111
|
+
aPosition: number
|
|
112
|
+
aTexCoord: number
|
|
113
|
+
uTransform: WebGLUniformLocation | null
|
|
114
|
+
uYOffset: WebGLUniformLocation | null
|
|
115
|
+
uYScale: WebGLUniformLocation | null
|
|
116
|
+
uCScale: WebGLUniformLocation | null
|
|
117
|
+
uCoeff: WebGLUniformLocation | null
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
interface Layout {
|
|
121
|
+
width: number
|
|
122
|
+
height: number
|
|
123
|
+
format: PreviewPixelFormat
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
const POSITIONS = new Float32Array([-1, -1, 1, -1, -1, 1, 1, 1])
|
|
127
|
+
// v = 0 is the first uploaded row, which is the top of the image, so it maps to NDC y = +1.
|
|
128
|
+
const TEX_COORDS = new Float32Array([0, 1, 1, 1, 0, 0, 1, 0])
|
|
129
|
+
|
|
130
|
+
class WebGlYuvRenderer implements YuvRenderer {
|
|
131
|
+
private readonly programs = new Map<PreviewPixelFormat, ProgramBundle>()
|
|
132
|
+
private readonly textures = new Map<string, WebGLTexture>()
|
|
133
|
+
private positionBuffer: WebGLBuffer | null = null
|
|
134
|
+
private texCoordBuffer: WebGLBuffer | null = null
|
|
135
|
+
private layout: Layout | null = null
|
|
136
|
+
private lost = false
|
|
137
|
+
private fit: PreviewFit
|
|
138
|
+
|
|
139
|
+
private readonly onContextLost = (event: Event) => {
|
|
140
|
+
// Without preventDefault the context is never restored, and the canvas would stay black for
|
|
141
|
+
// the rest of the document.
|
|
142
|
+
event.preventDefault()
|
|
143
|
+
this.lost = true
|
|
144
|
+
this.programs.clear()
|
|
145
|
+
this.textures.clear()
|
|
146
|
+
this.positionBuffer = null
|
|
147
|
+
this.texCoordBuffer = null
|
|
148
|
+
this.layout = null
|
|
149
|
+
previewTraceWarn("webgl_context_lost")
|
|
150
|
+
this.options.onError?.("webgl_context_lost")
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
private readonly onContextRestored = () => {
|
|
154
|
+
// Everything was invalidated by the loss; the next frame rebuilds it lazily.
|
|
155
|
+
this.lost = false
|
|
156
|
+
previewTrace("webgl_context_restored")
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
constructor(
|
|
160
|
+
private readonly canvas: HTMLCanvasElement,
|
|
161
|
+
private readonly gl: WebGLRenderingContext,
|
|
162
|
+
private readonly options: YuvRendererOptions,
|
|
163
|
+
) {
|
|
164
|
+
this.fit = options.fit ?? "contain"
|
|
165
|
+
canvas.addEventListener("webglcontextlost", this.onContextLost)
|
|
166
|
+
canvas.addEventListener("webglcontextrestored", this.onContextRestored)
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
get alive(): boolean {
|
|
170
|
+
return !this.lost && !this.gl.isContextLost()
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
setFit(fit: PreviewFit): void {
|
|
174
|
+
this.fit = fit
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* Upload the planes and submit one draw.
|
|
179
|
+
*
|
|
180
|
+
* The returned number is how long the SUBMIT took: uploads plus `drawArrays`, measured with
|
|
181
|
+
* `performance.now()`. It is NOT presentation latency and not a frame time — GL is asynchronous
|
|
182
|
+
* and the pixels reach the screen some unknown time later. Proving presentation would need
|
|
183
|
+
* `readPixels` or a fence, and both change what they measure.
|
|
184
|
+
*
|
|
185
|
+
* Returns null when there is no usable context.
|
|
186
|
+
*/
|
|
187
|
+
drawFrame(frame: ParsedFrame): number | null {
|
|
188
|
+
if (!this.alive) return null
|
|
189
|
+
const gl = this.gl
|
|
190
|
+
const startedAt = performance.now()
|
|
191
|
+
|
|
192
|
+
const bundle = this.programFor(frame.pixelFormat)
|
|
193
|
+
if (!bundle) return null
|
|
194
|
+
this.syncDrawingBuffer()
|
|
195
|
+
|
|
196
|
+
gl.useProgram(bundle.program)
|
|
197
|
+
// Tight packing: every plane row starts on a byte boundary, which the default alignment of 4
|
|
198
|
+
// would get wrong for any odd width.
|
|
199
|
+
gl.pixelStorei(gl.UNPACK_ALIGNMENT, 1)
|
|
200
|
+
|
|
201
|
+
const reallocate =
|
|
202
|
+
!this.layout ||
|
|
203
|
+
this.layout.width !== frame.width ||
|
|
204
|
+
this.layout.height !== frame.height ||
|
|
205
|
+
this.layout.format !== frame.pixelFormat
|
|
206
|
+
if (reallocate && this.layout && this.layout.format !== frame.pixelFormat) this.dropChromaTextures()
|
|
207
|
+
if (frame.pixelFormat === "nv12") {
|
|
208
|
+
if (!this.upload("y", 0, gl.LUMINANCE, frame.width, frame.height, frame.y, reallocate)) return null
|
|
209
|
+
if (!this.upload("uv", 1, gl.LUMINANCE_ALPHA, frame.chromaWidth, frame.chromaHeight, frame.uv, reallocate)) {
|
|
210
|
+
return null
|
|
211
|
+
}
|
|
212
|
+
} else {
|
|
213
|
+
if (!this.upload("y", 0, gl.LUMINANCE, frame.width, frame.height, frame.y, reallocate)) return null
|
|
214
|
+
if (!this.upload("u", 1, gl.LUMINANCE, frame.chromaWidth, frame.chromaHeight, frame.u, reallocate)) return null
|
|
215
|
+
if (!this.upload("v", 2, gl.LUMINANCE, frame.chromaWidth, frame.chromaHeight, frame.v, reallocate)) return null
|
|
216
|
+
}
|
|
217
|
+
if (reallocate) {
|
|
218
|
+
this.layout = {width: frame.width, height: frame.height, format: frame.pixelFormat}
|
|
219
|
+
this.bindSamplers(bundle, frame.pixelFormat)
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
const coefficients = MATRIX_COEFFICIENTS[frame.colorMatrix]
|
|
223
|
+
const range = RANGE_SCALES[frame.colorRange]
|
|
224
|
+
gl.uniform1f(bundle.uYOffset, range.yOffset)
|
|
225
|
+
gl.uniform1f(bundle.uYScale, range.yScale)
|
|
226
|
+
gl.uniform1f(bundle.uCScale, range.cScale)
|
|
227
|
+
gl.uniform4f(bundle.uCoeff, coefficients[0], coefficients[1], coefficients[2], coefficients[3])
|
|
228
|
+
gl.uniformMatrix2fv(bundle.uTransform, false, this.transformFor(frame))
|
|
229
|
+
|
|
230
|
+
gl.clearColor(0, 0, 0, 1)
|
|
231
|
+
gl.clear(gl.COLOR_BUFFER_BIT)
|
|
232
|
+
gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4)
|
|
233
|
+
return performance.now() - startedAt
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
dispose(): void {
|
|
237
|
+
this.canvas.removeEventListener("webglcontextlost", this.onContextLost)
|
|
238
|
+
this.canvas.removeEventListener("webglcontextrestored", this.onContextRestored)
|
|
239
|
+
const gl = this.gl
|
|
240
|
+
if (!gl.isContextLost()) {
|
|
241
|
+
for (const texture of this.textures.values()) gl.deleteTexture(texture)
|
|
242
|
+
for (const bundle of this.programs.values()) gl.deleteProgram(bundle.program)
|
|
243
|
+
if (this.positionBuffer) gl.deleteBuffer(this.positionBuffer)
|
|
244
|
+
if (this.texCoordBuffer) gl.deleteBuffer(this.texCoordBuffer)
|
|
245
|
+
}
|
|
246
|
+
this.textures.clear()
|
|
247
|
+
this.programs.clear()
|
|
248
|
+
this.positionBuffer = null
|
|
249
|
+
this.texCoordBuffer = null
|
|
250
|
+
this.layout = null
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
/** Scale the rotated frame to contain or cover the canvas without ever stretching it. */
|
|
254
|
+
private transformFor(frame: ParsedFrame): Float32Array {
|
|
255
|
+
const {cos, sin} = ROTATIONS[frame.rotationQuarters] ?? ROTATIONS[0]!
|
|
256
|
+
const quarterTurned = frame.rotationQuarters % 2 === 1
|
|
257
|
+
const frameWidth = quarterTurned ? frame.height : frame.width
|
|
258
|
+
const frameHeight = quarterTurned ? frame.width : frame.height
|
|
259
|
+
const canvasAspect = this.canvas.width / Math.max(1, this.canvas.height)
|
|
260
|
+
const frameAspect = frameWidth / Math.max(1, frameHeight)
|
|
261
|
+
let scaleX = 1
|
|
262
|
+
let scaleY = 1
|
|
263
|
+
const frameIsWider = frameAspect > canvasAspect
|
|
264
|
+
if (this.fit === "cover") {
|
|
265
|
+
if (frameIsWider) scaleX = frameAspect / canvasAspect
|
|
266
|
+
else scaleY = canvasAspect / frameAspect
|
|
267
|
+
} else if (frameIsWider) scaleY = canvasAspect / frameAspect
|
|
268
|
+
else scaleX = frameAspect / canvasAspect
|
|
269
|
+
// Column-major for uniformMatrix2fv: scale ∘ clockwise rotation.
|
|
270
|
+
return new Float32Array([scaleX * cos, -scaleY * sin, scaleX * sin, scaleY * cos])
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
private syncDrawingBuffer(): void {
|
|
274
|
+
const canvas = this.canvas
|
|
275
|
+
const ratio = Math.min(typeof window === "undefined" ? 1 : window.devicePixelRatio || 1, 2)
|
|
276
|
+
const width = Math.max(1, Math.round((canvas.clientWidth || canvas.width) * ratio))
|
|
277
|
+
const height = Math.max(1, Math.round((canvas.clientHeight || canvas.height) * ratio))
|
|
278
|
+
if (canvas.width !== width || canvas.height !== height) {
|
|
279
|
+
canvas.width = width
|
|
280
|
+
canvas.height = height
|
|
281
|
+
}
|
|
282
|
+
this.gl.viewport(0, 0, canvas.width, canvas.height)
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
private upload(
|
|
286
|
+
key: string,
|
|
287
|
+
unit: number,
|
|
288
|
+
format: number,
|
|
289
|
+
width: number,
|
|
290
|
+
height: number,
|
|
291
|
+
pixels: Uint8Array,
|
|
292
|
+
reallocate: boolean,
|
|
293
|
+
): boolean {
|
|
294
|
+
const gl = this.gl
|
|
295
|
+
let texture = this.textures.get(key)
|
|
296
|
+
if (!texture) {
|
|
297
|
+
const created = gl.createTexture()
|
|
298
|
+
if (!created) {
|
|
299
|
+
this.options.onError?.("texture_allocation_failed")
|
|
300
|
+
return false
|
|
301
|
+
}
|
|
302
|
+
texture = created
|
|
303
|
+
this.textures.set(key, texture)
|
|
304
|
+
gl.activeTexture(gl.TEXTURE0 + unit)
|
|
305
|
+
gl.bindTexture(gl.TEXTURE_2D, texture)
|
|
306
|
+
// No mipmaps and CLAMP_TO_EDGE, which is what WebGL1 requires for non-power-of-two sizes.
|
|
307
|
+
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE)
|
|
308
|
+
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE)
|
|
309
|
+
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR)
|
|
310
|
+
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR)
|
|
311
|
+
gl.texImage2D(gl.TEXTURE_2D, 0, format, width, height, 0, format, gl.UNSIGNED_BYTE, pixels)
|
|
312
|
+
return true
|
|
313
|
+
}
|
|
314
|
+
gl.activeTexture(gl.TEXTURE0 + unit)
|
|
315
|
+
gl.bindTexture(gl.TEXTURE_2D, texture)
|
|
316
|
+
// Reallocating every frame is the classic way to make a renderer look slower than it is.
|
|
317
|
+
if (reallocate) gl.texImage2D(gl.TEXTURE_2D, 0, format, width, height, 0, format, gl.UNSIGNED_BYTE, pixels)
|
|
318
|
+
else gl.texSubImage2D(gl.TEXTURE_2D, 0, 0, 0, width, height, format, gl.UNSIGNED_BYTE, pixels)
|
|
319
|
+
return true
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/** A format switch leaves the other layout's chroma textures allocated; drop them. */
|
|
323
|
+
private dropChromaTextures(): void {
|
|
324
|
+
for (const key of ["u", "v", "uv"]) {
|
|
325
|
+
const texture = this.textures.get(key)
|
|
326
|
+
if (!texture) continue
|
|
327
|
+
this.gl.deleteTexture(texture)
|
|
328
|
+
this.textures.delete(key)
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
private bindSamplers(bundle: ProgramBundle, format: PreviewPixelFormat): void {
|
|
333
|
+
const gl = this.gl
|
|
334
|
+
gl.uniform1i(gl.getUniformLocation(bundle.program, "uY"), 0)
|
|
335
|
+
if (format === "nv12") {
|
|
336
|
+
gl.uniform1i(gl.getUniformLocation(bundle.program, "uUv"), 1)
|
|
337
|
+
return
|
|
338
|
+
}
|
|
339
|
+
gl.uniform1i(gl.getUniformLocation(bundle.program, "uU"), 1)
|
|
340
|
+
gl.uniform1i(gl.getUniformLocation(bundle.program, "uV"), 2)
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
private programFor(format: PreviewPixelFormat): ProgramBundle | null {
|
|
344
|
+
const existing = this.programs.get(format)
|
|
345
|
+
if (existing) {
|
|
346
|
+
this.bindGeometry(existing)
|
|
347
|
+
return existing
|
|
348
|
+
}
|
|
349
|
+
const gl = this.gl
|
|
350
|
+
const program = this.link(VERTEX_SHADER, fragmentShader(format))
|
|
351
|
+
if (!program) return null
|
|
352
|
+
const bundle: ProgramBundle = {
|
|
353
|
+
program,
|
|
354
|
+
aPosition: gl.getAttribLocation(program, "aPosition"),
|
|
355
|
+
aTexCoord: gl.getAttribLocation(program, "aTexCoord"),
|
|
356
|
+
uTransform: gl.getUniformLocation(program, "uTransform"),
|
|
357
|
+
uYOffset: gl.getUniformLocation(program, "uYOffset"),
|
|
358
|
+
uYScale: gl.getUniformLocation(program, "uYScale"),
|
|
359
|
+
uCScale: gl.getUniformLocation(program, "uCScale"),
|
|
360
|
+
uCoeff: gl.getUniformLocation(program, "uCoeff"),
|
|
361
|
+
}
|
|
362
|
+
this.programs.set(format, bundle)
|
|
363
|
+
gl.useProgram(program)
|
|
364
|
+
this.bindSamplers(bundle, format)
|
|
365
|
+
this.bindGeometry(bundle)
|
|
366
|
+
return bundle
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
private bindGeometry(bundle: ProgramBundle): void {
|
|
370
|
+
const gl = this.gl
|
|
371
|
+
if (!this.positionBuffer) {
|
|
372
|
+
this.positionBuffer = gl.createBuffer()
|
|
373
|
+
gl.bindBuffer(gl.ARRAY_BUFFER, this.positionBuffer)
|
|
374
|
+
gl.bufferData(gl.ARRAY_BUFFER, POSITIONS, gl.STATIC_DRAW)
|
|
375
|
+
}
|
|
376
|
+
gl.bindBuffer(gl.ARRAY_BUFFER, this.positionBuffer)
|
|
377
|
+
gl.enableVertexAttribArray(bundle.aPosition)
|
|
378
|
+
gl.vertexAttribPointer(bundle.aPosition, 2, gl.FLOAT, false, 0, 0)
|
|
379
|
+
if (!this.texCoordBuffer) {
|
|
380
|
+
this.texCoordBuffer = gl.createBuffer()
|
|
381
|
+
gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer)
|
|
382
|
+
gl.bufferData(gl.ARRAY_BUFFER, TEX_COORDS, gl.STATIC_DRAW)
|
|
383
|
+
}
|
|
384
|
+
gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer)
|
|
385
|
+
gl.enableVertexAttribArray(bundle.aTexCoord)
|
|
386
|
+
gl.vertexAttribPointer(bundle.aTexCoord, 2, gl.FLOAT, false, 0, 0)
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
private link(vertexSource: string, fragmentSource: string): WebGLProgram | null {
|
|
390
|
+
const gl = this.gl
|
|
391
|
+
const vertex = this.compile(gl.VERTEX_SHADER, vertexSource)
|
|
392
|
+
const fragment = this.compile(gl.FRAGMENT_SHADER, fragmentSource)
|
|
393
|
+
if (!vertex || !fragment) return null
|
|
394
|
+
const program = gl.createProgram()
|
|
395
|
+
if (!program) {
|
|
396
|
+
this.options.onError?.("program_allocation_failed")
|
|
397
|
+
return null
|
|
398
|
+
}
|
|
399
|
+
gl.attachShader(program, vertex)
|
|
400
|
+
gl.attachShader(program, fragment)
|
|
401
|
+
gl.linkProgram(program)
|
|
402
|
+
// The shaders are owned by the program once linked; keeping them alive leaks on every restore.
|
|
403
|
+
gl.deleteShader(vertex)
|
|
404
|
+
gl.deleteShader(fragment)
|
|
405
|
+
if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {
|
|
406
|
+
const log = gl.getProgramInfoLog(program) ?? ""
|
|
407
|
+
gl.deleteProgram(program)
|
|
408
|
+
previewTraceWarn("webgl_program_link_failed", {log: log.slice(0, 120)})
|
|
409
|
+
this.options.onError?.(`program_link_failed: ${log.slice(0, 120)}`)
|
|
410
|
+
return null
|
|
411
|
+
}
|
|
412
|
+
return program
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
private compile(type: number, source: string): WebGLShader | null {
|
|
416
|
+
const gl = this.gl
|
|
417
|
+
const shader = gl.createShader(type)
|
|
418
|
+
if (!shader) {
|
|
419
|
+
this.options.onError?.("shader_allocation_failed")
|
|
420
|
+
return null
|
|
421
|
+
}
|
|
422
|
+
gl.shaderSource(shader, source)
|
|
423
|
+
gl.compileShader(shader)
|
|
424
|
+
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
|
|
425
|
+
const log = gl.getShaderInfoLog(shader) ?? ""
|
|
426
|
+
gl.deleteShader(shader)
|
|
427
|
+
previewTraceWarn("webgl_shader_compile_failed", {log: log.slice(0, 120)})
|
|
428
|
+
this.options.onError?.(`shader_compile_failed: ${log.slice(0, 120)}`)
|
|
429
|
+
return null
|
|
430
|
+
}
|
|
431
|
+
return shader
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
/** Create the renderer, or null when this WebView has no WebGL1 context to give. */
|
|
436
|
+
export function createYuvRenderer(canvas: HTMLCanvasElement, options: YuvRendererOptions = {}): YuvRenderer | null {
|
|
437
|
+
const attributes: WebGLContextAttributes = {
|
|
438
|
+
alpha: false,
|
|
439
|
+
antialias: false,
|
|
440
|
+
depth: false,
|
|
441
|
+
stencil: false,
|
|
442
|
+
preserveDrawingBuffer: false,
|
|
443
|
+
powerPreference: "low-power",
|
|
444
|
+
}
|
|
445
|
+
const gl =
|
|
446
|
+
(canvas.getContext("webgl", attributes) as WebGLRenderingContext | null) ??
|
|
447
|
+
(canvas.getContext("experimental-webgl", attributes) as WebGLRenderingContext | null)
|
|
448
|
+
if (!gl) {
|
|
449
|
+
options.onError?.("webgl_unavailable")
|
|
450
|
+
return null
|
|
451
|
+
}
|
|
452
|
+
return new WebGlYuvRenderer(canvas, gl, options)
|
|
453
|
+
}
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview useHistoryState — `useState` whose changes are browser history
|
|
3
|
+
* entries, so the phone's back gesture steps back through them.
|
|
4
|
+
*
|
|
5
|
+
* The host sends the back gesture (iOS edge swipe, Android back) to the
|
|
6
|
+
* miniapp's WebView history first, and minimizes the miniapp only once that
|
|
7
|
+
* history is empty. Screens, tabs and dialogs kept in plain React state never
|
|
8
|
+
* create history, so back skips past them and minimizes the whole miniapp.
|
|
9
|
+
* This hook keeps its value in `history.state` instead:
|
|
10
|
+
*
|
|
11
|
+
* - Setting a new value pushes a history entry. Back pops it, and the hook
|
|
12
|
+
* re-renders with the previous value.
|
|
13
|
+
* - Setting the value back to what it was before its latest change(s) goes
|
|
14
|
+
* back instead of pushing. Closing a dialog with its own button therefore
|
|
15
|
+
* consumes the dialog's entry, and the next back gesture does not reopen it.
|
|
16
|
+
*
|
|
17
|
+
* Routers in hash or browser mode (react-router's `HashRouter`, wouter's
|
|
18
|
+
* `useHashLocation`) already create history for pages; use this hook for UI
|
|
19
|
+
* state that is not a route. Every hook on the page shares one history entry
|
|
20
|
+
* per step, and `key` names the value within it, so components passing the
|
|
21
|
+
* same key share the same value. Values are stored as structured clones.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import {useCallback, useRef, useSyncExternalStore} from "react"
|
|
25
|
+
|
|
26
|
+
const STATE_KEY = "__mentraHistoryState"
|
|
27
|
+
|
|
28
|
+
interface Entry {
|
|
29
|
+
values: Record<string, unknown>
|
|
30
|
+
/** One record per entry this hook pushed, oldest first, with the value it replaced. */
|
|
31
|
+
trail: Array<{key: string; previous: unknown}>
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
const EMPTY: Entry = {values: {}, trail: []}
|
|
35
|
+
|
|
36
|
+
let cachedState: unknown = null
|
|
37
|
+
let cachedEntry: Entry = EMPTY
|
|
38
|
+
|
|
39
|
+
/** This hook's slice of the current entry, cached so snapshots stay referentially stable. */
|
|
40
|
+
function currentEntry(): Entry {
|
|
41
|
+
if (typeof window === "undefined") return EMPTY
|
|
42
|
+
const state: unknown = window.history.state
|
|
43
|
+
if (state !== cachedState) {
|
|
44
|
+
cachedState = state
|
|
45
|
+
cachedEntry = (state as Record<string, Entry | undefined> | null)?.[STATE_KEY] ?? EMPTY
|
|
46
|
+
}
|
|
47
|
+
return cachedEntry
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const listeners = new Set<() => void>()
|
|
51
|
+
let boundWindow: Window | null = null
|
|
52
|
+
// history.go() lands later, on popstate. A change applied before then would
|
|
53
|
+
// push on top of the entry being left, and the traversal would pop that push
|
|
54
|
+
// instead, so changes wait in order until the traversal lands.
|
|
55
|
+
let traversing = false
|
|
56
|
+
const pending: Array<() => void> = []
|
|
57
|
+
|
|
58
|
+
function notify() {
|
|
59
|
+
listeners.forEach((listener) => listener())
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function flush() {
|
|
63
|
+
while (!traversing && pending.length > 0) pending.shift()!()
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function onPopState() {
|
|
67
|
+
traversing = false
|
|
68
|
+
notify()
|
|
69
|
+
// Let every popstate listener, including the host's history bridge, see the
|
|
70
|
+
// landed entry before a waiting change pushes the next one.
|
|
71
|
+
setTimeout(flush, 0)
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function subscribe(listener: () => void): () => void {
|
|
75
|
+
if (boundWindow !== window) {
|
|
76
|
+
boundWindow?.removeEventListener("popstate", onPopState)
|
|
77
|
+
window.addEventListener("popstate", onPopState)
|
|
78
|
+
boundWindow = window
|
|
79
|
+
traversing = false
|
|
80
|
+
pending.length = 0
|
|
81
|
+
}
|
|
82
|
+
listeners.add(listener)
|
|
83
|
+
return () => listeners.delete(listener)
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function whenSettled(change: () => void) {
|
|
87
|
+
if (traversing || pending.length > 0) pending.push(change)
|
|
88
|
+
else change()
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function apply(key: string, initialValue: unknown, update: unknown) {
|
|
92
|
+
const {values, trail} = currentEntry()
|
|
93
|
+
const current = key in values ? values[key] : initialValue
|
|
94
|
+
const next = typeof update === "function" ? (update as (previous: unknown) => unknown)(current) : update
|
|
95
|
+
if (Object.is(current, next)) return
|
|
96
|
+
for (let index = trail.length - 1; index >= 0 && trail[index].key === key; index--) {
|
|
97
|
+
if (Object.is(trail[index].previous, next)) {
|
|
98
|
+
traversing = true
|
|
99
|
+
window.history.go(index - trail.length)
|
|
100
|
+
return
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
// Keep other fields (a router's index, the host's depth marker) on the new entry.
|
|
104
|
+
const state: unknown = window.history.state
|
|
105
|
+
const base = state !== null && typeof state === "object" ? state : {}
|
|
106
|
+
const entry: Entry = {values: {...values, [key]: next}, trail: [...trail, {key, previous: current}]}
|
|
107
|
+
window.history.pushState({...base, [STATE_KEY]: entry}, "")
|
|
108
|
+
notify()
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
export type SetHistoryState<T> = (value: T | ((previous: T) => T)) => void
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Like `useState`, but each change is a history entry that the phone's back
|
|
115
|
+
* gesture undoes. `key` names the value in history state and must be unique
|
|
116
|
+
* per piece of state on the page.
|
|
117
|
+
*
|
|
118
|
+
* ```tsx
|
|
119
|
+
* const [tab, setTab] = useHistoryState<"home" | "settings">("tab", "home")
|
|
120
|
+
* const [pickerOpen, setPickerOpen] = useHistoryState("languagePicker", false)
|
|
121
|
+
* ```
|
|
122
|
+
*/
|
|
123
|
+
export function useHistoryState<T>(key: string, initialValue: T): [T, SetHistoryState<T>] {
|
|
124
|
+
const initialRef = useRef(initialValue)
|
|
125
|
+
const read = () => {
|
|
126
|
+
const {values} = currentEntry()
|
|
127
|
+
return (key in values ? values[key] : initialRef.current) as T
|
|
128
|
+
}
|
|
129
|
+
const value = useSyncExternalStore(subscribe, read, () => initialRef.current)
|
|
130
|
+
const setValue = useCallback<SetHistoryState<T>>(
|
|
131
|
+
(update) => whenSettled(() => apply(key, initialRef.current, update)),
|
|
132
|
+
[key],
|
|
133
|
+
)
|
|
134
|
+
return [value, setValue]
|
|
135
|
+
}
|
package/src/session.ts
CHANGED
|
@@ -66,6 +66,8 @@ import {
|
|
|
66
66
|
* not zero. Populated on the "ready" event (null in `start()`).
|
|
67
67
|
*/
|
|
68
68
|
export interface DisplayCapabilities {
|
|
69
|
+
/** Legacy panel dimensions. Prefer drawable width/height when present. */
|
|
70
|
+
resolution?: {width: number; height: number}
|
|
69
71
|
/** Public drawable canvas in px — raw coordinate space for `display.render()` boxes. */
|
|
70
72
|
width?: number
|
|
71
73
|
height?: number
|
|
@@ -710,6 +712,7 @@ export class MiniappSession<TChannels extends object = any> {
|
|
|
710
712
|
const event: import("./modules/meeting").MeetingState = {
|
|
711
713
|
state,
|
|
712
714
|
muted: Boolean(payload.muted),
|
|
715
|
+
videoEnabled: typeof payload.videoEnabled === "boolean" ? payload.videoEnabled : undefined,
|
|
713
716
|
error: payload.error as string | undefined,
|
|
714
717
|
meetingUrl: payload.meetingUrl as string | undefined,
|
|
715
718
|
provider: payload.provider as import("./modules/meeting").MeetingProvider | undefined,
|
|
@@ -729,6 +732,11 @@ export class MiniappSession<TChannels extends object = any> {
|
|
|
729
732
|
return
|
|
730
733
|
}
|
|
731
734
|
|
|
735
|
+
case MiniappResponseType.STREAM_PREVIEW_STATUS: {
|
|
736
|
+
this.stream._applyPreviewStatus(payload)
|
|
737
|
+
return
|
|
738
|
+
}
|
|
739
|
+
|
|
732
740
|
case MiniappRequestType.PING: {
|
|
733
741
|
// Phone → miniapp keepalive ping. Auto-reply with PONG.
|
|
734
742
|
const pong: object = {type: MiniappResponseType.PONG}
|
package/src/ui/index.ts
CHANGED
|
@@ -93,6 +93,9 @@ export {useConnected} from "../react/useConnected"
|
|
|
93
93
|
export {useSafeArea} from "../react/useSafeArea"
|
|
94
94
|
export {useCapsuleHeaderStyle} from "../react/useCapsuleHeaderStyle"
|
|
95
95
|
export {useRpc, type RpcCallable} from "../react/useRpc"
|
|
96
|
+
export {useHistoryState, type SetHistoryState} from "../react/useHistoryState"
|
|
97
|
+
export {StreamPreview} from "../react/StreamPreview"
|
|
98
|
+
export type {StreamPreviewProps, StreamPreviewStatus, PreviewFit, PreviewConnectionState} from "../react/StreamPreview"
|
|
96
99
|
|
|
97
100
|
/**
|
|
98
101
|
* Type helper for declaring a typed `mentra` global inside a miniapp.
|