@mentra/miniapp 3.2.1-dev.314 → 3.3.0-beta.942

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/README.md +16 -0
  2. package/dist/background/index.d.ts +6 -3
  3. package/dist/background/index.d.ts.map +1 -1
  4. package/dist/background/index.js +1 -0
  5. package/dist/background/index.js.map +1 -1
  6. package/dist/background/register.d.ts +6 -0
  7. package/dist/background/register.d.ts.map +1 -1
  8. package/dist/background/register.js +19 -4
  9. package/dist/background/register.js.map +1 -1
  10. package/dist/index.d.ts +7 -4
  11. package/dist/index.d.ts.map +1 -1
  12. package/dist/index.js +2 -0
  13. package/dist/index.js.map +1 -1
  14. package/dist/modules/display.d.ts +26 -0
  15. package/dist/modules/display.d.ts.map +1 -1
  16. package/dist/modules/display.js +1 -0
  17. package/dist/modules/display.js.map +1 -1
  18. package/dist/modules/events.d.ts +8 -4
  19. package/dist/modules/events.d.ts.map +1 -1
  20. package/dist/modules/events.js +46 -6
  21. package/dist/modules/events.js.map +1 -1
  22. package/dist/modules/meeting.d.ts +129 -0
  23. package/dist/modules/meeting.d.ts.map +1 -1
  24. package/dist/modules/meeting.js +200 -14
  25. package/dist/modules/meeting.js.map +1 -1
  26. package/dist/modules/mic.d.ts +5 -4
  27. package/dist/modules/mic.d.ts.map +1 -1
  28. package/dist/modules/mic.js +5 -4
  29. package/dist/modules/mic.js.map +1 -1
  30. package/dist/modules/miniapps.d.ts +101 -4
  31. package/dist/modules/miniapps.d.ts.map +1 -1
  32. package/dist/modules/miniapps.js +59 -4
  33. package/dist/modules/miniapps.js.map +1 -1
  34. package/dist/modules/speaker.d.ts +2 -0
  35. package/dist/modules/speaker.d.ts.map +1 -1
  36. package/dist/modules/speaker.js +5 -0
  37. package/dist/modules/speaker.js.map +1 -1
  38. package/dist/modules/stream.d.ts +46 -0
  39. package/dist/modules/stream.d.ts.map +1 -1
  40. package/dist/modules/stream.js +107 -0
  41. package/dist/modules/stream.js.map +1 -1
  42. package/dist/protocol.d.ts +86 -1
  43. package/dist/protocol.d.ts.map +1 -1
  44. package/dist/protocol.js +82 -0
  45. package/dist/protocol.js.map +1 -1
  46. package/dist/react/StreamPreview.d.ts +41 -0
  47. package/dist/react/StreamPreview.d.ts.map +1 -0
  48. package/dist/react/StreamPreview.js +144 -0
  49. package/dist/react/StreamPreview.js.map +1 -0
  50. package/dist/react/index.d.ts +3 -0
  51. package/dist/react/index.d.ts.map +1 -1
  52. package/dist/react/index.js +2 -0
  53. package/dist/react/index.js.map +1 -1
  54. package/dist/react/stream-preview/PreviewConnection.d.ts +146 -0
  55. package/dist/react/stream-preview/PreviewConnection.d.ts.map +1 -0
  56. package/dist/react/stream-preview/PreviewConnection.js +493 -0
  57. package/dist/react/stream-preview/PreviewConnection.js.map +1 -0
  58. package/dist/react/stream-preview/controlChannel.d.ts +75 -0
  59. package/dist/react/stream-preview/controlChannel.d.ts.map +1 -0
  60. package/dist/react/stream-preview/controlChannel.js +55 -0
  61. package/dist/react/stream-preview/controlChannel.js.map +1 -0
  62. package/dist/react/stream-preview/creditLoop.d.ts +81 -0
  63. package/dist/react/stream-preview/creditLoop.d.ts.map +1 -0
  64. package/dist/react/stream-preview/creditLoop.js +146 -0
  65. package/dist/react/stream-preview/creditLoop.js.map +1 -0
  66. package/dist/react/stream-preview/protocol.d.ts +113 -0
  67. package/dist/react/stream-preview/protocol.d.ts.map +1 -0
  68. package/dist/react/stream-preview/protocol.js +148 -0
  69. package/dist/react/stream-preview/protocol.js.map +1 -0
  70. package/dist/react/stream-preview/resizeCoalescer.d.ts +38 -0
  71. package/dist/react/stream-preview/resizeCoalescer.d.ts.map +1 -0
  72. package/dist/react/stream-preview/resizeCoalescer.js +71 -0
  73. package/dist/react/stream-preview/resizeCoalescer.js.map +1 -0
  74. package/dist/react/stream-preview/testFrames.d.ts +38 -0
  75. package/dist/react/stream-preview/testFrames.d.ts.map +1 -0
  76. package/dist/react/stream-preview/testFrames.js +50 -0
  77. package/dist/react/stream-preview/testFrames.js.map +1 -0
  78. package/dist/react/stream-preview/tiers.d.ts +21 -0
  79. package/dist/react/stream-preview/tiers.d.ts.map +1 -0
  80. package/dist/react/stream-preview/tiers.js +33 -0
  81. package/dist/react/stream-preview/tiers.js.map +1 -0
  82. package/dist/react/stream-preview/trace.d.ts +23 -0
  83. package/dist/react/stream-preview/trace.d.ts.map +1 -0
  84. package/dist/react/stream-preview/trace.js +71 -0
  85. package/dist/react/stream-preview/trace.js.map +1 -0
  86. package/dist/react/stream-preview/transport.d.ts +79 -0
  87. package/dist/react/stream-preview/transport.d.ts.map +1 -0
  88. package/dist/react/stream-preview/transport.js +209 -0
  89. package/dist/react/stream-preview/transport.js.map +1 -0
  90. package/dist/react/stream-preview/yuvRenderer.d.ts +31 -0
  91. package/dist/react/stream-preview/yuvRenderer.d.ts.map +1 -0
  92. package/dist/react/stream-preview/yuvRenderer.js +398 -0
  93. package/dist/react/stream-preview/yuvRenderer.js.map +1 -0
  94. package/dist/react/useHistoryState.d.ts +35 -0
  95. package/dist/react/useHistoryState.d.ts.map +1 -0
  96. package/dist/react/useHistoryState.js +117 -0
  97. package/dist/react/useHistoryState.js.map +1 -0
  98. package/dist/session.d.ts +25 -0
  99. package/dist/session.d.ts.map +1 -1
  100. package/dist/session.js +61 -1
  101. package/dist/session.js.map +1 -1
  102. package/dist/ui/index.d.ts +3 -0
  103. package/dist/ui/index.d.ts.map +1 -1
  104. package/dist/ui/index.js +2 -0
  105. package/dist/ui/index.js.map +1 -1
  106. package/package.json +8 -3
  107. package/src/background/index.ts +28 -1
  108. package/src/background/register.ts +28 -6
  109. package/src/index.ts +29 -1
  110. package/src/modules/display.ts +29 -0
  111. package/src/modules/events.ts +51 -6
  112. package/src/modules/meeting.ts +292 -15
  113. package/src/modules/mic.ts +5 -4
  114. package/src/modules/miniapps.ts +145 -4
  115. package/src/modules/speaker.ts +10 -4
  116. package/src/modules/stream.ts +153 -1
  117. package/src/protocol.ts +125 -0
  118. package/src/react/StreamPreview.tsx +187 -0
  119. package/src/react/index.ts +3 -0
  120. package/src/react/stream-preview/PreviewConnection.ts +587 -0
  121. package/src/react/stream-preview/controlChannel.ts +119 -0
  122. package/src/react/stream-preview/creditLoop.ts +189 -0
  123. package/src/react/stream-preview/protocol.ts +227 -0
  124. package/src/react/stream-preview/resizeCoalescer.ts +88 -0
  125. package/src/react/stream-preview/testFrames.ts +86 -0
  126. package/src/react/stream-preview/tiers.ts +37 -0
  127. package/src/react/stream-preview/trace.ts +81 -0
  128. package/src/react/stream-preview/transport.ts +242 -0
  129. package/src/react/stream-preview/yuvRenderer.ts +453 -0
  130. package/src/react/useHistoryState.ts +135 -0
  131. package/src/session.ts +76 -0
  132. package/src/ui/index.ts +3 -0
@@ -0,0 +1,453 @@
1
+ /**
2
+ * WebGL1 YUV → RGB renderer for raw preview frames.
3
+ *
4
+ * WebGL1 only, on purpose: the Mentra App supports iOS 15.5 / Safari 15, where WebGL2, WebCodecs,
5
+ * `VideoFrame` and `OffscreenCanvas` are either absent or unreliable. Everything here is the
6
+ * 2013 feature set — `LUMINANCE` textures, one draw call, a fragment shader doing the colour
7
+ * conversion.
8
+ *
9
+ * Rules:
10
+ * - JavaScript never touches a pixel. Planes go to `texImage2D`/`texSubImage2D` as views.
11
+ * - `readPixels` is never called; it would stall the pipeline.
12
+ * - `drawFrame` times the SUBMIT, not the paint. See the comment on that method.
13
+ */
14
+
15
+ import type {ParsedFrame, PreviewColorMatrix, PreviewColorRange, PreviewPixelFormat} from "./protocol"
16
+ import {previewTrace, previewTraceWarn} from "./trace"
17
+
18
+ const VERTEX_SHADER = `
19
+ attribute vec2 aPosition;
20
+ attribute vec2 aTexCoord;
21
+ uniform mat2 uTransform;
22
+ varying vec2 vTexCoord;
23
+ void main() {
24
+ gl_Position = vec4(uTransform * aPosition, 0.0, 1.0);
25
+ vTexCoord = aTexCoord;
26
+ }
27
+ `
28
+
29
+ /**
30
+ * `uCoeff` is (Cr→R, Cb→G, Cr→G, Cb→B) for the frame's matrix; `uYOffset`/`uYScale`/`uCScale`
31
+ * carry the range. Both are uniforms rather than shader variants so a stream that switches
32
+ * colour metadata mid-run does not force a recompile.
33
+ */
34
+ function fragmentShader(format: PreviewPixelFormat): string {
35
+ const chroma =
36
+ format === "nv12"
37
+ ? `uniform sampler2D uUv;
38
+ vec2 chromaSample() {
39
+ vec4 texel = texture2D(uUv, vTexCoord);
40
+ return vec2(texel.r, texel.a);
41
+ }`
42
+ : `uniform sampler2D uU;
43
+ uniform sampler2D uV;
44
+ vec2 chromaSample() {
45
+ return vec2(texture2D(uU, vTexCoord).r, texture2D(uV, vTexCoord).r);
46
+ }`
47
+ return `
48
+ #ifdef GL_FRAGMENT_PRECISION_HIGH
49
+ precision highp float;
50
+ #else
51
+ precision mediump float;
52
+ #endif
53
+ varying vec2 vTexCoord;
54
+ uniform sampler2D uY;
55
+ uniform float uYOffset;
56
+ uniform float uYScale;
57
+ uniform float uCScale;
58
+ uniform vec4 uCoeff;
59
+ ${chroma}
60
+ void main() {
61
+ float y = (texture2D(uY, vTexCoord).r - uYOffset) * uYScale;
62
+ vec2 c = (chromaSample() - 0.5019608) * uCScale;
63
+ vec3 rgb = vec3(
64
+ y + uCoeff.x * c.y,
65
+ y + uCoeff.y * c.x + uCoeff.z * c.y,
66
+ y + uCoeff.w * c.x);
67
+ gl_FragColor = vec4(clamp(rgb, 0.0, 1.0), 1.0);
68
+ }
69
+ `
70
+ }
71
+
72
+ /** Standard BT.601 and BT.709 conversion coefficients, as (Cr→R, Cb→G, Cr→G, Cb→B). */
73
+ const MATRIX_COEFFICIENTS: Record<PreviewColorMatrix, readonly [number, number, number, number]> = {
74
+ bt601: [1.402, -0.344136, -0.714136, 1.772],
75
+ bt709: [1.5748, -0.187324, -0.468124, 1.8556],
76
+ }
77
+
78
+ /** Limited range is Y 16..235 and chroma 16..240; full range uses the whole byte. */
79
+ const RANGE_SCALES: Record<PreviewColorRange, {yOffset: number; yScale: number; cScale: number}> = {
80
+ limited: {yOffset: 16 / 255, yScale: 255 / 219, cScale: 255 / 224},
81
+ full: {yOffset: 0, yScale: 1, cScale: 1},
82
+ }
83
+
84
+ /** Clockwise quarter turns, as exact cos/sin so 90° stays 90°. */
85
+ const ROTATIONS: ReadonlyArray<{cos: number; sin: number}> = [
86
+ {cos: 1, sin: 0},
87
+ {cos: 0, sin: 1},
88
+ {cos: -1, sin: 0},
89
+ {cos: 0, sin: -1},
90
+ ]
91
+
92
+ /** `contain` letterboxes; `cover` fills the canvas and crops the overflow. Never stretches. */
93
+ export type PreviewFit = "contain" | "cover"
94
+
95
+ export interface YuvRendererOptions {
96
+ /** Context loss, a failed compile, a failed upload: anything that makes the canvas stop. */
97
+ onError?: (reason: string) => void
98
+ fit?: PreviewFit
99
+ }
100
+
101
+ export interface YuvRenderer {
102
+ /** False once the GL context is lost, until it is restored. */
103
+ readonly alive: boolean
104
+ setFit(fit: PreviewFit): void
105
+ drawFrame(frame: ParsedFrame): number | null
106
+ dispose(): void
107
+ }
108
+
109
+ interface ProgramBundle {
110
+ program: WebGLProgram
111
+ aPosition: number
112
+ aTexCoord: number
113
+ uTransform: WebGLUniformLocation | null
114
+ uYOffset: WebGLUniformLocation | null
115
+ uYScale: WebGLUniformLocation | null
116
+ uCScale: WebGLUniformLocation | null
117
+ uCoeff: WebGLUniformLocation | null
118
+ }
119
+
120
+ interface Layout {
121
+ width: number
122
+ height: number
123
+ format: PreviewPixelFormat
124
+ }
125
+
126
+ const POSITIONS = new Float32Array([-1, -1, 1, -1, -1, 1, 1, 1])
127
+ // v = 0 is the first uploaded row, which is the top of the image, so it maps to NDC y = +1.
128
+ const TEX_COORDS = new Float32Array([0, 1, 1, 1, 0, 0, 1, 0])
129
+
130
+ class WebGlYuvRenderer implements YuvRenderer {
131
+ private readonly programs = new Map<PreviewPixelFormat, ProgramBundle>()
132
+ private readonly textures = new Map<string, WebGLTexture>()
133
+ private positionBuffer: WebGLBuffer | null = null
134
+ private texCoordBuffer: WebGLBuffer | null = null
135
+ private layout: Layout | null = null
136
+ private lost = false
137
+ private fit: PreviewFit
138
+
139
+ private readonly onContextLost = (event: Event) => {
140
+ // Without preventDefault the context is never restored, and the canvas would stay black for
141
+ // the rest of the document.
142
+ event.preventDefault()
143
+ this.lost = true
144
+ this.programs.clear()
145
+ this.textures.clear()
146
+ this.positionBuffer = null
147
+ this.texCoordBuffer = null
148
+ this.layout = null
149
+ previewTraceWarn("webgl_context_lost")
150
+ this.options.onError?.("webgl_context_lost")
151
+ }
152
+
153
+ private readonly onContextRestored = () => {
154
+ // Everything was invalidated by the loss; the next frame rebuilds it lazily.
155
+ this.lost = false
156
+ previewTrace("webgl_context_restored")
157
+ }
158
+
159
+ constructor(
160
+ private readonly canvas: HTMLCanvasElement,
161
+ private readonly gl: WebGLRenderingContext,
162
+ private readonly options: YuvRendererOptions,
163
+ ) {
164
+ this.fit = options.fit ?? "contain"
165
+ canvas.addEventListener("webglcontextlost", this.onContextLost)
166
+ canvas.addEventListener("webglcontextrestored", this.onContextRestored)
167
+ }
168
+
169
+ get alive(): boolean {
170
+ return !this.lost && !this.gl.isContextLost()
171
+ }
172
+
173
+ setFit(fit: PreviewFit): void {
174
+ this.fit = fit
175
+ }
176
+
177
+ /**
178
+ * Upload the planes and submit one draw.
179
+ *
180
+ * The returned number is how long the SUBMIT took: uploads plus `drawArrays`, measured with
181
+ * `performance.now()`. It is NOT presentation latency and not a frame time — GL is asynchronous
182
+ * and the pixels reach the screen some unknown time later. Proving presentation would need
183
+ * `readPixels` or a fence, and both change what they measure.
184
+ *
185
+ * Returns null when there is no usable context.
186
+ */
187
+ drawFrame(frame: ParsedFrame): number | null {
188
+ if (!this.alive) return null
189
+ const gl = this.gl
190
+ const startedAt = performance.now()
191
+
192
+ const bundle = this.programFor(frame.pixelFormat)
193
+ if (!bundle) return null
194
+ this.syncDrawingBuffer()
195
+
196
+ gl.useProgram(bundle.program)
197
+ // Tight packing: every plane row starts on a byte boundary, which the default alignment of 4
198
+ // would get wrong for any odd width.
199
+ gl.pixelStorei(gl.UNPACK_ALIGNMENT, 1)
200
+
201
+ const reallocate =
202
+ !this.layout ||
203
+ this.layout.width !== frame.width ||
204
+ this.layout.height !== frame.height ||
205
+ this.layout.format !== frame.pixelFormat
206
+ if (reallocate && this.layout && this.layout.format !== frame.pixelFormat) this.dropChromaTextures()
207
+ if (frame.pixelFormat === "nv12") {
208
+ if (!this.upload("y", 0, gl.LUMINANCE, frame.width, frame.height, frame.y, reallocate)) return null
209
+ if (!this.upload("uv", 1, gl.LUMINANCE_ALPHA, frame.chromaWidth, frame.chromaHeight, frame.uv, reallocate)) {
210
+ return null
211
+ }
212
+ } else {
213
+ if (!this.upload("y", 0, gl.LUMINANCE, frame.width, frame.height, frame.y, reallocate)) return null
214
+ if (!this.upload("u", 1, gl.LUMINANCE, frame.chromaWidth, frame.chromaHeight, frame.u, reallocate)) return null
215
+ if (!this.upload("v", 2, gl.LUMINANCE, frame.chromaWidth, frame.chromaHeight, frame.v, reallocate)) return null
216
+ }
217
+ if (reallocate) {
218
+ this.layout = {width: frame.width, height: frame.height, format: frame.pixelFormat}
219
+ this.bindSamplers(bundle, frame.pixelFormat)
220
+ }
221
+
222
+ const coefficients = MATRIX_COEFFICIENTS[frame.colorMatrix]
223
+ const range = RANGE_SCALES[frame.colorRange]
224
+ gl.uniform1f(bundle.uYOffset, range.yOffset)
225
+ gl.uniform1f(bundle.uYScale, range.yScale)
226
+ gl.uniform1f(bundle.uCScale, range.cScale)
227
+ gl.uniform4f(bundle.uCoeff, coefficients[0], coefficients[1], coefficients[2], coefficients[3])
228
+ gl.uniformMatrix2fv(bundle.uTransform, false, this.transformFor(frame))
229
+
230
+ gl.clearColor(0, 0, 0, 1)
231
+ gl.clear(gl.COLOR_BUFFER_BIT)
232
+ gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4)
233
+ return performance.now() - startedAt
234
+ }
235
+
236
+ dispose(): void {
237
+ this.canvas.removeEventListener("webglcontextlost", this.onContextLost)
238
+ this.canvas.removeEventListener("webglcontextrestored", this.onContextRestored)
239
+ const gl = this.gl
240
+ if (!gl.isContextLost()) {
241
+ for (const texture of this.textures.values()) gl.deleteTexture(texture)
242
+ for (const bundle of this.programs.values()) gl.deleteProgram(bundle.program)
243
+ if (this.positionBuffer) gl.deleteBuffer(this.positionBuffer)
244
+ if (this.texCoordBuffer) gl.deleteBuffer(this.texCoordBuffer)
245
+ }
246
+ this.textures.clear()
247
+ this.programs.clear()
248
+ this.positionBuffer = null
249
+ this.texCoordBuffer = null
250
+ this.layout = null
251
+ }
252
+
253
+ /** Scale the rotated frame to contain or cover the canvas without ever stretching it. */
254
+ private transformFor(frame: ParsedFrame): Float32Array {
255
+ const {cos, sin} = ROTATIONS[frame.rotationQuarters] ?? ROTATIONS[0]!
256
+ const quarterTurned = frame.rotationQuarters % 2 === 1
257
+ const frameWidth = quarterTurned ? frame.height : frame.width
258
+ const frameHeight = quarterTurned ? frame.width : frame.height
259
+ const canvasAspect = this.canvas.width / Math.max(1, this.canvas.height)
260
+ const frameAspect = frameWidth / Math.max(1, frameHeight)
261
+ let scaleX = 1
262
+ let scaleY = 1
263
+ const frameIsWider = frameAspect > canvasAspect
264
+ if (this.fit === "cover") {
265
+ if (frameIsWider) scaleX = frameAspect / canvasAspect
266
+ else scaleY = canvasAspect / frameAspect
267
+ } else if (frameIsWider) scaleY = canvasAspect / frameAspect
268
+ else scaleX = frameAspect / canvasAspect
269
+ // Column-major for uniformMatrix2fv: scale ∘ clockwise rotation.
270
+ return new Float32Array([scaleX * cos, -scaleY * sin, scaleX * sin, scaleY * cos])
271
+ }
272
+
273
+ private syncDrawingBuffer(): void {
274
+ const canvas = this.canvas
275
+ const ratio = Math.min(typeof window === "undefined" ? 1 : window.devicePixelRatio || 1, 2)
276
+ const width = Math.max(1, Math.round((canvas.clientWidth || canvas.width) * ratio))
277
+ const height = Math.max(1, Math.round((canvas.clientHeight || canvas.height) * ratio))
278
+ if (canvas.width !== width || canvas.height !== height) {
279
+ canvas.width = width
280
+ canvas.height = height
281
+ }
282
+ this.gl.viewport(0, 0, canvas.width, canvas.height)
283
+ }
284
+
285
+ private upload(
286
+ key: string,
287
+ unit: number,
288
+ format: number,
289
+ width: number,
290
+ height: number,
291
+ pixels: Uint8Array,
292
+ reallocate: boolean,
293
+ ): boolean {
294
+ const gl = this.gl
295
+ let texture = this.textures.get(key)
296
+ if (!texture) {
297
+ const created = gl.createTexture()
298
+ if (!created) {
299
+ this.options.onError?.("texture_allocation_failed")
300
+ return false
301
+ }
302
+ texture = created
303
+ this.textures.set(key, texture)
304
+ gl.activeTexture(gl.TEXTURE0 + unit)
305
+ gl.bindTexture(gl.TEXTURE_2D, texture)
306
+ // No mipmaps and CLAMP_TO_EDGE, which is what WebGL1 requires for non-power-of-two sizes.
307
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE)
308
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE)
309
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR)
310
+ gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR)
311
+ gl.texImage2D(gl.TEXTURE_2D, 0, format, width, height, 0, format, gl.UNSIGNED_BYTE, pixels)
312
+ return true
313
+ }
314
+ gl.activeTexture(gl.TEXTURE0 + unit)
315
+ gl.bindTexture(gl.TEXTURE_2D, texture)
316
+ // Reallocating every frame is the classic way to make a renderer look slower than it is.
317
+ if (reallocate) gl.texImage2D(gl.TEXTURE_2D, 0, format, width, height, 0, format, gl.UNSIGNED_BYTE, pixels)
318
+ else gl.texSubImage2D(gl.TEXTURE_2D, 0, 0, 0, width, height, format, gl.UNSIGNED_BYTE, pixels)
319
+ return true
320
+ }
321
+
322
+ /** A format switch leaves the other layout's chroma textures allocated; drop them. */
323
+ private dropChromaTextures(): void {
324
+ for (const key of ["u", "v", "uv"]) {
325
+ const texture = this.textures.get(key)
326
+ if (!texture) continue
327
+ this.gl.deleteTexture(texture)
328
+ this.textures.delete(key)
329
+ }
330
+ }
331
+
332
+ private bindSamplers(bundle: ProgramBundle, format: PreviewPixelFormat): void {
333
+ const gl = this.gl
334
+ gl.uniform1i(gl.getUniformLocation(bundle.program, "uY"), 0)
335
+ if (format === "nv12") {
336
+ gl.uniform1i(gl.getUniformLocation(bundle.program, "uUv"), 1)
337
+ return
338
+ }
339
+ gl.uniform1i(gl.getUniformLocation(bundle.program, "uU"), 1)
340
+ gl.uniform1i(gl.getUniformLocation(bundle.program, "uV"), 2)
341
+ }
342
+
343
+ private programFor(format: PreviewPixelFormat): ProgramBundle | null {
344
+ const existing = this.programs.get(format)
345
+ if (existing) {
346
+ this.bindGeometry(existing)
347
+ return existing
348
+ }
349
+ const gl = this.gl
350
+ const program = this.link(VERTEX_SHADER, fragmentShader(format))
351
+ if (!program) return null
352
+ const bundle: ProgramBundle = {
353
+ program,
354
+ aPosition: gl.getAttribLocation(program, "aPosition"),
355
+ aTexCoord: gl.getAttribLocation(program, "aTexCoord"),
356
+ uTransform: gl.getUniformLocation(program, "uTransform"),
357
+ uYOffset: gl.getUniformLocation(program, "uYOffset"),
358
+ uYScale: gl.getUniformLocation(program, "uYScale"),
359
+ uCScale: gl.getUniformLocation(program, "uCScale"),
360
+ uCoeff: gl.getUniformLocation(program, "uCoeff"),
361
+ }
362
+ this.programs.set(format, bundle)
363
+ gl.useProgram(program)
364
+ this.bindSamplers(bundle, format)
365
+ this.bindGeometry(bundle)
366
+ return bundle
367
+ }
368
+
369
+ private bindGeometry(bundle: ProgramBundle): void {
370
+ const gl = this.gl
371
+ if (!this.positionBuffer) {
372
+ this.positionBuffer = gl.createBuffer()
373
+ gl.bindBuffer(gl.ARRAY_BUFFER, this.positionBuffer)
374
+ gl.bufferData(gl.ARRAY_BUFFER, POSITIONS, gl.STATIC_DRAW)
375
+ }
376
+ gl.bindBuffer(gl.ARRAY_BUFFER, this.positionBuffer)
377
+ gl.enableVertexAttribArray(bundle.aPosition)
378
+ gl.vertexAttribPointer(bundle.aPosition, 2, gl.FLOAT, false, 0, 0)
379
+ if (!this.texCoordBuffer) {
380
+ this.texCoordBuffer = gl.createBuffer()
381
+ gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer)
382
+ gl.bufferData(gl.ARRAY_BUFFER, TEX_COORDS, gl.STATIC_DRAW)
383
+ }
384
+ gl.bindBuffer(gl.ARRAY_BUFFER, this.texCoordBuffer)
385
+ gl.enableVertexAttribArray(bundle.aTexCoord)
386
+ gl.vertexAttribPointer(bundle.aTexCoord, 2, gl.FLOAT, false, 0, 0)
387
+ }
388
+
389
+ private link(vertexSource: string, fragmentSource: string): WebGLProgram | null {
390
+ const gl = this.gl
391
+ const vertex = this.compile(gl.VERTEX_SHADER, vertexSource)
392
+ const fragment = this.compile(gl.FRAGMENT_SHADER, fragmentSource)
393
+ if (!vertex || !fragment) return null
394
+ const program = gl.createProgram()
395
+ if (!program) {
396
+ this.options.onError?.("program_allocation_failed")
397
+ return null
398
+ }
399
+ gl.attachShader(program, vertex)
400
+ gl.attachShader(program, fragment)
401
+ gl.linkProgram(program)
402
+ // The shaders are owned by the program once linked; keeping them alive leaks on every restore.
403
+ gl.deleteShader(vertex)
404
+ gl.deleteShader(fragment)
405
+ if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {
406
+ const log = gl.getProgramInfoLog(program) ?? ""
407
+ gl.deleteProgram(program)
408
+ previewTraceWarn("webgl_program_link_failed", {log: log.slice(0, 120)})
409
+ this.options.onError?.(`program_link_failed: ${log.slice(0, 120)}`)
410
+ return null
411
+ }
412
+ return program
413
+ }
414
+
415
+ private compile(type: number, source: string): WebGLShader | null {
416
+ const gl = this.gl
417
+ const shader = gl.createShader(type)
418
+ if (!shader) {
419
+ this.options.onError?.("shader_allocation_failed")
420
+ return null
421
+ }
422
+ gl.shaderSource(shader, source)
423
+ gl.compileShader(shader)
424
+ if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
425
+ const log = gl.getShaderInfoLog(shader) ?? ""
426
+ gl.deleteShader(shader)
427
+ previewTraceWarn("webgl_shader_compile_failed", {log: log.slice(0, 120)})
428
+ this.options.onError?.(`shader_compile_failed: ${log.slice(0, 120)}`)
429
+ return null
430
+ }
431
+ return shader
432
+ }
433
+ }
434
+
435
+ /** Create the renderer, or null when this WebView has no WebGL1 context to give. */
436
+ export function createYuvRenderer(canvas: HTMLCanvasElement, options: YuvRendererOptions = {}): YuvRenderer | null {
437
+ const attributes: WebGLContextAttributes = {
438
+ alpha: false,
439
+ antialias: false,
440
+ depth: false,
441
+ stencil: false,
442
+ preserveDrawingBuffer: false,
443
+ powerPreference: "low-power",
444
+ }
445
+ const gl =
446
+ (canvas.getContext("webgl", attributes) as WebGLRenderingContext | null) ??
447
+ (canvas.getContext("experimental-webgl", attributes) as WebGLRenderingContext | null)
448
+ if (!gl) {
449
+ options.onError?.("webgl_unavailable")
450
+ return null
451
+ }
452
+ return new WebGlYuvRenderer(canvas, gl, options)
453
+ }
@@ -0,0 +1,135 @@
1
+ /**
2
+ * @fileoverview useHistoryState — `useState` whose changes are browser history
3
+ * entries, so the phone's back gesture steps back through them.
4
+ *
5
+ * The host sends the back gesture (iOS edge swipe, Android back) to the
6
+ * miniapp's WebView history first, and minimizes the miniapp only once that
7
+ * history is empty. Screens, tabs and dialogs kept in plain React state never
8
+ * create history, so back skips past them and minimizes the whole miniapp.
9
+ * This hook keeps its value in `history.state` instead:
10
+ *
11
+ * - Setting a new value pushes a history entry. Back pops it, and the hook
12
+ * re-renders with the previous value.
13
+ * - Setting the value back to what it was before its latest change(s) goes
14
+ * back instead of pushing. Closing a dialog with its own button therefore
15
+ * consumes the dialog's entry, and the next back gesture does not reopen it.
16
+ *
17
+ * Routers in hash or browser mode (react-router's `HashRouter`, wouter's
18
+ * `useHashLocation`) already create history for pages; use this hook for UI
19
+ * state that is not a route. Every hook on the page shares one history entry
20
+ * per step, and `key` names the value within it, so components passing the
21
+ * same key share the same value. Values are stored as structured clones.
22
+ */
23
+
24
+ import {useCallback, useRef, useSyncExternalStore} from "react"
25
+
26
+ const STATE_KEY = "__mentraHistoryState"
27
+
28
+ interface Entry {
29
+ values: Record<string, unknown>
30
+ /** One record per entry this hook pushed, oldest first, with the value it replaced. */
31
+ trail: Array<{key: string; previous: unknown}>
32
+ }
33
+
34
+ const EMPTY: Entry = {values: {}, trail: []}
35
+
36
+ let cachedState: unknown = null
37
+ let cachedEntry: Entry = EMPTY
38
+
39
+ /** This hook's slice of the current entry, cached so snapshots stay referentially stable. */
40
+ function currentEntry(): Entry {
41
+ if (typeof window === "undefined") return EMPTY
42
+ const state: unknown = window.history.state
43
+ if (state !== cachedState) {
44
+ cachedState = state
45
+ cachedEntry = (state as Record<string, Entry | undefined> | null)?.[STATE_KEY] ?? EMPTY
46
+ }
47
+ return cachedEntry
48
+ }
49
+
50
+ const listeners = new Set<() => void>()
51
+ let boundWindow: Window | null = null
52
+ // history.go() lands later, on popstate. A change applied before then would
53
+ // push on top of the entry being left, and the traversal would pop that push
54
+ // instead, so changes wait in order until the traversal lands.
55
+ let traversing = false
56
+ const pending: Array<() => void> = []
57
+
58
+ function notify() {
59
+ listeners.forEach((listener) => listener())
60
+ }
61
+
62
+ function flush() {
63
+ while (!traversing && pending.length > 0) pending.shift()!()
64
+ }
65
+
66
+ function onPopState() {
67
+ traversing = false
68
+ notify()
69
+ // Let every popstate listener, including the host's history bridge, see the
70
+ // landed entry before a waiting change pushes the next one.
71
+ setTimeout(flush, 0)
72
+ }
73
+
74
+ function subscribe(listener: () => void): () => void {
75
+ if (boundWindow !== window) {
76
+ boundWindow?.removeEventListener("popstate", onPopState)
77
+ window.addEventListener("popstate", onPopState)
78
+ boundWindow = window
79
+ traversing = false
80
+ pending.length = 0
81
+ }
82
+ listeners.add(listener)
83
+ return () => listeners.delete(listener)
84
+ }
85
+
86
+ function whenSettled(change: () => void) {
87
+ if (traversing || pending.length > 0) pending.push(change)
88
+ else change()
89
+ }
90
+
91
+ function apply(key: string, initialValue: unknown, update: unknown) {
92
+ const {values, trail} = currentEntry()
93
+ const current = key in values ? values[key] : initialValue
94
+ const next = typeof update === "function" ? (update as (previous: unknown) => unknown)(current) : update
95
+ if (Object.is(current, next)) return
96
+ for (let index = trail.length - 1; index >= 0 && trail[index].key === key; index--) {
97
+ if (Object.is(trail[index].previous, next)) {
98
+ traversing = true
99
+ window.history.go(index - trail.length)
100
+ return
101
+ }
102
+ }
103
+ // Keep other fields (a router's index, the host's depth marker) on the new entry.
104
+ const state: unknown = window.history.state
105
+ const base = state !== null && typeof state === "object" ? state : {}
106
+ const entry: Entry = {values: {...values, [key]: next}, trail: [...trail, {key, previous: current}]}
107
+ window.history.pushState({...base, [STATE_KEY]: entry}, "")
108
+ notify()
109
+ }
110
+
111
+ export type SetHistoryState<T> = (value: T | ((previous: T) => T)) => void
112
+
113
+ /**
114
+ * Like `useState`, but each change is a history entry that the phone's back
115
+ * gesture undoes. `key` names the value in history state and must be unique
116
+ * per piece of state on the page.
117
+ *
118
+ * ```tsx
119
+ * const [tab, setTab] = useHistoryState<"home" | "settings">("tab", "home")
120
+ * const [pickerOpen, setPickerOpen] = useHistoryState("languagePicker", false)
121
+ * ```
122
+ */
123
+ export function useHistoryState<T>(key: string, initialValue: T): [T, SetHistoryState<T>] {
124
+ const initialRef = useRef(initialValue)
125
+ const read = () => {
126
+ const {values} = currentEntry()
127
+ return (key in values ? values[key] : initialRef.current) as T
128
+ }
129
+ const value = useSyncExternalStore(subscribe, read, () => initialRef.current)
130
+ const setValue = useCallback<SetHistoryState<T>>(
131
+ (update) => whenSettled(() => apply(key, initialRef.current, update)),
132
+ [key],
133
+ )
134
+ return [value, setValue]
135
+ }