@myzonerocks/gosslens 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.ts ADDED
@@ -0,0 +1,2856 @@
1
+ // The web SDK over gosslens_web, the real bgfx-backed engine every
2
+ // other SDK already runs (Swift/Kotlin call the exact same frozen ABI
3
+ // through their own thin platform glue). This SDK owns only what the
4
+ // browser forces on it - camera capture through getUserMedia, decoding
5
+ // PNGs the core has no decoder for, driving the render loop - and hands
6
+ // everything else (the frame graph, all six beauty effects, mirror and
7
+ // rotation) straight to the engine.
8
+
9
+ export const GOSS_OK = 0;
10
+
11
+ export const enum GossDegradeLevel {
12
+ Full = 0,
13
+ ReducedMlCadence = 1,
14
+ SegmentationOff = 2,
15
+ BeautySimplified = 3,
16
+ Passthrough = 4,
17
+ }
18
+
19
+ export const enum GossBeautyEffect {
20
+ Smooth = 0,
21
+ Whiten = 1,
22
+ ThinFace = 2,
23
+ BigEye = 3,
24
+ Lipstick = 4,
25
+ Blush = 5,
26
+ }
27
+
28
+ /// Platform thermal pressure. No browser API surfaces device thermal
29
+ /// state, so web callers report nominal unless they know better.
30
+ export const enum GossThermal {
31
+ Nominal = 0,
32
+ Fair = 1,
33
+ Serious = 2,
34
+ Critical = 3,
35
+ }
36
+
37
+ /// Pixel layout of a submitted frame, mirroring the frozen C enum.
38
+ export const enum GossPixelFormat {
39
+ Nv12 = 0,
40
+ Nv21 = 1,
41
+ I420 = 2,
42
+ Bgra8 = 3,
43
+ Rgba8 = 4,
44
+ }
45
+
46
+ export const enum GossColorStandard {
47
+ Bt601 = 0,
48
+ Bt709 = 1,
49
+ Bt2020 = 2,
50
+ }
51
+
52
+ export const enum GossColorRange {
53
+ Video = 0,
54
+ Full = 1,
55
+ }
56
+
57
+ /// The live signals one tick evaluates a lens's compiled triggers
58
+ /// against. hasFace false means every face-driven signal reads as false
59
+ /// regardless of what blendshapes holds.
60
+ export interface GossLensSignals {
61
+ hasFace?: boolean;
62
+ handsPresent?: boolean;
63
+ tap?: boolean;
64
+ worldTrackingState?: number;
65
+ audioLevel?: number;
66
+ blendshapes?: Float32Array | readonly number[];
67
+ }
68
+
69
+ /// Frame-path pool bounds; omitted fields mean the built-in default.
70
+ export interface GossEngineConfig {
71
+ texturePoolCapacity?: number;
72
+ stagingPoolCapacity?: number;
73
+ }
74
+
75
+ /// Whole-pipeline frame budget; omitted means the built-in default (30 fps).
76
+ export interface GossSessionConfig {
77
+ frameBudgetUs?: number;
78
+ }
79
+
80
+ /// A guided-capture scan's progress: covered target viewpoints, whether it is
81
+ /// complete, the views captured and gaussians reconstructed so far, and the yaw
82
+ /// (radians) of the next uncovered target to steer the user toward.
83
+ export interface CaptureGuidance {
84
+ covered: number;
85
+ total: number;
86
+ complete: boolean;
87
+ viewCount: number;
88
+ splatCount: number;
89
+ nextYaw: number;
90
+ }
91
+
92
+ /// A high-resolution still capture, decoupled from the preview size. width
93
+ /// and height 0 capture at the submitted frame's own resolution; format is
94
+ /// PNG (0), JPEG (1) or HEIC (2); quality is 1..100 for the lossy formats.
95
+ export interface GossStillConfig {
96
+ width?: number;
97
+ height?: number;
98
+ supersample?: number;
99
+ format?: number;
100
+ quality?: number;
101
+ /** 0 = sRGB, 1 = Display-P3, 2 = Rec2020 - the gamut the file is tagged with. */
102
+ colorSpace?: number;
103
+ /** 8 or 16 bits per channel; 16 is the PNG high-bit-depth path. */
104
+ bitDepth?: number;
105
+ }
106
+
107
+ const FRAME_FLAG_MIRROR = 0x1;
108
+ const FRAME_ROTATION_SHIFT = 8;
109
+ const LENS_SIGNALS_BYTES = 232;
110
+ const GOSS_FACE_BLENDSHAPE_COUNT = 52;
111
+ export const GOSS_FACE_LANDMARK_COUNT = 478;
112
+ export const GOSS_FACE_MAX = 4;
113
+ const FACE_RESULT_BYTES = 5968;
114
+ export const GOSS_SEGMENTATION_MASK_SIDE = 256;
115
+
116
+ /// A face handed to submitFaces for the multi-face path. landmarks are frame
117
+ /// pixels, GOSS_FACE_LANDMARK_COUNT * 3 floats; presence defaults to 1 and a
118
+ /// face below the tracked threshold is dropped by the engine.
119
+ export interface GossFaceInput {
120
+ landmarks: Float32Array;
121
+ presence?: number;
122
+ blendshapes?: Float32Array;
123
+ frameSerial?: number;
124
+ timestampUs?: bigint;
125
+ }
126
+
127
+ /// One face read back from faceResultAt, the frozen goss_face_result laid out.
128
+ export interface GossFaceOut {
129
+ frameSerial: bigint;
130
+ timestampUs: bigint;
131
+ presence: number;
132
+ landmarkCount: number;
133
+ landmarks: Float32Array;
134
+ blendshapes: Float32Array;
135
+ }
136
+
137
+ export const GOSS_POSE_LANDMARK_COUNT = 33;
138
+ export const GOSS_BODY_MAX = 4;
139
+ const POSE_RESULT_BYTES = 688;
140
+
141
+ /// A body handed to submitBodies for the multi-person path. landmarks are
142
+ /// frame pixels, GOSS_POSE_LANDMARK_COUNT * 3 floats; presence defaults to 1
143
+ /// and a body below the tracked threshold is dropped by the engine.
144
+ export interface GossPoseInput {
145
+ landmarks: Float32Array;
146
+ presence?: number;
147
+ visibilities?: Float32Array;
148
+ presences?: Float32Array;
149
+ frameSerial?: number;
150
+ timestampUs?: bigint;
151
+ }
152
+
153
+ /// One body read back from bodyResultAt, the frozen goss_pose_result laid out.
154
+ export interface GossPoseOut {
155
+ frameSerial: bigint;
156
+ timestampUs: bigint;
157
+ presence: number;
158
+ landmarkCount: number;
159
+ landmarks: Float32Array;
160
+ visibilities: Float32Array;
161
+ presences: Float32Array;
162
+ }
163
+
164
+ export const GOSS_HAND_LANDMARK_COUNT = 21;
165
+ export const GOSS_HAND_MAX = 2;
166
+ const HAND_RESULT_BYTES = 560;
167
+ const HAND_ONE_BYTES = 268;
168
+
169
+ /// One tracked hand read back from handResult. handedness is the score it is a
170
+ /// right hand; gesture is the canned class index with its score; landmarks are
171
+ /// GOSS_HAND_LANDMARK_COUNT * 3 frame-pixel floats.
172
+ export interface GossHandOne {
173
+ presence: number;
174
+ handedness: number;
175
+ gesture: number;
176
+ gestureScore: number;
177
+ landmarks: Float32Array;
178
+ }
179
+
180
+ /// The hands read back from handResult, the frozen goss_hand_result laid out.
181
+ export interface GossHandOut {
182
+ frameSerial: bigint;
183
+ timestampUs: bigint;
184
+ hands: GossHandOne[];
185
+ }
186
+
187
+ /// A named attach point on the tracked face mesh, for faceRegion. The
188
+ /// left/right labels are the subject's own.
189
+ export enum GossFaceRegion {
190
+ Forehead = 0,
191
+ Glabella = 1,
192
+ NoseTip = 2,
193
+ Chin = 3,
194
+ LeftEye = 4,
195
+ RightEye = 5,
196
+ LeftCheek = 6,
197
+ RightCheek = 7,
198
+ LeftEar = 8,
199
+ RightEar = 9,
200
+ MouthCenter = 10,
201
+ LeftMouthCorner = 11,
202
+ RightMouthCorner = 12,
203
+ }
204
+
205
+ /// A named attach point on the tracked body skeleton, for bodyJoint. The
206
+ /// left/right labels are the subject's own.
207
+ export enum GossBodyJoint {
208
+ Head = 0,
209
+ LeftShoulder = 1,
210
+ RightShoulder = 2,
211
+ LeftElbow = 3,
212
+ RightElbow = 4,
213
+ LeftWrist = 5,
214
+ RightWrist = 6,
215
+ LeftHip = 7,
216
+ RightHip = 8,
217
+ LeftKnee = 9,
218
+ RightKnee = 10,
219
+ LeftAnkle = 11,
220
+ RightAnkle = 12,
221
+ }
222
+
223
+ /// A named attach point on a tracked hand, for handJoint. Palm is the
224
+ /// middle-finger knuckle, a stable palm-centre proxy.
225
+ export enum GossHandJoint {
226
+ Wrist = 0,
227
+ ThumbTip = 1,
228
+ IndexTip = 2,
229
+ MiddleTip = 3,
230
+ RingTip = 4,
231
+ PinkyTip = 5,
232
+ Palm = 6,
233
+ }
234
+
235
+ /// The segmentation mask channels a lens can name, in the engine's frozen
236
+ /// order: the derived person mask, then the multiclass model's own labels.
237
+ /// Index 0 (person) rides the subject mask; the rest upload as class masks.
238
+ export const GOSS_SEGMENTATION_CHANNELS = [
239
+ "person",
240
+ "background",
241
+ "hair",
242
+ "body_skin",
243
+ "face_skin",
244
+ "clothes",
245
+ "others",
246
+ ] as const;
247
+
248
+ export type GossCaptureState = "idle" | "running" | "denied" | "failed" | "interrupted";
249
+
250
+ export interface GossSessionEvents {
251
+ onState?(state: GossCaptureState): void;
252
+ onFps?(fps: number, renderedFrames: number, cameraFrames: number): void;
253
+ }
254
+
255
+ /// Emscripten's own Module surface, the pieces this SDK actually uses.
256
+ /// EXPORTED_RUNTIME_METHODS in build.zig's wasm-emscripten link step is
257
+ /// the source of truth for what's actually present at runtime.
258
+ interface EngineModule {
259
+ HEAPU8: Uint8Array;
260
+ HEAP16: Int16Array;
261
+ HEAP32: Int32Array;
262
+ HEAPU32: Uint32Array;
263
+ HEAPF32: Float32Array;
264
+ HEAPF64: Float64Array;
265
+ ccall(name: string, returnType: string | null, argTypes: string[], args: unknown[]): number;
266
+ ccall(name: string, returnType: string | null, argTypes: string[], args: unknown[], opts: { async: true }): Promise<number>;
267
+ getValue(ptr: number, type: string): number;
268
+ setValue(ptr: number, value: number, type: string): void;
269
+ stringToNewUTF8(value: string): number;
270
+ }
271
+
272
+ type EngineModuleFactory = (overrides?: Record<string, unknown>) => Promise<EngineModule>;
273
+
274
+ /// Decodes a fetched blob to raw RGBA bytes via a 2D canvas. Unlike
275
+ /// texImage2D (see the git history on this file - a real, hard-won
276
+ /// lesson from the old hand-rolled WebGL2 SDK this one replaces),
277
+ /// getImageData has always had simple, browser-consistent semantics:
278
+ /// row 0 is the visual top of the image, full stop. No DOM-source
279
+ /// orientation quirks to work around, because there's no texImage2D
280
+ /// involved at all - just plain bytes handed to the engine's own
281
+ /// texture upload, which owns its own orientation convention entirely
282
+ /// separately from WebGL's.
283
+ /// fit, when given, downscales (never upscales) so the decoded frame
284
+ /// fits within maxWidth/maxHeight - LUT and makeup textures pass
285
+ /// nothing and decode at native resolution; loadStillFrame passes the
286
+ /// canvas's own size, since a corpus photo can be far larger than a
287
+ /// real camera frame ever would be. The composite chain sizes every
288
+ /// offscreen target and the final swap-chain view rect off the
289
+ /// submitted frame's own dimensions, so a frame wider or taller than
290
+ /// the actual WebGL drawing buffer gets silently clipped by the GPU to
291
+ /// whatever corner overlaps it - real, found via a still photo (2400x
292
+ /// 3000) submitted straight through to a 1280x720 canvas, where only
293
+ /// the top-left ~13% ended up visible and every landmark-driven effect
294
+ /// (thin-face, big-eye, lipstick, blush) happened to warp a region
295
+ /// entirely outside that sliver, reading back as no change at all.
296
+ async function decodeImageRgba(
297
+ blob: Blob,
298
+ fit?: { maxWidth: number; maxHeight: number },
299
+ ): Promise<{ data: Uint8ClampedArray; width: number; height: number }> {
300
+ const bitmap = await createImageBitmap(blob);
301
+ const scale = fit ? Math.min(1, fit.maxWidth / bitmap.width, fit.maxHeight / bitmap.height) : 1;
302
+ const width = Math.round(bitmap.width * scale);
303
+ const height = Math.round(bitmap.height * scale);
304
+ const canvas = document.createElement("canvas");
305
+ canvas.width = width;
306
+ canvas.height = height;
307
+ const ctx = canvas.getContext("2d")!;
308
+ ctx.drawImage(bitmap, 0, 0, width, height);
309
+ const image = ctx.getImageData(0, 0, width, height);
310
+ return { data: image.data, width, height };
311
+ }
312
+
313
+ /// Chooses which gosslens_web.js build to load: the WebGPU one (bgfx's
314
+ /// WebGPU backend, Asyncify linked in) or the WebGL2 one (no Asyncify).
315
+ /// Two separate artifacts rather than a runtime toggle, since Asyncify
316
+ /// taxes the whole per-frame path, not just init. Checks for a real
317
+ /// working adapter, not just navigator.gpu's presence, which can exist
318
+ /// with no adapter behind it.
319
+ export async function pickEngineUrl(webgpuUrl: string | URL, webgl2Url: string | URL): Promise<string | URL> {
320
+ const gpu = (navigator as unknown as { gpu?: { requestAdapter(): Promise<unknown> } }).gpu;
321
+ if (!gpu) return webgl2Url;
322
+ try {
323
+ const adapter = await gpu.requestAdapter();
324
+ return adapter ? webgpuUrl : webgl2Url;
325
+ } catch {
326
+ return webgl2Url;
327
+ }
328
+ }
329
+
330
+ /// ABI bootstrap: the loaded wasm module plus the two free functions,
331
+ /// the same role Kotlin's Gosslens object plays around
332
+ /// System.loadLibrary. Module load needs a canvas here, unlike
333
+ /// Swift/Kotlin - a real platform difference, not an inconsistency.
334
+
335
+ /// The platform camera's tracking quality: 0 unavailable, 1
336
+ /// initializing, 2 tracking, 3 limited.
337
+ export interface GossWorldState {
338
+ trackingState: number;
339
+ worldFromCamera: ArrayLike<number>;
340
+ projection: ArrayLike<number>;
341
+ timestampUs: number;
342
+ }
343
+
344
+ export interface GossWorldPlane {
345
+ id: number;
346
+ pose: ArrayLike<number>;
347
+ extentX: number;
348
+ extentZ: number;
349
+ classification: number;
350
+ }
351
+
352
+ export interface GossWorldAnchorInput {
353
+ id: number;
354
+ pose: ArrayLike<number>;
355
+ }
356
+
357
+ export interface GossWorldLight {
358
+ ambientIntensity: number;
359
+ colorTemperatureKelvin: number;
360
+ }
361
+
362
+ export class Gosslens {
363
+ private constructor(
364
+ private readonly mod: EngineModule,
365
+ private readonly version: number,
366
+ ) {}
367
+
368
+ /// Any-thread. Compare the high 16 bits against the header's own
369
+ /// GOSS_ABI_MAJOR before creating anything - load() already has.
370
+ abiVersion(): number {
371
+ return this.version;
372
+ }
373
+
374
+ /// Loads gosslens_web.js and checks its ABI major version. A
375
+ /// dynamic import, not static: bun's bundler would otherwise inline
376
+ /// this file, breaking Emscripten's own import.meta.url-relative
377
+ /// fetch of gosslens_web.wasm sitting next to it.
378
+ static async load(canvas: HTMLCanvasElement, wasmJsUrl: string | URL): Promise<Gosslens> {
379
+ const imported = (await import(/* @vite-ignore */ String(wasmJsUrl))) as { default: EngineModuleFactory };
380
+ const mod = await imported.default({ canvas });
381
+ const version = mod.ccall("goss_abi_version", "number", [], []) >>> 0;
382
+ if (version >> 16 !== 0) throw new Error(`gosslens abi major mismatch: ${version >> 16}`);
383
+ return new Gosslens(mod, version);
384
+ }
385
+
386
+ /// The YCbCr to RGB conversion for a standard and range as one
387
+ /// column-major homogeneous matrix. Unused today (canvas always
388
+ /// yields RGBA already) - a real gap for any future debug/thumbnail
389
+ /// path, kept wrapped so that path doesn't start from a raw ccall.
390
+ yuvToRgb(colorStandard: GossColorStandard, colorRange: GossColorRange): Float32Array {
391
+ const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [64]);
392
+ this.mod.ccall("goss_color_yuv_to_rgb", "number", ["number", "number", "number"], [colorStandard, colorRange, ptr]);
393
+ const out = new Float32Array(16);
394
+ for (let i = 0; i < 16; i += 1) out[i] = this.mod.getValue(ptr + i * 4, "float");
395
+ this.mod.ccall("goss_free", null, ["number", "number"], [ptr, 64]);
396
+ return out;
397
+ }
398
+
399
+ /// Analytic two-bone inverse kinematics for a limb: root, the upper and lower
400
+ /// bone lengths, target, and pole (each [x, y, z]); returns the mid joint and
401
+ /// end. An out-of-reach target extends the limb straight at it.
402
+ solveTwoBoneIk(root: [number, number, number], upperLen: number, lowerLen: number, target: [number, number, number], pole: [number, number, number]): { mid: [number, number, number]; end: [number, number, number] } {
403
+ const rp = this.mod.ccall("goss_alloc", "number", ["number"], [12]) as number;
404
+ const tp = this.mod.ccall("goss_alloc", "number", ["number"], [12]) as number;
405
+ const pp = this.mod.ccall("goss_alloc", "number", ["number"], [12]) as number;
406
+ const mp = this.mod.ccall("goss_alloc", "number", ["number"], [12]) as number;
407
+ const ep = this.mod.ccall("goss_alloc", "number", ["number"], [12]) as number;
408
+ this.mod.HEAPF32.set(root, rp >> 2);
409
+ this.mod.HEAPF32.set(target, tp >> 2);
410
+ this.mod.HEAPF32.set(pole, pp >> 2);
411
+ this.mod.ccall("goss_solve_two_bone_ik", "number", ["number", "number", "number", "number", "number", "number", "number"], [rp, upperLen, lowerLen, tp, pp, mp, ep]);
412
+ const mw = mp >> 2;
413
+ const ew = ep >> 2;
414
+ const mid: [number, number, number] = [this.mod.HEAPF32[mw]!, this.mod.HEAPF32[mw + 1]!, this.mod.HEAPF32[mw + 2]!];
415
+ const end: [number, number, number] = [this.mod.HEAPF32[ew]!, this.mod.HEAPF32[ew + 1]!, this.mod.HEAPF32[ew + 2]!];
416
+ for (const p of [rp, tp, pp, mp, ep]) this.mod.ccall("goss_free", null, ["number", "number"], [p, 12]);
417
+ return { mid, end };
418
+ }
419
+
420
+ /// @internal - GossEngine/GossSession need the raw module to reach the ABI;
421
+ /// nothing outside this file should call ccall directly.
422
+ get module(): EngineModule {
423
+ return this.mod;
424
+ }
425
+ }
426
+
427
+ /// The best track a music snippet matched: its id and how many landmarks agreed.
428
+ export interface MusicMatch {
429
+ trackId: number;
430
+ votes: number;
431
+ }
432
+
433
+ /// Render-surface lifecycle: create/resize/render/read back. Confined
434
+ /// to the one canvas it was created against, matching GossSession/GossEngine's
435
+ /// single-thread confinement on every other SDK.
436
+ export class GossEngine {
437
+ private captureInFlight = false;
438
+ private canvas: HTMLCanvasElement | null = null;
439
+ /// Only set on the WebGL2 build - bgfx's WebGPU backend binds the
440
+ /// canvas to a 'webgpu' context instead, and a canvas can only ever
441
+ /// bind one context type for its lifetime. capturePixels() branches
442
+ /// on this: readPixels when set, goss_engine_capture_frame otherwise.
443
+ private gl: WebGL2RenderingContext | null = null;
444
+ /// bgfx's HTML5 backend keeps referencing the canvas selector string
445
+ /// for the renderer's life, so the engine owns it and frees it in
446
+ /// destroy() once goss_engine_destroy has torn the renderer down.
447
+ private selectorPtr = 0;
448
+ private selectorCapacity = 0;
449
+
450
+ private constructor(
451
+ private readonly mod: EngineModule,
452
+ readonly handle: number,
453
+ ) {}
454
+
455
+ static create(gosslens: Gosslens, config?: GossEngineConfig): GossEngine {
456
+ const mod = gosslens.module;
457
+ let configPtr = 0;
458
+ if (config) {
459
+ configPtr = mod.ccall("goss_alloc", "number", ["number"], [8]);
460
+ mod.setValue(configPtr, config.texturePoolCapacity ?? 0, "i32");
461
+ mod.setValue(configPtr + 4, config.stagingPoolCapacity ?? 0, "i32");
462
+ }
463
+ const engineOut = mod.ccall("goss_alloc", "number", ["number"], [4]);
464
+ const engineStatus = mod.ccall("goss_engine_create", "number", ["number", "number"], [configPtr, engineOut]);
465
+ const handle = mod.getValue(engineOut, "i32");
466
+ mod.ccall("goss_free", null, ["number", "number"], [engineOut, 4]);
467
+ if (configPtr !== 0) mod.ccall("goss_free", null, ["number", "number"], [configPtr, 8]);
468
+ if (engineStatus !== GOSS_OK) throw new Error(`engine create failed: ${engineStatus}`);
469
+ return new GossEngine(mod, handle);
470
+ }
471
+
472
+ /// @internal - GossSession needs the raw module to reach the ABI; nothing
473
+ /// outside this file should call ccall directly.
474
+ get module(): EngineModule {
475
+ return this.mod;
476
+ }
477
+
478
+ /// Brings the render backend up against canvas, which needs a stable
479
+ /// id: bgfx's own HTML5 backend resolves it via a #id selector string
480
+ /// (glcontext_html5.cpp), separate from the Module.canvas binding
481
+ /// Gosslens.load already made - both must agree.
482
+ async initRenderer(canvas: HTMLCanvasElement): Promise<void> {
483
+ if (!canvas.id) throw new Error("canvas needs a stable id for bgfx's own selector lookup");
484
+ const mod = this.mod;
485
+ // Encode the selector into engine-owned heap so its lifetime matches
486
+ // the renderer, not this call; a re-init frees the prior one first.
487
+ const selectorBytes = new TextEncoder().encode(`#${canvas.id}`);
488
+ if (this.selectorPtr !== 0) mod.ccall("goss_free", null, ["number", "number"], [this.selectorPtr, this.selectorCapacity]);
489
+ this.selectorCapacity = selectorBytes.length + 1;
490
+ this.selectorPtr = mod.ccall("goss_alloc", "number", ["number"], [this.selectorCapacity]);
491
+ mod.HEAPU8.set(selectorBytes, this.selectorPtr);
492
+ mod.HEAPU8[this.selectorPtr + selectorBytes.length] = 0;
493
+ const rendererDescPtr = mod.ccall("goss_alloc", "number", ["number"], [12]);
494
+ mod.setValue(rendererDescPtr, this.selectorPtr, "i32");
495
+ mod.setValue(rendererDescPtr + 4, canvas.width, "i32");
496
+ mod.setValue(rendererDescPtr + 8, canvas.height, "i32");
497
+ // bgfx's own HTML5 backend creates this canvas's WebGL2 context
498
+ // itself, via emscripten_webgl_create_context - passing
499
+ // webGLContextAttributes here has no effect regardless,
500
+ // preserveDrawingBuffer stays false. Worked around in capturePixels.
501
+ const rendererStatus = await mod.ccall("goss_engine_init_renderer", "number", ["number", "number"], [this.handle, rendererDescPtr], { async: true });
502
+ mod.ccall("goss_free", null, ["number", "number"], [rendererDescPtr, 12]);
503
+ if (rendererStatus !== GOSS_OK) throw new Error(`renderer init failed: ${rendererStatus}`);
504
+
505
+ // Emscripten's C++ side just created this canvas's own rendering
506
+ // context; a repeat getContext returns that same context. webgpu
507
+ // first: a webgpu-bound canvas answers a mismatched
508
+ // getContext("webgl2") with null, never the wrong context.
509
+ this.canvas = canvas;
510
+ this.gl = canvas.getContext("webgpu") ? null : canvas.getContext("webgl2");
511
+ }
512
+
513
+ resize(width: number, height: number): void {
514
+ if (!this.canvas) throw new Error("initRenderer first");
515
+ this.canvas.width = width;
516
+ this.canvas.height = height;
517
+ this.mod.ccall("goss_engine_resize", null, ["number", "number", "number"], [this.handle, width, height]);
518
+ }
519
+
520
+ /// A null session presents the clear color, matching every other
521
+ /// SDK's own goss_engine_render_frame contract.
522
+ renderFrame(session: GossSession | null): number {
523
+ return this.mod.ccall("goss_engine_render_frame", "number", ["number", "number"], [this.handle, session?.handle ?? 0]);
524
+ }
525
+
526
+ /// Compiles a text prompt into a GLF lens manifest on device. The result is
527
+ /// ordinary GLF the caller can inspect or pass to activateLens, and needs no
528
+ /// assets. A length probe sizes the buffer, then a fill call writes it.
529
+ compilePrompt(prompt: string): string {
530
+ const bytes = new TextEncoder().encode(prompt);
531
+ const promptPtr = this.mod.ccall("goss_alloc", "number", ["number"], [bytes.length || 1]) as number;
532
+ this.mod.HEAPU8.set(bytes, promptPtr);
533
+ const lenPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]) as number;
534
+ const view = () => new DataView(this.mod.HEAPU8.buffer, lenPtr, 4).getUint32(0, true);
535
+ this.mod.ccall("goss_compile_prompt", "number", ["number", "number", "number", "number", "number", "number"], [this.handle, promptPtr, bytes.length, 0, 0, lenPtr]);
536
+ const needed = view();
537
+ const outPtr = this.mod.ccall("goss_alloc", "number", ["number"], [needed || 1]) as number;
538
+ this.mod.ccall("goss_compile_prompt", "number", ["number", "number", "number", "number", "number", "number"], [this.handle, promptPtr, bytes.length, outPtr, needed, lenPtr]);
539
+ const written = view();
540
+ const json = new TextDecoder().decode(this.mod.HEAPU8.slice(outPtr, outPtr + written));
541
+ this.mod.ccall("goss_free", null, ["number", "number"], [outPtr, needed || 1]);
542
+ this.mod.ccall("goss_free", null, ["number", "number"], [lenPtr, 4]);
543
+ this.mod.ccall("goss_free", null, ["number", "number"], [promptPtr, bytes.length || 1]);
544
+ return json;
545
+ }
546
+
547
+ /// Composes an on-device generative-music track from a text prompt and returns
548
+ /// it as a mono 16-bit WAV. A non-zero seed varies the take; bars 0 uses the
549
+ /// default length. Deterministic, no model and no network.
550
+ generateSong(prompt: string, sampleRate = 48000, seed = 0, bars = 0): Uint8Array {
551
+ const bytes = new TextEncoder().encode(prompt);
552
+ const promptPtr = this.mod.ccall("goss_alloc", "number", ["number"], [bytes.length || 1]) as number;
553
+ this.mod.HEAPU8.set(bytes, promptPtr);
554
+ const lenPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]) as number;
555
+ const view = () => new DataView(this.mod.HEAPU8.buffer, lenPtr, 4).getUint32(0, true);
556
+ const args = ["number", "number", "number", "number", "number", "number", "number", "number", "number"];
557
+ this.mod.ccall("goss_engine_generate_song", "number", args, [this.handle, promptPtr, bytes.length, sampleRate, seed, bars, 0, 0, lenPtr]);
558
+ const needed = view();
559
+ const outPtr = this.mod.ccall("goss_alloc", "number", ["number"], [needed || 1]) as number;
560
+ this.mod.ccall("goss_engine_generate_song", "number", args, [this.handle, promptPtr, bytes.length, sampleRate, seed, bars, outPtr, needed, lenPtr]);
561
+ const written = view();
562
+ const wav = this.mod.HEAPU8.slice(outPtr, outPtr + written);
563
+ this.mod.ccall("goss_free", null, ["number", "number"], [outPtr, needed || 1]);
564
+ this.mod.ccall("goss_free", null, ["number", "number"], [lenPtr, 4]);
565
+ this.mod.ccall("goss_free", null, ["number", "number"], [promptPtr, bytes.length || 1]);
566
+ return wav;
567
+ }
568
+
569
+ /// Scans a width*height 8-bit luminance frame for an EAN-13 / UPC-A barcode,
570
+ /// returning its 13 digits or null when no checksum-valid symbol is found.
571
+ /// Purely algorithmic and deterministic, no model.
572
+ scanBarcode(luminance: Uint8Array, width: number, height: number): Uint8Array | null {
573
+ const lumPtr = this.mod.ccall("goss_alloc", "number", ["number"], [luminance.length || 1]) as number;
574
+ this.mod.HEAPU8.set(luminance, lumPtr);
575
+ const outPtr = this.mod.ccall("goss_alloc", "number", ["number"], [13]) as number;
576
+ const status = this.mod.ccall("goss_engine_scan_barcode", "number", ["number", "number", "number", "number", "number"], [this.handle, lumPtr, width, height, outPtr]) as number;
577
+ const digits = status === 0 ? this.mod.HEAPU8.slice(outPtr, outPtr + 13) : null;
578
+ this.mod.ccall("goss_free", null, ["number", "number"], [lumPtr, luminance.length || 1]);
579
+ this.mod.ccall("goss_free", null, ["number", "number"], [outPtr, 13]);
580
+ return digits;
581
+ }
582
+
583
+ /// Scans a width*height 8-bit luminance frame for a QR code and returns its
584
+ /// decoded payload bytes, or null when no QR decodes. Reed-Solomon error
585
+ /// correction, algorithmic and deterministic, no model.
586
+ scanQR(luminance: Uint8Array, width: number, height: number): Uint8Array | null {
587
+ const lumPtr = this.mod.ccall("goss_alloc", "number", ["number"], [luminance.length || 1]) as number;
588
+ this.mod.HEAPU8.set(luminance, lumPtr);
589
+ const lenPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]) as number;
590
+ const view = () => new DataView(this.mod.HEAPU8.buffer, lenPtr, 4).getUint32(0, true);
591
+ const args = ["number", "number", "number", "number", "number", "number", "number"];
592
+ const probe = this.mod.ccall("goss_engine_scan_qr", "number", args, [this.handle, lumPtr, width, height, 0, 0, lenPtr]) as number;
593
+ let out: Uint8Array | null = null;
594
+ if (probe === 0) {
595
+ const needed = view();
596
+ const outPtr = this.mod.ccall("goss_alloc", "number", ["number"], [needed || 1]) as number;
597
+ const fill = this.mod.ccall("goss_engine_scan_qr", "number", args, [this.handle, lumPtr, width, height, outPtr, needed, lenPtr]) as number;
598
+ if (fill === 0) out = this.mod.HEAPU8.slice(outPtr, outPtr + view());
599
+ this.mod.ccall("goss_free", null, ["number", "number"], [outPtr, needed || 1]);
600
+ }
601
+ this.mod.ccall("goss_free", null, ["number", "number"], [lumPtr, luminance.length || 1]);
602
+ this.mod.ccall("goss_free", null, ["number", "number"], [lenPtr, 4]);
603
+ return out;
604
+ }
605
+
606
+ /// Generates a QR code for a payload and renders it into a square 8-bit
607
+ /// luminance image (0 dark, 255 light); returns the pixels and the side
608
+ /// length. Algorithmic and deterministic, no model.
609
+ generateQR(payload: Uint8Array, moduleScale = 6, quietModules = 4): { image: Uint8Array; dim: number } | null {
610
+ const payPtr = this.mod.ccall("goss_alloc", "number", ["number"], [payload.length || 1]) as number;
611
+ this.mod.HEAPU8.set(payload, payPtr);
612
+ const dimPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]) as number;
613
+ const dimOf = () => new DataView(this.mod.HEAPU8.buffer, dimPtr, 4).getUint32(0, true);
614
+ const args = ["number", "number", "number", "number", "number", "number", "number", "number"];
615
+ let result: { image: Uint8Array; dim: number } | null = null;
616
+ if ((this.mod.ccall("goss_engine_generate_qr", "number", args, [this.handle, payPtr, payload.length, moduleScale, quietModules, 0, 0, dimPtr]) as number) === 0) {
617
+ const dim = dimOf();
618
+ const outPtr = this.mod.ccall("goss_alloc", "number", ["number"], [dim * dim || 1]) as number;
619
+ if ((this.mod.ccall("goss_engine_generate_qr", "number", args, [this.handle, payPtr, payload.length, moduleScale, quietModules, outPtr, dim * dim, dimPtr]) as number) === 0) {
620
+ result = { image: this.mod.HEAPU8.slice(outPtr, outPtr + dim * dim), dim };
621
+ }
622
+ this.mod.ccall("goss_free", null, ["number", "number"], [outPtr, dim * dim || 1]);
623
+ }
624
+ this.mod.ccall("goss_free", null, ["number", "number"], [payPtr, payload.length || 1]);
625
+ this.mod.ccall("goss_free", null, ["number", "number"], [dimPtr, 4]);
626
+ return result;
627
+ }
628
+
629
+ /// Ranks a media archive by semantic similarity. corpus holds count embedding
630
+ /// vectors of length dim contiguously and query is one dim-vector, all from
631
+ /// your own embedding model; returns the top k hits by cosine similarity. The
632
+ /// engine owns the exact search; any embedder feeds it.
633
+ mediaSearch(corpus: Float32Array, count: number, dim: number, query: Float32Array, k: number): { index: number; score: number }[] {
634
+ const corpusPtr = this.mod.ccall("goss_alloc", "number", ["number"], [corpus.length * 4 || 4]) as number;
635
+ this.mod.HEAPF32.set(corpus, corpusPtr / 4);
636
+ const queryPtr = this.mod.ccall("goss_alloc", "number", ["number"], [query.length * 4 || 4]) as number;
637
+ this.mod.HEAPF32.set(query, queryPtr / 4);
638
+ const idxPtr = this.mod.ccall("goss_alloc", "number", ["number"], [k * 4 || 4]) as number;
639
+ const scorePtr = this.mod.ccall("goss_alloc", "number", ["number"], [k * 4 || 4]) as number;
640
+ const countPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]) as number;
641
+ const args = ["number", "number", "number", "number", "number", "number", "number", "number", "number"];
642
+ const hits: { index: number; score: number }[] = [];
643
+ if ((this.mod.ccall("goss_engine_media_search", "number", args, [this.handle, corpusPtr, count, dim, queryPtr, k, idxPtr, scorePtr, countPtr]) as number) === 0) {
644
+ const n = new DataView(this.mod.HEAPU8.buffer, countPtr, 4).getUint32(0, true);
645
+ for (let i = 0; i < n; i++) {
646
+ const index = new DataView(this.mod.HEAPU8.buffer, idxPtr + i * 4, 4).getUint32(0, true);
647
+ const score = new DataView(this.mod.HEAPU8.buffer, scorePtr + i * 4, 4).getFloat32(0, true);
648
+ hits.push({ index, score });
649
+ }
650
+ }
651
+ this.mod.ccall("goss_free", null, ["number", "number"], [corpusPtr, corpus.length * 4 || 4]);
652
+ this.mod.ccall("goss_free", null, ["number", "number"], [queryPtr, query.length * 4 || 4]);
653
+ this.mod.ccall("goss_free", null, ["number", "number"], [idxPtr, k * 4 || 4]);
654
+ this.mod.ccall("goss_free", null, ["number", "number"], [scorePtr, k * 4 || 4]);
655
+ this.mod.ccall("goss_free", null, ["number", "number"], [countPtr, 4]);
656
+ return hits;
657
+ }
658
+
659
+ /// Seals a media blob for the on-device vault with ChaCha20-Poly1305 under a
660
+ /// 32-byte key and 12-byte nonce, binding aad; returns ciphertext then the
661
+ /// 16-byte tag. Keep the key in the platform keystore.
662
+ sealMedia(key: Uint8Array, nonce: Uint8Array, plaintext: Uint8Array, aad: Uint8Array = new Uint8Array(0)): Uint8Array | null {
663
+ return this.aeadMedia("goss_seal_media", key, nonce, plaintext, aad);
664
+ }
665
+
666
+ /// Opens a sealed vault blob back to plaintext under the same key, nonce and
667
+ /// aad. Returns null if authentication fails, so a tampered blob never decodes.
668
+ openMedia(key: Uint8Array, nonce: Uint8Array, sealed: Uint8Array, aad: Uint8Array = new Uint8Array(0)): Uint8Array | null {
669
+ return this.aeadMedia("goss_open_media", key, nonce, sealed, aad);
670
+ }
671
+
672
+ private aeadMedia(fn: string, key: Uint8Array, nonce: Uint8Array, input: Uint8Array, aad: Uint8Array): Uint8Array | null {
673
+ const keyPtr = this.mod.ccall("goss_alloc", "number", ["number"], [key.length || 1]) as number;
674
+ this.mod.HEAPU8.set(key, keyPtr);
675
+ const noncePtr = this.mod.ccall("goss_alloc", "number", ["number"], [nonce.length || 1]) as number;
676
+ this.mod.HEAPU8.set(nonce, noncePtr);
677
+ const inPtr = this.mod.ccall("goss_alloc", "number", ["number"], [input.length || 1]) as number;
678
+ this.mod.HEAPU8.set(input, inPtr);
679
+ const aadPtr = this.mod.ccall("goss_alloc", "number", ["number"], [aad.length || 1]) as number;
680
+ this.mod.HEAPU8.set(aad, aadPtr);
681
+ const lenPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]) as number;
682
+ const lenOf = () => new DataView(this.mod.HEAPU8.buffer, lenPtr, 4).getUint32(0, true);
683
+ const args = ["number", "number", "number", "number", "number", "number", "number", "number", "number"];
684
+ let out: Uint8Array | null = null;
685
+ if ((this.mod.ccall(fn, "number", args, [keyPtr, noncePtr, inPtr, input.length, aadPtr, aad.length, 0, 0, lenPtr]) as number) === 0) {
686
+ const needed = lenOf();
687
+ const outPtr = this.mod.ccall("goss_alloc", "number", ["number"], [needed || 1]) as number;
688
+ if ((this.mod.ccall(fn, "number", args, [keyPtr, noncePtr, inPtr, input.length, aadPtr, aad.length, outPtr, needed, lenPtr]) as number) === 0) {
689
+ out = this.mod.HEAPU8.slice(outPtr, outPtr + lenOf());
690
+ }
691
+ this.mod.ccall("goss_free", null, ["number", "number"], [outPtr, needed || 1]);
692
+ }
693
+ this.mod.ccall("goss_free", null, ["number", "number"], [keyPtr, key.length || 1]);
694
+ this.mod.ccall("goss_free", null, ["number", "number"], [noncePtr, nonce.length || 1]);
695
+ this.mod.ccall("goss_free", null, ["number", "number"], [inPtr, input.length || 1]);
696
+ this.mod.ccall("goss_free", null, ["number", "number"], [aadPtr, aad.length || 1]);
697
+ this.mod.ccall("goss_free", null, ["number", "number"], [lenPtr, 4]);
698
+ return out;
699
+ }
700
+
701
+ /// Picks the best frame of a burst: count luminance frames of width*height
702
+ /// frameStride bytes apart, scored by sharpness blended with a per-frame
703
+ /// openness score weighted by opennessWeight in 0..1. Returns the winning
704
+ /// frame index for best-take fusion.
705
+ bestTake(frames: Uint8Array, frameStride: number, count: number, width: number, height: number, openness: Float32Array, opennessWeight: number): number {
706
+ const framesPtr = this.mod.ccall("goss_alloc", "number", ["number"], [frames.length || 1]) as number;
707
+ this.mod.HEAPU8.set(frames, framesPtr);
708
+ const opennessPtr = this.mod.ccall("goss_alloc", "number", ["number"], [openness.length * 4 || 4]) as number;
709
+ this.mod.HEAPF32.set(openness, opennessPtr / 4);
710
+ const indexPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]) as number;
711
+ const args = ["number", "number", "number", "number", "number", "number", "number", "number", "number"];
712
+ let index = 0;
713
+ if ((this.mod.ccall("goss_engine_best_take", "number", args, [this.handle, framesPtr, frameStride, count, width, height, opennessPtr, opennessWeight, indexPtr]) as number) === 0) {
714
+ index = new DataView(this.mod.HEAPU8.buffer, indexPtr, 4).getUint32(0, true);
715
+ }
716
+ this.mod.ccall("goss_free", null, ["number", "number"], [framesPtr, frames.length || 1]);
717
+ this.mod.ccall("goss_free", null, ["number", "number"], [opennessPtr, openness.length * 4 || 4]);
718
+ this.mod.ccall("goss_free", null, ["number", "number"], [indexPtr, 4]);
719
+ return index;
720
+ }
721
+
722
+ /// Fingerprints a reference recording and registers it under trackId in the
723
+ /// engine's on-device music catalog. Samples are interleaved f32; re-adding a
724
+ /// trackId layers more landmarks in.
725
+ addMusicReference(trackId: number, samples: Float32Array, frameCount: number, sampleRate: number, channels: number): boolean {
726
+ const bytes = samples.length * 4;
727
+ const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [bytes || 4]) as number;
728
+ this.mod.HEAPF32.set(samples, ptr >> 2);
729
+ const status = this.mod.ccall(
730
+ "goss_engine_music_add_reference",
731
+ "number",
732
+ ["number", "number", "number", "number", "number", "number"],
733
+ [this.handle, trackId >>> 0, ptr, frameCount, sampleRate, channels],
734
+ );
735
+ this.mod.ccall("goss_free", null, ["number", "number"], [ptr, bytes || 4]);
736
+ return status === 0;
737
+ }
738
+
739
+ /// Empties the engine's music catalog.
740
+ clearMusicReferences(): void {
741
+ this.mod.ccall("goss_engine_music_clear_references", null, ["number"], [this.handle]);
742
+ }
743
+
744
+ /// Fingerprints a captured snippet and matches it against the catalog,
745
+ /// returning the best track and its landmark-agreement vote count, or null
746
+ /// below minVotes. A few seconds of noisy audio still identifies.
747
+ identifyMusic(samples: Float32Array, frameCount: number, sampleRate: number, channels: number, minVotes = 5): MusicMatch | null {
748
+ const bytes = samples.length * 4;
749
+ const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [bytes || 4]) as number;
750
+ const outPtr = this.mod.ccall("goss_alloc", "number", ["number"], [8]) as number;
751
+ this.mod.HEAPF32.set(samples, ptr >> 2);
752
+ const status = this.mod.ccall(
753
+ "goss_engine_music_identify",
754
+ "number",
755
+ ["number", "number", "number", "number", "number", "number", "number", "number"],
756
+ [this.handle, ptr, frameCount, sampleRate, channels, minVotes >>> 0, outPtr, outPtr + 4],
757
+ );
758
+ const dv = new DataView(this.mod.HEAPU8.buffer, outPtr, 8);
759
+ const trackId = dv.getUint32(0, true);
760
+ const votes = dv.getUint32(4, true);
761
+ this.mod.ccall("goss_free", null, ["number", "number"], [ptr, bytes || 4]);
762
+ this.mod.ccall("goss_free", null, ["number", "number"], [outPtr, 8]);
763
+ if (status !== 0 || votes === 0) return null;
764
+ return { trackId, votes };
765
+ }
766
+
767
+ /// Releases the persistent wrap the engine keeps per external live
768
+ /// texture handle - the pair of the native SDKs' renderToLiveTexture,
769
+ /// for a host retiring a publish surface before the engine goes away.
770
+ /// False for a handle with no live wrap.
771
+ releaseLiveTexture(nativeHandle: bigint): boolean {
772
+ return this.mod.ccall("goss_engine_release_live_texture", "number", ["number", "number"], [this.handle, nativeHandle]) === 0;
773
+ }
774
+
775
+ /// The two ways this SDK reads pixels back: bgfx's WebGL2 context
776
+ /// never preserves its drawing buffer, so readPixels runs right after
777
+ /// a fresh render; WebGPU has no sync equivalent, so
778
+ /// goss_engine_capture_frame runs async, mapping a GPU buffer.
779
+ private async capturePixels(session: GossSession | null): Promise<{ pixels: Uint8Array; width: number; height: number }> {
780
+ if (!this.canvas) throw new Error("initRenderer first");
781
+ if (this.gl) {
782
+ const gl = this.gl;
783
+ this.renderFrame(session);
784
+ const width = this.canvas.width;
785
+ const height = this.canvas.height;
786
+ const pixels = new Uint8Array(width * height * 4);
787
+ gl.readPixels(0, 0, width, height, gl.RGBA, gl.UNSIGNED_BYTE, pixels);
788
+ return { pixels, width, height };
789
+ }
790
+
791
+ const capacity = this.canvas.width * this.canvas.height * 4;
792
+ const dataPtr = this.mod.ccall("goss_alloc", "number", ["number"], [capacity]);
793
+ const outWidthPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]);
794
+ const outHeightPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]);
795
+ this.captureInFlight = true;
796
+ try {
797
+ const status = await this.mod.ccall(
798
+ "goss_engine_capture_frame",
799
+ "number",
800
+ ["number", "number", "number", "number", "number", "number"],
801
+ [this.handle, session?.handle ?? 0, dataPtr, capacity, outWidthPtr, outHeightPtr],
802
+ { async: true },
803
+ );
804
+ if (status !== GOSS_OK) throw new Error(`goss_engine_capture_frame failed: status ${status}`);
805
+ const width = this.mod.getValue(outWidthPtr, "i32");
806
+ const height = this.mod.getValue(outHeightPtr, "i32");
807
+ const pixels = this.mod.HEAPU8.slice(dataPtr, dataPtr + width * height * 4);
808
+ return { pixels, width, height };
809
+ } finally {
810
+ this.captureInFlight = false;
811
+ this.mod.ccall("goss_free", null, ["number", "number"], [dataPtr, capacity]);
812
+ this.mod.ccall("goss_free", null, ["number", "number"], [outWidthPtr, 4]);
813
+ this.mod.ccall("goss_free", null, ["number", "number"], [outHeightPtr, 4]);
814
+ }
815
+ }
816
+
817
+ /// goss_engine_capture_frame on the WebGPU build is an async
818
+ /// (Asyncify-suspending) ccall - calling renderFrame while one is in
819
+ /// flight would reenter the wasm module synchronously, which
820
+ /// Asyncify does not support while already suspended.
821
+ get isCaptureInFlight(): boolean {
822
+ return this.captureInFlight;
823
+ }
824
+
825
+ /// PNG-encodes the current frame. Test/debug tooling: a real image
826
+ /// beats a frame-sum heuristic for verifying a landmark-driven effect
827
+ /// actually landed where it should, not just that something changed
828
+ /// somewhere.
829
+ async captureFrame(session: GossSession | null): Promise<string> {
830
+ const { pixels, width: w, height: h } = await this.capturePixels(session);
831
+ const out = document.createElement("canvas");
832
+ out.width = w;
833
+ out.height = h;
834
+ const ctx = out.getContext("2d")!;
835
+ const imageData = ctx.createImageData(w, h);
836
+ if (this.gl) {
837
+ const rowBytes = w * 4;
838
+ for (let y = 0; y < h; y += 1) {
839
+ const srcStart = (h - 1 - y) * rowBytes;
840
+ imageData.data.set(pixels.subarray(srcStart, srcStart + rowBytes), y * rowBytes);
841
+ }
842
+ } else {
843
+ imageData.data.set(pixels);
844
+ }
845
+ ctx.putImageData(imageData, 0, 0);
846
+ return out.toDataURL("image/png");
847
+ }
848
+
849
+ /// The composited frame as packed bytes in a WebRTC format (BGRA or RGBA),
850
+ /// the supported per-frame output for a live source. On the web the canvas is
851
+ /// already a zero-copy source through captureStream(); reach for this only for
852
+ /// raw pixels. NV12 is the native encoders' path - captureStream handles web.
853
+ async captureLiveFrame(session: GossSession | null, format: GossPixelFormat = GossPixelFormat.Bgra8): Promise<{ pixels: Uint8Array; width: number; height: number }> {
854
+ if (format !== GossPixelFormat.Bgra8 && format !== GossPixelFormat.Rgba8) {
855
+ throw new Error("captureLiveFrame on web supports Bgra8 or Rgba8; use captureStream for a live track");
856
+ }
857
+ const { pixels, width, height } = await this.capturePixels(session);
858
+ if (format === GossPixelFormat.Bgra8) {
859
+ for (let i = 0; i + 3 < pixels.length; i += 4) {
860
+ const red = pixels[i];
861
+ pixels[i] = pixels[i + 2];
862
+ pixels[i + 2] = red;
863
+ }
864
+ }
865
+ return { pixels, width, height };
866
+ }
867
+
868
+ /// A high-resolution still of the composited frame at its own resolution
869
+ /// (width and height 0) or a requested one, decoupled from the preview
870
+ /// size, returned as the encoded image bytes. Wasm core only: the pure
871
+ /// WebGL path renders in JS and has no core encoder to reach.
872
+ async captureStill(session: GossSession | null, config: GossStillConfig = {}): Promise<Uint8Array> {
873
+ if (this.gl) throw new Error("captureStill needs the wasm renderer");
874
+ const cfgPtr = this.mod.ccall("goss_alloc", "number", ["number"], [28]);
875
+ this.mod.setValue(cfgPtr, config.width ?? 0, "i32");
876
+ this.mod.setValue(cfgPtr + 4, config.height ?? 0, "i32");
877
+ this.mod.setValue(cfgPtr + 8, config.supersample ?? 0, "i32");
878
+ this.mod.setValue(cfgPtr + 12, config.format ?? 0, "i32");
879
+ this.mod.setValue(cfgPtr + 16, config.quality ?? 0, "i32");
880
+ this.mod.setValue(cfgPtr + 20, config.colorSpace ?? 0, "i32");
881
+ this.mod.setValue(cfgPtr + 24, config.bitDepth ?? 8, "i32");
882
+ const lenPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]);
883
+ const widthPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]);
884
+ const heightPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]);
885
+ let dataPtr = 0;
886
+ let capacity = 0;
887
+ this.captureInFlight = true;
888
+ try {
889
+ // Probe for the encoded size, then capture into a buffer of that size.
890
+ const probeArgs = ["number", "number", "number", "number", "number", "number", "number", "number"];
891
+ const probeStatus = await this.mod.ccall(
892
+ "goss_engine_capture_still",
893
+ "number",
894
+ probeArgs,
895
+ [this.handle, session?.handle ?? 0, cfgPtr, 0, 0, lenPtr, widthPtr, heightPtr],
896
+ { async: true },
897
+ );
898
+ capacity = this.mod.getValue(lenPtr, "i32");
899
+ if (probeStatus === GOSS_OK && capacity === 0) return new Uint8Array(0);
900
+ if (capacity <= 0) throw new Error(`goss_engine_capture_still probe failed: status ${probeStatus}`);
901
+ dataPtr = this.mod.ccall("goss_alloc", "number", ["number"], [capacity]);
902
+ const status = await this.mod.ccall(
903
+ "goss_engine_capture_still",
904
+ "number",
905
+ probeArgs,
906
+ [this.handle, session?.handle ?? 0, cfgPtr, dataPtr, capacity, lenPtr, widthPtr, heightPtr],
907
+ { async: true },
908
+ );
909
+ if (status !== GOSS_OK) throw new Error(`goss_engine_capture_still failed: status ${status}`);
910
+ const encoded = this.mod.getValue(lenPtr, "i32");
911
+ return this.mod.HEAPU8.slice(dataPtr, dataPtr + encoded);
912
+ } finally {
913
+ this.captureInFlight = false;
914
+ this.mod.ccall("goss_free", null, ["number", "number"], [cfgPtr, 28]);
915
+ this.mod.ccall("goss_free", null, ["number", "number"], [lenPtr, 4]);
916
+ this.mod.ccall("goss_free", null, ["number", "number"], [widthPtr, 4]);
917
+ this.mod.ccall("goss_free", null, ["number", "number"], [heightPtr, 4]);
918
+ if (dataPtr !== 0) this.mod.ccall("goss_free", null, ["number", "number"], [dataPtr, capacity]);
919
+ }
920
+ }
921
+
922
+ /// The composited frame encoded as a deterministic PNG - the same
923
+ /// pixels, the same bytes - at the submitted frame's own resolution.
924
+ /// captureStill is the configurable superset (a chosen size, JPEG, or a
925
+ /// wider gamut); this is the plain PNG surface. Wasm core only.
926
+ async capturePhoto(session: GossSession | null): Promise<Uint8Array> {
927
+ return this.captureEncoded(
928
+ "goss_engine_capture_photo",
929
+ ["number", "number", "number", "number", "number", "number", "number"],
930
+ (dataPtr, capacity, lenPtr, widthPtr, heightPtr) => [this.handle, session?.handle ?? 0, dataPtr, capacity, lenPtr, widthPtr, heightPtr],
931
+ );
932
+ }
933
+
934
+ /// The composited frame as a platform photo: format 1 is JPEG at quality
935
+ /// 1..100 (the engine's own encoder, present on web too), format 2 is HEIC
936
+ /// (the native photo backend, GOSS_UNSUPPORTED on web). Lossy and not
937
+ /// bit-stable across runs, so capturePhoto stays the deterministic path.
938
+ async capturePhotoAs(session: GossSession | null, format: number, quality = 90): Promise<Uint8Array> {
939
+ return this.captureEncoded(
940
+ "goss_engine_capture_photo_as",
941
+ ["number", "number", "number", "number", "number", "number", "number", "number", "number"],
942
+ (dataPtr, capacity, lenPtr, widthPtr, heightPtr) => [this.handle, session?.handle ?? 0, format, quality, dataPtr, capacity, lenPtr, widthPtr, heightPtr],
943
+ );
944
+ }
945
+
946
+ /// The probe-then-capture the encoded-photo ABI ops share: render and encode
947
+ /// once into a one-byte buffer to learn the exact size (the encoders write
948
+ /// out_len before rejecting a too-small capacity), then capture into a buffer
949
+ /// of that size. Wasm core only, like captureStill; WebGL2 renders in JS.
950
+ private async captureEncoded(
951
+ callName: string,
952
+ argTypes: string[],
953
+ buildArgs: (dataPtr: number, capacity: number, lenPtr: number, widthPtr: number, heightPtr: number) => unknown[],
954
+ ): Promise<Uint8Array> {
955
+ if (this.gl) throw new Error(`${callName} needs the wasm renderer`);
956
+ const mod = this.mod;
957
+ const lenPtr = mod.ccall("goss_alloc", "number", ["number"], [4]);
958
+ const widthPtr = mod.ccall("goss_alloc", "number", ["number"], [4]);
959
+ const heightPtr = mod.ccall("goss_alloc", "number", ["number"], [4]);
960
+ // A one-byte buffer keeps the probe past the ABI's null-pointer guard so
961
+ // the encoder runs and reports the real size through lenPtr.
962
+ const probePtr = mod.ccall("goss_alloc", "number", ["number"], [1]);
963
+ let dataPtr = 0;
964
+ let capacity = 0;
965
+ this.captureInFlight = true;
966
+ try {
967
+ const probeStatus = await mod.ccall(callName, "number", argTypes, buildArgs(probePtr, 0, lenPtr, widthPtr, heightPtr), { async: true });
968
+ capacity = mod.getValue(lenPtr, "i32");
969
+ if (capacity === 0) return new Uint8Array(0);
970
+ if (capacity < 0) throw new Error(`${callName} probe failed: status ${probeStatus}`);
971
+ dataPtr = mod.ccall("goss_alloc", "number", ["number"], [capacity]);
972
+ const status = await mod.ccall(callName, "number", argTypes, buildArgs(dataPtr, capacity, lenPtr, widthPtr, heightPtr), { async: true });
973
+ if (status !== GOSS_OK) throw new Error(`${callName} failed: status ${status}`);
974
+ const encoded = mod.getValue(lenPtr, "i32");
975
+ return mod.HEAPU8.slice(dataPtr, dataPtr + encoded);
976
+ } finally {
977
+ this.captureInFlight = false;
978
+ mod.ccall("goss_free", null, ["number", "number"], [probePtr, 1]);
979
+ mod.ccall("goss_free", null, ["number", "number"], [lenPtr, 4]);
980
+ mod.ccall("goss_free", null, ["number", "number"], [widthPtr, 4]);
981
+ mod.ccall("goss_free", null, ["number", "number"], [heightPtr, 4]);
982
+ if (dataPtr !== 0) mod.ccall("goss_free", null, ["number", "number"], [dataPtr, capacity]);
983
+ }
984
+ }
985
+
986
+ async readCenterPixel(session: GossSession | null): Promise<Uint8Array> {
987
+ const { pixels, width, height } = await this.capturePixels(session);
988
+ const offset = (Math.floor(height / 2) * width + Math.floor(width / 2)) * 4;
989
+ return pixels.slice(offset, offset + 4);
990
+ }
991
+
992
+ /// Sums every RGBA byte over the whole canvas - courser but far more
993
+ /// robust than one fixed pixel, since a synthetic test pattern
994
+ /// (Chrome's fake capture device) is free to put its "lit" content
995
+ /// anywhere, leaving any single coordinate dark for long stretches.
996
+ async readFrameSum(session: GossSession | null): Promise<number> {
997
+ const { pixels } = await this.capturePixels(session);
998
+ let sum = 0;
999
+ for (const value of pixels) sum += value;
1000
+ return sum;
1001
+ }
1002
+
1003
+ destroy(): void {
1004
+ this.mod.ccall("goss_engine_destroy", null, ["number"], [this.handle]);
1005
+ // The renderer is gone now, so bgfx no longer references the selector.
1006
+ if (this.selectorPtr !== 0) this.mod.ccall("goss_free", null, ["number", "number"], [this.selectorPtr, this.selectorCapacity]);
1007
+ this.selectorPtr = 0;
1008
+ this.selectorCapacity = 0;
1009
+ }
1010
+ }
1011
+
1012
+ /// Declarative camera-hardware intent. The engine normalizes every field; the
1013
+ /// page reads it back and applies it via getUserMedia track constraints. Modes:
1014
+ /// flash 0 off/1 on/2 auto; focus 0 auto/1 locked/2 point; exposure 0 auto/1
1015
+ /// locked. Points normalized 0..1.
1016
+ export interface GossCameraControls {
1017
+ flashMode: number;
1018
+ torch: number;
1019
+ focusMode: number;
1020
+ exposureMode: number;
1021
+ focusPointX: number;
1022
+ focusPointY: number;
1023
+ exposureLinked: number;
1024
+ exposurePointX: number;
1025
+ exposurePointY: number;
1026
+ exposureBiasEv: number;
1027
+ zoomFactor: number;
1028
+ maxZoomFactor: number;
1029
+ mirrorSavePolicy: number;
1030
+ }
1031
+
1032
+ export interface GossRecordingPolicy {
1033
+ maxDurationMs: number;
1034
+ minClipMs: number;
1035
+ segmentMode: number;
1036
+ loopPlayback: number;
1037
+ speedPreset: number;
1038
+ micMuted: number;
1039
+ saveOriginal: number;
1040
+ stabilization: number;
1041
+ }
1042
+
1043
+ export interface GossCaptureUi {
1044
+ gridMode: number;
1045
+ levelIndicator: number;
1046
+ shutterMode: number;
1047
+ countdownS: number;
1048
+ nightMode: number;
1049
+ screenFlashMode: number;
1050
+ screenFlashIntensity: number;
1051
+ screenFlashWarmth: number;
1052
+ }
1053
+
1054
+ /// Per-preview runtime: frame submission, beauty, tracking, lens. Owns
1055
+ /// its own scratch allocations (frame descriptor, pixel buffer,
1056
+ /// landmarks) rather than one shared per-engine pool - matches every
1057
+ /// other SDK's own per-session confinement.
1058
+ export class GossSession {
1059
+ private worldScratchPtr = 0;
1060
+ private worldScratchLen = 0;
1061
+ private frameWidth = 0;
1062
+ private frameHeight = 0;
1063
+ /// Some cameras (certain external/virtual devices on macOS) hand the
1064
+ /// browser frames pre-rotated 180 degrees. Carried as a quarter-turn
1065
+ /// count on the submitted frame's own flags, the same mechanism
1066
+ /// every other SDK uses for sensor orientation.
1067
+ private videoFlipped = false;
1068
+ private whitenLutsLoaded = 0;
1069
+ private lipstickTextureLoaded = false;
1070
+ private blushTextureLoaded = false;
1071
+ /// Reused across frames, grown on resize rather than alloc/freed every
1072
+ /// tick - the frame descriptor is a fixed 32 bytes, the pixel buffer
1073
+ /// tracks the video's current resolution.
1074
+ private frameDescPtr: number;
1075
+ private framePixelsPtr = 0;
1076
+ private framePixelsCapacity = 0;
1077
+ /// Fixed capacity: GOSS_FACE_LANDMARK_COUNT never changes.
1078
+ private landmarksPtr: number;
1079
+ /// Fixed layout, reused every tick like the frame descriptor.
1080
+ private signalsPtr: number;
1081
+ /// Fixed capacity: the segmentation mask is always mask_side squared.
1082
+ private segmentationMaskPtr: number;
1083
+ /// A reusable wasm scratch the per-frame submit and readback paths slice
1084
+ /// instead of goss_alloc/goss_free each call; grown to the largest frame,
1085
+ /// freed in destroy() - no per-call wasm heap churn.
1086
+ private scratchPtr = 0;
1087
+ private scratchCapacity = 0;
1088
+
1089
+ private constructor(
1090
+ private readonly mod: EngineModule,
1091
+ readonly handle: number,
1092
+ ) {
1093
+ this.frameDescPtr = mod.ccall("goss_alloc", "number", ["number"], [32]);
1094
+ this.landmarksPtr = mod.ccall("goss_alloc", "number", ["number"], [GOSS_FACE_LANDMARK_COUNT * 3 * 4]);
1095
+ this.signalsPtr = mod.ccall("goss_alloc", "number", ["number"], [LENS_SIGNALS_BYTES]);
1096
+ this.segmentationMaskPtr = mod.ccall("goss_alloc", "number", ["number"], [GOSS_SEGMENTATION_MASK_SIDE * GOSS_SEGMENTATION_MASK_SIDE * 4]);
1097
+ }
1098
+
1099
+ /// The reusable scratch grown to at least `bytes`, returning its wasm
1100
+ /// pointer. Grows only when a larger frame arrives, so the steady path
1101
+ /// never touches the wasm allocator.
1102
+ private scratch(bytes: number): number {
1103
+ if (bytes > this.scratchCapacity) {
1104
+ if (this.scratchPtr !== 0) this.mod.ccall("goss_free", null, ["number", "number"], [this.scratchPtr, this.scratchCapacity]);
1105
+ this.scratchPtr = this.mod.ccall("goss_alloc", "number", ["number"], [bytes]) as number;
1106
+ this.scratchCapacity = bytes;
1107
+ }
1108
+ return this.scratchPtr;
1109
+ }
1110
+
1111
+ static create(engine: GossEngine, config?: GossSessionConfig): GossSession {
1112
+ const mod = engine.module;
1113
+ let configPtr = 0;
1114
+ if (config) {
1115
+ configPtr = mod.ccall("goss_alloc", "number", ["number"], [8]);
1116
+ mod.setValue(configPtr, config.frameBudgetUs ?? 0, "i32");
1117
+ mod.setValue(configPtr + 4, 0, "i32");
1118
+ }
1119
+ const sessionOut = mod.ccall("goss_alloc", "number", ["number"], [4]);
1120
+ const status = mod.ccall("goss_session_create", "number", ["number", "number", "number"], [engine.handle, configPtr, sessionOut]);
1121
+ const handle = mod.getValue(sessionOut, "i32");
1122
+ mod.ccall("goss_free", null, ["number", "number"], [sessionOut, 4]);
1123
+ if (configPtr !== 0) mod.ccall("goss_free", null, ["number", "number"], [configPtr, 8]);
1124
+ if (status !== GOSS_OK) throw new Error(`session create failed: ${status}`);
1125
+ return new GossSession(mod, handle);
1126
+ }
1127
+
1128
+ setWhiten(amount: number): void {
1129
+ this.setBeauty(GossBeautyEffect.Whiten, this.whitenLutsLoaded === 4 ? amount : 0);
1130
+ }
1131
+
1132
+ setSmooth(amount: number): void {
1133
+ this.setBeauty(GossBeautyEffect.Smooth, amount);
1134
+ }
1135
+
1136
+ setThinFace(amount: number): void {
1137
+ this.setBeauty(GossBeautyEffect.ThinFace, amount);
1138
+ }
1139
+
1140
+ setBigEye(amount: number): void {
1141
+ this.setBeauty(GossBeautyEffect.BigEye, amount);
1142
+ }
1143
+
1144
+ setLipstick(amount: number): void {
1145
+ this.setBeauty(GossBeautyEffect.Lipstick, this.lipstickTextureLoaded ? amount : 0);
1146
+ }
1147
+
1148
+ setBlush(amount: number): void {
1149
+ this.setBeauty(GossBeautyEffect.Blush, this.blushTextureLoaded ? amount : 0);
1150
+ }
1151
+
1152
+ setBeauty(effect: GossBeautyEffect, amount: number): void {
1153
+ this.mod.ccall("goss_session_set_beauty", "number", ["number", "number", "number"], [this.handle, effect, amount]);
1154
+ }
1155
+
1156
+ /// Activates a lens from its manifest JSON directly (goss_session_
1157
+ /// activate_lens, not the directory-based variant) - the only
1158
+ /// activation path this build actually supports: has_file_io is
1159
+ /// comptime-false for every wasm target, so goss_session_activate_lens_
1160
+ /// from_directory always reports unsupported here, and shader.pass/
1161
+ /// lut.pass/blend.pass nodes need compiled resources a bundle
1162
+ /// directory would provide that this SDK has no way to supply yet.
1163
+ /// A lens built entirely from beauty.* nodes (beauty-baseline, say)
1164
+ /// activates and runs for real regardless, since those go through
1165
+ /// applyWebBeautyChain's own embedded shaders, not a per-lens one.
1166
+ activateLens(manifestJson: string): void {
1167
+ const bytes = new TextEncoder().encode(manifestJson);
1168
+ const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [bytes.length]);
1169
+ this.mod.HEAPU8.set(bytes, ptr);
1170
+ this.mod.ccall("goss_session_activate_lens", "number", ["number", "number", "number"], [this.handle, ptr, bytes.length]);
1171
+ this.mod.ccall("goss_free", null, ["number", "number"], [ptr, bytes.length]);
1172
+ }
1173
+
1174
+ deactivateLens(): void {
1175
+ this.mod.ccall("goss_session_deactivate_lens", null, ["number"], [this.handle]);
1176
+ }
1177
+
1178
+ /// Advances the active lens's triggers/param ramps by dtUs, evaluating
1179
+ /// them against signals - omitted fields read as false/zero, so a bare
1180
+ /// tickLens(dtUs) only fires triggers with no `when` gate.
1181
+ tickLens(dtUs: number, signals: GossLensSignals = {}): void {
1182
+ const ptr = this.signalsPtr;
1183
+ this.mod.HEAPU8.fill(0, ptr, ptr + LENS_SIGNALS_BYTES);
1184
+ this.mod.HEAPU8[ptr] = signals.hasFace ? 1 : 0;
1185
+ this.mod.HEAPU8[ptr + 1] = signals.handsPresent ? 1 : 0;
1186
+ this.mod.HEAPU8[ptr + 2] = signals.tap ? 1 : 0;
1187
+ this.mod.setValue(ptr + 8, signals.worldTrackingState ?? 0, "double");
1188
+ this.mod.setValue(ptr + 16, signals.audioLevel ?? 0, "double");
1189
+ if (signals.blendshapes) {
1190
+ const base = (ptr + 24) >> 2;
1191
+ const count = Math.min(GOSS_FACE_BLENDSHAPE_COUNT, signals.blendshapes.length);
1192
+ for (let at = 0; at < count; at += 1) this.mod.HEAPF32[base + at] = signals.blendshapes[at]!;
1193
+ }
1194
+ this.mod.ccall("goss_session_tick_lens", "number", ["number", "number", "number"], [this.handle, dtUs, ptr]);
1195
+ }
1196
+
1197
+ /// Reads a live parameter of the active lens by name, including whatever a
1198
+ /// script node last wrote. Null with no active lens or no such parameter.
1199
+ parameterValue(name: string): number | null {
1200
+ const bytes = new TextEncoder().encode(name);
1201
+ const namePtr = this.mod.ccall("goss_alloc", "number", ["number"], [bytes.length]);
1202
+ this.mod.HEAPU8.set(bytes, namePtr);
1203
+ const outPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]);
1204
+ const status = this.mod.ccall("goss_session_parameter_value", "number", ["number", "number", "number", "number"], [this.handle, namePtr, bytes.length, outPtr]);
1205
+ const value = status === 0 ? this.mod.getValue(outPtr, "float") : null;
1206
+ this.mod.ccall("goss_free", null, ["number", "number"], [namePtr, bytes.length]);
1207
+ this.mod.ccall("goss_free", null, ["number", "number"], [outPtr, 4]);
1208
+ return value;
1209
+ }
1210
+
1211
+ /// Raycasts a normalized screen point (0..1, origin top-left) against the
1212
+ /// tracked ground plane, returning the world hit position [x, y, z], or null
1213
+ /// until world tracking is live and the ray meets the plane. A tap-to-place
1214
+ /// lens polls this and drops an anchor at the hit.
1215
+ hitTest(x: number, y: number): [number, number, number] | null {
1216
+ const outPtr = this.mod.ccall("goss_alloc", "number", ["number"], [12]);
1217
+ const status = this.mod.ccall("goss_session_hit_test", "number", ["number", "number", "number", "number"], [this.handle, x, y, outPtr]);
1218
+ const w = outPtr >> 2;
1219
+ const hit: [number, number, number] | null = status === 0 ? [this.mod.HEAPF32[w]!, this.mod.HEAPF32[w + 1]!, this.mod.HEAPF32[w + 2]!] : null;
1220
+ this.mod.ccall("goss_free", null, ["number", "number"], [outPtr, 12]);
1221
+ return hit;
1222
+ }
1223
+
1224
+ /// Submits the device's pre-scanned world mesh (scene reconstruction, a VPS
1225
+ /// scan) in world space: vertices are xyz triples and indices name three
1226
+ /// vertices per triangle. Passing empty arrays clears the stored mesh.
1227
+ submitWorldMesh(vertices: Float32Array, indices: Uint32Array): void {
1228
+ const vPtr = this.mod.ccall("goss_alloc", "number", ["number"], [vertices.length * 4 || 4]) as number;
1229
+ this.mod.HEAPF32.set(vertices, vPtr / 4);
1230
+ const iPtr = this.mod.ccall("goss_alloc", "number", ["number"], [indices.length * 4 || 4]) as number;
1231
+ this.mod.HEAPU32.set(indices, iPtr / 4);
1232
+ const args = ["number", "number", "number", "number", "number"];
1233
+ this.mod.ccall("goss_session_submit_world_mesh", "number", args, [this.handle, vPtr, vertices.length / 3, iPtr, indices.length]);
1234
+ this.mod.ccall("goss_free", null, ["number", "number"], [vPtr, vertices.length * 4 || 4]);
1235
+ this.mod.ccall("goss_free", null, ["number", "number"], [iPtr, indices.length * 4 || 4]);
1236
+ }
1237
+
1238
+ /// Casts a world-space ray against the submitted world mesh, returning the
1239
+ /// nearest surface hit `{ point, distance }`, or null when no mesh is
1240
+ /// submitted or the ray misses. A tap-to-place lens anchors content there.
1241
+ raycastWorldMesh(origin: [number, number, number], direction: [number, number, number]): { point: [number, number, number]; distance: number } | null {
1242
+ const oPtr = this.mod.ccall("goss_alloc", "number", ["number"], [12]) as number;
1243
+ this.mod.HEAPF32.set(origin, oPtr / 4);
1244
+ const dPtr = this.mod.ccall("goss_alloc", "number", ["number"], [12]) as number;
1245
+ this.mod.HEAPF32.set(direction, dPtr / 4);
1246
+ const pPtr = this.mod.ccall("goss_alloc", "number", ["number"], [12]) as number;
1247
+ const distPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]) as number;
1248
+ const args = ["number", "number", "number", "number", "number"];
1249
+ const status = this.mod.ccall("goss_session_raycast_world_mesh", "number", args, [this.handle, oPtr, dPtr, pPtr, distPtr]) as number;
1250
+ const pw = pPtr >> 2;
1251
+ const result = status === 0
1252
+ ? { point: [this.mod.HEAPF32[pw]!, this.mod.HEAPF32[pw + 1]!, this.mod.HEAPF32[pw + 2]!] as [number, number, number], distance: this.mod.HEAPF32[distPtr >> 2]! }
1253
+ : null;
1254
+ this.mod.ccall("goss_free", null, ["number", "number"], [oPtr, 12]);
1255
+ this.mod.ccall("goss_free", null, ["number", "number"], [dPtr, 12]);
1256
+ this.mod.ccall("goss_free", null, ["number", "number"], [pPtr, 12]);
1257
+ this.mod.ccall("goss_free", null, ["number", "number"], [distPtr, 4]);
1258
+ return result;
1259
+ }
1260
+
1261
+ /// Feeds interleaved f32 PCM into the session's own level and beat
1262
+ /// analysis, which drives the audio.level and audio.beat lens triggers.
1263
+ /// samples holds frameCount * channels floats; timestampUs is carried for a
1264
+ /// muxed recording track, unused by the web preview path.
1265
+ submitAudio(samples: Float32Array, frameCount: number, sampleRate: number, channels: number, timestampUs: bigint = 0n): void {
1266
+ const bytes = samples.length * 4;
1267
+ const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [bytes]) as number;
1268
+ this.mod.HEAPF32.set(samples, ptr >> 2);
1269
+ this.mod.ccall(
1270
+ "goss_session_submit_audio",
1271
+ "number",
1272
+ ["number", "number", "number", "number", "number", "number"],
1273
+ [this.handle, ptr, frameCount, sampleRate, channels, timestampUs],
1274
+ );
1275
+ this.mod.ccall("goss_free", null, ["number", "number"], [ptr, bytes]);
1276
+ }
1277
+
1278
+ /// Pulls the next block of mixed lens audio (frames interleaved s16) that
1279
+ /// play_sound triggers produced, for the page to feed into WebAudio.
1280
+ pullAudio(frames: number): Int16Array {
1281
+ const byteLen = frames * 2;
1282
+ const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [byteLen]);
1283
+ this.mod.ccall("goss_session_pull_audio", "number", ["number", "number", "number"], [this.handle, ptr, frames]);
1284
+ const out = new Int16Array(this.mod.HEAP16.buffer, ptr, frames).slice();
1285
+ this.mod.ccall("goss_free", null, ["number", "number"], [ptr, byteLen]);
1286
+ return out;
1287
+ }
1288
+
1289
+ /// Folds the active lens sound into the caller's outgoing call/live track:
1290
+ /// `mic` (interleaved f32 at `sampleRate`/`channels`, or null for silence)
1291
+ /// summed with the 48 kHz mono lens mixer resampled to that rate; returns the
1292
+ /// mixed interleaved s16. Advances the mixer once, replacing `pullAudio`.
1293
+ mixOutputAudio(mic: Float32Array | null, frameCount: number, sampleRate: number, channels: number): Int16Array {
1294
+ const outLen = frameCount * channels;
1295
+ const outBytes = outLen * 2;
1296
+ const outPtr = this.mod.ccall("goss_alloc", "number", ["number"], [outBytes]);
1297
+ let micPtr = 0;
1298
+ const micBytes = mic ? mic.length * 4 : 0;
1299
+ if (mic) {
1300
+ micPtr = this.mod.ccall("goss_alloc", "number", ["number"], [micBytes]);
1301
+ this.mod.HEAPF32.set(mic, micPtr >> 2);
1302
+ }
1303
+ this.mod.ccall(
1304
+ "goss_session_mix_output_audio",
1305
+ "number",
1306
+ ["number", "number", "number", "number", "number", "number"],
1307
+ [this.handle, micPtr, outPtr, frameCount, sampleRate, channels],
1308
+ );
1309
+ const out = new Int16Array(this.mod.HEAP16.buffer, outPtr, outLen).slice();
1310
+ this.mod.ccall("goss_free", null, ["number", "number"], [outPtr, outBytes]);
1311
+ if (mic) this.mod.ccall("goss_free", null, ["number", "number"], [micPtr, micBytes]);
1312
+ return out;
1313
+ }
1314
+
1315
+ /// Stores validated camera-hardware intent; the engine normalizes it. Read it
1316
+ /// back with `cameraControls` and apply it via getUserMedia track constraints.
1317
+ setCameraControls(c: GossCameraControls): void {
1318
+ const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [56]) as number;
1319
+ const w = ptr >> 2;
1320
+ this.mod.HEAP32[w] = c.flashMode; this.mod.HEAP32[w + 1] = c.torch;
1321
+ this.mod.HEAP32[w + 2] = c.focusMode; this.mod.HEAP32[w + 3] = c.exposureMode;
1322
+ this.mod.HEAPF32[w + 4] = c.focusPointX; this.mod.HEAPF32[w + 5] = c.focusPointY;
1323
+ this.mod.HEAP32[w + 6] = c.exposureLinked;
1324
+ this.mod.HEAPF32[w + 7] = c.exposurePointX; this.mod.HEAPF32[w + 8] = c.exposurePointY;
1325
+ this.mod.HEAPF32[w + 9] = c.exposureBiasEv; this.mod.HEAPF32[w + 10] = c.zoomFactor;
1326
+ this.mod.HEAPF32[w + 11] = c.maxZoomFactor; this.mod.HEAP32[w + 12] = c.mirrorSavePolicy;
1327
+ this.mod.HEAP32[w + 13] = 0;
1328
+ this.mod.ccall("goss_session_set_camera_controls", "number", ["number", "number"], [this.handle, ptr]);
1329
+ this.mod.ccall("goss_free", null, ["number", "number"], [ptr, 56]);
1330
+ }
1331
+
1332
+ /// The normalized camera controls for the page to apply to the media track.
1333
+ cameraControls(): GossCameraControls {
1334
+ const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [56]) as number;
1335
+ this.mod.ccall("goss_session_camera_controls", "number", ["number", "number"], [this.handle, ptr]);
1336
+ const w = ptr >> 2;
1337
+ const c: GossCameraControls = {
1338
+ flashMode: this.mod.HEAP32[w], torch: this.mod.HEAP32[w + 1],
1339
+ focusMode: this.mod.HEAP32[w + 2], exposureMode: this.mod.HEAP32[w + 3],
1340
+ focusPointX: this.mod.HEAPF32[w + 4], focusPointY: this.mod.HEAPF32[w + 5],
1341
+ exposureLinked: this.mod.HEAP32[w + 6],
1342
+ exposurePointX: this.mod.HEAPF32[w + 7], exposurePointY: this.mod.HEAPF32[w + 8],
1343
+ exposureBiasEv: this.mod.HEAPF32[w + 9], zoomFactor: this.mod.HEAPF32[w + 10],
1344
+ maxZoomFactor: this.mod.HEAPF32[w + 11], mirrorSavePolicy: this.mod.HEAP32[w + 12],
1345
+ };
1346
+ this.mod.ccall("goss_free", null, ["number", "number"], [ptr, 56]);
1347
+ return c;
1348
+ }
1349
+
1350
+ /// Stores the recording policy the app applies to MediaRecorder. The engine
1351
+ /// normalizes it; read it back with `recordingPolicy`.
1352
+ setRecordingPolicy(p: GossRecordingPolicy): void {
1353
+ const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [40]) as number;
1354
+ const w = ptr >> 2;
1355
+ this.mod.HEAP32[w] = p.maxDurationMs; this.mod.HEAP32[w + 1] = p.minClipMs;
1356
+ this.mod.HEAP32[w + 2] = p.segmentMode; this.mod.HEAP32[w + 3] = p.loopPlayback;
1357
+ this.mod.HEAP32[w + 4] = p.speedPreset; this.mod.HEAP32[w + 5] = p.micMuted;
1358
+ this.mod.HEAP32[w + 6] = p.saveOriginal; this.mod.HEAP32[w + 7] = p.stabilization;
1359
+ this.mod.HEAP32[w + 8] = 0; this.mod.HEAP32[w + 9] = 0;
1360
+ this.mod.ccall("goss_session_set_recording_policy", "number", ["number", "number"], [this.handle, ptr]);
1361
+ this.mod.ccall("goss_free", null, ["number", "number"], [ptr, 40]);
1362
+ }
1363
+
1364
+ recordingPolicy(): GossRecordingPolicy {
1365
+ const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [40]) as number;
1366
+ this.mod.ccall("goss_session_recording_policy", "number", ["number", "number"], [this.handle, ptr]);
1367
+ const w = ptr >> 2;
1368
+ const p: GossRecordingPolicy = {
1369
+ maxDurationMs: this.mod.HEAP32[w], minClipMs: this.mod.HEAP32[w + 1],
1370
+ segmentMode: this.mod.HEAP32[w + 2], loopPlayback: this.mod.HEAP32[w + 3],
1371
+ speedPreset: this.mod.HEAP32[w + 4], micMuted: this.mod.HEAP32[w + 5],
1372
+ saveOriginal: this.mod.HEAP32[w + 6], stabilization: this.mod.HEAP32[w + 7],
1373
+ };
1374
+ this.mod.ccall("goss_free", null, ["number", "number"], [ptr, 40]);
1375
+ return p;
1376
+ }
1377
+
1378
+ /// Stores the capture-UI intent the page renders (grid, timer, night mode, the
1379
+ /// front-screen flash). The engine normalizes it; read it back with `captureUi`.
1380
+ setCaptureUi(u: GossCaptureUi): void {
1381
+ const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [40]) as number;
1382
+ const w = ptr >> 2;
1383
+ this.mod.HEAP32[w] = u.gridMode; this.mod.HEAP32[w + 1] = u.levelIndicator;
1384
+ this.mod.HEAP32[w + 2] = u.shutterMode; this.mod.HEAP32[w + 3] = u.countdownS;
1385
+ this.mod.HEAP32[w + 4] = u.nightMode; this.mod.HEAP32[w + 5] = u.screenFlashMode;
1386
+ this.mod.HEAPF32[w + 6] = u.screenFlashIntensity; this.mod.HEAPF32[w + 7] = u.screenFlashWarmth;
1387
+ this.mod.HEAP32[w + 8] = 0; this.mod.HEAP32[w + 9] = 0;
1388
+ this.mod.ccall("goss_session_set_capture_ui", "number", ["number", "number"], [this.handle, ptr]);
1389
+ this.mod.ccall("goss_free", null, ["number", "number"], [ptr, 40]);
1390
+ }
1391
+
1392
+ captureUi(): GossCaptureUi {
1393
+ const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [40]) as number;
1394
+ this.mod.ccall("goss_session_capture_ui", "number", ["number", "number"], [this.handle, ptr]);
1395
+ const w = ptr >> 2;
1396
+ const u: GossCaptureUi = {
1397
+ gridMode: this.mod.HEAP32[w], levelIndicator: this.mod.HEAP32[w + 1],
1398
+ shutterMode: this.mod.HEAP32[w + 2], countdownS: this.mod.HEAP32[w + 3],
1399
+ nightMode: this.mod.HEAP32[w + 4], screenFlashMode: this.mod.HEAP32[w + 5],
1400
+ screenFlashIntensity: this.mod.HEAPF32[w + 6], screenFlashWarmth: this.mod.HEAPF32[w + 7],
1401
+ };
1402
+ this.mod.ccall("goss_free", null, ["number", "number"], [ptr, 40]);
1403
+ return u;
1404
+ }
1405
+
1406
+ /// Fires a named event the next `tickLens` delivers to the lens's
1407
+ /// `event('name')` triggers for one tick.
1408
+ fireEvent(name: string): void {
1409
+ const bytes = new TextEncoder().encode(name);
1410
+ if (bytes.length === 0) return;
1411
+ const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [bytes.length]) as number;
1412
+ this.mod.HEAPU8.set(bytes, ptr);
1413
+ this.mod.ccall("goss_session_fire_event", "number", ["number", "number", "number"], [this.handle, ptr, bytes.length]);
1414
+ this.mod.ccall("goss_free", null, ["number", "number"], [ptr, bytes.length]);
1415
+ }
1416
+
1417
+ /// Allowlists a bring-your-own model by its 32-byte SHA-256 digest, so a net
1418
+ /// whose digest is not listed is refused when a tracker or segmenter is
1419
+ /// enabled. With none set, any model loads. Call before enabling the worker.
1420
+ allowModelDigest(digest: Uint8Array): void {
1421
+ if (digest.length !== 32) throw new Error("a model digest is 32 bytes");
1422
+ const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [32]) as number;
1423
+ this.mod.HEAPU8.set(digest, ptr);
1424
+ this.mod.ccall("goss_session_allow_model_digest", "number", ["number", "number"], [this.handle, ptr]);
1425
+ this.mod.ccall("goss_free", null, ["number", "number"], [ptr, 32]);
1426
+ }
1427
+
1428
+ /// Clears the model allowlist; with none set, any model loads again.
1429
+ clearModelAllowlist(): void {
1430
+ this.mod.ccall("goss_session_clear_model_allowlist", "number", ["number"], [this.handle]);
1431
+ }
1432
+
1433
+ private withName(name: string, fn: (ptr: number, len: number) => void): void {
1434
+ const bytes = new TextEncoder().encode(name);
1435
+ if (bytes.length === 0) return;
1436
+ const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [bytes.length]) as number;
1437
+ this.mod.HEAPU8.set(bytes, ptr);
1438
+ fn(ptr, bytes.length);
1439
+ this.mod.ccall("goss_free", null, ["number", "number"], [ptr, bytes.length]);
1440
+ }
1441
+
1442
+ /// Registers a named RGBA source for multi-source composition (Duet, Stitch,
1443
+ /// live grids). The camera is the implicit source 0.
1444
+ defineSource(name: string): void {
1445
+ this.withName(name, (ptr, len) =>
1446
+ this.mod.ccall("goss_session_define_source", "number", ["number", "number", "number"], [this.handle, ptr, len]));
1447
+ }
1448
+
1449
+ removeSource(name: string): void {
1450
+ this.withName(name, (ptr, len) =>
1451
+ this.mod.ccall("goss_session_remove_source", "number", ["number", "number", "number"], [this.handle, ptr, len]));
1452
+ }
1453
+
1454
+ /// Uploads one RGBA/BGRA frame into a named source (pixelFormat 3 BGRA, 4 RGBA).
1455
+ submitSourceFrameRgba(name: string, rgba: Uint8Array, width: number, height: number, stride: number, pixelFormat: GossPixelFormat = GossPixelFormat.Rgba8): void {
1456
+ const byteLen = stride * height;
1457
+ const rgbaPtr = this.mod.ccall("goss_alloc", "number", ["number"], [byteLen]) as number;
1458
+ this.mod.HEAPU8.set(rgba.subarray(0, byteLen), rgbaPtr);
1459
+ this.mod.setValue(this.frameDescPtr, width, "i32");
1460
+ this.mod.setValue(this.frameDescPtr + 4, height, "i32");
1461
+ this.mod.setValue(this.frameDescPtr + 8, pixelFormat, "i32");
1462
+ this.mod.setValue(this.frameDescPtr + 12, 0, "i32");
1463
+ this.mod.setValue(this.frameDescPtr + 16, 1, "i32");
1464
+ this.mod.setValue(this.frameDescPtr + 20, 0, "i32");
1465
+ this.mod.setValue(this.frameDescPtr + 24, 0, "i32");
1466
+ this.mod.setValue(this.frameDescPtr + 28, 0, "i32");
1467
+ this.withName(name, (ptr, len) =>
1468
+ this.mod.ccall("goss_session_submit_source_frame_rgba_copy", "number", ["number", "number", "number", "number", "number", "number"], [this.handle, ptr, len, this.frameDescPtr, rgbaPtr, stride]));
1469
+ this.mod.ccall("goss_free", null, ["number", "number"], [rgbaPtr, byteLen]);
1470
+ }
1471
+
1472
+ /// Arranges the camera and named sources: 0 custom, 1 side-by-side, 2 top-bottom, 3 pip, 4 grid.
1473
+ setLayout(arrangement: number): void {
1474
+ this.mod.ccall("goss_session_set_layout", "number", ["number", "number"], [this.handle, arrangement]);
1475
+ }
1476
+
1477
+ /// Upper-body pose mode: while enabled the tracked pose reports only the upper
1478
+ /// body; the lower-body joints (knees down) read absent.
1479
+ setPoseUpperBody(enabled: boolean): void {
1480
+ this.mod.ccall("goss_session_set_pose_upper_body", "number", ["number", "number"], [this.handle, enabled ? 1 : 0]);
1481
+ }
1482
+
1483
+ clearLayout(): void {
1484
+ this.mod.ccall("goss_session_clear_layout", "number", ["number"], [this.handle]);
1485
+ }
1486
+
1487
+ /// Sets a source's composite blend: opacity, key mode (0 none, 1 matte, 2
1488
+ /// chroma), chroma color, similarity. The name "camera" is the base.
1489
+ setSourceComposite(name: string, opacity = 1, keyMode = 0, chroma: [number, number, number] = [0, 0, 0], similarity = 0): void {
1490
+ this.withName(name, (ptr, len) =>
1491
+ this.mod.ccall("goss_session_set_source_composite", "number", ["number", "number", "number", "number", "number", "number", "number", "number", "number"], [this.handle, ptr, len, opacity, keyMode, chroma[0], chroma[1], chroma[2], similarity]));
1492
+ }
1493
+
1494
+ /// Uploads a per-source matte for key mode 3 (the red channel is the mask), so
1495
+ /// an opaque guest is keyed to a subject without a baked alpha.
1496
+ submitSourceMask(name: string, rgba: Uint8Array | Uint8ClampedArray, width: number, height: number): void {
1497
+ const rgbaPtr = this.mod.ccall("goss_alloc", "number", ["number"], [rgba.length]) as number;
1498
+ this.mod.HEAPU8.set(rgba, rgbaPtr);
1499
+ this.withName(name, (ptr, len) =>
1500
+ this.mod.ccall("goss_session_submit_source_mask", "number", ["number", "number", "number", "number", "number", "number"], [this.handle, ptr, len, rgbaPtr, width, height]));
1501
+ this.mod.ccall("goss_free", null, ["number", "number"], [rgbaPtr, rgba.length]);
1502
+ }
1503
+
1504
+ /// Runs the engine's own segmenter on a source's frames so its key mode 3
1505
+ /// matte is computed on-device (a virtual background for a remote guest). The
1506
+ /// model is the selfie/hair net enableSegmentation takes; an empty model tears
1507
+ /// the source segmenter down.
1508
+ enableSourceSegmentation(name: string, model: Uint8Array, threads: number): void {
1509
+ const modelPtr = model.length > 0 ? (this.mod.ccall("goss_alloc", "number", ["number"], [model.length]) as number) : 0;
1510
+ if (modelPtr) this.mod.HEAPU8.set(model, modelPtr);
1511
+ this.withName(name, (ptr, len) =>
1512
+ this.mod.ccall("goss_session_enable_source_segmentation", "number", ["number", "number", "number", "number", "number", "number"], [this.handle, ptr, len, modelPtr, model.length, threads]));
1513
+ if (modelPtr) this.mod.ccall("goss_free", null, ["number", "number"], [modelPtr, model.length]);
1514
+ }
1515
+
1516
+ /// Defines a screen-share source whose frame letterboxes to fit its cell.
1517
+ defineScreenShare(name: string): void {
1518
+ this.withName(name, (ptr, len) =>
1519
+ this.mod.ccall("goss_session_define_screen_share", "number", ["number", "number", "number"], [this.handle, ptr, len]));
1520
+ }
1521
+
1522
+ /// Feeds a location fix for on-device geo.in_region membership; the location never leaves the engine.
1523
+ submitLocation(latitude: number, longitude: number, accuracyM: number, timestampUs: number): void {
1524
+ this.mod.ccall("goss_session_submit_location", "number", ["number", "number", "number", "number", "number"], [this.handle, latitude, longitude, accuracyM, timestampUs]);
1525
+ }
1526
+
1527
+ /// Sets the geofence circle the app derives from a lens's intended place.
1528
+ setGeofence(latitude: number, longitude: number, radiusM: number): void {
1529
+ this.mod.ccall("goss_session_set_geofence", "number", ["number", "number", "number", "number"], [this.handle, latitude, longitude, radiusM]);
1530
+ }
1531
+
1532
+ clearGeofence(): void {
1533
+ this.mod.ccall("goss_session_clear_geofence", "number", ["number"], [this.handle]);
1534
+ }
1535
+
1536
+ /// Sets the geofence to an axis-aligned lat/lon box.
1537
+ setGeofenceBBox(minLat: number, minLon: number, maxLat: number, maxLon: number): void {
1538
+ this.mod.ccall("goss_session_set_geofence_bbox", "number", ["number", "number", "number", "number", "number"], [this.handle, minLat, minLon, maxLat, maxLon]);
1539
+ }
1540
+
1541
+ /// Sets the geofence to a polygon ring of [latitude, longitude] pairs, three
1542
+ /// to 64 vertices.
1543
+ setGeofencePolygon(vertices: [number, number][]): void {
1544
+ const bytes = vertices.length * 2 * 8;
1545
+ const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [bytes]) as number;
1546
+ const base = ptr >> 3;
1547
+ for (let i = 0; i < vertices.length; i += 1) {
1548
+ this.mod.HEAPF64[base + i * 2] = vertices[i]![0];
1549
+ this.mod.HEAPF64[base + i * 2 + 1] = vertices[i]![1];
1550
+ }
1551
+ this.mod.ccall("goss_session_set_geofence_polygon", "number", ["number", "number", "number"], [this.handle, ptr, vertices.length]);
1552
+ this.mod.ccall("goss_free", null, ["number", "number"], [ptr, bytes]);
1553
+ }
1554
+
1555
+ /// Adds a named circular geofence, so a lens fires geo.in_region('name') for
1556
+ /// its own place among several; re-adding a name replaces its region.
1557
+ setNamedGeofence(name: string, latitude: number, longitude: number, radiusM: number): void {
1558
+ const bytes = new TextEncoder().encode(name);
1559
+ if (bytes.length === 0) return;
1560
+ const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [bytes.length]) as number;
1561
+ this.mod.HEAPU8.set(bytes, ptr);
1562
+ this.mod.ccall("goss_session_set_named_geofence", "number", ["number", "number", "number", "number", "number", "number"], [this.handle, ptr, bytes.length, latitude, longitude, radiusM]);
1563
+ this.mod.ccall("goss_free", null, ["number", "number"], [ptr, bytes.length]);
1564
+ }
1565
+
1566
+ /// Adds a named polygon geofence (a ring of [latitude, longitude] pairs, three
1567
+ /// or more), the non-circular counterpart of setNamedGeofence; re-adding a name
1568
+ /// replaces its region.
1569
+ setNamedGeofencePolygon(name: string, vertices: [number, number][]): void {
1570
+ const nameBytes = new TextEncoder().encode(name);
1571
+ if (nameBytes.length === 0) return;
1572
+ const namePtr = this.mod.ccall("goss_alloc", "number", ["number"], [nameBytes.length]) as number;
1573
+ this.mod.HEAPU8.set(nameBytes, namePtr);
1574
+ const coordBytes = vertices.length * 2 * 8;
1575
+ const coordPtr = this.mod.ccall("goss_alloc", "number", ["number"], [coordBytes]) as number;
1576
+ const base = coordPtr >> 3;
1577
+ for (let i = 0; i < vertices.length; i += 1) {
1578
+ this.mod.HEAPF64[base + i * 2] = vertices[i]![0];
1579
+ this.mod.HEAPF64[base + i * 2 + 1] = vertices[i]![1];
1580
+ }
1581
+ this.mod.ccall("goss_session_set_named_geofence_polygon", "number", ["number", "number", "number", "number", "number"], [this.handle, namePtr, nameBytes.length, coordPtr, vertices.length]);
1582
+ this.mod.ccall("goss_free", null, ["number", "number"], [namePtr, nameBytes.length]);
1583
+ this.mod.ccall("goss_free", null, ["number", "number"], [coordPtr, coordBytes]);
1584
+ }
1585
+
1586
+ /// Clears every named geofence; the default geofence is untouched.
1587
+ clearNamedGeofences(): void {
1588
+ this.mod.ccall("goss_session_clear_named_geofences", "number", ["number"], [this.handle]);
1589
+ }
1590
+
1591
+ /// Sets the worst fix accuracy (meters) that still counts as inside a region;
1592
+ /// zero clears the gate.
1593
+ setGeoAccuracy(maxAccuracyM: number): void {
1594
+ this.mod.ccall("goss_session_set_geo_accuracy", "number", ["number", "number"], [this.handle, maxAccuracyM]);
1595
+ }
1596
+
1597
+ /// Sets the color and half-width (normalized units) the next stroke opens with.
1598
+ setBrushStyle(r: number, g: number, b: number, a: number, width: number): void {
1599
+ this.mod.ccall("goss_session_brush_set_style", "number", ["number", "number", "number", "number", "number", "number"], [this.handle, r, g, b, a, width]);
1600
+ }
1601
+
1602
+ /// Uploads the RGBA sprite a stamp-mode stroke (brush mode 4) lays along its
1603
+ /// length, an emoji or icon the host rasterizes.
1604
+ setBrushStamp(rgba: Uint8ClampedArray | Uint8Array, width: number, height: number): void {
1605
+ const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [rgba.length]) as number;
1606
+ this.mod.HEAPU8.set(rgba, ptr);
1607
+ this.mod.ccall("goss_session_brush_set_stamp", "number", ["number", "number", "number", "number"], [this.handle, ptr, width, height]);
1608
+ this.mod.ccall("goss_free", null, ["number", "number"], [ptr, rgba.length]);
1609
+ }
1610
+
1611
+ /// Opens a stroke in the current style. A fresh stroke drops the redo stack.
1612
+ beginStroke(): void {
1613
+ this.mod.ccall("goss_session_brush_begin", "number", ["number"], [this.handle]);
1614
+ }
1615
+
1616
+ /// Adds a point to the open stroke, in normalized screen space (0..1).
1617
+ addStrokePoint(x: number, y: number): void {
1618
+ this.mod.ccall("goss_session_brush_point", "number", ["number", "number", "number"], [this.handle, x, y]);
1619
+ }
1620
+
1621
+ /// Commits the open stroke. A stroke of fewer than two points is dropped.
1622
+ endStroke(): void {
1623
+ this.mod.ccall("goss_session_brush_end", "number", ["number"], [this.handle]);
1624
+ }
1625
+
1626
+ undoStroke(): void {
1627
+ this.mod.ccall("goss_session_brush_undo", "number", ["number"], [this.handle]);
1628
+ }
1629
+
1630
+ redoStroke(): void {
1631
+ this.mod.ccall("goss_session_brush_redo", "number", ["number"], [this.handle]);
1632
+ }
1633
+
1634
+ clearStrokes(): void {
1635
+ this.mod.ccall("goss_session_brush_clear", "number", ["number"], [this.handle]);
1636
+ }
1637
+
1638
+ /// The brush preset the next stroke opens with: 0 pen, 1 highlighter, 2 marker, 3 neon.
1639
+ setBrushMode(mode: number): void {
1640
+ this.mod.ccall("goss_session_brush_set_mode", "number", ["number", "number"], [this.handle, mode]);
1641
+ }
1642
+
1643
+ /// Erases committed strokes within `radius` (normalized units) of the point
1644
+ /// and returns how many were removed.
1645
+ eraseStrokes(x: number, y: number, radius: number): number {
1646
+ const outPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]) as number;
1647
+ this.mod.ccall("goss_session_brush_erase_at", "number", ["number", "number", "number", "number", "number"], [this.handle, x, y, radius, outPtr]);
1648
+ const removed = this.mod.HEAP32[outPtr >> 2]!;
1649
+ this.mod.ccall("goss_free", null, ["number", "number"], [outPtr, 4]);
1650
+ return removed;
1651
+ }
1652
+
1653
+ /// The world-anchored brush. Points are pushed in the world frame world
1654
+ /// tracking reports; the engine projects and draws them so a stroke stays
1655
+ /// fixed in the scene. Nothing draws without live world tracking.
1656
+ setARBrushStyle(r: number, g: number, b: number, a: number, width: number): void {
1657
+ this.mod.ccall("goss_session_ar_brush_set_style", "number", ["number", "number", "number", "number", "number", "number"], [this.handle, r, g, b, a, width]);
1658
+ }
1659
+
1660
+ setARBrushMode(mode: number): void {
1661
+ this.mod.ccall("goss_session_ar_brush_set_mode", "number", ["number", "number"], [this.handle, mode]);
1662
+ }
1663
+
1664
+ beginARStroke(): void {
1665
+ this.mod.ccall("goss_session_ar_brush_begin", "number", ["number"], [this.handle]);
1666
+ }
1667
+
1668
+ addARStrokePoint(x: number, y: number, z: number): void {
1669
+ this.mod.ccall("goss_session_ar_brush_point", "number", ["number", "number", "number", "number"], [this.handle, x, y, z]);
1670
+ }
1671
+
1672
+ endARStroke(): void {
1673
+ this.mod.ccall("goss_session_ar_brush_end", "number", ["number"], [this.handle]);
1674
+ }
1675
+
1676
+ undoARStroke(): void {
1677
+ this.mod.ccall("goss_session_ar_brush_undo", "number", ["number"], [this.handle]);
1678
+ }
1679
+
1680
+ clearARStrokes(): void {
1681
+ this.mod.ccall("goss_session_ar_brush_clear", "number", ["number"], [this.handle]);
1682
+ }
1683
+
1684
+ /** Feeds one screen touch event so the engine recognizes the gestures a lens
1685
+ * reacts to. phase is 0 began, 1 moved, 2 ended, 3 cancelled; pointerId names
1686
+ * the finger; x and y are normalized 0..1 over the frame. */
1687
+ touch(phase: number, pointerId: number, x: number, y: number): void {
1688
+ this.mod.ccall("goss_session_touch", "number", ["number", "number", "number", "number", "number"], [this.handle, phase, pointerId, x, y]);
1689
+ }
1690
+
1691
+ /** Drains one haptic a haptic trigger queued this tick, or null when none
1692
+ * remain. Call in a loop after tickLens and buzz the device for each. The
1693
+ * style is 0 light..7 failure; intensity is a 0..1 hint. */
1694
+ pullHaptic(): { style: number; intensity: number } | null {
1695
+ const stylePtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]);
1696
+ const intensityPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]);
1697
+ const status = this.mod.ccall("goss_session_pull_haptic", "number", ["number", "number", "number"], [this.handle, stylePtr, intensityPtr]);
1698
+ const result = status === 0 ? { style: this.mod.getValue(stylePtr, "i32"), intensity: this.mod.getValue(intensityPtr, "float") } : null;
1699
+ this.mod.ccall("goss_free", null, ["number", "number"], [stylePtr, 4]);
1700
+ this.mod.ccall("goss_free", null, ["number", "number"], [intensityPtr, 4]);
1701
+ return result;
1702
+ }
1703
+
1704
+ grab(x: number, y: number, z: number): void {
1705
+ this.mod.ccall("goss_session_grab", "number", ["number", "number", "number", "number"], [this.handle, x, y, z]);
1706
+ }
1707
+
1708
+ release(): void {
1709
+ this.mod.ccall("goss_session_release", "number", ["number"], [this.handle]);
1710
+ }
1711
+
1712
+ addCollider(x: number, y: number, z: number): void {
1713
+ this.mod.ccall("goss_session_add_collider", "number", ["number", "number", "number", "number"], [this.handle, x, y, z]);
1714
+ }
1715
+
1716
+ eraseCollider(x: number, y: number, z: number, radius: number): void {
1717
+ this.mod.ccall("goss_session_erase_collider", "number", ["number", "number", "number", "number", "number"], [this.handle, x, y, z, radius]);
1718
+ }
1719
+
1720
+ /// Releases one solver hair by the id the physics world assigned it,
1721
+ /// pairing the acquire a hair lens performs at activation, so a hair
1722
+ /// can retire mid-session without tearing the physics world down.
1723
+ /// False with no physics world or for an unknown id.
1724
+ physicsHairRemove(hairId: number): boolean {
1725
+ return this.mod.ccall("goss_physics_hair_remove", "number", ["number", "number"], [this.handle, hairId]) === 0;
1726
+ }
1727
+
1728
+ /// Pulls the finished brush ribbon (x, y, r, g, b, a per vertex) for the
1729
+ /// renderer. Queries the float count, then reads it out of a scratch buffer.
1730
+ brushVertices(): Float32Array {
1731
+ const countPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]) as number;
1732
+ this.mod.ccall("goss_session_brush_vertices", "number", ["number", "number", "number", "number"], [this.handle, 0, 0, countPtr]);
1733
+ const count = this.mod.HEAP32[countPtr >> 2]!;
1734
+ if (count <= 0) {
1735
+ this.mod.ccall("goss_free", null, ["number", "number"], [countPtr, 4]);
1736
+ return new Float32Array(0);
1737
+ }
1738
+ const bytes = count * 4;
1739
+ const outPtr = this.mod.ccall("goss_alloc", "number", ["number"], [bytes]) as number;
1740
+ this.mod.ccall("goss_session_brush_vertices", "number", ["number", "number", "number", "number"], [this.handle, outPtr, count, countPtr]);
1741
+ const written = this.mod.HEAP32[countPtr >> 2]!;
1742
+ const out = new Float32Array(this.mod.HEAPF32.buffer, outPtr, written).slice();
1743
+ this.mod.ccall("goss_free", null, ["number", "number"], [outPtr, bytes]);
1744
+ this.mod.ccall("goss_free", null, ["number", "number"], [countPtr, 4]);
1745
+ return out;
1746
+ }
1747
+
1748
+ setVideoFlip(enabled: boolean): void {
1749
+ this.videoFlipped = enabled;
1750
+ }
1751
+
1752
+ isVideoFlipped(): boolean {
1753
+ return this.videoFlipped;
1754
+ }
1755
+
1756
+ /// landmarks are raw tracker output - x, y in sourceWidth/sourceHeight
1757
+ /// pixels (whatever resolution the caller's own tracking pass ran
1758
+ /// at, which need not match the live video's own resolution), z in
1759
+ /// the same relative scale, three floats per point, matching
1760
+ /// goss_face_result's own convention. Scaled here to the frame
1761
+ /// currently being rendered - the engine's own contour math expects
1762
+ /// "frame pixels" of the frame it's compositing, not of whatever
1763
+ /// analysis resolution tracking happened to use. Null clears
1764
+ /// tracking (no face this frame).
1765
+ setFaceLandmarks(landmarks: Float32Array | null, sourceWidth: number, sourceHeight: number): void {
1766
+ if (!landmarks || landmarks.length === 0 || this.frameWidth === 0) {
1767
+ this.mod.ccall("goss_session_set_face_landmarks", "number", ["number", "number", "number"], [this.handle, 0, 0]);
1768
+ return;
1769
+ }
1770
+ const scaleX = this.frameWidth / sourceWidth;
1771
+ const scaleY = this.frameHeight / sourceHeight;
1772
+ const pointCount = landmarks.length / 3;
1773
+ const base = this.landmarksPtr >> 2;
1774
+ for (let at = 0; at < pointCount; at += 1) {
1775
+ this.mod.HEAPF32[base + at * 3] = landmarks[at * 3]! * scaleX;
1776
+ this.mod.HEAPF32[base + at * 3 + 1] = landmarks[at * 3 + 1]! * scaleY;
1777
+ this.mod.HEAPF32[base + at * 3 + 2] = landmarks[at * 3 + 2]!;
1778
+ }
1779
+ this.mod.ccall(
1780
+ "goss_session_set_face_landmarks",
1781
+ "number",
1782
+ ["number", "number", "number"],
1783
+ [this.handle, this.landmarksPtr, pointCount],
1784
+ );
1785
+ }
1786
+
1787
+ /// Submits the faces tracked this frame for the multi-face path. landmarks
1788
+ /// are frame pixels, GOSS_FACE_LANDMARK_COUNT * 3 floats; presence defaults
1789
+ /// to 1. An empty array clears the path; faces past GOSS_FACE_MAX or below
1790
+ /// the tracked presence are dropped.
1791
+ submitFaces(faces: GossFaceInput[]): void {
1792
+ if (faces.length === 0) {
1793
+ this.mod.ccall("goss_session_submit_faces", "number", ["number", "number", "number"], [this.handle, 0, 0]);
1794
+ return;
1795
+ }
1796
+ const bytes = faces.length * FACE_RESULT_BYTES;
1797
+ const ptr = this.scratch(bytes);
1798
+ const dv = new DataView(this.mod.HEAPU8.buffer, ptr, bytes);
1799
+ for (let i = 0; i < faces.length; i += 1) {
1800
+ const off = i * FACE_RESULT_BYTES;
1801
+ const base = ptr + off;
1802
+ const f = faces[i]!;
1803
+ const count = f.landmarks.length / 3;
1804
+ dv.setBigUint64(off, BigInt(f.frameSerial ?? 0), true);
1805
+ dv.setBigInt64(off + 8, BigInt(f.timestampUs ?? 0), true);
1806
+ dv.setFloat32(off + 16, f.presence ?? 1, true);
1807
+ dv.setUint32(off + 20, count, true);
1808
+ this.mod.HEAPF32.set(f.landmarks, (base + 24) >> 2);
1809
+ const bsStart = (base + 24 + GOSS_FACE_LANDMARK_COUNT * 3 * 4) >> 2;
1810
+ this.mod.HEAPF32.fill(0, bsStart, bsStart + GOSS_FACE_BLENDSHAPE_COUNT);
1811
+ if (f.blendshapes) this.mod.HEAPF32.set(f.blendshapes, bsStart);
1812
+ }
1813
+ this.mod.ccall("goss_session_submit_faces", "number", ["number", "number", "number"], [this.handle, ptr, faces.length]);
1814
+ }
1815
+
1816
+ /// The number of faces the last submitFaces kept, zero to GOSS_FACE_MAX.
1817
+ faceCount(): number {
1818
+ const ptr = this.scratch(4);
1819
+ this.mod.ccall("goss_session_face_count", "number", ["number", "number"], [this.handle, ptr]);
1820
+ return this.mod.HEAP32[ptr >> 2]!;
1821
+ }
1822
+
1823
+ /// Reads the index-th submitted face, or null once index reaches faceCount,
1824
+ /// so a caller loops zero to faceCount to visit every face.
1825
+ faceResultAt(index: number): GossFaceOut | null {
1826
+ const ptr = this.scratch(FACE_RESULT_BYTES);
1827
+ const status = this.mod.ccall("goss_session_face_result_at", "number", ["number", "number", "number"], [this.handle, index, ptr]) as number;
1828
+ if (status !== 0) return null;
1829
+ const dv = new DataView(this.mod.HEAPU8.buffer, ptr, FACE_RESULT_BYTES);
1830
+ const lmStart = (ptr + 24) >> 2;
1831
+ const bsStart = lmStart + GOSS_FACE_LANDMARK_COUNT * 3;
1832
+ const out: GossFaceOut = {
1833
+ frameSerial: dv.getBigUint64(0, true),
1834
+ timestampUs: dv.getBigInt64(8, true),
1835
+ presence: dv.getFloat32(16, true),
1836
+ landmarkCount: dv.getUint32(20, true),
1837
+ landmarks: this.mod.HEAPF32.slice(lmStart, bsStart),
1838
+ blendshapes: this.mod.HEAPF32.slice(bsStart, bsStart + GOSS_FACE_BLENDSHAPE_COUNT),
1839
+ };
1840
+ return out;
1841
+ }
1842
+
1843
+ /// The stable track id of the index-th face, an integer that stays with the
1844
+ /// same person across frames as the submission order shuffles, or null once
1845
+ /// index reaches faceCount.
1846
+ faceTrackId(index: number): number | null {
1847
+ const ptr = this.scratch(4);
1848
+ const status = this.mod.ccall("goss_session_face_track_id", "number", ["number", "number", "number"], [this.handle, index, ptr]) as number;
1849
+ if (status !== 0) return null;
1850
+ return this.mod.HEAPU32[ptr >> 2]!;
1851
+ }
1852
+
1853
+ /// Submits the bodies tracked this frame for the multi-person path, so a
1854
+ /// lens can instance effects across every body. An empty array clears the
1855
+ /// path; bodies past GOSS_BODY_MAX are ignored.
1856
+ submitBodies(bodies: GossPoseInput[]): void {
1857
+ if (bodies.length === 0) {
1858
+ this.mod.ccall("goss_session_submit_bodies", "number", ["number", "number", "number"], [this.handle, 0, 0]);
1859
+ return;
1860
+ }
1861
+ const bytes = bodies.length * POSE_RESULT_BYTES;
1862
+ const ptr = this.scratch(bytes);
1863
+ const dv = new DataView(this.mod.HEAPU8.buffer, ptr, bytes);
1864
+ for (let i = 0; i < bodies.length; i += 1) {
1865
+ const off = i * POSE_RESULT_BYTES;
1866
+ const base = ptr + off;
1867
+ const b = bodies[i]!;
1868
+ const count = b.landmarks.length / 3;
1869
+ dv.setBigUint64(off, BigInt(b.frameSerial ?? 0), true);
1870
+ dv.setBigInt64(off + 8, BigInt(b.timestampUs ?? 0), true);
1871
+ dv.setFloat32(off + 16, b.presence ?? 1, true);
1872
+ dv.setUint32(off + 20, count, true);
1873
+ this.mod.HEAPF32.set(b.landmarks, (base + 24) >> 2);
1874
+ const visStart = (base + 24 + GOSS_POSE_LANDMARK_COUNT * 3 * 4) >> 2;
1875
+ this.mod.HEAPF32.fill(0, visStart, visStart + GOSS_POSE_LANDMARK_COUNT * 2);
1876
+ if (b.visibilities) this.mod.HEAPF32.set(b.visibilities, visStart);
1877
+ if (b.presences) this.mod.HEAPF32.set(b.presences, visStart + GOSS_POSE_LANDMARK_COUNT);
1878
+ }
1879
+ this.mod.ccall("goss_session_submit_bodies", "number", ["number", "number", "number"], [this.handle, ptr, bodies.length]);
1880
+ }
1881
+
1882
+ /// Submits one frame's depth map from the host AR backend (WebXR depth-
1883
+ /// sensing): width by height metres per pixel, row major, with the near and
1884
+ /// far metres that bound it. An empty array clears it. Kept for depth
1885
+ /// occlusion against the rendered content.
1886
+ submitDepth(depth: Float32Array, width: number, height: number, near: number, far: number): void {
1887
+ if (depth.length === 0) {
1888
+ this.mod.ccall("goss_session_submit_depth", "number", ["number", "number", "number", "number", "number", "number"], [this.handle, 0, 0, 0, 0, 0]);
1889
+ return;
1890
+ }
1891
+ const bytes = depth.length * 4;
1892
+ const ptr = this.scratch(bytes);
1893
+ this.mod.HEAPF32.set(depth, ptr >> 2);
1894
+ this.mod.ccall("goss_session_submit_depth", "number", ["number", "number", "number", "number", "number", "number"], [this.handle, ptr, width, height, near, far]);
1895
+ }
1896
+
1897
+ /// Submits the camera intrinsics an undistort.pass corrects for: the focal
1898
+ /// lengths and principal point in pixels of the submitted frame, and the
1899
+ /// radial distortion coefficients (k1, k2 read). An empty array or zero focal
1900
+ /// length clears them, leaving an undistort.pass inert.
1901
+ submitCameraIntrinsics(fx: number, fy: number, cx: number, cy: number, distortion: Float32Array): void {
1902
+ if (distortion.length === 0) {
1903
+ this.mod.ccall("goss_session_submit_camera_intrinsics", "number", ["number", "number", "number", "number", "number", "number", "number"], [this.handle, 0, 0, 0, 0, 0, 0]);
1904
+ return;
1905
+ }
1906
+ const ptr = this.scratch(distortion.length * 4);
1907
+ this.mod.HEAPF32.set(distortion, ptr >> 2);
1908
+ this.mod.ccall("goss_session_submit_camera_intrinsics", "number", ["number", "number", "number", "number", "number", "number", "number"], [this.handle, fx, fy, cx, cy, ptr, distortion.length]);
1909
+ }
1910
+
1911
+ /// Submits one device gravity sample with its timestamp in microseconds. A
1912
+ /// rolling.pass reads the image-plane motion derived from consecutive samples
1913
+ /// to correct rolling-shutter skew; feed one per frame from the IMU. A
1914
+ /// near-zero vector clears the stream, leaving a rolling.pass inert.
1915
+ submitOrientation(gravityX: number, gravityY: number, gravityZ: number, timestampUs: bigint = 0n): void {
1916
+ this.mod.ccall("goss_session_submit_orientation", "number", ["number", "number", "number", "number", "number"], [this.handle, gravityX, gravityY, gravityZ, timestampUs]);
1917
+ }
1918
+
1919
+ /// Feeds a host info value keyed by name, the rail an info sticker reads: a
1920
+ /// text.2d node with a matching content_source shows the latest value each
1921
+ /// frame (a time, a place, a sensor reading). A null value clears the key.
1922
+ setInfo(key: string, value: string | null): void {
1923
+ const keyBytes = new TextEncoder().encode(key);
1924
+ const keyPtr = this.mod.ccall("goss_alloc", "number", ["number"], [keyBytes.length || 1]) as number;
1925
+ this.mod.HEAPU8.set(keyBytes, keyPtr);
1926
+ let valuePtr = 0;
1927
+ let valueLen = 0;
1928
+ let valueBytes: Uint8Array | null = null;
1929
+ if (value !== null) {
1930
+ valueBytes = new TextEncoder().encode(value);
1931
+ valueLen = valueBytes.length;
1932
+ valuePtr = this.mod.ccall("goss_alloc", "number", ["number"], [valueLen || 1]) as number;
1933
+ this.mod.HEAPU8.set(valueBytes, valuePtr);
1934
+ }
1935
+ this.mod.ccall("goss_session_set_info", "number", ["number", "number", "number", "number", "number"], [this.handle, keyPtr, keyBytes.length, valuePtr, valueLen]);
1936
+ this.mod.ccall("goss_free", null, ["number", "number"], [keyPtr, keyBytes.length || 1]);
1937
+ if (valuePtr !== 0) this.mod.ccall("goss_free", null, ["number", "number"], [valuePtr, valueLen || 1]);
1938
+ }
1939
+
1940
+ /// Serializes the active lens's parameter state to a blob a connected lens
1941
+ /// publishes so the cloud syncs it to peers, or null with no lens.
1942
+ snapshotLensState(): Uint8Array | null {
1943
+ const lenPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]) as number;
1944
+ const view = () => new DataView(this.mod.HEAPU8.buffer, lenPtr, 4).getUint32(0, true);
1945
+ let out: Uint8Array | null = null;
1946
+ if ((this.mod.ccall("goss_session_snapshot_lens_state", "number", ["number", "number", "number", "number"], [this.handle, 0, 0, lenPtr]) as number) === 0) {
1947
+ const needed = view();
1948
+ const outPtr = this.mod.ccall("goss_alloc", "number", ["number"], [needed || 1]) as number;
1949
+ if ((this.mod.ccall("goss_session_snapshot_lens_state", "number", ["number", "number", "number", "number"], [this.handle, outPtr, needed, lenPtr]) as number) === 0) {
1950
+ out = this.mod.HEAPU8.slice(outPtr, outPtr + view());
1951
+ }
1952
+ this.mod.ccall("goss_free", null, ["number", "number"], [outPtr, needed || 1]);
1953
+ }
1954
+ this.mod.ccall("goss_free", null, ["number", "number"], [lenPtr, 4]);
1955
+ return out;
1956
+ }
1957
+
1958
+ /// Applies a peer's lens-state blob to the active lens, clamping each value
1959
+ /// into its parameter so two runtimes on the same lens converge.
1960
+ applyLensState(blob: Uint8Array): void {
1961
+ const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [blob.length || 1]) as number;
1962
+ this.mod.HEAPU8.set(blob, ptr);
1963
+ this.mod.ccall("goss_session_apply_lens_state", "number", ["number", "number", "number"], [this.handle, ptr, blob.length]);
1964
+ this.mod.ccall("goss_free", null, ["number", "number"], [ptr, blob.length || 1]);
1965
+ }
1966
+
1967
+ /// The active lens's content-provenance manifest as JSON (producer, lens,
1968
+ /// whether the frame is model-generated or edited, and the operations), for the
1969
+ /// host to bind to a capture per C2PA. Null with no active lens.
1970
+ captureProvenance(): string | null {
1971
+ const lenPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]) as number;
1972
+ const view = () => new DataView(this.mod.HEAPU8.buffer, lenPtr, 4).getUint32(0, true);
1973
+ let json: string | null = null;
1974
+ if ((this.mod.ccall("goss_session_capture_provenance", "number", ["number", "number", "number", "number"], [this.handle, 0, 0, lenPtr]) as number) === 0) {
1975
+ const needed = view();
1976
+ const outPtr = this.mod.ccall("goss_alloc", "number", ["number"], [needed || 1]) as number;
1977
+ if ((this.mod.ccall("goss_session_capture_provenance", "number", ["number", "number", "number", "number"], [this.handle, outPtr, needed, lenPtr]) as number) === 0) {
1978
+ json = new TextDecoder().decode(this.mod.HEAPU8.slice(outPtr, outPtr + view()));
1979
+ }
1980
+ this.mod.ccall("goss_free", null, ["number", "number"], [outPtr, needed || 1]);
1981
+ }
1982
+ this.mod.ccall("goss_free", null, ["number", "number"], [lenPtr, 4]);
1983
+ return json;
1984
+ }
1985
+
1986
+ /// Captures the current viewpoint (the last submitted world pose and depth) into
1987
+ /// a guided scan, back-projecting the depth into a deterministic gaussian
1988
+ /// reconstruction, and returns the scan's coverage so the app can steer the user
1989
+ /// to the next uncovered viewpoint. Reset with resetCapture.
1990
+ captureView(): CaptureGuidance {
1991
+ const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [24]) as number;
1992
+ this.mod.ccall("goss_session_capture_view", "number", ["number", "number"], [this.handle, ptr]);
1993
+ const guidance: CaptureGuidance = {
1994
+ covered: this.mod.getValue(ptr, "i32") >>> 0,
1995
+ total: this.mod.getValue(ptr + 4, "i32") >>> 0,
1996
+ complete: this.mod.getValue(ptr + 8, "i32") !== 0,
1997
+ viewCount: this.mod.getValue(ptr + 12, "i32") >>> 0,
1998
+ splatCount: this.mod.getValue(ptr + 16, "i32") >>> 0,
1999
+ nextYaw: this.mod.getValue(ptr + 20, "float"),
2000
+ };
2001
+ this.mod.ccall("goss_free", null, ["number", "number"], [ptr, 24]);
2002
+ return guidance;
2003
+ }
2004
+
2005
+ /// Clears a guided-capture scan: its covered targets, poses, and reconstruction.
2006
+ resetCapture(): void {
2007
+ this.mod.ccall("goss_session_reset_capture", "number", ["number"], [this.handle]);
2008
+ }
2009
+
2010
+ /// Enables or disables on-device dubbing: when on, a dub-bound audio.infer node
2011
+ /// synthesizes its decoded caption or translation to speech and plays it into
2012
+ /// the lens mixer. Off by default; a host turns it on for a voice-over.
2013
+ setDubbing(enabled: boolean): void {
2014
+ this.mod.ccall("goss_session_set_dubbing", "number", ["number", "number"], [this.handle, enabled ? 1 : 0]);
2015
+ }
2016
+
2017
+ /// The latest caption an audio.infer node decoded, by the node's id, or null
2018
+ /// when that node has no caption binding or nothing decoded yet. On-device ASR
2019
+ /// the app can draw as a live subtitle. A length probe sizes the buffer.
2020
+ captionText(nodeId: string): string | null {
2021
+ const id = new TextEncoder().encode(nodeId);
2022
+ const idPtr = this.mod.ccall("goss_alloc", "number", ["number"], [id.length || 1]) as number;
2023
+ this.mod.HEAPU8.set(id, idPtr);
2024
+ const lenPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]) as number;
2025
+ const readLen = () => new DataView(this.mod.HEAPU8.buffer, lenPtr, 4).getUint32(0, true);
2026
+ const args = ["number", "number", "number", "number", "number", "number"];
2027
+ const probe = this.mod.ccall("goss_session_caption_text", "number", args, [this.handle, idPtr, id.length, 0, 0, lenPtr]) as number;
2028
+ const needed = readLen();
2029
+ let result: string | null = null;
2030
+ if (probe === 0 && needed > 0) {
2031
+ const outPtr = this.mod.ccall("goss_alloc", "number", ["number"], [needed]) as number;
2032
+ const status = this.mod.ccall("goss_session_caption_text", "number", args, [this.handle, idPtr, id.length, outPtr, needed, lenPtr]) as number;
2033
+ const written = readLen();
2034
+ if (status === 0) result = new TextDecoder().decode(this.mod.HEAPU8.slice(outPtr, outPtr + written));
2035
+ this.mod.ccall("goss_free", null, ["number", "number"], [outPtr, needed]);
2036
+ }
2037
+ this.mod.ccall("goss_free", null, ["number", "number"], [lenPtr, 4]);
2038
+ this.mod.ccall("goss_free", null, ["number", "number"], [idPtr, id.length || 1]);
2039
+ return result;
2040
+ }
2041
+
2042
+ /// The recent diarized caption segment at index (0 the newest), or null when
2043
+ /// the index is past the segments held: the times it spanned, the speaker who
2044
+ /// spoke it, and its text, a speaker-tagged transcript for diarized subtitles.
2045
+ captionSegment(index: number): { startUs: bigint; endUs: bigint; speaker: number; text: string } | null {
2046
+ const segPtr = this.mod.ccall("goss_alloc", "number", ["number"], [24]) as number;
2047
+ const status = this.mod.ccall("goss_session_caption_segment", "number", ["number", "number", "number"], [this.handle, index, segPtr]) as number;
2048
+ if (status !== 0) {
2049
+ this.mod.ccall("goss_free", null, ["number", "number"], [segPtr, 24]);
2050
+ return null;
2051
+ }
2052
+ const dv = new DataView(this.mod.HEAPU8.buffer, segPtr, 24);
2053
+ const startUs = dv.getBigInt64(0, true);
2054
+ const endUs = dv.getBigInt64(8, true);
2055
+ const speaker = dv.getUint32(16, true);
2056
+ const textLen = dv.getUint32(20, true);
2057
+ this.mod.ccall("goss_free", null, ["number", "number"], [segPtr, 24]);
2058
+ let text = "";
2059
+ if (textLen > 0) {
2060
+ const lenPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]) as number;
2061
+ const outPtr = this.mod.ccall("goss_alloc", "number", ["number"], [textLen]) as number;
2062
+ const st = this.mod.ccall("goss_session_caption_segment_text", "number", ["number", "number", "number", "number", "number"], [this.handle, index, outPtr, textLen, lenPtr]) as number;
2063
+ const written = new DataView(this.mod.HEAPU8.buffer, lenPtr, 4).getUint32(0, true);
2064
+ if (st === 0) text = new TextDecoder().decode(this.mod.HEAPU8.slice(outPtr, outPtr + written));
2065
+ this.mod.ccall("goss_free", null, ["number", "number"], [outPtr, textLen]);
2066
+ this.mod.ccall("goss_free", null, ["number", "number"], [lenPtr, 4]);
2067
+ }
2068
+ return { startUs, endUs, speaker, text };
2069
+ }
2070
+
2071
+ /// Segments a host-provided still image through the running segmenter: rgba
2072
+ /// is width by height RGBA8 pixels, row major. The mask reaches the active
2073
+ /// lens the way a camera frame's would.
2074
+ submitSegmentationImage(rgba: Uint8Array, width: number, height: number): void {
2075
+ const ptr = this.scratch(rgba.length);
2076
+ this.mod.HEAPU8.set(rgba, ptr);
2077
+ this.mod.ccall("goss_session_submit_segmentation_image", "number", ["number", "number", "number", "number"], [this.handle, ptr, width, height]);
2078
+ }
2079
+
2080
+ /// Submits one RGBA exposure of an HDR bracket (width by height RGBA8, row
2081
+ /// major), converted to NV12 and fed to bracket-source temporal.fuse nodes; the
2082
+ /// fusion publishes once the ring holds a full bracket.
2083
+ submitFrameBracketRgba(rgba: Uint8Array, width: number, height: number): void {
2084
+ const ptr = this.scratch(rgba.length);
2085
+ this.mod.HEAPU8.set(rgba, ptr);
2086
+ this.mod.ccall("goss_session_submit_frame_bracket_rgba", "number", ["number", "number", "number", "number"], [this.handle, ptr, width, height]);
2087
+ }
2088
+
2089
+ /// Submits one NV12 exposure of an HDR bracket, fed to bracket-source
2090
+ /// temporal.fuse nodes. y and uv are the plane bytes as submitFrameCopy takes.
2091
+ submitFrameBracket(y: Uint8Array, yStride: number, uv: Uint8Array, uvStride: number, width: number, height: number, colorStandard = 1, colorRange = 0): void {
2092
+ const yPtr = this.mod.ccall("goss_alloc", "number", ["number"], [y.length]) as number;
2093
+ this.mod.HEAPU8.set(y, yPtr);
2094
+ const uvPtr = this.mod.ccall("goss_alloc", "number", ["number"], [uv.length]) as number;
2095
+ this.mod.HEAPU8.set(uv, uvPtr);
2096
+ this.mod.setValue(this.frameDescPtr, width, "i32");
2097
+ this.mod.setValue(this.frameDescPtr + 4, height, "i32");
2098
+ this.mod.setValue(this.frameDescPtr + 8, 0, "i32");
2099
+ this.mod.setValue(this.frameDescPtr + 12, colorStandard, "i32");
2100
+ this.mod.setValue(this.frameDescPtr + 16, colorRange, "i32");
2101
+ this.mod.setValue(this.frameDescPtr + 20, 0, "i32");
2102
+ this.mod.setValue(this.frameDescPtr + 24, 0, "i32");
2103
+ this.mod.setValue(this.frameDescPtr + 28, 0, "i32");
2104
+ this.mod.ccall("goss_session_submit_frame_bracket", "number", ["number", "number", "number", "number", "number", "number"], [this.handle, this.frameDescPtr, yPtr, yStride, uvPtr, uvStride]);
2105
+ this.mod.ccall("goss_free", null, ["number", "number"], [yPtr, y.length]);
2106
+ this.mod.ccall("goss_free", null, ["number", "number"], [uvPtr, uv.length]);
2107
+ }
2108
+
2109
+ private submitPlanes(sym: string, y: Uint8Array, yStride: number, uv: Uint8Array, uvStride: number, width: number, height: number, colorStandard: number, colorRange: number): void {
2110
+ const yPtr = this.mod.ccall("goss_alloc", "number", ["number"], [y.length]) as number;
2111
+ this.mod.HEAPU8.set(y, yPtr);
2112
+ const uvPtr = this.mod.ccall("goss_alloc", "number", ["number"], [uv.length]) as number;
2113
+ this.mod.HEAPU8.set(uv, uvPtr);
2114
+ this.mod.setValue(this.frameDescPtr, width, "i32");
2115
+ this.mod.setValue(this.frameDescPtr + 4, height, "i32");
2116
+ this.mod.setValue(this.frameDescPtr + 8, 0, "i32");
2117
+ this.mod.setValue(this.frameDescPtr + 12, colorStandard, "i32");
2118
+ this.mod.setValue(this.frameDescPtr + 16, colorRange, "i32");
2119
+ this.mod.setValue(this.frameDescPtr + 20, 0, "i32");
2120
+ this.mod.setValue(this.frameDescPtr + 24, 0, "i32");
2121
+ this.mod.setValue(this.frameDescPtr + 28, 0, "i32");
2122
+ this.mod.ccall(sym, "number", ["number", "number", "number", "number", "number", "number"], [this.handle, this.frameDescPtr, yPtr, yStride, uvPtr, uvStride]);
2123
+ this.mod.ccall("goss_free", null, ["number", "number"], [yPtr, y.length]);
2124
+ this.mod.ccall("goss_free", null, ["number", "number"], [uvPtr, uv.length]);
2125
+ }
2126
+
2127
+ /// Feeds one NV12 frame to the in-engine trackers only (no render), for a host
2128
+ /// that renders elsewhere but wants the engine's face/hand/pose tracking. Web
2129
+ /// usually tracks in its own worker and feeds results through the producer path.
2130
+ trackFrame(y: Uint8Array, yStride: number, uv: Uint8Array, uvStride: number, width: number, height: number, colorStandard = 1, colorRange = 0): void {
2131
+ this.submitPlanes("goss_session_track_frame", y, yStride, uv, uvStride, width, height, colorStandard, colorRange);
2132
+ }
2133
+
2134
+ /// Submits one NV12 frame the engine copies into its own buffer (the caller may
2135
+ /// reuse the planes at once), the copy counterpart of submitFrameRgbaCopy.
2136
+ submitFrameCopy(y: Uint8Array, yStride: number, uv: Uint8Array, uvStride: number, width: number, height: number, colorStandard = 1, colorRange = 0): void {
2137
+ this.submitPlanes("goss_session_submit_frame_copy", y, yStride, uv, uvStride, width, height, colorStandard, colorRange);
2138
+ }
2139
+
2140
+ /// Submits one NV12 frame of a driving performance for avatar reenactment, so
2141
+ /// the tracked avatar is posed from this source instead of the camera.
2142
+ submitAvatarSource(y: Uint8Array, yStride: number, uv: Uint8Array, uvStride: number, width: number, height: number, colorStandard = 1, colorRange = 0): void {
2143
+ this.submitPlanes("goss_session_submit_avatar_source", y, yStride, uv, uvStride, width, height, colorStandard, colorRange);
2144
+ }
2145
+
2146
+ /// Reads the in-engine face tracker's single-face result, or null when no face
2147
+ /// is tracked; faceResultAt visits every face on the multi-face path.
2148
+ faceResult(): GossFaceOut | null {
2149
+ const ptr = this.scratch(FACE_RESULT_BYTES);
2150
+ if (this.mod.ccall("goss_session_face_result", "number", ["number", "number"], [this.handle, ptr]) !== 0) return null;
2151
+ const dv = new DataView(this.mod.HEAPU8.buffer, ptr, FACE_RESULT_BYTES);
2152
+ const lmStart = (ptr + 24) >> 2;
2153
+ const bsStart = lmStart + GOSS_FACE_LANDMARK_COUNT * 3;
2154
+ return {
2155
+ frameSerial: dv.getBigUint64(0, true),
2156
+ timestampUs: dv.getBigInt64(8, true),
2157
+ presence: dv.getFloat32(16, true),
2158
+ landmarkCount: dv.getUint32(20, true),
2159
+ landmarks: this.mod.HEAPF32.slice(lmStart, bsStart),
2160
+ blendshapes: this.mod.HEAPF32.slice(bsStart, bsStart + GOSS_FACE_BLENDSHAPE_COUNT),
2161
+ };
2162
+ }
2163
+
2164
+ /// The tracked head pose as a 4x4 column-major matrix (16 floats), or null when
2165
+ /// no face is tracked, for anchoring 3D content to the head.
2166
+ facePose(): Float32Array | null {
2167
+ const ptr = this.scratch(64);
2168
+ if (this.mod.ccall("goss_session_face_pose", "number", ["number", "number"], [this.handle, ptr]) !== 0) return null;
2169
+ return this.mod.HEAPF32.slice(ptr >> 2, (ptr >> 2) + 16);
2170
+ }
2171
+
2172
+ /// Reads the in-engine hand tracker's result, or null when no hand is tracked;
2173
+ /// hands past handCount are omitted.
2174
+ handResult(): GossHandOut | null {
2175
+ const ptr = this.scratch(HAND_RESULT_BYTES);
2176
+ if (this.mod.ccall("goss_session_hand_result", "number", ["number", "number"], [this.handle, ptr]) !== 0) return null;
2177
+ const dv = new DataView(this.mod.HEAPU8.buffer, ptr, HAND_RESULT_BYTES);
2178
+ const count = dv.getUint32(16, true);
2179
+ const hands: GossHandOne[] = [];
2180
+ for (let i = 0; i < count && i < GOSS_HAND_MAX; i += 1) {
2181
+ const base = ptr + 24 + i * HAND_ONE_BYTES;
2182
+ const lm = (base + 16) >> 2;
2183
+ hands.push({
2184
+ presence: dv.getFloat32(24 + i * HAND_ONE_BYTES, true),
2185
+ handedness: dv.getFloat32(24 + i * HAND_ONE_BYTES + 4, true),
2186
+ gesture: dv.getUint32(24 + i * HAND_ONE_BYTES + 8, true),
2187
+ gestureScore: dv.getFloat32(24 + i * HAND_ONE_BYTES + 12, true),
2188
+ landmarks: this.mod.HEAPF32.slice(lm, lm + GOSS_HAND_LANDMARK_COUNT * 3),
2189
+ });
2190
+ }
2191
+ return { frameSerial: dv.getBigUint64(0, true), timestampUs: dv.getBigInt64(8, true), hands };
2192
+ }
2193
+
2194
+ /// Reads the in-engine pose tracker's single-body result, or null when no body
2195
+ /// is tracked; bodyResultAt visits every body on the multi-person path.
2196
+ poseResult(): GossPoseOut | null {
2197
+ const ptr = this.scratch(POSE_RESULT_BYTES);
2198
+ if (this.mod.ccall("goss_session_pose_result", "number", ["number", "number"], [this.handle, ptr]) !== 0) return null;
2199
+ const dv = new DataView(this.mod.HEAPU8.buffer, ptr, POSE_RESULT_BYTES);
2200
+ const lmStart = (ptr + 24) >> 2;
2201
+ const visStart = lmStart + GOSS_POSE_LANDMARK_COUNT * 3;
2202
+ const presStart = visStart + GOSS_POSE_LANDMARK_COUNT;
2203
+ return {
2204
+ frameSerial: dv.getBigUint64(0, true),
2205
+ timestampUs: dv.getBigInt64(8, true),
2206
+ presence: dv.getFloat32(16, true),
2207
+ landmarkCount: dv.getUint32(20, true),
2208
+ landmarks: this.mod.HEAPF32.slice(lmStart, visStart),
2209
+ visibilities: this.mod.HEAPF32.slice(visStart, presStart),
2210
+ presences: this.mod.HEAPF32.slice(presStart, presStart + GOSS_POSE_LANDMARK_COUNT),
2211
+ };
2212
+ }
2213
+
2214
+ /// Samples a reference photo's makeup color per face part, so a tint.pass
2215
+ /// with a reference source paints the live face in that color. rgba is width
2216
+ /// by height RGBA8; landmarks is the reference face's 478 x, y, z points. An
2217
+ /// empty landmarks array clears the reference.
2218
+ setMakeupReference(rgba: Uint8Array, width: number, height: number, landmarks: Float32Array): void {
2219
+ if (landmarks.length === 0) {
2220
+ this.mod.ccall("goss_session_set_makeup_reference", "number", ["number", "number", "number", "number", "number", "number"], [this.handle, 0, 0, 0, 0, 0]);
2221
+ return;
2222
+ }
2223
+ const rptr = this.mod.ccall("goss_alloc", "number", ["number"], [rgba.length]) as number;
2224
+ this.mod.HEAPU8.set(rgba, rptr);
2225
+ const lbytes = landmarks.length * 4;
2226
+ const lptr = this.mod.ccall("goss_alloc", "number", ["number"], [lbytes]) as number;
2227
+ this.mod.HEAPF32.set(landmarks, lptr >> 2);
2228
+ this.mod.ccall("goss_session_set_makeup_reference", "number", ["number", "number", "number", "number", "number", "number"], [this.handle, rptr, width, height, lptr, landmarks.length / 3]);
2229
+ this.mod.ccall("goss_free", null, ["number", "number"], [rptr, rgba.length]);
2230
+ this.mod.ccall("goss_free", null, ["number", "number"], [lptr, lbytes]);
2231
+ }
2232
+
2233
+ /// The number of bodies the last submitBodies kept, zero to GOSS_BODY_MAX.
2234
+ bodyCount(): number {
2235
+ const ptr = this.scratch(4);
2236
+ this.mod.ccall("goss_session_body_count", "number", ["number", "number"], [this.handle, ptr]);
2237
+ return this.mod.HEAP32[ptr >> 2]!;
2238
+ }
2239
+
2240
+ /// Reads the index-th submitted body, or null once index reaches bodyCount,
2241
+ /// so a caller loops zero to bodyCount to visit every body.
2242
+ bodyResultAt(index: number): GossPoseOut | null {
2243
+ const ptr = this.scratch(POSE_RESULT_BYTES);
2244
+ const status = this.mod.ccall("goss_session_body_result_at", "number", ["number", "number", "number"], [this.handle, index, ptr]) as number;
2245
+ if (status !== 0) return null;
2246
+ const dv = new DataView(this.mod.HEAPU8.buffer, ptr, POSE_RESULT_BYTES);
2247
+ const lmStart = (ptr + 24) >> 2;
2248
+ const visStart = lmStart + GOSS_POSE_LANDMARK_COUNT * 3;
2249
+ const presStart = visStart + GOSS_POSE_LANDMARK_COUNT;
2250
+ const out: GossPoseOut = {
2251
+ frameSerial: dv.getBigUint64(0, true),
2252
+ timestampUs: dv.getBigInt64(8, true),
2253
+ presence: dv.getFloat32(16, true),
2254
+ landmarkCount: dv.getUint32(20, true),
2255
+ landmarks: this.mod.HEAPF32.slice(lmStart, visStart),
2256
+ visibilities: this.mod.HEAPF32.slice(visStart, presStart),
2257
+ presences: this.mod.HEAPF32.slice(presStart, presStart + GOSS_POSE_LANDMARK_COUNT),
2258
+ };
2259
+ return out;
2260
+ }
2261
+
2262
+ /// The tracked point (x, y in frame pixels, z in the same scale) of a named
2263
+ /// face region, or null until a face is tracked.
2264
+ faceRegion(region: GossFaceRegion): [number, number, number] | null {
2265
+ const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [12]) as number;
2266
+ const status = this.mod.ccall("goss_session_face_region", "number", ["number", "number", "number"], [this.handle, region, ptr]) as number;
2267
+ if (status !== 0) {
2268
+ this.mod.ccall("goss_free", null, ["number", "number"], [ptr, 12]);
2269
+ return null;
2270
+ }
2271
+ const w = ptr >> 2;
2272
+ const out: [number, number, number] = [this.mod.HEAPF32[w], this.mod.HEAPF32[w + 1], this.mod.HEAPF32[w + 2]];
2273
+ this.mod.ccall("goss_free", null, ["number", "number"], [ptr, 12]);
2274
+ return out;
2275
+ }
2276
+
2277
+ /// The tracked point (x, y in frame pixels, z in the same scale) of a named
2278
+ /// body skeleton joint, or null until a body is tracked.
2279
+ bodyJoint(joint: GossBodyJoint): [number, number, number] | null {
2280
+ const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [12]) as number;
2281
+ const status = this.mod.ccall("goss_session_body_joint", "number", ["number", "number", "number"], [this.handle, joint, ptr]) as number;
2282
+ if (status !== 0) {
2283
+ this.mod.ccall("goss_free", null, ["number", "number"], [ptr, 12]);
2284
+ return null;
2285
+ }
2286
+ const w = ptr >> 2;
2287
+ const out: [number, number, number] = [this.mod.HEAPF32[w], this.mod.HEAPF32[w + 1], this.mod.HEAPF32[w + 2]];
2288
+ this.mod.ccall("goss_free", null, ["number", "number"], [ptr, 12]);
2289
+ return out;
2290
+ }
2291
+
2292
+ /// The tracked point (x, y in frame pixels, z in the same scale) of a named
2293
+ /// joint on the handIndex-th tracked hand, or null until that hand is tracked.
2294
+ handJoint(joint: GossHandJoint, handIndex = 0): [number, number, number] | null {
2295
+ const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [12]) as number;
2296
+ const status = this.mod.ccall("goss_session_hand_joint", "number", ["number", "number", "number", "number"], [this.handle, handIndex, joint, ptr]) as number;
2297
+ if (status !== 0) {
2298
+ this.mod.ccall("goss_free", null, ["number", "number"], [ptr, 12]);
2299
+ return null;
2300
+ }
2301
+ const w = ptr >> 2;
2302
+ const out: [number, number, number] = [this.mod.HEAPF32[w], this.mod.HEAPF32[w + 1], this.mod.HEAPF32[w + 2]];
2303
+ this.mod.ccall("goss_free", null, ["number", "number"], [ptr, 12]);
2304
+ return out;
2305
+ }
2306
+
2307
+ /// Runs the in-engine segmenter on the camera frames: model is any square RGB
2308
+ /// segmenter (portrait or a multi-class scene net, up to 32 classes), read for
2309
+ /// its own tensor dimensions; threads is the worker count. A build with no
2310
+ /// inference stack returns unsupported and setSegmentationMask is used instead.
2311
+ enableSegmentation(model: Uint8Array, threads: number): void {
2312
+ const modelPtr = model.length > 0 ? (this.mod.ccall("goss_alloc", "number", ["number"], [model.length]) as number) : 0;
2313
+ if (modelPtr) this.mod.HEAPU8.set(model, modelPtr);
2314
+ this.mod.ccall("goss_session_enable_segmentation", "number", ["number", "number", "number", "number"], [this.handle, modelPtr, model.length, threads]);
2315
+ if (modelPtr) this.mod.ccall("goss_free", null, ["number", "number"], [modelPtr, model.length]);
2316
+ }
2317
+
2318
+ /// Tears the in-engine segmenter down; the subject and class channels go empty.
2319
+ disableSegmentation(): void {
2320
+ this.mod.ccall("goss_session_disable_segmentation", "number", ["number"], [this.handle]);
2321
+ }
2322
+
2323
+ private enableTracker(sym: string, task: Uint8Array, threads: number): void {
2324
+ const ptr = task.length > 0 ? (this.mod.ccall("goss_alloc", "number", ["number"], [task.length]) as number) : 0;
2325
+ if (ptr) this.mod.HEAPU8.set(task, ptr);
2326
+ this.mod.ccall(sym, "number", ["number", "number", "number", "number"], [this.handle, ptr, task.length, threads]);
2327
+ if (ptr) this.mod.ccall("goss_free", null, ["number", "number"], [ptr, task.length]);
2328
+ }
2329
+
2330
+ /// Enables the in-engine face tracker: task is a mediapipe face-landmarker
2331
+ /// .task, threads the worker count. Web usually feeds faces through the
2332
+ /// producer path (submitFaces); a build with no inference stack returns
2333
+ /// unsupported.
2334
+ enableFaceTracking(task: Uint8Array, threads: number): void {
2335
+ this.enableTracker("goss_session_enable_face_tracking", task, threads);
2336
+ }
2337
+
2338
+ /// Tears the in-engine face tracker down.
2339
+ disableFaceTracking(): void {
2340
+ this.mod.ccall("goss_session_disable_face_tracking", null, ["number"], [this.handle]);
2341
+ }
2342
+
2343
+ /// Enables the in-engine hand tracker from a hand-landmarker .task.
2344
+ enableHandTracking(task: Uint8Array, threads: number): void {
2345
+ this.enableTracker("goss_session_enable_hand_tracking", task, threads);
2346
+ }
2347
+
2348
+ /// Tears the in-engine hand tracker down.
2349
+ disableHandTracking(): void {
2350
+ this.mod.ccall("goss_session_disable_hand_tracking", null, ["number"], [this.handle]);
2351
+ }
2352
+
2353
+ /// Enables the in-engine pose tracker from a pose-landmarker .task.
2354
+ enablePoseTracking(task: Uint8Array, threads: number): void {
2355
+ this.enableTracker("goss_session_enable_pose_tracking", task, threads);
2356
+ }
2357
+
2358
+ /// Tears the in-engine pose tracker down.
2359
+ disablePoseTracking(): void {
2360
+ this.mod.ccall("goss_session_disable_pose_tracking", null, ["number"], [this.handle]);
2361
+ }
2362
+
2363
+ /// Turns the in-engine beauty pass on. resourcePath points at the gpupixel
2364
+ /// resource directory the native build ships; on web it is unused, so an
2365
+ /// empty string is fine. A build without the beauty stack leaves it off.
2366
+ enableBeauty(resourcePath = ""): void {
2367
+ const bytes = new TextEncoder().encode(resourcePath);
2368
+ const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [bytes.length + 1]) as number;
2369
+ this.mod.HEAPU8.set(bytes, ptr);
2370
+ this.mod.HEAPU8[ptr + bytes.length] = 0;
2371
+ this.mod.ccall("goss_session_enable_beauty", "number", ["number", "number"], [this.handle, ptr]);
2372
+ this.mod.ccall("goss_free", null, ["number", "number"], [ptr, bytes.length + 1]);
2373
+ }
2374
+
2375
+ /// Turns the in-engine beauty pass off.
2376
+ disableBeauty(): void {
2377
+ this.mod.ccall("goss_session_disable_beauty", null, ["number"], [this.handle]);
2378
+ }
2379
+
2380
+ /// Beautifies one RGBA frame on the CPU and returns a new RGBA buffer the same
2381
+ /// size; the smoothing follows the beauty parameters set on the session. A
2382
+ /// build without the beauty stack returns the input unchanged.
2383
+ beautifyFrame(rgba: Uint8Array | Uint8ClampedArray, width: number, height: number): Uint8Array {
2384
+ const inPtr = this.mod.ccall("goss_alloc", "number", ["number"], [rgba.length]) as number;
2385
+ const outPtr = this.mod.ccall("goss_alloc", "number", ["number"], [rgba.length]) as number;
2386
+ this.mod.HEAPU8.set(rgba, inPtr);
2387
+ this.mod.ccall("goss_session_beautify_frame", "number", ["number", "number", "number", "number", "number"], [this.handle, inPtr, width, height, outPtr]);
2388
+ const out = this.mod.HEAPU8.slice(outPtr, outPtr + rgba.length);
2389
+ this.mod.ccall("goss_free", null, ["number", "number"], [inPtr, rgba.length]);
2390
+ this.mod.ccall("goss_free", null, ["number", "number"], [outPtr, rgba.length]);
2391
+ return out;
2392
+ }
2393
+
2394
+ /// Feeds a segmentation mask (GOSS_SEGMENTATION_MASK_SIDE squared floats,
2395
+ /// from a GossSegmenter) into the session as the subject texture the blend
2396
+ /// and mask channels sample. Null clears it (no subject this frame).
2397
+ setSegmentationMask(mask: Float32Array | null): void {
2398
+ const count = GOSS_SEGMENTATION_MASK_SIDE * GOSS_SEGMENTATION_MASK_SIDE;
2399
+ if (!mask || mask.length < count) {
2400
+ this.mod.ccall("goss_session_set_segmentation_mask", "number", ["number", "number", "number"], [this.handle, 0, 0]);
2401
+ return;
2402
+ }
2403
+ this.mod.HEAPF32.set(mask.subarray(0, count), this.segmentationMaskPtr >> 2);
2404
+ this.mod.ccall(
2405
+ "goss_session_set_segmentation_mask",
2406
+ "number",
2407
+ ["number", "number", "number"],
2408
+ [this.handle, this.segmentationMaskPtr, count],
2409
+ );
2410
+ }
2411
+
2412
+ /// The class channels the active lens samples, as a bitmask over
2413
+ /// GOSS_SEGMENTATION_CHANNELS. Upload exactly these with setSegmentationClassMask
2414
+ /// each frame; zero means only the subject mask is wanted.
2415
+ segmentationChannels(): number {
2416
+ return this.mod.ccall("goss_session_segmentation_channels", "number", ["number"], [this.handle]);
2417
+ }
2418
+
2419
+ /// Feeds one class channel's mask (from a GossSegmenter's classMask) as the
2420
+ /// texture that channel's passes sample. channel indexes
2421
+ /// GOSS_SEGMENTATION_CHANNELS; channel 0 (person) goes through
2422
+ /// setSegmentationMask, which clears the classes, so upload these after.
2423
+ setSegmentationClassMask(channel: number, mask: Float32Array | null): void {
2424
+ const count = GOSS_SEGMENTATION_MASK_SIDE * GOSS_SEGMENTATION_MASK_SIDE;
2425
+ if (!mask || mask.length < count) {
2426
+ this.mod.ccall("goss_session_set_segmentation_class_mask", "number", ["number", "number", "number", "number"], [this.handle, channel, 0, 0]);
2427
+ return;
2428
+ }
2429
+ this.mod.HEAPF32.set(mask.subarray(0, count), this.segmentationMaskPtr >> 2);
2430
+ this.mod.ccall(
2431
+ "goss_session_set_segmentation_class_mask",
2432
+ "number",
2433
+ ["number", "number", "number", "number"],
2434
+ [this.handle, channel, this.segmentationMaskPtr, count],
2435
+ );
2436
+ }
2437
+
2438
+ /// Uploads one of whiten's four lookup textures directly - slot 0
2439
+ /// gray, 1 origin, 2 skin, 3 custom. loadWhitenLuts is the sugar most
2440
+ /// callers want; this is the raw upload it calls internally.
2441
+ setBeautyLut(slot: number, rgba: Uint8ClampedArray | Uint8Array, width: number, height: number): void {
2442
+ const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [rgba.length]);
2443
+ this.mod.HEAPU8.set(rgba, ptr);
2444
+ this.mod.ccall(
2445
+ "goss_session_set_beauty_lut",
2446
+ "number",
2447
+ ["number", "number", "number", "number", "number"],
2448
+ [this.handle, slot, ptr, width, height],
2449
+ );
2450
+ this.mod.ccall("goss_free", null, ["number", "number"], [ptr, rgba.length]);
2451
+ }
2452
+
2453
+ /// Uploads lipstick's or blush's own source image directly.
2454
+ /// loadMakeupTextures is the sugar most callers want; this is the raw
2455
+ /// upload it calls internally.
2456
+ setBeautyMakeupTexture(effect: GossBeautyEffect, rgba: Uint8ClampedArray | Uint8Array, width: number, height: number): void {
2457
+ const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [rgba.length]);
2458
+ this.mod.HEAPU8.set(rgba, ptr);
2459
+ this.mod.ccall(
2460
+ "goss_session_set_beauty_makeup_texture",
2461
+ "number",
2462
+ ["number", "number", "number", "number", "number"],
2463
+ [this.handle, effect, ptr, width, height],
2464
+ );
2465
+ this.mod.ccall("goss_free", null, ["number", "number"], [ptr, rgba.length]);
2466
+ }
2467
+
2468
+ /// Fetches the four whiten lookup textures (gray/origin/skin/custom),
2469
+ /// relative to lutBaseUrl. Safe to call once after construction;
2470
+ /// setWhiten stays a no-op until this resolves.
2471
+ async loadWhitenLuts(lutBaseUrl: string | URL): Promise<void> {
2472
+ const names = ["lookup_gray", "lookup_origin", "lookup_skin", "lookup_light"];
2473
+ const images = await Promise.all(
2474
+ names.map((name) => fetch(new URL(`${name}.png`, lutBaseUrl)).then((r) => r.blob()).then(decodeImageRgba)),
2475
+ );
2476
+ images.forEach((image, slot) => {
2477
+ this.setBeautyLut(slot, image.data, image.width, image.height);
2478
+ this.whitenLutsLoaded += 1;
2479
+ });
2480
+ }
2481
+
2482
+ /// Fetches mouth.png/blusher.png, relative to baseUrl. Safe to call
2483
+ /// once after construction; setLipstick/setBlush stay a no-op until
2484
+ /// this resolves.
2485
+ async loadMakeupTextures(baseUrl: string | URL): Promise<void> {
2486
+ const [mouth, blusher] = await Promise.all(
2487
+ ["mouth.png", "blusher.png"].map((name) => fetch(new URL(name, baseUrl)).then((r) => r.blob()).then(decodeImageRgba)),
2488
+ );
2489
+ for (const [effect, image] of [
2490
+ [GossBeautyEffect.Lipstick, mouth],
2491
+ [GossBeautyEffect.Blush, blusher],
2492
+ ] as const) {
2493
+ this.setBeautyMakeupTexture(effect, image.data, image.width, image.height);
2494
+ }
2495
+ this.lipstickTextureLoaded = true;
2496
+ this.blushTextureLoaded = true;
2497
+ }
2498
+
2499
+ private ensureFramePixels(byteLength: number): void {
2500
+ if (this.framePixelsCapacity >= byteLength) return;
2501
+ if (this.framePixelsPtr !== 0) this.mod.ccall("goss_free", null, ["number", "number"], [this.framePixelsPtr, this.framePixelsCapacity]);
2502
+ this.framePixelsPtr = this.mod.ccall("goss_alloc", "number", ["number"], [byteLength]);
2503
+ this.framePixelsCapacity = byteLength;
2504
+ }
2505
+
2506
+ /// rotationDegrees omitted means the setVideoFlip state decides (a
2507
+ /// flipped source is a 180-degree turn); timestampUs omitted means
2508
+ /// now.
2509
+ submitFrameRgbaCopy(rgba: Uint8ClampedArray | Uint8Array, stride: number, width: number, height: number, pixelFormat: GossPixelFormat = GossPixelFormat.Rgba8, rotationDegrees?: number, mirrored = false, timestampUs?: number): void {
2510
+ this.frameWidth = width;
2511
+ this.frameHeight = height;
2512
+ const byteLength = stride * height;
2513
+ this.ensureFramePixels(byteLength);
2514
+ this.mod.HEAPU8.set(rgba.subarray(0, byteLength), this.framePixelsPtr);
2515
+
2516
+ const rotationQuarters = ((rotationDegrees ?? (this.videoFlipped ? 180 : 0)) / 90) & 3;
2517
+ const flags = (mirrored ? FRAME_FLAG_MIRROR : 0) | (rotationQuarters << FRAME_ROTATION_SHIFT);
2518
+ this.mod.setValue(this.frameDescPtr, width, "i32");
2519
+ this.mod.setValue(this.frameDescPtr + 4, height, "i32");
2520
+ this.mod.setValue(this.frameDescPtr + 8, pixelFormat, "i32");
2521
+ this.mod.setValue(this.frameDescPtr + 12, 0, "i32");
2522
+ this.mod.setValue(this.frameDescPtr + 16, 0, "i32");
2523
+ this.mod.setValue(this.frameDescPtr + 20, flags, "i32");
2524
+ const stampUs = timestampUs ?? Math.round(performance.now() * 1000);
2525
+ this.mod.setValue(this.frameDescPtr + 24, stampUs >>> 0, "i32");
2526
+ this.mod.setValue(this.frameDescPtr + 28, Math.floor(stampUs / 4294967296), "i32");
2527
+
2528
+ this.mod.ccall(
2529
+ "goss_session_submit_frame_rgba_copy",
2530
+ "number",
2531
+ ["number", "number", "number", "number"],
2532
+ [this.handle, this.frameDescPtr, this.framePixelsPtr, stride],
2533
+ );
2534
+ }
2535
+
2536
+ /// The web selfie path: runs each selfie-source splat.cloud once over one
2537
+ /// RGBA8 still (row major), so a photoreal avatar is generated from one photo
2538
+ /// and then held off the live camera. Reuses the per-session frame staging.
2539
+ submitAvatarSourceRgba(rgba: Uint8ClampedArray | Uint8Array, width: number, height: number): void {
2540
+ const byteLength = width * 4 * height;
2541
+ this.ensureFramePixels(byteLength);
2542
+ this.mod.HEAPU8.set(rgba.subarray(0, byteLength), this.framePixelsPtr);
2543
+ this.mod.ccall(
2544
+ "goss_session_submit_avatar_source_rgba",
2545
+ "number",
2546
+ ["number", "number", "number"],
2547
+ [this.handle, this.framePixelsPtr, width, height],
2548
+ );
2549
+ }
2550
+
2551
+ /// Feeds the platform's world understanding into the session: camera
2552
+ /// pose and projection (column-major float16 arrays), tracked planes,
2553
+ /// anchors, and the light estimate. Drives the world.tracking_state
2554
+ /// trigger and world-anchored lens content.
2555
+ submitWorld(state: GossWorldState, planes: GossWorldPlane[] = [], anchors: GossWorldAnchorInput[] = [], light?: GossWorldLight): void {
2556
+ const stateBytes = 144;
2557
+ const planeBytes = 88;
2558
+ const anchorBytes = 72;
2559
+ const lightBytes = 8;
2560
+ const total = stateBytes + planes.length * planeBytes + anchors.length * anchorBytes + lightBytes;
2561
+ this.ensureWorldScratch(total);
2562
+ const base = this.worldScratchPtr;
2563
+ const heap = this.mod.HEAPU8;
2564
+ const view = new DataView(heap.buffer, base, total);
2565
+
2566
+ view.setUint32(0, state.trackingState, true);
2567
+ for (let i = 0; i < 16; i++) view.setFloat32(4 + i * 4, state.worldFromCamera[i], true);
2568
+ for (let i = 0; i < 16; i++) view.setFloat32(68 + i * 4, state.projection[i], true);
2569
+ view.setBigInt64(136, BigInt(Math.round(state.timestampUs)), true);
2570
+
2571
+ let at = stateBytes;
2572
+ const planesPtr = planes.length > 0 ? base + at : 0;
2573
+ for (const plane of planes) {
2574
+ view.setBigUint64(at, BigInt(plane.id), true);
2575
+ for (let i = 0; i < 16; i++) view.setFloat32(at + 8 + i * 4, plane.pose[i], true);
2576
+ view.setFloat32(at + 72, plane.extentX, true);
2577
+ view.setFloat32(at + 76, plane.extentZ, true);
2578
+ view.setUint32(at + 80, plane.classification, true);
2579
+ at += planeBytes;
2580
+ }
2581
+ const anchorsPtr = anchors.length > 0 ? base + at : 0;
2582
+ for (const anchorInput of anchors) {
2583
+ view.setBigUint64(at, BigInt(anchorInput.id), true);
2584
+ for (let i = 0; i < 16; i++) view.setFloat32(at + 8 + i * 4, anchorInput.pose[i], true);
2585
+ at += anchorBytes;
2586
+ }
2587
+ let lightPtr = 0;
2588
+ if (light) {
2589
+ lightPtr = base + at;
2590
+ view.setFloat32(at, light.ambientIntensity, true);
2591
+ view.setFloat32(at + 4, light.colorTemperatureKelvin, true);
2592
+ }
2593
+
2594
+ this.mod.ccall(
2595
+ "goss_session_submit_world",
2596
+ "number",
2597
+ ["number", "number", "number", "number", "number", "number", "number"],
2598
+ [this.handle, base, planesPtr, planes.length, anchorsPtr, anchors.length, lightPtr],
2599
+ );
2600
+ }
2601
+
2602
+ private ensureWorldScratch(byteLength: number): void {
2603
+ if (this.worldScratchLen >= byteLength) return;
2604
+ if (this.worldScratchPtr !== 0) this.mod.ccall("goss_free", null, ["number", "number"], [this.worldScratchPtr, this.worldScratchLen]);
2605
+ this.worldScratchPtr = this.mod.ccall("goss_alloc", "number", ["number"], [byteLength]);
2606
+ this.worldScratchLen = byteLength;
2607
+ }
2608
+
2609
+ /// Reports one finished frame: measured whole-pipeline time plus
2610
+ /// thermal pressure (nominal by default - no browser API surfaces
2611
+ /// device thermal state). Returns the degradation level in effect
2612
+ /// for the next frame.
2613
+ reportFrame(frameTimeUs: number, thermal: GossThermal = GossThermal.Nominal): GossDegradeLevel {
2614
+ return this.mod.ccall("goss_session_report_frame", "number", ["number", "number", "number"], [this.handle, frameTimeUs, thermal]);
2615
+ }
2616
+
2617
+ degradeLevel(): GossDegradeLevel {
2618
+ return this.mod.ccall("goss_session_degrade_level", "number", ["number"], [this.handle]);
2619
+ }
2620
+
2621
+ destroy(): void {
2622
+ // frameDescPtr is allocated at construction and only ever nulled here,
2623
+ // so it doubles as the guard that makes a second destroy() a no-op.
2624
+ if (this.frameDescPtr === 0) return;
2625
+ this.mod.ccall("goss_session_destroy", null, ["number"], [this.handle]);
2626
+ // goss_session_destroy owns only the native session; every scratch
2627
+ // buffer the session allocated is our heap and freed here.
2628
+ const free = (ptr: number, size: number) => {
2629
+ if (ptr !== 0 && size !== 0) this.mod.ccall("goss_free", null, ["number", "number"], [ptr, size]);
2630
+ };
2631
+ free(this.frameDescPtr, 32);
2632
+ free(this.landmarksPtr, GOSS_FACE_LANDMARK_COUNT * 3 * 4);
2633
+ free(this.signalsPtr, LENS_SIGNALS_BYTES);
2634
+ free(this.segmentationMaskPtr, GOSS_SEGMENTATION_MASK_SIDE * GOSS_SEGMENTATION_MASK_SIDE * 4);
2635
+ free(this.framePixelsPtr, this.framePixelsCapacity);
2636
+ free(this.worldScratchPtr, this.worldScratchLen);
2637
+ free(this.scratchPtr, this.scratchCapacity);
2638
+ this.frameDescPtr = 0;
2639
+ this.landmarksPtr = 0;
2640
+ this.signalsPtr = 0;
2641
+ this.segmentationMaskPtr = 0;
2642
+ this.framePixelsPtr = 0;
2643
+ this.framePixelsCapacity = 0;
2644
+ this.worldScratchPtr = 0;
2645
+ this.worldScratchLen = 0;
2646
+ this.scratchPtr = 0;
2647
+ this.scratchCapacity = 0;
2648
+ }
2649
+ }
2650
+
2651
+ /// The SDK-facing orchestrator: capture loop, video element, DOM
2652
+ /// events. Composes Gosslens/GossEngine/GossSession rather than being one of
2653
+ /// them - the same relationship CameraController/PreviewViewController
2654
+ /// have to GossEngine/GossSession on iOS, not a fourth ABI-shaped type.
2655
+ export class GossPreviewSession {
2656
+ readonly video = document.createElement("video");
2657
+ state: GossCaptureState = "idle";
2658
+
2659
+ private stream: MediaStream | null = null;
2660
+ private raf = 0;
2661
+ private lastTick = 0;
2662
+ private fpsWindowStart = 0;
2663
+ private fpsWindowFrames = 0;
2664
+ private renderedFrames = 0;
2665
+ private cameraFrames = 0;
2666
+ private lastVideoTime = -1;
2667
+ private scratchCanvas = document.createElement("canvas");
2668
+ private scratchCtx: CanvasRenderingContext2D;
2669
+
2670
+ private constructor(
2671
+ readonly gosslens: Gosslens,
2672
+ readonly engine: GossEngine,
2673
+ readonly session: GossSession,
2674
+ private events: GossSessionEvents,
2675
+ ) {
2676
+ this.scratchCtx = this.scratchCanvas.getContext("2d", { willReadFrequently: true })!;
2677
+ }
2678
+
2679
+ static async create(canvas: HTMLCanvasElement, wasmJsUrl: string | URL, events: GossSessionEvents = {}): Promise<GossPreviewSession> {
2680
+ const gosslens = await Gosslens.load(canvas, wasmJsUrl);
2681
+ const engine = GossEngine.create(gosslens);
2682
+ await engine.initRenderer(canvas);
2683
+ const session = GossSession.create(engine);
2684
+ return new GossPreviewSession(gosslens, engine, session, events);
2685
+ }
2686
+
2687
+ abiVersion(): number {
2688
+ return this.gosslens.abiVersion();
2689
+ }
2690
+
2691
+ private setState(state: GossCaptureState): void {
2692
+ this.state = state;
2693
+ this.events.onState?.(state);
2694
+ }
2695
+
2696
+ setWhiten(amount: number): void {
2697
+ this.session.setWhiten(amount);
2698
+ }
2699
+
2700
+ setSmooth(amount: number): void {
2701
+ this.session.setSmooth(amount);
2702
+ }
2703
+
2704
+ setThinFace(amount: number): void {
2705
+ this.session.setThinFace(amount);
2706
+ }
2707
+
2708
+ setBigEye(amount: number): void {
2709
+ this.session.setBigEye(amount);
2710
+ }
2711
+
2712
+ setLipstick(amount: number): void {
2713
+ this.session.setLipstick(amount);
2714
+ }
2715
+
2716
+ setBlush(amount: number): void {
2717
+ this.session.setBlush(amount);
2718
+ }
2719
+
2720
+ activateLens(manifestJson: string): void {
2721
+ this.session.activateLens(manifestJson);
2722
+ }
2723
+
2724
+ deactivateLens(): void {
2725
+ this.session.deactivateLens();
2726
+ }
2727
+
2728
+ tickLens(dtUs: number, signals: GossLensSignals = {}): void {
2729
+ this.session.tickLens(dtUs, signals);
2730
+ }
2731
+
2732
+ setVideoFlip(enabled: boolean): void {
2733
+ this.session.setVideoFlip(enabled);
2734
+ }
2735
+
2736
+ isVideoFlipped(): boolean {
2737
+ return this.session.isVideoFlipped();
2738
+ }
2739
+
2740
+ setFaceLandmarks(landmarks: Float32Array | null, sourceWidth: number, sourceHeight: number): void {
2741
+ this.session.setFaceLandmarks(landmarks, sourceWidth, sourceHeight);
2742
+ }
2743
+
2744
+ loadWhitenLuts(lutBaseUrl: string | URL): Promise<void> {
2745
+ return this.session.loadWhitenLuts(lutBaseUrl);
2746
+ }
2747
+
2748
+ loadMakeupTextures(baseUrl: string | URL): Promise<void> {
2749
+ return this.session.loadMakeupTextures(baseUrl);
2750
+ }
2751
+
2752
+ /// Uploads a still image directly into the frame the engine renders,
2753
+ /// bypassing the video element - freezeCamera() first stops tick()
2754
+ /// from re-submitting over it. Test/demo tooling only: skin-smoothing's
2755
+ /// content-adaptive blend needs a real face to prove, not a fake one.
2756
+ async loadStillFrame(url: string): Promise<void> {
2757
+ const image = await decodeImageRgba(await (await fetch(url)).blob(), {
2758
+ maxWidth: this.scratchCanvas.width,
2759
+ maxHeight: this.scratchCanvas.height,
2760
+ });
2761
+ // Not mirrored: a loaded test photo isn't a front camera, and
2762
+ // setLandmarksFromStill tracks this same unmirrored image - mirroring
2763
+ // only the background here would leave the tracked landmarks
2764
+ // pointing at the wrong side of the now-mirrored face.
2765
+ this.session.submitFrameRgbaCopy(image.data, image.width * 4, image.width, image.height);
2766
+ }
2767
+
2768
+ async start(): Promise<void> {
2769
+ try {
2770
+ this.stream = await navigator.mediaDevices.getUserMedia({
2771
+ video: { width: { ideal: 1280 }, height: { ideal: 720 } },
2772
+ audio: false,
2773
+ });
2774
+ } catch (err) {
2775
+ this.setState(err instanceof DOMException && err.name === "NotAllowedError" ? "denied" : "failed");
2776
+ return;
2777
+ }
2778
+ const track = this.stream.getVideoTracks()[0];
2779
+ track.addEventListener("mute", () => this.setState("interrupted"));
2780
+ track.addEventListener("unmute", () => this.setState("running"));
2781
+ track.addEventListener("ended", () => this.setState("failed"));
2782
+
2783
+ this.video.srcObject = this.stream;
2784
+ this.video.muted = true;
2785
+ this.video.playsInline = true;
2786
+ await this.video.play();
2787
+ this.setState("running");
2788
+ this.fpsWindowStart = performance.now();
2789
+ this.lastTick = performance.now();
2790
+ this.tick();
2791
+ }
2792
+
2793
+ stop(): void {
2794
+ cancelAnimationFrame(this.raf);
2795
+ this.stream?.getTracks().forEach((track) => track.stop());
2796
+ this.stream = null;
2797
+ this.setState("idle");
2798
+ }
2799
+
2800
+ private tick = (): void => {
2801
+ this.raf = requestAnimationFrame(this.tick);
2802
+ if (this.engine.isCaptureInFlight) return;
2803
+ const now = performance.now();
2804
+ const frameTimeUs = Math.max(0, Math.round((now - this.lastTick) * 1000));
2805
+ this.lastTick = now;
2806
+ this.session.reportFrame(frameTimeUs);
2807
+
2808
+ if (this.video.readyState >= 2 && this.video.currentTime !== this.lastVideoTime) {
2809
+ this.lastVideoTime = this.video.currentTime;
2810
+ this.cameraFrames += 1;
2811
+ const width = this.video.videoWidth;
2812
+ const height = this.video.videoHeight;
2813
+ this.scratchCanvas.width = width;
2814
+ this.scratchCanvas.height = height;
2815
+ this.scratchCtx.drawImage(this.video, 0, 0, width, height);
2816
+ const pixels = this.scratchCtx.getImageData(0, 0, width, height);
2817
+ // Not mirrored here - the demo page's own CSS mirrors the canvas
2818
+ // for display, so the engine keeps working in the camera's real,
2819
+ // unmirrored coordinate space (matching tracking, which analyzes
2820
+ // this same unmirrored buffer).
2821
+ this.session.submitFrameRgbaCopy(pixels.data, width * 4, width, height);
2822
+ }
2823
+
2824
+ const status = this.engine.renderFrame(this.session);
2825
+ if (status === GOSS_OK) {
2826
+ this.renderedFrames += 1;
2827
+ this.fpsWindowFrames += 1;
2828
+ }
2829
+
2830
+ if (now - this.fpsWindowStart >= 1000) {
2831
+ const fps = (this.fpsWindowFrames * 1000) / (now - this.fpsWindowStart);
2832
+ this.events.onFps?.(fps, this.renderedFrames, this.cameraFrames);
2833
+ this.fpsWindowStart = now;
2834
+ this.fpsWindowFrames = 0;
2835
+ }
2836
+ };
2837
+
2838
+ degradeLevel(): GossDegradeLevel {
2839
+ return this.session.degradeLevel();
2840
+ }
2841
+
2842
+ captureFrame(): Promise<string> {
2843
+ return this.engine.captureFrame(this.session);
2844
+ }
2845
+
2846
+ readCenterPixel(): Promise<Uint8Array> {
2847
+ return this.engine.readCenterPixel(this.session);
2848
+ }
2849
+
2850
+ readFrameSum(): Promise<number> {
2851
+ return this.engine.readFrameSum(this.session);
2852
+ }
2853
+ }
2854
+
2855
+ export { GossWebXRWorldSource } from "./world";
2856
+ export type { GossXRFrameLike } from "./world";