@myzonerocks/gosslens 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +96 -0
- package/dist/src/index.d.ts +529 -0
- package/dist/src/index.d.ts.map +1 -0
- package/dist/src/index.js +2360 -0
- package/dist/src/index.js.map +1 -0
- package/dist/src/tracking.d.ts +109 -0
- package/dist/src/tracking.d.ts.map +1 -0
- package/dist/src/tracking.js +426 -0
- package/dist/src/tracking.js.map +1 -0
- package/dist/src/world.d.ts +38 -0
- package/dist/src/world.d.ts.map +1 -0
- package/dist/src/world.js +67 -0
- package/dist/src/world.js.map +1 -0
- package/dist/test/index.test.d.ts +2 -0
- package/dist/test/index.test.d.ts.map +1 -0
- package/dist/test/index.test.js +42 -0
- package/dist/test/index.test.js.map +1 -0
- package/dist/test/tracking.test.d.ts +2 -0
- package/dist/test/tracking.test.d.ts.map +1 -0
- package/dist/test/tracking.test.js +85 -0
- package/dist/test/tracking.test.js.map +1 -0
- package/package.json +49 -0
- package/src/index.ts +2856 -0
- package/src/tracking.ts +493 -0
- package/src/world.ts +108 -0
|
@@ -0,0 +1,2360 @@
|
|
|
1
|
+
// The web SDK over gosslens_web, the real bgfx-backed engine every
|
|
2
|
+
// other SDK already runs (Swift/Kotlin call the exact same frozen ABI
|
|
3
|
+
// through their own thin platform glue). This SDK owns only what the
|
|
4
|
+
// browser forces on it - camera capture through getUserMedia, decoding
|
|
5
|
+
// PNGs the core has no decoder for, driving the render loop - and hands
|
|
6
|
+
// everything else (the frame graph, all six beauty effects, mirror and
|
|
7
|
+
// rotation) straight to the engine.
|
|
8
|
+
export const GOSS_OK = 0;
|
|
9
|
+
const FRAME_FLAG_MIRROR = 0x1;
|
|
10
|
+
const FRAME_ROTATION_SHIFT = 8;
|
|
11
|
+
const LENS_SIGNALS_BYTES = 232;
|
|
12
|
+
const GOSS_FACE_BLENDSHAPE_COUNT = 52;
|
|
13
|
+
export const GOSS_FACE_LANDMARK_COUNT = 478;
|
|
14
|
+
export const GOSS_FACE_MAX = 4;
|
|
15
|
+
const FACE_RESULT_BYTES = 5968;
|
|
16
|
+
export const GOSS_SEGMENTATION_MASK_SIDE = 256;
|
|
17
|
+
export const GOSS_POSE_LANDMARK_COUNT = 33;
|
|
18
|
+
export const GOSS_BODY_MAX = 4;
|
|
19
|
+
const POSE_RESULT_BYTES = 688;
|
|
20
|
+
export const GOSS_HAND_LANDMARK_COUNT = 21;
|
|
21
|
+
export const GOSS_HAND_MAX = 2;
|
|
22
|
+
const HAND_RESULT_BYTES = 560;
|
|
23
|
+
const HAND_ONE_BYTES = 268;
|
|
24
|
+
/// A named attach point on the tracked face mesh, for faceRegion. The
|
|
25
|
+
/// left/right labels are the subject's own.
|
|
26
|
+
export var GossFaceRegion;
|
|
27
|
+
(function (GossFaceRegion) {
|
|
28
|
+
GossFaceRegion[GossFaceRegion["Forehead"] = 0] = "Forehead";
|
|
29
|
+
GossFaceRegion[GossFaceRegion["Glabella"] = 1] = "Glabella";
|
|
30
|
+
GossFaceRegion[GossFaceRegion["NoseTip"] = 2] = "NoseTip";
|
|
31
|
+
GossFaceRegion[GossFaceRegion["Chin"] = 3] = "Chin";
|
|
32
|
+
GossFaceRegion[GossFaceRegion["LeftEye"] = 4] = "LeftEye";
|
|
33
|
+
GossFaceRegion[GossFaceRegion["RightEye"] = 5] = "RightEye";
|
|
34
|
+
GossFaceRegion[GossFaceRegion["LeftCheek"] = 6] = "LeftCheek";
|
|
35
|
+
GossFaceRegion[GossFaceRegion["RightCheek"] = 7] = "RightCheek";
|
|
36
|
+
GossFaceRegion[GossFaceRegion["LeftEar"] = 8] = "LeftEar";
|
|
37
|
+
GossFaceRegion[GossFaceRegion["RightEar"] = 9] = "RightEar";
|
|
38
|
+
GossFaceRegion[GossFaceRegion["MouthCenter"] = 10] = "MouthCenter";
|
|
39
|
+
GossFaceRegion[GossFaceRegion["LeftMouthCorner"] = 11] = "LeftMouthCorner";
|
|
40
|
+
GossFaceRegion[GossFaceRegion["RightMouthCorner"] = 12] = "RightMouthCorner";
|
|
41
|
+
})(GossFaceRegion || (GossFaceRegion = {}));
|
|
42
|
+
/// A named attach point on the tracked body skeleton, for bodyJoint. The
|
|
43
|
+
/// left/right labels are the subject's own.
|
|
44
|
+
export var GossBodyJoint;
|
|
45
|
+
(function (GossBodyJoint) {
|
|
46
|
+
GossBodyJoint[GossBodyJoint["Head"] = 0] = "Head";
|
|
47
|
+
GossBodyJoint[GossBodyJoint["LeftShoulder"] = 1] = "LeftShoulder";
|
|
48
|
+
GossBodyJoint[GossBodyJoint["RightShoulder"] = 2] = "RightShoulder";
|
|
49
|
+
GossBodyJoint[GossBodyJoint["LeftElbow"] = 3] = "LeftElbow";
|
|
50
|
+
GossBodyJoint[GossBodyJoint["RightElbow"] = 4] = "RightElbow";
|
|
51
|
+
GossBodyJoint[GossBodyJoint["LeftWrist"] = 5] = "LeftWrist";
|
|
52
|
+
GossBodyJoint[GossBodyJoint["RightWrist"] = 6] = "RightWrist";
|
|
53
|
+
GossBodyJoint[GossBodyJoint["LeftHip"] = 7] = "LeftHip";
|
|
54
|
+
GossBodyJoint[GossBodyJoint["RightHip"] = 8] = "RightHip";
|
|
55
|
+
GossBodyJoint[GossBodyJoint["LeftKnee"] = 9] = "LeftKnee";
|
|
56
|
+
GossBodyJoint[GossBodyJoint["RightKnee"] = 10] = "RightKnee";
|
|
57
|
+
GossBodyJoint[GossBodyJoint["LeftAnkle"] = 11] = "LeftAnkle";
|
|
58
|
+
GossBodyJoint[GossBodyJoint["RightAnkle"] = 12] = "RightAnkle";
|
|
59
|
+
})(GossBodyJoint || (GossBodyJoint = {}));
|
|
60
|
+
/// A named attach point on a tracked hand, for handJoint. Palm is the
|
|
61
|
+
/// middle-finger knuckle, a stable palm-centre proxy.
|
|
62
|
+
export var GossHandJoint;
|
|
63
|
+
(function (GossHandJoint) {
|
|
64
|
+
GossHandJoint[GossHandJoint["Wrist"] = 0] = "Wrist";
|
|
65
|
+
GossHandJoint[GossHandJoint["ThumbTip"] = 1] = "ThumbTip";
|
|
66
|
+
GossHandJoint[GossHandJoint["IndexTip"] = 2] = "IndexTip";
|
|
67
|
+
GossHandJoint[GossHandJoint["MiddleTip"] = 3] = "MiddleTip";
|
|
68
|
+
GossHandJoint[GossHandJoint["RingTip"] = 4] = "RingTip";
|
|
69
|
+
GossHandJoint[GossHandJoint["PinkyTip"] = 5] = "PinkyTip";
|
|
70
|
+
GossHandJoint[GossHandJoint["Palm"] = 6] = "Palm";
|
|
71
|
+
})(GossHandJoint || (GossHandJoint = {}));
|
|
72
|
+
/// The segmentation mask channels a lens can name, in the engine's frozen
|
|
73
|
+
/// order: the derived person mask, then the multiclass model's own labels.
|
|
74
|
+
/// Index 0 (person) rides the subject mask; the rest upload as class masks.
|
|
75
|
+
export const GOSS_SEGMENTATION_CHANNELS = [
|
|
76
|
+
"person",
|
|
77
|
+
"background",
|
|
78
|
+
"hair",
|
|
79
|
+
"body_skin",
|
|
80
|
+
"face_skin",
|
|
81
|
+
"clothes",
|
|
82
|
+
"others",
|
|
83
|
+
];
|
|
84
|
+
/// Decodes a fetched blob to raw RGBA bytes via a 2D canvas. Unlike
|
|
85
|
+
/// texImage2D (see the git history on this file - a real, hard-won
|
|
86
|
+
/// lesson from the old hand-rolled WebGL2 SDK this one replaces),
|
|
87
|
+
/// getImageData has always had simple, browser-consistent semantics:
|
|
88
|
+
/// row 0 is the visual top of the image, full stop. No DOM-source
|
|
89
|
+
/// orientation quirks to work around, because there's no texImage2D
|
|
90
|
+
/// involved at all - just plain bytes handed to the engine's own
|
|
91
|
+
/// texture upload, which owns its own orientation convention entirely
|
|
92
|
+
/// separately from WebGL's.
|
|
93
|
+
/// fit, when given, downscales (never upscales) so the decoded frame
|
|
94
|
+
/// fits within maxWidth/maxHeight - LUT and makeup textures pass
|
|
95
|
+
/// nothing and decode at native resolution; loadStillFrame passes the
|
|
96
|
+
/// canvas's own size, since a corpus photo can be far larger than a
|
|
97
|
+
/// real camera frame ever would be. The composite chain sizes every
|
|
98
|
+
/// offscreen target and the final swap-chain view rect off the
|
|
99
|
+
/// submitted frame's own dimensions, so a frame wider or taller than
|
|
100
|
+
/// the actual WebGL drawing buffer gets silently clipped by the GPU to
|
|
101
|
+
/// whatever corner overlaps it - real, found via a still photo (2400x
|
|
102
|
+
/// 3000) submitted straight through to a 1280x720 canvas, where only
|
|
103
|
+
/// the top-left ~13% ended up visible and every landmark-driven effect
|
|
104
|
+
/// (thin-face, big-eye, lipstick, blush) happened to warp a region
|
|
105
|
+
/// entirely outside that sliver, reading back as no change at all.
|
|
106
|
+
async function decodeImageRgba(blob, fit) {
|
|
107
|
+
const bitmap = await createImageBitmap(blob);
|
|
108
|
+
const scale = fit ? Math.min(1, fit.maxWidth / bitmap.width, fit.maxHeight / bitmap.height) : 1;
|
|
109
|
+
const width = Math.round(bitmap.width * scale);
|
|
110
|
+
const height = Math.round(bitmap.height * scale);
|
|
111
|
+
const canvas = document.createElement("canvas");
|
|
112
|
+
canvas.width = width;
|
|
113
|
+
canvas.height = height;
|
|
114
|
+
const ctx = canvas.getContext("2d");
|
|
115
|
+
ctx.drawImage(bitmap, 0, 0, width, height);
|
|
116
|
+
const image = ctx.getImageData(0, 0, width, height);
|
|
117
|
+
return { data: image.data, width, height };
|
|
118
|
+
}
|
|
119
|
+
/// Chooses which gosslens_web.js build to load: the WebGPU one (bgfx's
|
|
120
|
+
/// WebGPU backend, Asyncify linked in) or the WebGL2 one (no Asyncify).
|
|
121
|
+
/// Two separate artifacts rather than a runtime toggle, since Asyncify
|
|
122
|
+
/// taxes the whole per-frame path, not just init. Checks for a real
|
|
123
|
+
/// working adapter, not just navigator.gpu's presence, which can exist
|
|
124
|
+
/// with no adapter behind it.
|
|
125
|
+
export async function pickEngineUrl(webgpuUrl, webgl2Url) {
|
|
126
|
+
const gpu = navigator.gpu;
|
|
127
|
+
if (!gpu)
|
|
128
|
+
return webgl2Url;
|
|
129
|
+
try {
|
|
130
|
+
const adapter = await gpu.requestAdapter();
|
|
131
|
+
return adapter ? webgpuUrl : webgl2Url;
|
|
132
|
+
}
|
|
133
|
+
catch {
|
|
134
|
+
return webgl2Url;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
export class Gosslens {
|
|
138
|
+
mod;
|
|
139
|
+
version;
|
|
140
|
+
constructor(mod, version) {
|
|
141
|
+
this.mod = mod;
|
|
142
|
+
this.version = version;
|
|
143
|
+
}
|
|
144
|
+
/// Any-thread. Compare the high 16 bits against the header's own
|
|
145
|
+
/// GOSS_ABI_MAJOR before creating anything - load() already has.
|
|
146
|
+
abiVersion() {
|
|
147
|
+
return this.version;
|
|
148
|
+
}
|
|
149
|
+
/// Loads gosslens_web.js and checks its ABI major version. A
|
|
150
|
+
/// dynamic import, not static: bun's bundler would otherwise inline
|
|
151
|
+
/// this file, breaking Emscripten's own import.meta.url-relative
|
|
152
|
+
/// fetch of gosslens_web.wasm sitting next to it.
|
|
153
|
+
static async load(canvas, wasmJsUrl) {
|
|
154
|
+
const imported = (await import(/* @vite-ignore */ String(wasmJsUrl)));
|
|
155
|
+
const mod = await imported.default({ canvas });
|
|
156
|
+
const version = mod.ccall("goss_abi_version", "number", [], []) >>> 0;
|
|
157
|
+
if (version >> 16 !== 0)
|
|
158
|
+
throw new Error(`gosslens abi major mismatch: ${version >> 16}`);
|
|
159
|
+
return new Gosslens(mod, version);
|
|
160
|
+
}
|
|
161
|
+
/// The YCbCr to RGB conversion for a standard and range as one
|
|
162
|
+
/// column-major homogeneous matrix. Unused today (canvas always
|
|
163
|
+
/// yields RGBA already) - a real gap for any future debug/thumbnail
|
|
164
|
+
/// path, kept wrapped so that path doesn't start from a raw ccall.
|
|
165
|
+
yuvToRgb(colorStandard, colorRange) {
|
|
166
|
+
const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [64]);
|
|
167
|
+
this.mod.ccall("goss_color_yuv_to_rgb", "number", ["number", "number", "number"], [colorStandard, colorRange, ptr]);
|
|
168
|
+
const out = new Float32Array(16);
|
|
169
|
+
for (let i = 0; i < 16; i += 1)
|
|
170
|
+
out[i] = this.mod.getValue(ptr + i * 4, "float");
|
|
171
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [ptr, 64]);
|
|
172
|
+
return out;
|
|
173
|
+
}
|
|
174
|
+
/// Analytic two-bone inverse kinematics for a limb: root, the upper and lower
|
|
175
|
+
/// bone lengths, target, and pole (each [x, y, z]); returns the mid joint and
|
|
176
|
+
/// end. An out-of-reach target extends the limb straight at it.
|
|
177
|
+
solveTwoBoneIk(root, upperLen, lowerLen, target, pole) {
|
|
178
|
+
const rp = this.mod.ccall("goss_alloc", "number", ["number"], [12]);
|
|
179
|
+
const tp = this.mod.ccall("goss_alloc", "number", ["number"], [12]);
|
|
180
|
+
const pp = this.mod.ccall("goss_alloc", "number", ["number"], [12]);
|
|
181
|
+
const mp = this.mod.ccall("goss_alloc", "number", ["number"], [12]);
|
|
182
|
+
const ep = this.mod.ccall("goss_alloc", "number", ["number"], [12]);
|
|
183
|
+
this.mod.HEAPF32.set(root, rp >> 2);
|
|
184
|
+
this.mod.HEAPF32.set(target, tp >> 2);
|
|
185
|
+
this.mod.HEAPF32.set(pole, pp >> 2);
|
|
186
|
+
this.mod.ccall("goss_solve_two_bone_ik", "number", ["number", "number", "number", "number", "number", "number", "number"], [rp, upperLen, lowerLen, tp, pp, mp, ep]);
|
|
187
|
+
const mw = mp >> 2;
|
|
188
|
+
const ew = ep >> 2;
|
|
189
|
+
const mid = [this.mod.HEAPF32[mw], this.mod.HEAPF32[mw + 1], this.mod.HEAPF32[mw + 2]];
|
|
190
|
+
const end = [this.mod.HEAPF32[ew], this.mod.HEAPF32[ew + 1], this.mod.HEAPF32[ew + 2]];
|
|
191
|
+
for (const p of [rp, tp, pp, mp, ep])
|
|
192
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [p, 12]);
|
|
193
|
+
return { mid, end };
|
|
194
|
+
}
|
|
195
|
+
/// @internal - GossEngine/GossSession need the raw module to reach the ABI;
|
|
196
|
+
/// nothing outside this file should call ccall directly.
|
|
197
|
+
get module() {
|
|
198
|
+
return this.mod;
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
/// Render-surface lifecycle: create/resize/render/read back. Confined
|
|
202
|
+
/// to the one canvas it was created against, matching GossSession/GossEngine's
|
|
203
|
+
/// single-thread confinement on every other SDK.
|
|
204
|
+
export class GossEngine {
|
|
205
|
+
mod;
|
|
206
|
+
handle;
|
|
207
|
+
captureInFlight = false;
|
|
208
|
+
canvas = null;
|
|
209
|
+
/// Only set on the WebGL2 build - bgfx's WebGPU backend binds the
|
|
210
|
+
/// canvas to a 'webgpu' context instead, and a canvas can only ever
|
|
211
|
+
/// bind one context type for its lifetime. capturePixels() branches
|
|
212
|
+
/// on this: readPixels when set, goss_engine_capture_frame otherwise.
|
|
213
|
+
gl = null;
|
|
214
|
+
/// bgfx's HTML5 backend keeps referencing the canvas selector string
|
|
215
|
+
/// for the renderer's life, so the engine owns it and frees it in
|
|
216
|
+
/// destroy() once goss_engine_destroy has torn the renderer down.
|
|
217
|
+
selectorPtr = 0;
|
|
218
|
+
selectorCapacity = 0;
|
|
219
|
+
constructor(mod, handle) {
|
|
220
|
+
this.mod = mod;
|
|
221
|
+
this.handle = handle;
|
|
222
|
+
}
|
|
223
|
+
static create(gosslens, config) {
|
|
224
|
+
const mod = gosslens.module;
|
|
225
|
+
let configPtr = 0;
|
|
226
|
+
if (config) {
|
|
227
|
+
configPtr = mod.ccall("goss_alloc", "number", ["number"], [8]);
|
|
228
|
+
mod.setValue(configPtr, config.texturePoolCapacity ?? 0, "i32");
|
|
229
|
+
mod.setValue(configPtr + 4, config.stagingPoolCapacity ?? 0, "i32");
|
|
230
|
+
}
|
|
231
|
+
const engineOut = mod.ccall("goss_alloc", "number", ["number"], [4]);
|
|
232
|
+
const engineStatus = mod.ccall("goss_engine_create", "number", ["number", "number"], [configPtr, engineOut]);
|
|
233
|
+
const handle = mod.getValue(engineOut, "i32");
|
|
234
|
+
mod.ccall("goss_free", null, ["number", "number"], [engineOut, 4]);
|
|
235
|
+
if (configPtr !== 0)
|
|
236
|
+
mod.ccall("goss_free", null, ["number", "number"], [configPtr, 8]);
|
|
237
|
+
if (engineStatus !== GOSS_OK)
|
|
238
|
+
throw new Error(`engine create failed: ${engineStatus}`);
|
|
239
|
+
return new GossEngine(mod, handle);
|
|
240
|
+
}
|
|
241
|
+
/// @internal - GossSession needs the raw module to reach the ABI; nothing
|
|
242
|
+
/// outside this file should call ccall directly.
|
|
243
|
+
get module() {
|
|
244
|
+
return this.mod;
|
|
245
|
+
}
|
|
246
|
+
/// Brings the render backend up against canvas, which needs a stable
|
|
247
|
+
/// id: bgfx's own HTML5 backend resolves it via a #id selector string
|
|
248
|
+
/// (glcontext_html5.cpp), separate from the Module.canvas binding
|
|
249
|
+
/// Gosslens.load already made - both must agree.
|
|
250
|
+
async initRenderer(canvas) {
|
|
251
|
+
if (!canvas.id)
|
|
252
|
+
throw new Error("canvas needs a stable id for bgfx's own selector lookup");
|
|
253
|
+
const mod = this.mod;
|
|
254
|
+
// Encode the selector into engine-owned heap so its lifetime matches
|
|
255
|
+
// the renderer, not this call; a re-init frees the prior one first.
|
|
256
|
+
const selectorBytes = new TextEncoder().encode(`#${canvas.id}`);
|
|
257
|
+
if (this.selectorPtr !== 0)
|
|
258
|
+
mod.ccall("goss_free", null, ["number", "number"], [this.selectorPtr, this.selectorCapacity]);
|
|
259
|
+
this.selectorCapacity = selectorBytes.length + 1;
|
|
260
|
+
this.selectorPtr = mod.ccall("goss_alloc", "number", ["number"], [this.selectorCapacity]);
|
|
261
|
+
mod.HEAPU8.set(selectorBytes, this.selectorPtr);
|
|
262
|
+
mod.HEAPU8[this.selectorPtr + selectorBytes.length] = 0;
|
|
263
|
+
const rendererDescPtr = mod.ccall("goss_alloc", "number", ["number"], [12]);
|
|
264
|
+
mod.setValue(rendererDescPtr, this.selectorPtr, "i32");
|
|
265
|
+
mod.setValue(rendererDescPtr + 4, canvas.width, "i32");
|
|
266
|
+
mod.setValue(rendererDescPtr + 8, canvas.height, "i32");
|
|
267
|
+
// bgfx's own HTML5 backend creates this canvas's WebGL2 context
|
|
268
|
+
// itself, via emscripten_webgl_create_context - passing
|
|
269
|
+
// webGLContextAttributes here has no effect regardless,
|
|
270
|
+
// preserveDrawingBuffer stays false. Worked around in capturePixels.
|
|
271
|
+
const rendererStatus = await mod.ccall("goss_engine_init_renderer", "number", ["number", "number"], [this.handle, rendererDescPtr], { async: true });
|
|
272
|
+
mod.ccall("goss_free", null, ["number", "number"], [rendererDescPtr, 12]);
|
|
273
|
+
if (rendererStatus !== GOSS_OK)
|
|
274
|
+
throw new Error(`renderer init failed: ${rendererStatus}`);
|
|
275
|
+
// Emscripten's C++ side just created this canvas's own rendering
|
|
276
|
+
// context; a repeat getContext returns that same context. webgpu
|
|
277
|
+
// first: a webgpu-bound canvas answers a mismatched
|
|
278
|
+
// getContext("webgl2") with null, never the wrong context.
|
|
279
|
+
this.canvas = canvas;
|
|
280
|
+
this.gl = canvas.getContext("webgpu") ? null : canvas.getContext("webgl2");
|
|
281
|
+
}
|
|
282
|
+
resize(width, height) {
|
|
283
|
+
if (!this.canvas)
|
|
284
|
+
throw new Error("initRenderer first");
|
|
285
|
+
this.canvas.width = width;
|
|
286
|
+
this.canvas.height = height;
|
|
287
|
+
this.mod.ccall("goss_engine_resize", null, ["number", "number", "number"], [this.handle, width, height]);
|
|
288
|
+
}
|
|
289
|
+
/// A null session presents the clear color, matching every other
|
|
290
|
+
/// SDK's own goss_engine_render_frame contract.
|
|
291
|
+
renderFrame(session) {
|
|
292
|
+
return this.mod.ccall("goss_engine_render_frame", "number", ["number", "number"], [this.handle, session?.handle ?? 0]);
|
|
293
|
+
}
|
|
294
|
+
/// Compiles a text prompt into a GLF lens manifest on device. The result is
|
|
295
|
+
/// ordinary GLF the caller can inspect or pass to activateLens, and needs no
|
|
296
|
+
/// assets. A length probe sizes the buffer, then a fill call writes it.
|
|
297
|
+
compilePrompt(prompt) {
|
|
298
|
+
const bytes = new TextEncoder().encode(prompt);
|
|
299
|
+
const promptPtr = this.mod.ccall("goss_alloc", "number", ["number"], [bytes.length || 1]);
|
|
300
|
+
this.mod.HEAPU8.set(bytes, promptPtr);
|
|
301
|
+
const lenPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]);
|
|
302
|
+
const view = () => new DataView(this.mod.HEAPU8.buffer, lenPtr, 4).getUint32(0, true);
|
|
303
|
+
this.mod.ccall("goss_compile_prompt", "number", ["number", "number", "number", "number", "number", "number"], [this.handle, promptPtr, bytes.length, 0, 0, lenPtr]);
|
|
304
|
+
const needed = view();
|
|
305
|
+
const outPtr = this.mod.ccall("goss_alloc", "number", ["number"], [needed || 1]);
|
|
306
|
+
this.mod.ccall("goss_compile_prompt", "number", ["number", "number", "number", "number", "number", "number"], [this.handle, promptPtr, bytes.length, outPtr, needed, lenPtr]);
|
|
307
|
+
const written = view();
|
|
308
|
+
const json = new TextDecoder().decode(this.mod.HEAPU8.slice(outPtr, outPtr + written));
|
|
309
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [outPtr, needed || 1]);
|
|
310
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [lenPtr, 4]);
|
|
311
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [promptPtr, bytes.length || 1]);
|
|
312
|
+
return json;
|
|
313
|
+
}
|
|
314
|
+
/// Composes an on-device generative-music track from a text prompt and returns
|
|
315
|
+
/// it as a mono 16-bit WAV. A non-zero seed varies the take; bars 0 uses the
|
|
316
|
+
/// default length. Deterministic, no model and no network.
|
|
317
|
+
generateSong(prompt, sampleRate = 48000, seed = 0, bars = 0) {
|
|
318
|
+
const bytes = new TextEncoder().encode(prompt);
|
|
319
|
+
const promptPtr = this.mod.ccall("goss_alloc", "number", ["number"], [bytes.length || 1]);
|
|
320
|
+
this.mod.HEAPU8.set(bytes, promptPtr);
|
|
321
|
+
const lenPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]);
|
|
322
|
+
const view = () => new DataView(this.mod.HEAPU8.buffer, lenPtr, 4).getUint32(0, true);
|
|
323
|
+
const args = ["number", "number", "number", "number", "number", "number", "number", "number", "number"];
|
|
324
|
+
this.mod.ccall("goss_engine_generate_song", "number", args, [this.handle, promptPtr, bytes.length, sampleRate, seed, bars, 0, 0, lenPtr]);
|
|
325
|
+
const needed = view();
|
|
326
|
+
const outPtr = this.mod.ccall("goss_alloc", "number", ["number"], [needed || 1]);
|
|
327
|
+
this.mod.ccall("goss_engine_generate_song", "number", args, [this.handle, promptPtr, bytes.length, sampleRate, seed, bars, outPtr, needed, lenPtr]);
|
|
328
|
+
const written = view();
|
|
329
|
+
const wav = this.mod.HEAPU8.slice(outPtr, outPtr + written);
|
|
330
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [outPtr, needed || 1]);
|
|
331
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [lenPtr, 4]);
|
|
332
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [promptPtr, bytes.length || 1]);
|
|
333
|
+
return wav;
|
|
334
|
+
}
|
|
335
|
+
/// Scans a width*height 8-bit luminance frame for an EAN-13 / UPC-A barcode,
|
|
336
|
+
/// returning its 13 digits or null when no checksum-valid symbol is found.
|
|
337
|
+
/// Purely algorithmic and deterministic, no model.
|
|
338
|
+
scanBarcode(luminance, width, height) {
|
|
339
|
+
const lumPtr = this.mod.ccall("goss_alloc", "number", ["number"], [luminance.length || 1]);
|
|
340
|
+
this.mod.HEAPU8.set(luminance, lumPtr);
|
|
341
|
+
const outPtr = this.mod.ccall("goss_alloc", "number", ["number"], [13]);
|
|
342
|
+
const status = this.mod.ccall("goss_engine_scan_barcode", "number", ["number", "number", "number", "number", "number"], [this.handle, lumPtr, width, height, outPtr]);
|
|
343
|
+
const digits = status === 0 ? this.mod.HEAPU8.slice(outPtr, outPtr + 13) : null;
|
|
344
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [lumPtr, luminance.length || 1]);
|
|
345
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [outPtr, 13]);
|
|
346
|
+
return digits;
|
|
347
|
+
}
|
|
348
|
+
/// Scans a width*height 8-bit luminance frame for a QR code and returns its
|
|
349
|
+
/// decoded payload bytes, or null when no QR decodes. Reed-Solomon error
|
|
350
|
+
/// correction, algorithmic and deterministic, no model.
|
|
351
|
+
scanQR(luminance, width, height) {
|
|
352
|
+
const lumPtr = this.mod.ccall("goss_alloc", "number", ["number"], [luminance.length || 1]);
|
|
353
|
+
this.mod.HEAPU8.set(luminance, lumPtr);
|
|
354
|
+
const lenPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]);
|
|
355
|
+
const view = () => new DataView(this.mod.HEAPU8.buffer, lenPtr, 4).getUint32(0, true);
|
|
356
|
+
const args = ["number", "number", "number", "number", "number", "number", "number"];
|
|
357
|
+
const probe = this.mod.ccall("goss_engine_scan_qr", "number", args, [this.handle, lumPtr, width, height, 0, 0, lenPtr]);
|
|
358
|
+
let out = null;
|
|
359
|
+
if (probe === 0) {
|
|
360
|
+
const needed = view();
|
|
361
|
+
const outPtr = this.mod.ccall("goss_alloc", "number", ["number"], [needed || 1]);
|
|
362
|
+
const fill = this.mod.ccall("goss_engine_scan_qr", "number", args, [this.handle, lumPtr, width, height, outPtr, needed, lenPtr]);
|
|
363
|
+
if (fill === 0)
|
|
364
|
+
out = this.mod.HEAPU8.slice(outPtr, outPtr + view());
|
|
365
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [outPtr, needed || 1]);
|
|
366
|
+
}
|
|
367
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [lumPtr, luminance.length || 1]);
|
|
368
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [lenPtr, 4]);
|
|
369
|
+
return out;
|
|
370
|
+
}
|
|
371
|
+
/// Generates a QR code for a payload and renders it into a square 8-bit
|
|
372
|
+
/// luminance image (0 dark, 255 light); returns the pixels and the side
|
|
373
|
+
/// length. Algorithmic and deterministic, no model.
|
|
374
|
+
generateQR(payload, moduleScale = 6, quietModules = 4) {
|
|
375
|
+
const payPtr = this.mod.ccall("goss_alloc", "number", ["number"], [payload.length || 1]);
|
|
376
|
+
this.mod.HEAPU8.set(payload, payPtr);
|
|
377
|
+
const dimPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]);
|
|
378
|
+
const dimOf = () => new DataView(this.mod.HEAPU8.buffer, dimPtr, 4).getUint32(0, true);
|
|
379
|
+
const args = ["number", "number", "number", "number", "number", "number", "number", "number"];
|
|
380
|
+
let result = null;
|
|
381
|
+
if (this.mod.ccall("goss_engine_generate_qr", "number", args, [this.handle, payPtr, payload.length, moduleScale, quietModules, 0, 0, dimPtr]) === 0) {
|
|
382
|
+
const dim = dimOf();
|
|
383
|
+
const outPtr = this.mod.ccall("goss_alloc", "number", ["number"], [dim * dim || 1]);
|
|
384
|
+
if (this.mod.ccall("goss_engine_generate_qr", "number", args, [this.handle, payPtr, payload.length, moduleScale, quietModules, outPtr, dim * dim, dimPtr]) === 0) {
|
|
385
|
+
result = { image: this.mod.HEAPU8.slice(outPtr, outPtr + dim * dim), dim };
|
|
386
|
+
}
|
|
387
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [outPtr, dim * dim || 1]);
|
|
388
|
+
}
|
|
389
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [payPtr, payload.length || 1]);
|
|
390
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [dimPtr, 4]);
|
|
391
|
+
return result;
|
|
392
|
+
}
|
|
393
|
+
/// Ranks a media archive by semantic similarity. corpus holds count embedding
|
|
394
|
+
/// vectors of length dim contiguously and query is one dim-vector, all from
|
|
395
|
+
/// your own embedding model; returns the top k hits by cosine similarity. The
|
|
396
|
+
/// engine owns the exact search; any embedder feeds it.
|
|
397
|
+
mediaSearch(corpus, count, dim, query, k) {
|
|
398
|
+
const corpusPtr = this.mod.ccall("goss_alloc", "number", ["number"], [corpus.length * 4 || 4]);
|
|
399
|
+
this.mod.HEAPF32.set(corpus, corpusPtr / 4);
|
|
400
|
+
const queryPtr = this.mod.ccall("goss_alloc", "number", ["number"], [query.length * 4 || 4]);
|
|
401
|
+
this.mod.HEAPF32.set(query, queryPtr / 4);
|
|
402
|
+
const idxPtr = this.mod.ccall("goss_alloc", "number", ["number"], [k * 4 || 4]);
|
|
403
|
+
const scorePtr = this.mod.ccall("goss_alloc", "number", ["number"], [k * 4 || 4]);
|
|
404
|
+
const countPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]);
|
|
405
|
+
const args = ["number", "number", "number", "number", "number", "number", "number", "number", "number"];
|
|
406
|
+
const hits = [];
|
|
407
|
+
if (this.mod.ccall("goss_engine_media_search", "number", args, [this.handle, corpusPtr, count, dim, queryPtr, k, idxPtr, scorePtr, countPtr]) === 0) {
|
|
408
|
+
const n = new DataView(this.mod.HEAPU8.buffer, countPtr, 4).getUint32(0, true);
|
|
409
|
+
for (let i = 0; i < n; i++) {
|
|
410
|
+
const index = new DataView(this.mod.HEAPU8.buffer, idxPtr + i * 4, 4).getUint32(0, true);
|
|
411
|
+
const score = new DataView(this.mod.HEAPU8.buffer, scorePtr + i * 4, 4).getFloat32(0, true);
|
|
412
|
+
hits.push({ index, score });
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [corpusPtr, corpus.length * 4 || 4]);
|
|
416
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [queryPtr, query.length * 4 || 4]);
|
|
417
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [idxPtr, k * 4 || 4]);
|
|
418
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [scorePtr, k * 4 || 4]);
|
|
419
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [countPtr, 4]);
|
|
420
|
+
return hits;
|
|
421
|
+
}
|
|
422
|
+
/// Seals a media blob for the on-device vault with ChaCha20-Poly1305 under a
|
|
423
|
+
/// 32-byte key and 12-byte nonce, binding aad; returns ciphertext then the
|
|
424
|
+
/// 16-byte tag. Keep the key in the platform keystore.
|
|
425
|
+
sealMedia(key, nonce, plaintext, aad = new Uint8Array(0)) {
|
|
426
|
+
return this.aeadMedia("goss_seal_media", key, nonce, plaintext, aad);
|
|
427
|
+
}
|
|
428
|
+
/// Opens a sealed vault blob back to plaintext under the same key, nonce and
|
|
429
|
+
/// aad. Returns null if authentication fails, so a tampered blob never decodes.
|
|
430
|
+
openMedia(key, nonce, sealed, aad = new Uint8Array(0)) {
|
|
431
|
+
return this.aeadMedia("goss_open_media", key, nonce, sealed, aad);
|
|
432
|
+
}
|
|
433
|
+
aeadMedia(fn, key, nonce, input, aad) {
|
|
434
|
+
const keyPtr = this.mod.ccall("goss_alloc", "number", ["number"], [key.length || 1]);
|
|
435
|
+
this.mod.HEAPU8.set(key, keyPtr);
|
|
436
|
+
const noncePtr = this.mod.ccall("goss_alloc", "number", ["number"], [nonce.length || 1]);
|
|
437
|
+
this.mod.HEAPU8.set(nonce, noncePtr);
|
|
438
|
+
const inPtr = this.mod.ccall("goss_alloc", "number", ["number"], [input.length || 1]);
|
|
439
|
+
this.mod.HEAPU8.set(input, inPtr);
|
|
440
|
+
const aadPtr = this.mod.ccall("goss_alloc", "number", ["number"], [aad.length || 1]);
|
|
441
|
+
this.mod.HEAPU8.set(aad, aadPtr);
|
|
442
|
+
const lenPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]);
|
|
443
|
+
const lenOf = () => new DataView(this.mod.HEAPU8.buffer, lenPtr, 4).getUint32(0, true);
|
|
444
|
+
const args = ["number", "number", "number", "number", "number", "number", "number", "number", "number"];
|
|
445
|
+
let out = null;
|
|
446
|
+
if (this.mod.ccall(fn, "number", args, [keyPtr, noncePtr, inPtr, input.length, aadPtr, aad.length, 0, 0, lenPtr]) === 0) {
|
|
447
|
+
const needed = lenOf();
|
|
448
|
+
const outPtr = this.mod.ccall("goss_alloc", "number", ["number"], [needed || 1]);
|
|
449
|
+
if (this.mod.ccall(fn, "number", args, [keyPtr, noncePtr, inPtr, input.length, aadPtr, aad.length, outPtr, needed, lenPtr]) === 0) {
|
|
450
|
+
out = this.mod.HEAPU8.slice(outPtr, outPtr + lenOf());
|
|
451
|
+
}
|
|
452
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [outPtr, needed || 1]);
|
|
453
|
+
}
|
|
454
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [keyPtr, key.length || 1]);
|
|
455
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [noncePtr, nonce.length || 1]);
|
|
456
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [inPtr, input.length || 1]);
|
|
457
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [aadPtr, aad.length || 1]);
|
|
458
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [lenPtr, 4]);
|
|
459
|
+
return out;
|
|
460
|
+
}
|
|
461
|
+
/// Picks the best frame of a burst: count luminance frames of width*height
|
|
462
|
+
/// frameStride bytes apart, scored by sharpness blended with a per-frame
|
|
463
|
+
/// openness score weighted by opennessWeight in 0..1. Returns the winning
|
|
464
|
+
/// frame index for best-take fusion.
|
|
465
|
+
bestTake(frames, frameStride, count, width, height, openness, opennessWeight) {
|
|
466
|
+
const framesPtr = this.mod.ccall("goss_alloc", "number", ["number"], [frames.length || 1]);
|
|
467
|
+
this.mod.HEAPU8.set(frames, framesPtr);
|
|
468
|
+
const opennessPtr = this.mod.ccall("goss_alloc", "number", ["number"], [openness.length * 4 || 4]);
|
|
469
|
+
this.mod.HEAPF32.set(openness, opennessPtr / 4);
|
|
470
|
+
const indexPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]);
|
|
471
|
+
const args = ["number", "number", "number", "number", "number", "number", "number", "number", "number"];
|
|
472
|
+
let index = 0;
|
|
473
|
+
if (this.mod.ccall("goss_engine_best_take", "number", args, [this.handle, framesPtr, frameStride, count, width, height, opennessPtr, opennessWeight, indexPtr]) === 0) {
|
|
474
|
+
index = new DataView(this.mod.HEAPU8.buffer, indexPtr, 4).getUint32(0, true);
|
|
475
|
+
}
|
|
476
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [framesPtr, frames.length || 1]);
|
|
477
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [opennessPtr, openness.length * 4 || 4]);
|
|
478
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [indexPtr, 4]);
|
|
479
|
+
return index;
|
|
480
|
+
}
|
|
481
|
+
/// Fingerprints a reference recording and registers it under trackId in the
|
|
482
|
+
/// engine's on-device music catalog. Samples are interleaved f32; re-adding a
|
|
483
|
+
/// trackId layers more landmarks in.
|
|
484
|
+
addMusicReference(trackId, samples, frameCount, sampleRate, channels) {
|
|
485
|
+
const bytes = samples.length * 4;
|
|
486
|
+
const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [bytes || 4]);
|
|
487
|
+
this.mod.HEAPF32.set(samples, ptr >> 2);
|
|
488
|
+
const status = this.mod.ccall("goss_engine_music_add_reference", "number", ["number", "number", "number", "number", "number", "number"], [this.handle, trackId >>> 0, ptr, frameCount, sampleRate, channels]);
|
|
489
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [ptr, bytes || 4]);
|
|
490
|
+
return status === 0;
|
|
491
|
+
}
|
|
492
|
+
/// Empties the engine's music catalog.
|
|
493
|
+
clearMusicReferences() {
|
|
494
|
+
this.mod.ccall("goss_engine_music_clear_references", null, ["number"], [this.handle]);
|
|
495
|
+
}
|
|
496
|
+
/// Fingerprints a captured snippet and matches it against the catalog,
|
|
497
|
+
/// returning the best track and its landmark-agreement vote count, or null
|
|
498
|
+
/// below minVotes. A few seconds of noisy audio still identifies.
|
|
499
|
+
identifyMusic(samples, frameCount, sampleRate, channels, minVotes = 5) {
|
|
500
|
+
const bytes = samples.length * 4;
|
|
501
|
+
const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [bytes || 4]);
|
|
502
|
+
const outPtr = this.mod.ccall("goss_alloc", "number", ["number"], [8]);
|
|
503
|
+
this.mod.HEAPF32.set(samples, ptr >> 2);
|
|
504
|
+
const status = this.mod.ccall("goss_engine_music_identify", "number", ["number", "number", "number", "number", "number", "number", "number", "number"], [this.handle, ptr, frameCount, sampleRate, channels, minVotes >>> 0, outPtr, outPtr + 4]);
|
|
505
|
+
const dv = new DataView(this.mod.HEAPU8.buffer, outPtr, 8);
|
|
506
|
+
const trackId = dv.getUint32(0, true);
|
|
507
|
+
const votes = dv.getUint32(4, true);
|
|
508
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [ptr, bytes || 4]);
|
|
509
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [outPtr, 8]);
|
|
510
|
+
if (status !== 0 || votes === 0)
|
|
511
|
+
return null;
|
|
512
|
+
return { trackId, votes };
|
|
513
|
+
}
|
|
514
|
+
/// Releases the persistent wrap the engine keeps per external live
|
|
515
|
+
/// texture handle - the pair of the native SDKs' renderToLiveTexture,
|
|
516
|
+
/// for a host retiring a publish surface before the engine goes away.
|
|
517
|
+
/// False for a handle with no live wrap.
|
|
518
|
+
releaseLiveTexture(nativeHandle) {
|
|
519
|
+
return this.mod.ccall("goss_engine_release_live_texture", "number", ["number", "number"], [this.handle, nativeHandle]) === 0;
|
|
520
|
+
}
|
|
521
|
+
/// The two ways this SDK reads pixels back: bgfx's WebGL2 context
|
|
522
|
+
/// never preserves its drawing buffer, so readPixels runs right after
|
|
523
|
+
/// a fresh render; WebGPU has no sync equivalent, so
|
|
524
|
+
/// goss_engine_capture_frame runs async, mapping a GPU buffer.
|
|
525
|
+
async capturePixels(session) {
|
|
526
|
+
if (!this.canvas)
|
|
527
|
+
throw new Error("initRenderer first");
|
|
528
|
+
if (this.gl) {
|
|
529
|
+
const gl = this.gl;
|
|
530
|
+
this.renderFrame(session);
|
|
531
|
+
const width = this.canvas.width;
|
|
532
|
+
const height = this.canvas.height;
|
|
533
|
+
const pixels = new Uint8Array(width * height * 4);
|
|
534
|
+
gl.readPixels(0, 0, width, height, gl.RGBA, gl.UNSIGNED_BYTE, pixels);
|
|
535
|
+
return { pixels, width, height };
|
|
536
|
+
}
|
|
537
|
+
const capacity = this.canvas.width * this.canvas.height * 4;
|
|
538
|
+
const dataPtr = this.mod.ccall("goss_alloc", "number", ["number"], [capacity]);
|
|
539
|
+
const outWidthPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]);
|
|
540
|
+
const outHeightPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]);
|
|
541
|
+
this.captureInFlight = true;
|
|
542
|
+
try {
|
|
543
|
+
const status = await this.mod.ccall("goss_engine_capture_frame", "number", ["number", "number", "number", "number", "number", "number"], [this.handle, session?.handle ?? 0, dataPtr, capacity, outWidthPtr, outHeightPtr], { async: true });
|
|
544
|
+
if (status !== GOSS_OK)
|
|
545
|
+
throw new Error(`goss_engine_capture_frame failed: status ${status}`);
|
|
546
|
+
const width = this.mod.getValue(outWidthPtr, "i32");
|
|
547
|
+
const height = this.mod.getValue(outHeightPtr, "i32");
|
|
548
|
+
const pixels = this.mod.HEAPU8.slice(dataPtr, dataPtr + width * height * 4);
|
|
549
|
+
return { pixels, width, height };
|
|
550
|
+
}
|
|
551
|
+
finally {
|
|
552
|
+
this.captureInFlight = false;
|
|
553
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [dataPtr, capacity]);
|
|
554
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [outWidthPtr, 4]);
|
|
555
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [outHeightPtr, 4]);
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
/// goss_engine_capture_frame on the WebGPU build is an async
|
|
559
|
+
/// (Asyncify-suspending) ccall - calling renderFrame while one is in
|
|
560
|
+
/// flight would reenter the wasm module synchronously, which
|
|
561
|
+
/// Asyncify does not support while already suspended.
|
|
562
|
+
get isCaptureInFlight() {
|
|
563
|
+
return this.captureInFlight;
|
|
564
|
+
}
|
|
565
|
+
/// PNG-encodes the current frame. Test/debug tooling: a real image
|
|
566
|
+
/// beats a frame-sum heuristic for verifying a landmark-driven effect
|
|
567
|
+
/// actually landed where it should, not just that something changed
|
|
568
|
+
/// somewhere.
|
|
569
|
+
async captureFrame(session) {
|
|
570
|
+
const { pixels, width: w, height: h } = await this.capturePixels(session);
|
|
571
|
+
const out = document.createElement("canvas");
|
|
572
|
+
out.width = w;
|
|
573
|
+
out.height = h;
|
|
574
|
+
const ctx = out.getContext("2d");
|
|
575
|
+
const imageData = ctx.createImageData(w, h);
|
|
576
|
+
if (this.gl) {
|
|
577
|
+
const rowBytes = w * 4;
|
|
578
|
+
for (let y = 0; y < h; y += 1) {
|
|
579
|
+
const srcStart = (h - 1 - y) * rowBytes;
|
|
580
|
+
imageData.data.set(pixels.subarray(srcStart, srcStart + rowBytes), y * rowBytes);
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
else {
|
|
584
|
+
imageData.data.set(pixels);
|
|
585
|
+
}
|
|
586
|
+
ctx.putImageData(imageData, 0, 0);
|
|
587
|
+
return out.toDataURL("image/png");
|
|
588
|
+
}
|
|
589
|
+
/// The composited frame as packed bytes in a WebRTC format (BGRA or RGBA),
|
|
590
|
+
/// the supported per-frame output for a live source. On the web the canvas is
|
|
591
|
+
/// already a zero-copy source through captureStream(); reach for this only for
|
|
592
|
+
/// raw pixels. NV12 is the native encoders' path - captureStream handles web.
|
|
593
|
+
async captureLiveFrame(session, format = 3 /* GossPixelFormat.Bgra8 */) {
|
|
594
|
+
if (format !== 3 /* GossPixelFormat.Bgra8 */ && format !== 4 /* GossPixelFormat.Rgba8 */) {
|
|
595
|
+
throw new Error("captureLiveFrame on web supports Bgra8 or Rgba8; use captureStream for a live track");
|
|
596
|
+
}
|
|
597
|
+
const { pixels, width, height } = await this.capturePixels(session);
|
|
598
|
+
if (format === 3 /* GossPixelFormat.Bgra8 */) {
|
|
599
|
+
for (let i = 0; i + 3 < pixels.length; i += 4) {
|
|
600
|
+
const red = pixels[i];
|
|
601
|
+
pixels[i] = pixels[i + 2];
|
|
602
|
+
pixels[i + 2] = red;
|
|
603
|
+
}
|
|
604
|
+
}
|
|
605
|
+
return { pixels, width, height };
|
|
606
|
+
}
|
|
607
|
+
/// A high-resolution still of the composited frame at its own resolution
|
|
608
|
+
/// (width and height 0) or a requested one, decoupled from the preview
|
|
609
|
+
/// size, returned as the encoded image bytes. Wasm core only: the pure
|
|
610
|
+
/// WebGL path renders in JS and has no core encoder to reach.
|
|
611
|
+
async captureStill(session, config = {}) {
|
|
612
|
+
if (this.gl)
|
|
613
|
+
throw new Error("captureStill needs the wasm renderer");
|
|
614
|
+
const cfgPtr = this.mod.ccall("goss_alloc", "number", ["number"], [28]);
|
|
615
|
+
this.mod.setValue(cfgPtr, config.width ?? 0, "i32");
|
|
616
|
+
this.mod.setValue(cfgPtr + 4, config.height ?? 0, "i32");
|
|
617
|
+
this.mod.setValue(cfgPtr + 8, config.supersample ?? 0, "i32");
|
|
618
|
+
this.mod.setValue(cfgPtr + 12, config.format ?? 0, "i32");
|
|
619
|
+
this.mod.setValue(cfgPtr + 16, config.quality ?? 0, "i32");
|
|
620
|
+
this.mod.setValue(cfgPtr + 20, config.colorSpace ?? 0, "i32");
|
|
621
|
+
this.mod.setValue(cfgPtr + 24, config.bitDepth ?? 8, "i32");
|
|
622
|
+
const lenPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]);
|
|
623
|
+
const widthPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]);
|
|
624
|
+
const heightPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]);
|
|
625
|
+
let dataPtr = 0;
|
|
626
|
+
let capacity = 0;
|
|
627
|
+
this.captureInFlight = true;
|
|
628
|
+
try {
|
|
629
|
+
// Probe for the encoded size, then capture into a buffer of that size.
|
|
630
|
+
const probeArgs = ["number", "number", "number", "number", "number", "number", "number", "number"];
|
|
631
|
+
const probeStatus = await this.mod.ccall("goss_engine_capture_still", "number", probeArgs, [this.handle, session?.handle ?? 0, cfgPtr, 0, 0, lenPtr, widthPtr, heightPtr], { async: true });
|
|
632
|
+
capacity = this.mod.getValue(lenPtr, "i32");
|
|
633
|
+
if (probeStatus === GOSS_OK && capacity === 0)
|
|
634
|
+
return new Uint8Array(0);
|
|
635
|
+
if (capacity <= 0)
|
|
636
|
+
throw new Error(`goss_engine_capture_still probe failed: status ${probeStatus}`);
|
|
637
|
+
dataPtr = this.mod.ccall("goss_alloc", "number", ["number"], [capacity]);
|
|
638
|
+
const status = await this.mod.ccall("goss_engine_capture_still", "number", probeArgs, [this.handle, session?.handle ?? 0, cfgPtr, dataPtr, capacity, lenPtr, widthPtr, heightPtr], { async: true });
|
|
639
|
+
if (status !== GOSS_OK)
|
|
640
|
+
throw new Error(`goss_engine_capture_still failed: status ${status}`);
|
|
641
|
+
const encoded = this.mod.getValue(lenPtr, "i32");
|
|
642
|
+
return this.mod.HEAPU8.slice(dataPtr, dataPtr + encoded);
|
|
643
|
+
}
|
|
644
|
+
finally {
|
|
645
|
+
this.captureInFlight = false;
|
|
646
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [cfgPtr, 28]);
|
|
647
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [lenPtr, 4]);
|
|
648
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [widthPtr, 4]);
|
|
649
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [heightPtr, 4]);
|
|
650
|
+
if (dataPtr !== 0)
|
|
651
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [dataPtr, capacity]);
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
/// The composited frame encoded as a deterministic PNG - the same
|
|
655
|
+
/// pixels, the same bytes - at the submitted frame's own resolution.
|
|
656
|
+
/// captureStill is the configurable superset (a chosen size, JPEG, or a
|
|
657
|
+
/// wider gamut); this is the plain PNG surface. Wasm core only.
|
|
658
|
+
async capturePhoto(session) {
|
|
659
|
+
return this.captureEncoded("goss_engine_capture_photo", ["number", "number", "number", "number", "number", "number", "number"], (dataPtr, capacity, lenPtr, widthPtr, heightPtr) => [this.handle, session?.handle ?? 0, dataPtr, capacity, lenPtr, widthPtr, heightPtr]);
|
|
660
|
+
}
|
|
661
|
+
/// The composited frame as a platform photo: format 1 is JPEG at quality
|
|
662
|
+
/// 1..100 (the engine's own encoder, present on web too), format 2 is HEIC
|
|
663
|
+
/// (the native photo backend, GOSS_UNSUPPORTED on web). Lossy and not
|
|
664
|
+
/// bit-stable across runs, so capturePhoto stays the deterministic path.
|
|
665
|
+
async capturePhotoAs(session, format, quality = 90) {
|
|
666
|
+
return this.captureEncoded("goss_engine_capture_photo_as", ["number", "number", "number", "number", "number", "number", "number", "number", "number"], (dataPtr, capacity, lenPtr, widthPtr, heightPtr) => [this.handle, session?.handle ?? 0, format, quality, dataPtr, capacity, lenPtr, widthPtr, heightPtr]);
|
|
667
|
+
}
|
|
668
|
+
/// The probe-then-capture the encoded-photo ABI ops share: render and encode
|
|
669
|
+
/// once into a one-byte buffer to learn the exact size (the encoders write
|
|
670
|
+
/// out_len before rejecting a too-small capacity), then capture into a buffer
|
|
671
|
+
/// of that size. Wasm core only, like captureStill; WebGL2 renders in JS.
|
|
672
|
+
async captureEncoded(callName, argTypes, buildArgs) {
|
|
673
|
+
if (this.gl)
|
|
674
|
+
throw new Error(`${callName} needs the wasm renderer`);
|
|
675
|
+
const mod = this.mod;
|
|
676
|
+
const lenPtr = mod.ccall("goss_alloc", "number", ["number"], [4]);
|
|
677
|
+
const widthPtr = mod.ccall("goss_alloc", "number", ["number"], [4]);
|
|
678
|
+
const heightPtr = mod.ccall("goss_alloc", "number", ["number"], [4]);
|
|
679
|
+
// A one-byte buffer keeps the probe past the ABI's null-pointer guard so
|
|
680
|
+
// the encoder runs and reports the real size through lenPtr.
|
|
681
|
+
const probePtr = mod.ccall("goss_alloc", "number", ["number"], [1]);
|
|
682
|
+
let dataPtr = 0;
|
|
683
|
+
let capacity = 0;
|
|
684
|
+
this.captureInFlight = true;
|
|
685
|
+
try {
|
|
686
|
+
const probeStatus = await mod.ccall(callName, "number", argTypes, buildArgs(probePtr, 0, lenPtr, widthPtr, heightPtr), { async: true });
|
|
687
|
+
capacity = mod.getValue(lenPtr, "i32");
|
|
688
|
+
if (capacity === 0)
|
|
689
|
+
return new Uint8Array(0);
|
|
690
|
+
if (capacity < 0)
|
|
691
|
+
throw new Error(`${callName} probe failed: status ${probeStatus}`);
|
|
692
|
+
dataPtr = mod.ccall("goss_alloc", "number", ["number"], [capacity]);
|
|
693
|
+
const status = await mod.ccall(callName, "number", argTypes, buildArgs(dataPtr, capacity, lenPtr, widthPtr, heightPtr), { async: true });
|
|
694
|
+
if (status !== GOSS_OK)
|
|
695
|
+
throw new Error(`${callName} failed: status ${status}`);
|
|
696
|
+
const encoded = mod.getValue(lenPtr, "i32");
|
|
697
|
+
return mod.HEAPU8.slice(dataPtr, dataPtr + encoded);
|
|
698
|
+
}
|
|
699
|
+
finally {
|
|
700
|
+
this.captureInFlight = false;
|
|
701
|
+
mod.ccall("goss_free", null, ["number", "number"], [probePtr, 1]);
|
|
702
|
+
mod.ccall("goss_free", null, ["number", "number"], [lenPtr, 4]);
|
|
703
|
+
mod.ccall("goss_free", null, ["number", "number"], [widthPtr, 4]);
|
|
704
|
+
mod.ccall("goss_free", null, ["number", "number"], [heightPtr, 4]);
|
|
705
|
+
if (dataPtr !== 0)
|
|
706
|
+
mod.ccall("goss_free", null, ["number", "number"], [dataPtr, capacity]);
|
|
707
|
+
}
|
|
708
|
+
}
|
|
709
|
+
async readCenterPixel(session) {
|
|
710
|
+
const { pixels, width, height } = await this.capturePixels(session);
|
|
711
|
+
const offset = (Math.floor(height / 2) * width + Math.floor(width / 2)) * 4;
|
|
712
|
+
return pixels.slice(offset, offset + 4);
|
|
713
|
+
}
|
|
714
|
+
/// Sums every RGBA byte over the whole canvas - courser but far more
|
|
715
|
+
/// robust than one fixed pixel, since a synthetic test pattern
|
|
716
|
+
/// (Chrome's fake capture device) is free to put its "lit" content
|
|
717
|
+
/// anywhere, leaving any single coordinate dark for long stretches.
|
|
718
|
+
async readFrameSum(session) {
|
|
719
|
+
const { pixels } = await this.capturePixels(session);
|
|
720
|
+
let sum = 0;
|
|
721
|
+
for (const value of pixels)
|
|
722
|
+
sum += value;
|
|
723
|
+
return sum;
|
|
724
|
+
}
|
|
725
|
+
destroy() {
|
|
726
|
+
this.mod.ccall("goss_engine_destroy", null, ["number"], [this.handle]);
|
|
727
|
+
// The renderer is gone now, so bgfx no longer references the selector.
|
|
728
|
+
if (this.selectorPtr !== 0)
|
|
729
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [this.selectorPtr, this.selectorCapacity]);
|
|
730
|
+
this.selectorPtr = 0;
|
|
731
|
+
this.selectorCapacity = 0;
|
|
732
|
+
}
|
|
733
|
+
}
|
|
734
|
+
/// Per-preview runtime: frame submission, beauty, tracking, lens. Owns
|
|
735
|
+
/// its own scratch allocations (frame descriptor, pixel buffer,
|
|
736
|
+
/// landmarks) rather than one shared per-engine pool - matches every
|
|
737
|
+
/// other SDK's own per-session confinement.
|
|
738
|
+
export class GossSession {
|
|
739
|
+
mod;
|
|
740
|
+
handle;
|
|
741
|
+
worldScratchPtr = 0;
|
|
742
|
+
worldScratchLen = 0;
|
|
743
|
+
frameWidth = 0;
|
|
744
|
+
frameHeight = 0;
|
|
745
|
+
/// Some cameras (certain external/virtual devices on macOS) hand the
|
|
746
|
+
/// browser frames pre-rotated 180 degrees. Carried as a quarter-turn
|
|
747
|
+
/// count on the submitted frame's own flags, the same mechanism
|
|
748
|
+
/// every other SDK uses for sensor orientation.
|
|
749
|
+
videoFlipped = false;
|
|
750
|
+
whitenLutsLoaded = 0;
|
|
751
|
+
lipstickTextureLoaded = false;
|
|
752
|
+
blushTextureLoaded = false;
|
|
753
|
+
/// Reused across frames, grown on resize rather than alloc/freed every
|
|
754
|
+
/// tick - the frame descriptor is a fixed 32 bytes, the pixel buffer
|
|
755
|
+
/// tracks the video's current resolution.
|
|
756
|
+
frameDescPtr;
|
|
757
|
+
framePixelsPtr = 0;
|
|
758
|
+
framePixelsCapacity = 0;
|
|
759
|
+
/// Fixed capacity: GOSS_FACE_LANDMARK_COUNT never changes.
|
|
760
|
+
landmarksPtr;
|
|
761
|
+
/// Fixed layout, reused every tick like the frame descriptor.
|
|
762
|
+
signalsPtr;
|
|
763
|
+
/// Fixed capacity: the segmentation mask is always mask_side squared.
|
|
764
|
+
segmentationMaskPtr;
|
|
765
|
+
/// A reusable wasm scratch the per-frame submit and readback paths slice
|
|
766
|
+
/// instead of goss_alloc/goss_free each call; grown to the largest frame,
|
|
767
|
+
/// freed in destroy() - no per-call wasm heap churn.
|
|
768
|
+
scratchPtr = 0;
|
|
769
|
+
scratchCapacity = 0;
|
|
770
|
+
constructor(mod, handle) {
|
|
771
|
+
this.mod = mod;
|
|
772
|
+
this.handle = handle;
|
|
773
|
+
this.frameDescPtr = mod.ccall("goss_alloc", "number", ["number"], [32]);
|
|
774
|
+
this.landmarksPtr = mod.ccall("goss_alloc", "number", ["number"], [GOSS_FACE_LANDMARK_COUNT * 3 * 4]);
|
|
775
|
+
this.signalsPtr = mod.ccall("goss_alloc", "number", ["number"], [LENS_SIGNALS_BYTES]);
|
|
776
|
+
this.segmentationMaskPtr = mod.ccall("goss_alloc", "number", ["number"], [GOSS_SEGMENTATION_MASK_SIDE * GOSS_SEGMENTATION_MASK_SIDE * 4]);
|
|
777
|
+
}
|
|
778
|
+
/// The reusable scratch grown to at least `bytes`, returning its wasm
|
|
779
|
+
/// pointer. Grows only when a larger frame arrives, so the steady path
|
|
780
|
+
/// never touches the wasm allocator.
|
|
781
|
+
scratch(bytes) {
|
|
782
|
+
if (bytes > this.scratchCapacity) {
|
|
783
|
+
if (this.scratchPtr !== 0)
|
|
784
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [this.scratchPtr, this.scratchCapacity]);
|
|
785
|
+
this.scratchPtr = this.mod.ccall("goss_alloc", "number", ["number"], [bytes]);
|
|
786
|
+
this.scratchCapacity = bytes;
|
|
787
|
+
}
|
|
788
|
+
return this.scratchPtr;
|
|
789
|
+
}
|
|
790
|
+
static create(engine, config) {
|
|
791
|
+
const mod = engine.module;
|
|
792
|
+
let configPtr = 0;
|
|
793
|
+
if (config) {
|
|
794
|
+
configPtr = mod.ccall("goss_alloc", "number", ["number"], [8]);
|
|
795
|
+
mod.setValue(configPtr, config.frameBudgetUs ?? 0, "i32");
|
|
796
|
+
mod.setValue(configPtr + 4, 0, "i32");
|
|
797
|
+
}
|
|
798
|
+
const sessionOut = mod.ccall("goss_alloc", "number", ["number"], [4]);
|
|
799
|
+
const status = mod.ccall("goss_session_create", "number", ["number", "number", "number"], [engine.handle, configPtr, sessionOut]);
|
|
800
|
+
const handle = mod.getValue(sessionOut, "i32");
|
|
801
|
+
mod.ccall("goss_free", null, ["number", "number"], [sessionOut, 4]);
|
|
802
|
+
if (configPtr !== 0)
|
|
803
|
+
mod.ccall("goss_free", null, ["number", "number"], [configPtr, 8]);
|
|
804
|
+
if (status !== GOSS_OK)
|
|
805
|
+
throw new Error(`session create failed: ${status}`);
|
|
806
|
+
return new GossSession(mod, handle);
|
|
807
|
+
}
|
|
808
|
+
setWhiten(amount) {
|
|
809
|
+
this.setBeauty(1 /* GossBeautyEffect.Whiten */, this.whitenLutsLoaded === 4 ? amount : 0);
|
|
810
|
+
}
|
|
811
|
+
setSmooth(amount) {
|
|
812
|
+
this.setBeauty(0 /* GossBeautyEffect.Smooth */, amount);
|
|
813
|
+
}
|
|
814
|
+
setThinFace(amount) {
|
|
815
|
+
this.setBeauty(2 /* GossBeautyEffect.ThinFace */, amount);
|
|
816
|
+
}
|
|
817
|
+
setBigEye(amount) {
|
|
818
|
+
this.setBeauty(3 /* GossBeautyEffect.BigEye */, amount);
|
|
819
|
+
}
|
|
820
|
+
setLipstick(amount) {
|
|
821
|
+
this.setBeauty(4 /* GossBeautyEffect.Lipstick */, this.lipstickTextureLoaded ? amount : 0);
|
|
822
|
+
}
|
|
823
|
+
setBlush(amount) {
|
|
824
|
+
this.setBeauty(5 /* GossBeautyEffect.Blush */, this.blushTextureLoaded ? amount : 0);
|
|
825
|
+
}
|
|
826
|
+
setBeauty(effect, amount) {
|
|
827
|
+
this.mod.ccall("goss_session_set_beauty", "number", ["number", "number", "number"], [this.handle, effect, amount]);
|
|
828
|
+
}
|
|
829
|
+
/// Activates a lens from its manifest JSON directly (goss_session_
|
|
830
|
+
/// activate_lens, not the directory-based variant) - the only
|
|
831
|
+
/// activation path this build actually supports: has_file_io is
|
|
832
|
+
/// comptime-false for every wasm target, so goss_session_activate_lens_
|
|
833
|
+
/// from_directory always reports unsupported here, and shader.pass/
|
|
834
|
+
/// lut.pass/blend.pass nodes need compiled resources a bundle
|
|
835
|
+
/// directory would provide that this SDK has no way to supply yet.
|
|
836
|
+
/// A lens built entirely from beauty.* nodes (beauty-baseline, say)
|
|
837
|
+
/// activates and runs for real regardless, since those go through
|
|
838
|
+
/// applyWebBeautyChain's own embedded shaders, not a per-lens one.
|
|
839
|
+
activateLens(manifestJson) {
|
|
840
|
+
const bytes = new TextEncoder().encode(manifestJson);
|
|
841
|
+
const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [bytes.length]);
|
|
842
|
+
this.mod.HEAPU8.set(bytes, ptr);
|
|
843
|
+
this.mod.ccall("goss_session_activate_lens", "number", ["number", "number", "number"], [this.handle, ptr, bytes.length]);
|
|
844
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [ptr, bytes.length]);
|
|
845
|
+
}
|
|
846
|
+
deactivateLens() {
|
|
847
|
+
this.mod.ccall("goss_session_deactivate_lens", null, ["number"], [this.handle]);
|
|
848
|
+
}
|
|
849
|
+
/// Advances the active lens's triggers/param ramps by dtUs, evaluating
|
|
850
|
+
/// them against signals - omitted fields read as false/zero, so a bare
|
|
851
|
+
/// tickLens(dtUs) only fires triggers with no `when` gate.
|
|
852
|
+
tickLens(dtUs, signals = {}) {
|
|
853
|
+
const ptr = this.signalsPtr;
|
|
854
|
+
this.mod.HEAPU8.fill(0, ptr, ptr + LENS_SIGNALS_BYTES);
|
|
855
|
+
this.mod.HEAPU8[ptr] = signals.hasFace ? 1 : 0;
|
|
856
|
+
this.mod.HEAPU8[ptr + 1] = signals.handsPresent ? 1 : 0;
|
|
857
|
+
this.mod.HEAPU8[ptr + 2] = signals.tap ? 1 : 0;
|
|
858
|
+
this.mod.setValue(ptr + 8, signals.worldTrackingState ?? 0, "double");
|
|
859
|
+
this.mod.setValue(ptr + 16, signals.audioLevel ?? 0, "double");
|
|
860
|
+
if (signals.blendshapes) {
|
|
861
|
+
const base = (ptr + 24) >> 2;
|
|
862
|
+
const count = Math.min(GOSS_FACE_BLENDSHAPE_COUNT, signals.blendshapes.length);
|
|
863
|
+
for (let at = 0; at < count; at += 1)
|
|
864
|
+
this.mod.HEAPF32[base + at] = signals.blendshapes[at];
|
|
865
|
+
}
|
|
866
|
+
this.mod.ccall("goss_session_tick_lens", "number", ["number", "number", "number"], [this.handle, dtUs, ptr]);
|
|
867
|
+
}
|
|
868
|
+
/// Reads a live parameter of the active lens by name, including whatever a
|
|
869
|
+
/// script node last wrote. Null with no active lens or no such parameter.
|
|
870
|
+
parameterValue(name) {
|
|
871
|
+
const bytes = new TextEncoder().encode(name);
|
|
872
|
+
const namePtr = this.mod.ccall("goss_alloc", "number", ["number"], [bytes.length]);
|
|
873
|
+
this.mod.HEAPU8.set(bytes, namePtr);
|
|
874
|
+
const outPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]);
|
|
875
|
+
const status = this.mod.ccall("goss_session_parameter_value", "number", ["number", "number", "number", "number"], [this.handle, namePtr, bytes.length, outPtr]);
|
|
876
|
+
const value = status === 0 ? this.mod.getValue(outPtr, "float") : null;
|
|
877
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [namePtr, bytes.length]);
|
|
878
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [outPtr, 4]);
|
|
879
|
+
return value;
|
|
880
|
+
}
|
|
881
|
+
/// Raycasts a normalized screen point (0..1, origin top-left) against the
|
|
882
|
+
/// tracked ground plane, returning the world hit position [x, y, z], or null
|
|
883
|
+
/// until world tracking is live and the ray meets the plane. A tap-to-place
|
|
884
|
+
/// lens polls this and drops an anchor at the hit.
|
|
885
|
+
hitTest(x, y) {
|
|
886
|
+
const outPtr = this.mod.ccall("goss_alloc", "number", ["number"], [12]);
|
|
887
|
+
const status = this.mod.ccall("goss_session_hit_test", "number", ["number", "number", "number", "number"], [this.handle, x, y, outPtr]);
|
|
888
|
+
const w = outPtr >> 2;
|
|
889
|
+
const hit = status === 0 ? [this.mod.HEAPF32[w], this.mod.HEAPF32[w + 1], this.mod.HEAPF32[w + 2]] : null;
|
|
890
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [outPtr, 12]);
|
|
891
|
+
return hit;
|
|
892
|
+
}
|
|
893
|
+
/// Submits the device's pre-scanned world mesh (scene reconstruction, a VPS
|
|
894
|
+
/// scan) in world space: vertices are xyz triples and indices name three
|
|
895
|
+
/// vertices per triangle. Passing empty arrays clears the stored mesh.
|
|
896
|
+
submitWorldMesh(vertices, indices) {
|
|
897
|
+
const vPtr = this.mod.ccall("goss_alloc", "number", ["number"], [vertices.length * 4 || 4]);
|
|
898
|
+
this.mod.HEAPF32.set(vertices, vPtr / 4);
|
|
899
|
+
const iPtr = this.mod.ccall("goss_alloc", "number", ["number"], [indices.length * 4 || 4]);
|
|
900
|
+
this.mod.HEAPU32.set(indices, iPtr / 4);
|
|
901
|
+
const args = ["number", "number", "number", "number", "number"];
|
|
902
|
+
this.mod.ccall("goss_session_submit_world_mesh", "number", args, [this.handle, vPtr, vertices.length / 3, iPtr, indices.length]);
|
|
903
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [vPtr, vertices.length * 4 || 4]);
|
|
904
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [iPtr, indices.length * 4 || 4]);
|
|
905
|
+
}
|
|
906
|
+
/// Casts a world-space ray against the submitted world mesh, returning the
|
|
907
|
+
/// nearest surface hit `{ point, distance }`, or null when no mesh is
|
|
908
|
+
/// submitted or the ray misses. A tap-to-place lens anchors content there.
|
|
909
|
+
raycastWorldMesh(origin, direction) {
|
|
910
|
+
const oPtr = this.mod.ccall("goss_alloc", "number", ["number"], [12]);
|
|
911
|
+
this.mod.HEAPF32.set(origin, oPtr / 4);
|
|
912
|
+
const dPtr = this.mod.ccall("goss_alloc", "number", ["number"], [12]);
|
|
913
|
+
this.mod.HEAPF32.set(direction, dPtr / 4);
|
|
914
|
+
const pPtr = this.mod.ccall("goss_alloc", "number", ["number"], [12]);
|
|
915
|
+
const distPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]);
|
|
916
|
+
const args = ["number", "number", "number", "number", "number"];
|
|
917
|
+
const status = this.mod.ccall("goss_session_raycast_world_mesh", "number", args, [this.handle, oPtr, dPtr, pPtr, distPtr]);
|
|
918
|
+
const pw = pPtr >> 2;
|
|
919
|
+
const result = status === 0
|
|
920
|
+
? { point: [this.mod.HEAPF32[pw], this.mod.HEAPF32[pw + 1], this.mod.HEAPF32[pw + 2]], distance: this.mod.HEAPF32[distPtr >> 2] }
|
|
921
|
+
: null;
|
|
922
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [oPtr, 12]);
|
|
923
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [dPtr, 12]);
|
|
924
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [pPtr, 12]);
|
|
925
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [distPtr, 4]);
|
|
926
|
+
return result;
|
|
927
|
+
}
|
|
928
|
+
/// Feeds interleaved f32 PCM into the session's own level and beat
|
|
929
|
+
/// analysis, which drives the audio.level and audio.beat lens triggers.
|
|
930
|
+
/// samples holds frameCount * channels floats; timestampUs is carried for a
|
|
931
|
+
/// muxed recording track, unused by the web preview path.
|
|
932
|
+
submitAudio(samples, frameCount, sampleRate, channels, timestampUs = 0n) {
|
|
933
|
+
const bytes = samples.length * 4;
|
|
934
|
+
const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [bytes]);
|
|
935
|
+
this.mod.HEAPF32.set(samples, ptr >> 2);
|
|
936
|
+
this.mod.ccall("goss_session_submit_audio", "number", ["number", "number", "number", "number", "number", "number"], [this.handle, ptr, frameCount, sampleRate, channels, timestampUs]);
|
|
937
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [ptr, bytes]);
|
|
938
|
+
}
|
|
939
|
+
/// Pulls the next block of mixed lens audio (frames interleaved s16) that
|
|
940
|
+
/// play_sound triggers produced, for the page to feed into WebAudio.
|
|
941
|
+
pullAudio(frames) {
|
|
942
|
+
const byteLen = frames * 2;
|
|
943
|
+
const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [byteLen]);
|
|
944
|
+
this.mod.ccall("goss_session_pull_audio", "number", ["number", "number", "number"], [this.handle, ptr, frames]);
|
|
945
|
+
const out = new Int16Array(this.mod.HEAP16.buffer, ptr, frames).slice();
|
|
946
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [ptr, byteLen]);
|
|
947
|
+
return out;
|
|
948
|
+
}
|
|
949
|
+
/// Folds the active lens sound into the caller's outgoing call/live track:
|
|
950
|
+
/// `mic` (interleaved f32 at `sampleRate`/`channels`, or null for silence)
|
|
951
|
+
/// summed with the 48 kHz mono lens mixer resampled to that rate; returns the
|
|
952
|
+
/// mixed interleaved s16. Advances the mixer once, replacing `pullAudio`.
|
|
953
|
+
mixOutputAudio(mic, frameCount, sampleRate, channels) {
|
|
954
|
+
const outLen = frameCount * channels;
|
|
955
|
+
const outBytes = outLen * 2;
|
|
956
|
+
const outPtr = this.mod.ccall("goss_alloc", "number", ["number"], [outBytes]);
|
|
957
|
+
let micPtr = 0;
|
|
958
|
+
const micBytes = mic ? mic.length * 4 : 0;
|
|
959
|
+
if (mic) {
|
|
960
|
+
micPtr = this.mod.ccall("goss_alloc", "number", ["number"], [micBytes]);
|
|
961
|
+
this.mod.HEAPF32.set(mic, micPtr >> 2);
|
|
962
|
+
}
|
|
963
|
+
this.mod.ccall("goss_session_mix_output_audio", "number", ["number", "number", "number", "number", "number", "number"], [this.handle, micPtr, outPtr, frameCount, sampleRate, channels]);
|
|
964
|
+
const out = new Int16Array(this.mod.HEAP16.buffer, outPtr, outLen).slice();
|
|
965
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [outPtr, outBytes]);
|
|
966
|
+
if (mic)
|
|
967
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [micPtr, micBytes]);
|
|
968
|
+
return out;
|
|
969
|
+
}
|
|
970
|
+
/// Stores validated camera-hardware intent; the engine normalizes it. Read it
|
|
971
|
+
/// back with `cameraControls` and apply it via getUserMedia track constraints.
|
|
972
|
+
setCameraControls(c) {
|
|
973
|
+
const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [56]);
|
|
974
|
+
const w = ptr >> 2;
|
|
975
|
+
this.mod.HEAP32[w] = c.flashMode;
|
|
976
|
+
this.mod.HEAP32[w + 1] = c.torch;
|
|
977
|
+
this.mod.HEAP32[w + 2] = c.focusMode;
|
|
978
|
+
this.mod.HEAP32[w + 3] = c.exposureMode;
|
|
979
|
+
this.mod.HEAPF32[w + 4] = c.focusPointX;
|
|
980
|
+
this.mod.HEAPF32[w + 5] = c.focusPointY;
|
|
981
|
+
this.mod.HEAP32[w + 6] = c.exposureLinked;
|
|
982
|
+
this.mod.HEAPF32[w + 7] = c.exposurePointX;
|
|
983
|
+
this.mod.HEAPF32[w + 8] = c.exposurePointY;
|
|
984
|
+
this.mod.HEAPF32[w + 9] = c.exposureBiasEv;
|
|
985
|
+
this.mod.HEAPF32[w + 10] = c.zoomFactor;
|
|
986
|
+
this.mod.HEAPF32[w + 11] = c.maxZoomFactor;
|
|
987
|
+
this.mod.HEAP32[w + 12] = c.mirrorSavePolicy;
|
|
988
|
+
this.mod.HEAP32[w + 13] = 0;
|
|
989
|
+
this.mod.ccall("goss_session_set_camera_controls", "number", ["number", "number"], [this.handle, ptr]);
|
|
990
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [ptr, 56]);
|
|
991
|
+
}
|
|
992
|
+
/// The normalized camera controls for the page to apply to the media track.
|
|
993
|
+
cameraControls() {
|
|
994
|
+
const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [56]);
|
|
995
|
+
this.mod.ccall("goss_session_camera_controls", "number", ["number", "number"], [this.handle, ptr]);
|
|
996
|
+
const w = ptr >> 2;
|
|
997
|
+
const c = {
|
|
998
|
+
flashMode: this.mod.HEAP32[w], torch: this.mod.HEAP32[w + 1],
|
|
999
|
+
focusMode: this.mod.HEAP32[w + 2], exposureMode: this.mod.HEAP32[w + 3],
|
|
1000
|
+
focusPointX: this.mod.HEAPF32[w + 4], focusPointY: this.mod.HEAPF32[w + 5],
|
|
1001
|
+
exposureLinked: this.mod.HEAP32[w + 6],
|
|
1002
|
+
exposurePointX: this.mod.HEAPF32[w + 7], exposurePointY: this.mod.HEAPF32[w + 8],
|
|
1003
|
+
exposureBiasEv: this.mod.HEAPF32[w + 9], zoomFactor: this.mod.HEAPF32[w + 10],
|
|
1004
|
+
maxZoomFactor: this.mod.HEAPF32[w + 11], mirrorSavePolicy: this.mod.HEAP32[w + 12],
|
|
1005
|
+
};
|
|
1006
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [ptr, 56]);
|
|
1007
|
+
return c;
|
|
1008
|
+
}
|
|
1009
|
+
/// Stores the recording policy the app applies to MediaRecorder. The engine
|
|
1010
|
+
/// normalizes it; read it back with `recordingPolicy`.
|
|
1011
|
+
setRecordingPolicy(p) {
|
|
1012
|
+
const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [40]);
|
|
1013
|
+
const w = ptr >> 2;
|
|
1014
|
+
this.mod.HEAP32[w] = p.maxDurationMs;
|
|
1015
|
+
this.mod.HEAP32[w + 1] = p.minClipMs;
|
|
1016
|
+
this.mod.HEAP32[w + 2] = p.segmentMode;
|
|
1017
|
+
this.mod.HEAP32[w + 3] = p.loopPlayback;
|
|
1018
|
+
this.mod.HEAP32[w + 4] = p.speedPreset;
|
|
1019
|
+
this.mod.HEAP32[w + 5] = p.micMuted;
|
|
1020
|
+
this.mod.HEAP32[w + 6] = p.saveOriginal;
|
|
1021
|
+
this.mod.HEAP32[w + 7] = p.stabilization;
|
|
1022
|
+
this.mod.HEAP32[w + 8] = 0;
|
|
1023
|
+
this.mod.HEAP32[w + 9] = 0;
|
|
1024
|
+
this.mod.ccall("goss_session_set_recording_policy", "number", ["number", "number"], [this.handle, ptr]);
|
|
1025
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [ptr, 40]);
|
|
1026
|
+
}
|
|
1027
|
+
recordingPolicy() {
|
|
1028
|
+
const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [40]);
|
|
1029
|
+
this.mod.ccall("goss_session_recording_policy", "number", ["number", "number"], [this.handle, ptr]);
|
|
1030
|
+
const w = ptr >> 2;
|
|
1031
|
+
const p = {
|
|
1032
|
+
maxDurationMs: this.mod.HEAP32[w], minClipMs: this.mod.HEAP32[w + 1],
|
|
1033
|
+
segmentMode: this.mod.HEAP32[w + 2], loopPlayback: this.mod.HEAP32[w + 3],
|
|
1034
|
+
speedPreset: this.mod.HEAP32[w + 4], micMuted: this.mod.HEAP32[w + 5],
|
|
1035
|
+
saveOriginal: this.mod.HEAP32[w + 6], stabilization: this.mod.HEAP32[w + 7],
|
|
1036
|
+
};
|
|
1037
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [ptr, 40]);
|
|
1038
|
+
return p;
|
|
1039
|
+
}
|
|
1040
|
+
/// Stores the capture-UI intent the page renders (grid, timer, night mode, the
|
|
1041
|
+
/// front-screen flash). The engine normalizes it; read it back with `captureUi`.
|
|
1042
|
+
setCaptureUi(u) {
|
|
1043
|
+
const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [40]);
|
|
1044
|
+
const w = ptr >> 2;
|
|
1045
|
+
this.mod.HEAP32[w] = u.gridMode;
|
|
1046
|
+
this.mod.HEAP32[w + 1] = u.levelIndicator;
|
|
1047
|
+
this.mod.HEAP32[w + 2] = u.shutterMode;
|
|
1048
|
+
this.mod.HEAP32[w + 3] = u.countdownS;
|
|
1049
|
+
this.mod.HEAP32[w + 4] = u.nightMode;
|
|
1050
|
+
this.mod.HEAP32[w + 5] = u.screenFlashMode;
|
|
1051
|
+
this.mod.HEAPF32[w + 6] = u.screenFlashIntensity;
|
|
1052
|
+
this.mod.HEAPF32[w + 7] = u.screenFlashWarmth;
|
|
1053
|
+
this.mod.HEAP32[w + 8] = 0;
|
|
1054
|
+
this.mod.HEAP32[w + 9] = 0;
|
|
1055
|
+
this.mod.ccall("goss_session_set_capture_ui", "number", ["number", "number"], [this.handle, ptr]);
|
|
1056
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [ptr, 40]);
|
|
1057
|
+
}
|
|
1058
|
+
captureUi() {
|
|
1059
|
+
const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [40]);
|
|
1060
|
+
this.mod.ccall("goss_session_capture_ui", "number", ["number", "number"], [this.handle, ptr]);
|
|
1061
|
+
const w = ptr >> 2;
|
|
1062
|
+
const u = {
|
|
1063
|
+
gridMode: this.mod.HEAP32[w], levelIndicator: this.mod.HEAP32[w + 1],
|
|
1064
|
+
shutterMode: this.mod.HEAP32[w + 2], countdownS: this.mod.HEAP32[w + 3],
|
|
1065
|
+
nightMode: this.mod.HEAP32[w + 4], screenFlashMode: this.mod.HEAP32[w + 5],
|
|
1066
|
+
screenFlashIntensity: this.mod.HEAPF32[w + 6], screenFlashWarmth: this.mod.HEAPF32[w + 7],
|
|
1067
|
+
};
|
|
1068
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [ptr, 40]);
|
|
1069
|
+
return u;
|
|
1070
|
+
}
|
|
1071
|
+
/// Fires a named event the next `tickLens` delivers to the lens's
|
|
1072
|
+
/// `event('name')` triggers for one tick.
|
|
1073
|
+
fireEvent(name) {
|
|
1074
|
+
const bytes = new TextEncoder().encode(name);
|
|
1075
|
+
if (bytes.length === 0)
|
|
1076
|
+
return;
|
|
1077
|
+
const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [bytes.length]);
|
|
1078
|
+
this.mod.HEAPU8.set(bytes, ptr);
|
|
1079
|
+
this.mod.ccall("goss_session_fire_event", "number", ["number", "number", "number"], [this.handle, ptr, bytes.length]);
|
|
1080
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [ptr, bytes.length]);
|
|
1081
|
+
}
|
|
1082
|
+
/// Allowlists a bring-your-own model by its 32-byte SHA-256 digest, so a net
|
|
1083
|
+
/// whose digest is not listed is refused when a tracker or segmenter is
|
|
1084
|
+
/// enabled. With none set, any model loads. Call before enabling the worker.
|
|
1085
|
+
allowModelDigest(digest) {
|
|
1086
|
+
if (digest.length !== 32)
|
|
1087
|
+
throw new Error("a model digest is 32 bytes");
|
|
1088
|
+
const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [32]);
|
|
1089
|
+
this.mod.HEAPU8.set(digest, ptr);
|
|
1090
|
+
this.mod.ccall("goss_session_allow_model_digest", "number", ["number", "number"], [this.handle, ptr]);
|
|
1091
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [ptr, 32]);
|
|
1092
|
+
}
|
|
1093
|
+
/// Clears the model allowlist; with none set, any model loads again.
|
|
1094
|
+
clearModelAllowlist() {
|
|
1095
|
+
this.mod.ccall("goss_session_clear_model_allowlist", "number", ["number"], [this.handle]);
|
|
1096
|
+
}
|
|
1097
|
+
withName(name, fn) {
|
|
1098
|
+
const bytes = new TextEncoder().encode(name);
|
|
1099
|
+
if (bytes.length === 0)
|
|
1100
|
+
return;
|
|
1101
|
+
const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [bytes.length]);
|
|
1102
|
+
this.mod.HEAPU8.set(bytes, ptr);
|
|
1103
|
+
fn(ptr, bytes.length);
|
|
1104
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [ptr, bytes.length]);
|
|
1105
|
+
}
|
|
1106
|
+
/// Registers a named RGBA source for multi-source composition (Duet, Stitch,
|
|
1107
|
+
/// live grids). The camera is the implicit source 0.
|
|
1108
|
+
defineSource(name) {
|
|
1109
|
+
this.withName(name, (ptr, len) => this.mod.ccall("goss_session_define_source", "number", ["number", "number", "number"], [this.handle, ptr, len]));
|
|
1110
|
+
}
|
|
1111
|
+
removeSource(name) {
|
|
1112
|
+
this.withName(name, (ptr, len) => this.mod.ccall("goss_session_remove_source", "number", ["number", "number", "number"], [this.handle, ptr, len]));
|
|
1113
|
+
}
|
|
1114
|
+
/// Uploads one RGBA/BGRA frame into a named source (pixelFormat 3 BGRA, 4 RGBA).
|
|
1115
|
+
submitSourceFrameRgba(name, rgba, width, height, stride, pixelFormat = 4 /* GossPixelFormat.Rgba8 */) {
|
|
1116
|
+
const byteLen = stride * height;
|
|
1117
|
+
const rgbaPtr = this.mod.ccall("goss_alloc", "number", ["number"], [byteLen]);
|
|
1118
|
+
this.mod.HEAPU8.set(rgba.subarray(0, byteLen), rgbaPtr);
|
|
1119
|
+
this.mod.setValue(this.frameDescPtr, width, "i32");
|
|
1120
|
+
this.mod.setValue(this.frameDescPtr + 4, height, "i32");
|
|
1121
|
+
this.mod.setValue(this.frameDescPtr + 8, pixelFormat, "i32");
|
|
1122
|
+
this.mod.setValue(this.frameDescPtr + 12, 0, "i32");
|
|
1123
|
+
this.mod.setValue(this.frameDescPtr + 16, 1, "i32");
|
|
1124
|
+
this.mod.setValue(this.frameDescPtr + 20, 0, "i32");
|
|
1125
|
+
this.mod.setValue(this.frameDescPtr + 24, 0, "i32");
|
|
1126
|
+
this.mod.setValue(this.frameDescPtr + 28, 0, "i32");
|
|
1127
|
+
this.withName(name, (ptr, len) => this.mod.ccall("goss_session_submit_source_frame_rgba_copy", "number", ["number", "number", "number", "number", "number", "number"], [this.handle, ptr, len, this.frameDescPtr, rgbaPtr, stride]));
|
|
1128
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [rgbaPtr, byteLen]);
|
|
1129
|
+
}
|
|
1130
|
+
/// Arranges the camera and named sources: 0 custom, 1 side-by-side, 2 top-bottom, 3 pip, 4 grid.
|
|
1131
|
+
setLayout(arrangement) {
|
|
1132
|
+
this.mod.ccall("goss_session_set_layout", "number", ["number", "number"], [this.handle, arrangement]);
|
|
1133
|
+
}
|
|
1134
|
+
/// Upper-body pose mode: while enabled the tracked pose reports only the upper
|
|
1135
|
+
/// body; the lower-body joints (knees down) read absent.
|
|
1136
|
+
setPoseUpperBody(enabled) {
|
|
1137
|
+
this.mod.ccall("goss_session_set_pose_upper_body", "number", ["number", "number"], [this.handle, enabled ? 1 : 0]);
|
|
1138
|
+
}
|
|
1139
|
+
clearLayout() {
|
|
1140
|
+
this.mod.ccall("goss_session_clear_layout", "number", ["number"], [this.handle]);
|
|
1141
|
+
}
|
|
1142
|
+
/// Sets a source's composite blend: opacity, key mode (0 none, 1 matte, 2
|
|
1143
|
+
/// chroma), chroma color, similarity. The name "camera" is the base.
|
|
1144
|
+
setSourceComposite(name, opacity = 1, keyMode = 0, chroma = [0, 0, 0], similarity = 0) {
|
|
1145
|
+
this.withName(name, (ptr, len) => this.mod.ccall("goss_session_set_source_composite", "number", ["number", "number", "number", "number", "number", "number", "number", "number", "number"], [this.handle, ptr, len, opacity, keyMode, chroma[0], chroma[1], chroma[2], similarity]));
|
|
1146
|
+
}
|
|
1147
|
+
/// Uploads a per-source matte for key mode 3 (the red channel is the mask), so
|
|
1148
|
+
/// an opaque guest is keyed to a subject without a baked alpha.
|
|
1149
|
+
submitSourceMask(name, rgba, width, height) {
|
|
1150
|
+
const rgbaPtr = this.mod.ccall("goss_alloc", "number", ["number"], [rgba.length]);
|
|
1151
|
+
this.mod.HEAPU8.set(rgba, rgbaPtr);
|
|
1152
|
+
this.withName(name, (ptr, len) => this.mod.ccall("goss_session_submit_source_mask", "number", ["number", "number", "number", "number", "number", "number"], [this.handle, ptr, len, rgbaPtr, width, height]));
|
|
1153
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [rgbaPtr, rgba.length]);
|
|
1154
|
+
}
|
|
1155
|
+
/// Runs the engine's own segmenter on a source's frames so its key mode 3
|
|
1156
|
+
/// matte is computed on-device (a virtual background for a remote guest). The
|
|
1157
|
+
/// model is the selfie/hair net enableSegmentation takes; an empty model tears
|
|
1158
|
+
/// the source segmenter down.
|
|
1159
|
+
enableSourceSegmentation(name, model, threads) {
|
|
1160
|
+
const modelPtr = model.length > 0 ? this.mod.ccall("goss_alloc", "number", ["number"], [model.length]) : 0;
|
|
1161
|
+
if (modelPtr)
|
|
1162
|
+
this.mod.HEAPU8.set(model, modelPtr);
|
|
1163
|
+
this.withName(name, (ptr, len) => this.mod.ccall("goss_session_enable_source_segmentation", "number", ["number", "number", "number", "number", "number", "number"], [this.handle, ptr, len, modelPtr, model.length, threads]));
|
|
1164
|
+
if (modelPtr)
|
|
1165
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [modelPtr, model.length]);
|
|
1166
|
+
}
|
|
1167
|
+
/// Defines a screen-share source whose frame letterboxes to fit its cell.
|
|
1168
|
+
defineScreenShare(name) {
|
|
1169
|
+
this.withName(name, (ptr, len) => this.mod.ccall("goss_session_define_screen_share", "number", ["number", "number", "number"], [this.handle, ptr, len]));
|
|
1170
|
+
}
|
|
1171
|
+
/// Feeds a location fix for on-device geo.in_region membership; the location never leaves the engine.
|
|
1172
|
+
submitLocation(latitude, longitude, accuracyM, timestampUs) {
|
|
1173
|
+
this.mod.ccall("goss_session_submit_location", "number", ["number", "number", "number", "number", "number"], [this.handle, latitude, longitude, accuracyM, timestampUs]);
|
|
1174
|
+
}
|
|
1175
|
+
/// Sets the geofence circle the app derives from a lens's intended place.
|
|
1176
|
+
setGeofence(latitude, longitude, radiusM) {
|
|
1177
|
+
this.mod.ccall("goss_session_set_geofence", "number", ["number", "number", "number", "number"], [this.handle, latitude, longitude, radiusM]);
|
|
1178
|
+
}
|
|
1179
|
+
clearGeofence() {
|
|
1180
|
+
this.mod.ccall("goss_session_clear_geofence", "number", ["number"], [this.handle]);
|
|
1181
|
+
}
|
|
1182
|
+
/// Sets the geofence to an axis-aligned lat/lon box.
|
|
1183
|
+
setGeofenceBBox(minLat, minLon, maxLat, maxLon) {
|
|
1184
|
+
this.mod.ccall("goss_session_set_geofence_bbox", "number", ["number", "number", "number", "number", "number"], [this.handle, minLat, minLon, maxLat, maxLon]);
|
|
1185
|
+
}
|
|
1186
|
+
/// Sets the geofence to a polygon ring of [latitude, longitude] pairs, three
|
|
1187
|
+
/// to 64 vertices.
|
|
1188
|
+
setGeofencePolygon(vertices) {
|
|
1189
|
+
const bytes = vertices.length * 2 * 8;
|
|
1190
|
+
const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [bytes]);
|
|
1191
|
+
const base = ptr >> 3;
|
|
1192
|
+
for (let i = 0; i < vertices.length; i += 1) {
|
|
1193
|
+
this.mod.HEAPF64[base + i * 2] = vertices[i][0];
|
|
1194
|
+
this.mod.HEAPF64[base + i * 2 + 1] = vertices[i][1];
|
|
1195
|
+
}
|
|
1196
|
+
this.mod.ccall("goss_session_set_geofence_polygon", "number", ["number", "number", "number"], [this.handle, ptr, vertices.length]);
|
|
1197
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [ptr, bytes]);
|
|
1198
|
+
}
|
|
1199
|
+
/// Adds a named circular geofence, so a lens fires geo.in_region('name') for
|
|
1200
|
+
/// its own place among several; re-adding a name replaces its region.
|
|
1201
|
+
setNamedGeofence(name, latitude, longitude, radiusM) {
|
|
1202
|
+
const bytes = new TextEncoder().encode(name);
|
|
1203
|
+
if (bytes.length === 0)
|
|
1204
|
+
return;
|
|
1205
|
+
const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [bytes.length]);
|
|
1206
|
+
this.mod.HEAPU8.set(bytes, ptr);
|
|
1207
|
+
this.mod.ccall("goss_session_set_named_geofence", "number", ["number", "number", "number", "number", "number", "number"], [this.handle, ptr, bytes.length, latitude, longitude, radiusM]);
|
|
1208
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [ptr, bytes.length]);
|
|
1209
|
+
}
|
|
1210
|
+
/// Adds a named polygon geofence (a ring of [latitude, longitude] pairs, three
|
|
1211
|
+
/// or more), the non-circular counterpart of setNamedGeofence; re-adding a name
|
|
1212
|
+
/// replaces its region.
|
|
1213
|
+
setNamedGeofencePolygon(name, vertices) {
|
|
1214
|
+
const nameBytes = new TextEncoder().encode(name);
|
|
1215
|
+
if (nameBytes.length === 0)
|
|
1216
|
+
return;
|
|
1217
|
+
const namePtr = this.mod.ccall("goss_alloc", "number", ["number"], [nameBytes.length]);
|
|
1218
|
+
this.mod.HEAPU8.set(nameBytes, namePtr);
|
|
1219
|
+
const coordBytes = vertices.length * 2 * 8;
|
|
1220
|
+
const coordPtr = this.mod.ccall("goss_alloc", "number", ["number"], [coordBytes]);
|
|
1221
|
+
const base = coordPtr >> 3;
|
|
1222
|
+
for (let i = 0; i < vertices.length; i += 1) {
|
|
1223
|
+
this.mod.HEAPF64[base + i * 2] = vertices[i][0];
|
|
1224
|
+
this.mod.HEAPF64[base + i * 2 + 1] = vertices[i][1];
|
|
1225
|
+
}
|
|
1226
|
+
this.mod.ccall("goss_session_set_named_geofence_polygon", "number", ["number", "number", "number", "number", "number"], [this.handle, namePtr, nameBytes.length, coordPtr, vertices.length]);
|
|
1227
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [namePtr, nameBytes.length]);
|
|
1228
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [coordPtr, coordBytes]);
|
|
1229
|
+
}
|
|
1230
|
+
/// Clears every named geofence; the default geofence is untouched.
|
|
1231
|
+
clearNamedGeofences() {
|
|
1232
|
+
this.mod.ccall("goss_session_clear_named_geofences", "number", ["number"], [this.handle]);
|
|
1233
|
+
}
|
|
1234
|
+
/// Sets the worst fix accuracy (meters) that still counts as inside a region;
|
|
1235
|
+
/// zero clears the gate.
|
|
1236
|
+
setGeoAccuracy(maxAccuracyM) {
|
|
1237
|
+
this.mod.ccall("goss_session_set_geo_accuracy", "number", ["number", "number"], [this.handle, maxAccuracyM]);
|
|
1238
|
+
}
|
|
1239
|
+
/// Sets the color and half-width (normalized units) the next stroke opens with.
|
|
1240
|
+
setBrushStyle(r, g, b, a, width) {
|
|
1241
|
+
this.mod.ccall("goss_session_brush_set_style", "number", ["number", "number", "number", "number", "number", "number"], [this.handle, r, g, b, a, width]);
|
|
1242
|
+
}
|
|
1243
|
+
/// Uploads the RGBA sprite a stamp-mode stroke (brush mode 4) lays along its
|
|
1244
|
+
/// length, an emoji or icon the host rasterizes.
|
|
1245
|
+
setBrushStamp(rgba, width, height) {
|
|
1246
|
+
const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [rgba.length]);
|
|
1247
|
+
this.mod.HEAPU8.set(rgba, ptr);
|
|
1248
|
+
this.mod.ccall("goss_session_brush_set_stamp", "number", ["number", "number", "number", "number"], [this.handle, ptr, width, height]);
|
|
1249
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [ptr, rgba.length]);
|
|
1250
|
+
}
|
|
1251
|
+
/// Opens a stroke in the current style. A fresh stroke drops the redo stack.
|
|
1252
|
+
beginStroke() {
|
|
1253
|
+
this.mod.ccall("goss_session_brush_begin", "number", ["number"], [this.handle]);
|
|
1254
|
+
}
|
|
1255
|
+
/// Adds a point to the open stroke, in normalized screen space (0..1).
|
|
1256
|
+
addStrokePoint(x, y) {
|
|
1257
|
+
this.mod.ccall("goss_session_brush_point", "number", ["number", "number", "number"], [this.handle, x, y]);
|
|
1258
|
+
}
|
|
1259
|
+
/// Commits the open stroke. A stroke of fewer than two points is dropped.
|
|
1260
|
+
endStroke() {
|
|
1261
|
+
this.mod.ccall("goss_session_brush_end", "number", ["number"], [this.handle]);
|
|
1262
|
+
}
|
|
1263
|
+
undoStroke() {
|
|
1264
|
+
this.mod.ccall("goss_session_brush_undo", "number", ["number"], [this.handle]);
|
|
1265
|
+
}
|
|
1266
|
+
redoStroke() {
|
|
1267
|
+
this.mod.ccall("goss_session_brush_redo", "number", ["number"], [this.handle]);
|
|
1268
|
+
}
|
|
1269
|
+
clearStrokes() {
|
|
1270
|
+
this.mod.ccall("goss_session_brush_clear", "number", ["number"], [this.handle]);
|
|
1271
|
+
}
|
|
1272
|
+
/// The brush preset the next stroke opens with: 0 pen, 1 highlighter, 2 marker, 3 neon.
|
|
1273
|
+
setBrushMode(mode) {
|
|
1274
|
+
this.mod.ccall("goss_session_brush_set_mode", "number", ["number", "number"], [this.handle, mode]);
|
|
1275
|
+
}
|
|
1276
|
+
/// Erases committed strokes within `radius` (normalized units) of the point
|
|
1277
|
+
/// and returns how many were removed.
|
|
1278
|
+
eraseStrokes(x, y, radius) {
|
|
1279
|
+
const outPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]);
|
|
1280
|
+
this.mod.ccall("goss_session_brush_erase_at", "number", ["number", "number", "number", "number", "number"], [this.handle, x, y, radius, outPtr]);
|
|
1281
|
+
const removed = this.mod.HEAP32[outPtr >> 2];
|
|
1282
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [outPtr, 4]);
|
|
1283
|
+
return removed;
|
|
1284
|
+
}
|
|
1285
|
+
/// The world-anchored brush. Points are pushed in the world frame world
|
|
1286
|
+
/// tracking reports; the engine projects and draws them so a stroke stays
|
|
1287
|
+
/// fixed in the scene. Nothing draws without live world tracking.
|
|
1288
|
+
setARBrushStyle(r, g, b, a, width) {
|
|
1289
|
+
this.mod.ccall("goss_session_ar_brush_set_style", "number", ["number", "number", "number", "number", "number", "number"], [this.handle, r, g, b, a, width]);
|
|
1290
|
+
}
|
|
1291
|
+
setARBrushMode(mode) {
|
|
1292
|
+
this.mod.ccall("goss_session_ar_brush_set_mode", "number", ["number", "number"], [this.handle, mode]);
|
|
1293
|
+
}
|
|
1294
|
+
beginARStroke() {
|
|
1295
|
+
this.mod.ccall("goss_session_ar_brush_begin", "number", ["number"], [this.handle]);
|
|
1296
|
+
}
|
|
1297
|
+
addARStrokePoint(x, y, z) {
|
|
1298
|
+
this.mod.ccall("goss_session_ar_brush_point", "number", ["number", "number", "number", "number"], [this.handle, x, y, z]);
|
|
1299
|
+
}
|
|
1300
|
+
endARStroke() {
|
|
1301
|
+
this.mod.ccall("goss_session_ar_brush_end", "number", ["number"], [this.handle]);
|
|
1302
|
+
}
|
|
1303
|
+
undoARStroke() {
|
|
1304
|
+
this.mod.ccall("goss_session_ar_brush_undo", "number", ["number"], [this.handle]);
|
|
1305
|
+
}
|
|
1306
|
+
clearARStrokes() {
|
|
1307
|
+
this.mod.ccall("goss_session_ar_brush_clear", "number", ["number"], [this.handle]);
|
|
1308
|
+
}
|
|
1309
|
+
/** Feeds one screen touch event so the engine recognizes the gestures a lens
|
|
1310
|
+
* reacts to. phase is 0 began, 1 moved, 2 ended, 3 cancelled; pointerId names
|
|
1311
|
+
* the finger; x and y are normalized 0..1 over the frame. */
|
|
1312
|
+
touch(phase, pointerId, x, y) {
|
|
1313
|
+
this.mod.ccall("goss_session_touch", "number", ["number", "number", "number", "number", "number"], [this.handle, phase, pointerId, x, y]);
|
|
1314
|
+
}
|
|
1315
|
+
/** Drains one haptic a haptic trigger queued this tick, or null when none
|
|
1316
|
+
* remain. Call in a loop after tickLens and buzz the device for each. The
|
|
1317
|
+
* style is 0 light..7 failure; intensity is a 0..1 hint. */
|
|
1318
|
+
pullHaptic() {
|
|
1319
|
+
const stylePtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]);
|
|
1320
|
+
const intensityPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]);
|
|
1321
|
+
const status = this.mod.ccall("goss_session_pull_haptic", "number", ["number", "number", "number"], [this.handle, stylePtr, intensityPtr]);
|
|
1322
|
+
const result = status === 0 ? { style: this.mod.getValue(stylePtr, "i32"), intensity: this.mod.getValue(intensityPtr, "float") } : null;
|
|
1323
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [stylePtr, 4]);
|
|
1324
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [intensityPtr, 4]);
|
|
1325
|
+
return result;
|
|
1326
|
+
}
|
|
1327
|
+
grab(x, y, z) {
|
|
1328
|
+
this.mod.ccall("goss_session_grab", "number", ["number", "number", "number", "number"], [this.handle, x, y, z]);
|
|
1329
|
+
}
|
|
1330
|
+
release() {
|
|
1331
|
+
this.mod.ccall("goss_session_release", "number", ["number"], [this.handle]);
|
|
1332
|
+
}
|
|
1333
|
+
addCollider(x, y, z) {
|
|
1334
|
+
this.mod.ccall("goss_session_add_collider", "number", ["number", "number", "number", "number"], [this.handle, x, y, z]);
|
|
1335
|
+
}
|
|
1336
|
+
eraseCollider(x, y, z, radius) {
|
|
1337
|
+
this.mod.ccall("goss_session_erase_collider", "number", ["number", "number", "number", "number", "number"], [this.handle, x, y, z, radius]);
|
|
1338
|
+
}
|
|
1339
|
+
/// Releases one solver hair by the id the physics world assigned it,
|
|
1340
|
+
/// pairing the acquire a hair lens performs at activation, so a hair
|
|
1341
|
+
/// can retire mid-session without tearing the physics world down.
|
|
1342
|
+
/// False with no physics world or for an unknown id.
|
|
1343
|
+
physicsHairRemove(hairId) {
|
|
1344
|
+
return this.mod.ccall("goss_physics_hair_remove", "number", ["number", "number"], [this.handle, hairId]) === 0;
|
|
1345
|
+
}
|
|
1346
|
+
/// Pulls the finished brush ribbon (x, y, r, g, b, a per vertex) for the
|
|
1347
|
+
/// renderer. Queries the float count, then reads it out of a scratch buffer.
|
|
1348
|
+
brushVertices() {
|
|
1349
|
+
const countPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]);
|
|
1350
|
+
this.mod.ccall("goss_session_brush_vertices", "number", ["number", "number", "number", "number"], [this.handle, 0, 0, countPtr]);
|
|
1351
|
+
const count = this.mod.HEAP32[countPtr >> 2];
|
|
1352
|
+
if (count <= 0) {
|
|
1353
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [countPtr, 4]);
|
|
1354
|
+
return new Float32Array(0);
|
|
1355
|
+
}
|
|
1356
|
+
const bytes = count * 4;
|
|
1357
|
+
const outPtr = this.mod.ccall("goss_alloc", "number", ["number"], [bytes]);
|
|
1358
|
+
this.mod.ccall("goss_session_brush_vertices", "number", ["number", "number", "number", "number"], [this.handle, outPtr, count, countPtr]);
|
|
1359
|
+
const written = this.mod.HEAP32[countPtr >> 2];
|
|
1360
|
+
const out = new Float32Array(this.mod.HEAPF32.buffer, outPtr, written).slice();
|
|
1361
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [outPtr, bytes]);
|
|
1362
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [countPtr, 4]);
|
|
1363
|
+
return out;
|
|
1364
|
+
}
|
|
1365
|
+
setVideoFlip(enabled) {
|
|
1366
|
+
this.videoFlipped = enabled;
|
|
1367
|
+
}
|
|
1368
|
+
isVideoFlipped() {
|
|
1369
|
+
return this.videoFlipped;
|
|
1370
|
+
}
|
|
1371
|
+
/// landmarks are raw tracker output - x, y in sourceWidth/sourceHeight
|
|
1372
|
+
/// pixels (whatever resolution the caller's own tracking pass ran
|
|
1373
|
+
/// at, which need not match the live video's own resolution), z in
|
|
1374
|
+
/// the same relative scale, three floats per point, matching
|
|
1375
|
+
/// goss_face_result's own convention. Scaled here to the frame
|
|
1376
|
+
/// currently being rendered - the engine's own contour math expects
|
|
1377
|
+
/// "frame pixels" of the frame it's compositing, not of whatever
|
|
1378
|
+
/// analysis resolution tracking happened to use. Null clears
|
|
1379
|
+
/// tracking (no face this frame).
|
|
1380
|
+
setFaceLandmarks(landmarks, sourceWidth, sourceHeight) {
|
|
1381
|
+
if (!landmarks || landmarks.length === 0 || this.frameWidth === 0) {
|
|
1382
|
+
this.mod.ccall("goss_session_set_face_landmarks", "number", ["number", "number", "number"], [this.handle, 0, 0]);
|
|
1383
|
+
return;
|
|
1384
|
+
}
|
|
1385
|
+
const scaleX = this.frameWidth / sourceWidth;
|
|
1386
|
+
const scaleY = this.frameHeight / sourceHeight;
|
|
1387
|
+
const pointCount = landmarks.length / 3;
|
|
1388
|
+
const base = this.landmarksPtr >> 2;
|
|
1389
|
+
for (let at = 0; at < pointCount; at += 1) {
|
|
1390
|
+
this.mod.HEAPF32[base + at * 3] = landmarks[at * 3] * scaleX;
|
|
1391
|
+
this.mod.HEAPF32[base + at * 3 + 1] = landmarks[at * 3 + 1] * scaleY;
|
|
1392
|
+
this.mod.HEAPF32[base + at * 3 + 2] = landmarks[at * 3 + 2];
|
|
1393
|
+
}
|
|
1394
|
+
this.mod.ccall("goss_session_set_face_landmarks", "number", ["number", "number", "number"], [this.handle, this.landmarksPtr, pointCount]);
|
|
1395
|
+
}
|
|
1396
|
+
/// Submits the faces tracked this frame for the multi-face path. landmarks
|
|
1397
|
+
/// are frame pixels, GOSS_FACE_LANDMARK_COUNT * 3 floats; presence defaults
|
|
1398
|
+
/// to 1. An empty array clears the path; faces past GOSS_FACE_MAX or below
|
|
1399
|
+
/// the tracked presence are dropped.
|
|
1400
|
+
submitFaces(faces) {
|
|
1401
|
+
if (faces.length === 0) {
|
|
1402
|
+
this.mod.ccall("goss_session_submit_faces", "number", ["number", "number", "number"], [this.handle, 0, 0]);
|
|
1403
|
+
return;
|
|
1404
|
+
}
|
|
1405
|
+
const bytes = faces.length * FACE_RESULT_BYTES;
|
|
1406
|
+
const ptr = this.scratch(bytes);
|
|
1407
|
+
const dv = new DataView(this.mod.HEAPU8.buffer, ptr, bytes);
|
|
1408
|
+
for (let i = 0; i < faces.length; i += 1) {
|
|
1409
|
+
const off = i * FACE_RESULT_BYTES;
|
|
1410
|
+
const base = ptr + off;
|
|
1411
|
+
const f = faces[i];
|
|
1412
|
+
const count = f.landmarks.length / 3;
|
|
1413
|
+
dv.setBigUint64(off, BigInt(f.frameSerial ?? 0), true);
|
|
1414
|
+
dv.setBigInt64(off + 8, BigInt(f.timestampUs ?? 0), true);
|
|
1415
|
+
dv.setFloat32(off + 16, f.presence ?? 1, true);
|
|
1416
|
+
dv.setUint32(off + 20, count, true);
|
|
1417
|
+
this.mod.HEAPF32.set(f.landmarks, (base + 24) >> 2);
|
|
1418
|
+
const bsStart = (base + 24 + GOSS_FACE_LANDMARK_COUNT * 3 * 4) >> 2;
|
|
1419
|
+
this.mod.HEAPF32.fill(0, bsStart, bsStart + GOSS_FACE_BLENDSHAPE_COUNT);
|
|
1420
|
+
if (f.blendshapes)
|
|
1421
|
+
this.mod.HEAPF32.set(f.blendshapes, bsStart);
|
|
1422
|
+
}
|
|
1423
|
+
this.mod.ccall("goss_session_submit_faces", "number", ["number", "number", "number"], [this.handle, ptr, faces.length]);
|
|
1424
|
+
}
|
|
1425
|
+
/// The number of faces the last submitFaces kept, zero to GOSS_FACE_MAX.
|
|
1426
|
+
faceCount() {
|
|
1427
|
+
const ptr = this.scratch(4);
|
|
1428
|
+
this.mod.ccall("goss_session_face_count", "number", ["number", "number"], [this.handle, ptr]);
|
|
1429
|
+
return this.mod.HEAP32[ptr >> 2];
|
|
1430
|
+
}
|
|
1431
|
+
/// Reads the index-th submitted face, or null once index reaches faceCount,
|
|
1432
|
+
/// so a caller loops zero to faceCount to visit every face.
|
|
1433
|
+
faceResultAt(index) {
|
|
1434
|
+
const ptr = this.scratch(FACE_RESULT_BYTES);
|
|
1435
|
+
const status = this.mod.ccall("goss_session_face_result_at", "number", ["number", "number", "number"], [this.handle, index, ptr]);
|
|
1436
|
+
if (status !== 0)
|
|
1437
|
+
return null;
|
|
1438
|
+
const dv = new DataView(this.mod.HEAPU8.buffer, ptr, FACE_RESULT_BYTES);
|
|
1439
|
+
const lmStart = (ptr + 24) >> 2;
|
|
1440
|
+
const bsStart = lmStart + GOSS_FACE_LANDMARK_COUNT * 3;
|
|
1441
|
+
const out = {
|
|
1442
|
+
frameSerial: dv.getBigUint64(0, true),
|
|
1443
|
+
timestampUs: dv.getBigInt64(8, true),
|
|
1444
|
+
presence: dv.getFloat32(16, true),
|
|
1445
|
+
landmarkCount: dv.getUint32(20, true),
|
|
1446
|
+
landmarks: this.mod.HEAPF32.slice(lmStart, bsStart),
|
|
1447
|
+
blendshapes: this.mod.HEAPF32.slice(bsStart, bsStart + GOSS_FACE_BLENDSHAPE_COUNT),
|
|
1448
|
+
};
|
|
1449
|
+
return out;
|
|
1450
|
+
}
|
|
1451
|
+
/// The stable track id of the index-th face, an integer that stays with the
|
|
1452
|
+
/// same person across frames as the submission order shuffles, or null once
|
|
1453
|
+
/// index reaches faceCount.
|
|
1454
|
+
faceTrackId(index) {
|
|
1455
|
+
const ptr = this.scratch(4);
|
|
1456
|
+
const status = this.mod.ccall("goss_session_face_track_id", "number", ["number", "number", "number"], [this.handle, index, ptr]);
|
|
1457
|
+
if (status !== 0)
|
|
1458
|
+
return null;
|
|
1459
|
+
return this.mod.HEAPU32[ptr >> 2];
|
|
1460
|
+
}
|
|
1461
|
+
/// Submits the bodies tracked this frame for the multi-person path, so a
|
|
1462
|
+
/// lens can instance effects across every body. An empty array clears the
|
|
1463
|
+
/// path; bodies past GOSS_BODY_MAX are ignored.
|
|
1464
|
+
submitBodies(bodies) {
|
|
1465
|
+
if (bodies.length === 0) {
|
|
1466
|
+
this.mod.ccall("goss_session_submit_bodies", "number", ["number", "number", "number"], [this.handle, 0, 0]);
|
|
1467
|
+
return;
|
|
1468
|
+
}
|
|
1469
|
+
const bytes = bodies.length * POSE_RESULT_BYTES;
|
|
1470
|
+
const ptr = this.scratch(bytes);
|
|
1471
|
+
const dv = new DataView(this.mod.HEAPU8.buffer, ptr, bytes);
|
|
1472
|
+
for (let i = 0; i < bodies.length; i += 1) {
|
|
1473
|
+
const off = i * POSE_RESULT_BYTES;
|
|
1474
|
+
const base = ptr + off;
|
|
1475
|
+
const b = bodies[i];
|
|
1476
|
+
const count = b.landmarks.length / 3;
|
|
1477
|
+
dv.setBigUint64(off, BigInt(b.frameSerial ?? 0), true);
|
|
1478
|
+
dv.setBigInt64(off + 8, BigInt(b.timestampUs ?? 0), true);
|
|
1479
|
+
dv.setFloat32(off + 16, b.presence ?? 1, true);
|
|
1480
|
+
dv.setUint32(off + 20, count, true);
|
|
1481
|
+
this.mod.HEAPF32.set(b.landmarks, (base + 24) >> 2);
|
|
1482
|
+
const visStart = (base + 24 + GOSS_POSE_LANDMARK_COUNT * 3 * 4) >> 2;
|
|
1483
|
+
this.mod.HEAPF32.fill(0, visStart, visStart + GOSS_POSE_LANDMARK_COUNT * 2);
|
|
1484
|
+
if (b.visibilities)
|
|
1485
|
+
this.mod.HEAPF32.set(b.visibilities, visStart);
|
|
1486
|
+
if (b.presences)
|
|
1487
|
+
this.mod.HEAPF32.set(b.presences, visStart + GOSS_POSE_LANDMARK_COUNT);
|
|
1488
|
+
}
|
|
1489
|
+
this.mod.ccall("goss_session_submit_bodies", "number", ["number", "number", "number"], [this.handle, ptr, bodies.length]);
|
|
1490
|
+
}
|
|
1491
|
+
/// Submits one frame's depth map from the host AR backend (WebXR depth-
|
|
1492
|
+
/// sensing): width by height metres per pixel, row major, with the near and
|
|
1493
|
+
/// far metres that bound it. An empty array clears it. Kept for depth
|
|
1494
|
+
/// occlusion against the rendered content.
|
|
1495
|
+
submitDepth(depth, width, height, near, far) {
|
|
1496
|
+
if (depth.length === 0) {
|
|
1497
|
+
this.mod.ccall("goss_session_submit_depth", "number", ["number", "number", "number", "number", "number", "number"], [this.handle, 0, 0, 0, 0, 0]);
|
|
1498
|
+
return;
|
|
1499
|
+
}
|
|
1500
|
+
const bytes = depth.length * 4;
|
|
1501
|
+
const ptr = this.scratch(bytes);
|
|
1502
|
+
this.mod.HEAPF32.set(depth, ptr >> 2);
|
|
1503
|
+
this.mod.ccall("goss_session_submit_depth", "number", ["number", "number", "number", "number", "number", "number"], [this.handle, ptr, width, height, near, far]);
|
|
1504
|
+
}
|
|
1505
|
+
/// Submits the camera intrinsics an undistort.pass corrects for: the focal
|
|
1506
|
+
/// lengths and principal point in pixels of the submitted frame, and the
|
|
1507
|
+
/// radial distortion coefficients (k1, k2 read). An empty array or zero focal
|
|
1508
|
+
/// length clears them, leaving an undistort.pass inert.
|
|
1509
|
+
submitCameraIntrinsics(fx, fy, cx, cy, distortion) {
|
|
1510
|
+
if (distortion.length === 0) {
|
|
1511
|
+
this.mod.ccall("goss_session_submit_camera_intrinsics", "number", ["number", "number", "number", "number", "number", "number", "number"], [this.handle, 0, 0, 0, 0, 0, 0]);
|
|
1512
|
+
return;
|
|
1513
|
+
}
|
|
1514
|
+
const ptr = this.scratch(distortion.length * 4);
|
|
1515
|
+
this.mod.HEAPF32.set(distortion, ptr >> 2);
|
|
1516
|
+
this.mod.ccall("goss_session_submit_camera_intrinsics", "number", ["number", "number", "number", "number", "number", "number", "number"], [this.handle, fx, fy, cx, cy, ptr, distortion.length]);
|
|
1517
|
+
}
|
|
1518
|
+
/// Submits one device gravity sample with its timestamp in microseconds. A
|
|
1519
|
+
/// rolling.pass reads the image-plane motion derived from consecutive samples
|
|
1520
|
+
/// to correct rolling-shutter skew; feed one per frame from the IMU. A
|
|
1521
|
+
/// near-zero vector clears the stream, leaving a rolling.pass inert.
|
|
1522
|
+
submitOrientation(gravityX, gravityY, gravityZ, timestampUs = 0n) {
|
|
1523
|
+
this.mod.ccall("goss_session_submit_orientation", "number", ["number", "number", "number", "number", "number"], [this.handle, gravityX, gravityY, gravityZ, timestampUs]);
|
|
1524
|
+
}
|
|
1525
|
+
/// Feeds a host info value keyed by name, the rail an info sticker reads: a
|
|
1526
|
+
/// text.2d node with a matching content_source shows the latest value each
|
|
1527
|
+
/// frame (a time, a place, a sensor reading). A null value clears the key.
|
|
1528
|
+
setInfo(key, value) {
|
|
1529
|
+
const keyBytes = new TextEncoder().encode(key);
|
|
1530
|
+
const keyPtr = this.mod.ccall("goss_alloc", "number", ["number"], [keyBytes.length || 1]);
|
|
1531
|
+
this.mod.HEAPU8.set(keyBytes, keyPtr);
|
|
1532
|
+
let valuePtr = 0;
|
|
1533
|
+
let valueLen = 0;
|
|
1534
|
+
let valueBytes = null;
|
|
1535
|
+
if (value !== null) {
|
|
1536
|
+
valueBytes = new TextEncoder().encode(value);
|
|
1537
|
+
valueLen = valueBytes.length;
|
|
1538
|
+
valuePtr = this.mod.ccall("goss_alloc", "number", ["number"], [valueLen || 1]);
|
|
1539
|
+
this.mod.HEAPU8.set(valueBytes, valuePtr);
|
|
1540
|
+
}
|
|
1541
|
+
this.mod.ccall("goss_session_set_info", "number", ["number", "number", "number", "number", "number"], [this.handle, keyPtr, keyBytes.length, valuePtr, valueLen]);
|
|
1542
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [keyPtr, keyBytes.length || 1]);
|
|
1543
|
+
if (valuePtr !== 0)
|
|
1544
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [valuePtr, valueLen || 1]);
|
|
1545
|
+
}
|
|
1546
|
+
/// Serializes the active lens's parameter state to a blob a connected lens
|
|
1547
|
+
/// publishes so the cloud syncs it to peers, or null with no lens.
|
|
1548
|
+
snapshotLensState() {
|
|
1549
|
+
const lenPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]);
|
|
1550
|
+
const view = () => new DataView(this.mod.HEAPU8.buffer, lenPtr, 4).getUint32(0, true);
|
|
1551
|
+
let out = null;
|
|
1552
|
+
if (this.mod.ccall("goss_session_snapshot_lens_state", "number", ["number", "number", "number", "number"], [this.handle, 0, 0, lenPtr]) === 0) {
|
|
1553
|
+
const needed = view();
|
|
1554
|
+
const outPtr = this.mod.ccall("goss_alloc", "number", ["number"], [needed || 1]);
|
|
1555
|
+
if (this.mod.ccall("goss_session_snapshot_lens_state", "number", ["number", "number", "number", "number"], [this.handle, outPtr, needed, lenPtr]) === 0) {
|
|
1556
|
+
out = this.mod.HEAPU8.slice(outPtr, outPtr + view());
|
|
1557
|
+
}
|
|
1558
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [outPtr, needed || 1]);
|
|
1559
|
+
}
|
|
1560
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [lenPtr, 4]);
|
|
1561
|
+
return out;
|
|
1562
|
+
}
|
|
1563
|
+
/// Applies a peer's lens-state blob to the active lens, clamping each value
|
|
1564
|
+
/// into its parameter so two runtimes on the same lens converge.
|
|
1565
|
+
applyLensState(blob) {
|
|
1566
|
+
const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [blob.length || 1]);
|
|
1567
|
+
this.mod.HEAPU8.set(blob, ptr);
|
|
1568
|
+
this.mod.ccall("goss_session_apply_lens_state", "number", ["number", "number", "number"], [this.handle, ptr, blob.length]);
|
|
1569
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [ptr, blob.length || 1]);
|
|
1570
|
+
}
|
|
1571
|
+
/// The active lens's content-provenance manifest as JSON (producer, lens,
|
|
1572
|
+
/// whether the frame is model-generated or edited, and the operations), for the
|
|
1573
|
+
/// host to bind to a capture per C2PA. Null with no active lens.
|
|
1574
|
+
captureProvenance() {
|
|
1575
|
+
const lenPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]);
|
|
1576
|
+
const view = () => new DataView(this.mod.HEAPU8.buffer, lenPtr, 4).getUint32(0, true);
|
|
1577
|
+
let json = null;
|
|
1578
|
+
if (this.mod.ccall("goss_session_capture_provenance", "number", ["number", "number", "number", "number"], [this.handle, 0, 0, lenPtr]) === 0) {
|
|
1579
|
+
const needed = view();
|
|
1580
|
+
const outPtr = this.mod.ccall("goss_alloc", "number", ["number"], [needed || 1]);
|
|
1581
|
+
if (this.mod.ccall("goss_session_capture_provenance", "number", ["number", "number", "number", "number"], [this.handle, outPtr, needed, lenPtr]) === 0) {
|
|
1582
|
+
json = new TextDecoder().decode(this.mod.HEAPU8.slice(outPtr, outPtr + view()));
|
|
1583
|
+
}
|
|
1584
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [outPtr, needed || 1]);
|
|
1585
|
+
}
|
|
1586
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [lenPtr, 4]);
|
|
1587
|
+
return json;
|
|
1588
|
+
}
|
|
1589
|
+
/// Captures the current viewpoint (the last submitted world pose and depth) into
|
|
1590
|
+
/// a guided scan, back-projecting the depth into a deterministic gaussian
|
|
1591
|
+
/// reconstruction, and returns the scan's coverage so the app can steer the user
|
|
1592
|
+
/// to the next uncovered viewpoint. Reset with resetCapture.
|
|
1593
|
+
captureView() {
|
|
1594
|
+
const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [24]);
|
|
1595
|
+
this.mod.ccall("goss_session_capture_view", "number", ["number", "number"], [this.handle, ptr]);
|
|
1596
|
+
const guidance = {
|
|
1597
|
+
covered: this.mod.getValue(ptr, "i32") >>> 0,
|
|
1598
|
+
total: this.mod.getValue(ptr + 4, "i32") >>> 0,
|
|
1599
|
+
complete: this.mod.getValue(ptr + 8, "i32") !== 0,
|
|
1600
|
+
viewCount: this.mod.getValue(ptr + 12, "i32") >>> 0,
|
|
1601
|
+
splatCount: this.mod.getValue(ptr + 16, "i32") >>> 0,
|
|
1602
|
+
nextYaw: this.mod.getValue(ptr + 20, "float"),
|
|
1603
|
+
};
|
|
1604
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [ptr, 24]);
|
|
1605
|
+
return guidance;
|
|
1606
|
+
}
|
|
1607
|
+
/// Clears a guided-capture scan: its covered targets, poses, and reconstruction.
|
|
1608
|
+
resetCapture() {
|
|
1609
|
+
this.mod.ccall("goss_session_reset_capture", "number", ["number"], [this.handle]);
|
|
1610
|
+
}
|
|
1611
|
+
/// Enables or disables on-device dubbing: when on, a dub-bound audio.infer node
|
|
1612
|
+
/// synthesizes its decoded caption or translation to speech and plays it into
|
|
1613
|
+
/// the lens mixer. Off by default; a host turns it on for a voice-over.
|
|
1614
|
+
setDubbing(enabled) {
|
|
1615
|
+
this.mod.ccall("goss_session_set_dubbing", "number", ["number", "number"], [this.handle, enabled ? 1 : 0]);
|
|
1616
|
+
}
|
|
1617
|
+
/// The latest caption an audio.infer node decoded, by the node's id, or null
|
|
1618
|
+
/// when that node has no caption binding or nothing decoded yet. On-device ASR
|
|
1619
|
+
/// the app can draw as a live subtitle. A length probe sizes the buffer.
|
|
1620
|
+
captionText(nodeId) {
|
|
1621
|
+
const id = new TextEncoder().encode(nodeId);
|
|
1622
|
+
const idPtr = this.mod.ccall("goss_alloc", "number", ["number"], [id.length || 1]);
|
|
1623
|
+
this.mod.HEAPU8.set(id, idPtr);
|
|
1624
|
+
const lenPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]);
|
|
1625
|
+
const readLen = () => new DataView(this.mod.HEAPU8.buffer, lenPtr, 4).getUint32(0, true);
|
|
1626
|
+
const args = ["number", "number", "number", "number", "number", "number"];
|
|
1627
|
+
const probe = this.mod.ccall("goss_session_caption_text", "number", args, [this.handle, idPtr, id.length, 0, 0, lenPtr]);
|
|
1628
|
+
const needed = readLen();
|
|
1629
|
+
let result = null;
|
|
1630
|
+
if (probe === 0 && needed > 0) {
|
|
1631
|
+
const outPtr = this.mod.ccall("goss_alloc", "number", ["number"], [needed]);
|
|
1632
|
+
const status = this.mod.ccall("goss_session_caption_text", "number", args, [this.handle, idPtr, id.length, outPtr, needed, lenPtr]);
|
|
1633
|
+
const written = readLen();
|
|
1634
|
+
if (status === 0)
|
|
1635
|
+
result = new TextDecoder().decode(this.mod.HEAPU8.slice(outPtr, outPtr + written));
|
|
1636
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [outPtr, needed]);
|
|
1637
|
+
}
|
|
1638
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [lenPtr, 4]);
|
|
1639
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [idPtr, id.length || 1]);
|
|
1640
|
+
return result;
|
|
1641
|
+
}
|
|
1642
|
+
/// The recent diarized caption segment at index (0 the newest), or null when
|
|
1643
|
+
/// the index is past the segments held: the times it spanned, the speaker who
|
|
1644
|
+
/// spoke it, and its text, a speaker-tagged transcript for diarized subtitles.
|
|
1645
|
+
captionSegment(index) {
|
|
1646
|
+
const segPtr = this.mod.ccall("goss_alloc", "number", ["number"], [24]);
|
|
1647
|
+
const status = this.mod.ccall("goss_session_caption_segment", "number", ["number", "number", "number"], [this.handle, index, segPtr]);
|
|
1648
|
+
if (status !== 0) {
|
|
1649
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [segPtr, 24]);
|
|
1650
|
+
return null;
|
|
1651
|
+
}
|
|
1652
|
+
const dv = new DataView(this.mod.HEAPU8.buffer, segPtr, 24);
|
|
1653
|
+
const startUs = dv.getBigInt64(0, true);
|
|
1654
|
+
const endUs = dv.getBigInt64(8, true);
|
|
1655
|
+
const speaker = dv.getUint32(16, true);
|
|
1656
|
+
const textLen = dv.getUint32(20, true);
|
|
1657
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [segPtr, 24]);
|
|
1658
|
+
let text = "";
|
|
1659
|
+
if (textLen > 0) {
|
|
1660
|
+
const lenPtr = this.mod.ccall("goss_alloc", "number", ["number"], [4]);
|
|
1661
|
+
const outPtr = this.mod.ccall("goss_alloc", "number", ["number"], [textLen]);
|
|
1662
|
+
const st = this.mod.ccall("goss_session_caption_segment_text", "number", ["number", "number", "number", "number", "number"], [this.handle, index, outPtr, textLen, lenPtr]);
|
|
1663
|
+
const written = new DataView(this.mod.HEAPU8.buffer, lenPtr, 4).getUint32(0, true);
|
|
1664
|
+
if (st === 0)
|
|
1665
|
+
text = new TextDecoder().decode(this.mod.HEAPU8.slice(outPtr, outPtr + written));
|
|
1666
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [outPtr, textLen]);
|
|
1667
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [lenPtr, 4]);
|
|
1668
|
+
}
|
|
1669
|
+
return { startUs, endUs, speaker, text };
|
|
1670
|
+
}
|
|
1671
|
+
/// Segments a host-provided still image through the running segmenter: rgba
|
|
1672
|
+
/// is width by height RGBA8 pixels, row major. The mask reaches the active
|
|
1673
|
+
/// lens the way a camera frame's would.
|
|
1674
|
+
submitSegmentationImage(rgba, width, height) {
|
|
1675
|
+
const ptr = this.scratch(rgba.length);
|
|
1676
|
+
this.mod.HEAPU8.set(rgba, ptr);
|
|
1677
|
+
this.mod.ccall("goss_session_submit_segmentation_image", "number", ["number", "number", "number", "number"], [this.handle, ptr, width, height]);
|
|
1678
|
+
}
|
|
1679
|
+
/// Submits one RGBA exposure of an HDR bracket (width by height RGBA8, row
|
|
1680
|
+
/// major), converted to NV12 and fed to bracket-source temporal.fuse nodes; the
|
|
1681
|
+
/// fusion publishes once the ring holds a full bracket.
|
|
1682
|
+
submitFrameBracketRgba(rgba, width, height) {
|
|
1683
|
+
const ptr = this.scratch(rgba.length);
|
|
1684
|
+
this.mod.HEAPU8.set(rgba, ptr);
|
|
1685
|
+
this.mod.ccall("goss_session_submit_frame_bracket_rgba", "number", ["number", "number", "number", "number"], [this.handle, ptr, width, height]);
|
|
1686
|
+
}
|
|
1687
|
+
/// Submits one NV12 exposure of an HDR bracket, fed to bracket-source
|
|
1688
|
+
/// temporal.fuse nodes. y and uv are the plane bytes as submitFrameCopy takes.
|
|
1689
|
+
submitFrameBracket(y, yStride, uv, uvStride, width, height, colorStandard = 1, colorRange = 0) {
|
|
1690
|
+
const yPtr = this.mod.ccall("goss_alloc", "number", ["number"], [y.length]);
|
|
1691
|
+
this.mod.HEAPU8.set(y, yPtr);
|
|
1692
|
+
const uvPtr = this.mod.ccall("goss_alloc", "number", ["number"], [uv.length]);
|
|
1693
|
+
this.mod.HEAPU8.set(uv, uvPtr);
|
|
1694
|
+
this.mod.setValue(this.frameDescPtr, width, "i32");
|
|
1695
|
+
this.mod.setValue(this.frameDescPtr + 4, height, "i32");
|
|
1696
|
+
this.mod.setValue(this.frameDescPtr + 8, 0, "i32");
|
|
1697
|
+
this.mod.setValue(this.frameDescPtr + 12, colorStandard, "i32");
|
|
1698
|
+
this.mod.setValue(this.frameDescPtr + 16, colorRange, "i32");
|
|
1699
|
+
this.mod.setValue(this.frameDescPtr + 20, 0, "i32");
|
|
1700
|
+
this.mod.setValue(this.frameDescPtr + 24, 0, "i32");
|
|
1701
|
+
this.mod.setValue(this.frameDescPtr + 28, 0, "i32");
|
|
1702
|
+
this.mod.ccall("goss_session_submit_frame_bracket", "number", ["number", "number", "number", "number", "number", "number"], [this.handle, this.frameDescPtr, yPtr, yStride, uvPtr, uvStride]);
|
|
1703
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [yPtr, y.length]);
|
|
1704
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [uvPtr, uv.length]);
|
|
1705
|
+
}
|
|
1706
|
+
submitPlanes(sym, y, yStride, uv, uvStride, width, height, colorStandard, colorRange) {
|
|
1707
|
+
const yPtr = this.mod.ccall("goss_alloc", "number", ["number"], [y.length]);
|
|
1708
|
+
this.mod.HEAPU8.set(y, yPtr);
|
|
1709
|
+
const uvPtr = this.mod.ccall("goss_alloc", "number", ["number"], [uv.length]);
|
|
1710
|
+
this.mod.HEAPU8.set(uv, uvPtr);
|
|
1711
|
+
this.mod.setValue(this.frameDescPtr, width, "i32");
|
|
1712
|
+
this.mod.setValue(this.frameDescPtr + 4, height, "i32");
|
|
1713
|
+
this.mod.setValue(this.frameDescPtr + 8, 0, "i32");
|
|
1714
|
+
this.mod.setValue(this.frameDescPtr + 12, colorStandard, "i32");
|
|
1715
|
+
this.mod.setValue(this.frameDescPtr + 16, colorRange, "i32");
|
|
1716
|
+
this.mod.setValue(this.frameDescPtr + 20, 0, "i32");
|
|
1717
|
+
this.mod.setValue(this.frameDescPtr + 24, 0, "i32");
|
|
1718
|
+
this.mod.setValue(this.frameDescPtr + 28, 0, "i32");
|
|
1719
|
+
this.mod.ccall(sym, "number", ["number", "number", "number", "number", "number", "number"], [this.handle, this.frameDescPtr, yPtr, yStride, uvPtr, uvStride]);
|
|
1720
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [yPtr, y.length]);
|
|
1721
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [uvPtr, uv.length]);
|
|
1722
|
+
}
|
|
1723
|
+
/// Feeds one NV12 frame to the in-engine trackers only (no render), for a host
|
|
1724
|
+
/// that renders elsewhere but wants the engine's face/hand/pose tracking. Web
|
|
1725
|
+
/// usually tracks in its own worker and feeds results through the producer path.
|
|
1726
|
+
trackFrame(y, yStride, uv, uvStride, width, height, colorStandard = 1, colorRange = 0) {
|
|
1727
|
+
this.submitPlanes("goss_session_track_frame", y, yStride, uv, uvStride, width, height, colorStandard, colorRange);
|
|
1728
|
+
}
|
|
1729
|
+
/// Submits one NV12 frame the engine copies into its own buffer (the caller may
|
|
1730
|
+
/// reuse the planes at once), the copy counterpart of submitFrameRgbaCopy.
|
|
1731
|
+
submitFrameCopy(y, yStride, uv, uvStride, width, height, colorStandard = 1, colorRange = 0) {
|
|
1732
|
+
this.submitPlanes("goss_session_submit_frame_copy", y, yStride, uv, uvStride, width, height, colorStandard, colorRange);
|
|
1733
|
+
}
|
|
1734
|
+
/// Submits one NV12 frame of a driving performance for avatar reenactment, so
|
|
1735
|
+
/// the tracked avatar is posed from this source instead of the camera.
|
|
1736
|
+
submitAvatarSource(y, yStride, uv, uvStride, width, height, colorStandard = 1, colorRange = 0) {
|
|
1737
|
+
this.submitPlanes("goss_session_submit_avatar_source", y, yStride, uv, uvStride, width, height, colorStandard, colorRange);
|
|
1738
|
+
}
|
|
1739
|
+
/// Reads the in-engine face tracker's single-face result, or null when no face
|
|
1740
|
+
/// is tracked; faceResultAt visits every face on the multi-face path.
|
|
1741
|
+
faceResult() {
|
|
1742
|
+
const ptr = this.scratch(FACE_RESULT_BYTES);
|
|
1743
|
+
if (this.mod.ccall("goss_session_face_result", "number", ["number", "number"], [this.handle, ptr]) !== 0)
|
|
1744
|
+
return null;
|
|
1745
|
+
const dv = new DataView(this.mod.HEAPU8.buffer, ptr, FACE_RESULT_BYTES);
|
|
1746
|
+
const lmStart = (ptr + 24) >> 2;
|
|
1747
|
+
const bsStart = lmStart + GOSS_FACE_LANDMARK_COUNT * 3;
|
|
1748
|
+
return {
|
|
1749
|
+
frameSerial: dv.getBigUint64(0, true),
|
|
1750
|
+
timestampUs: dv.getBigInt64(8, true),
|
|
1751
|
+
presence: dv.getFloat32(16, true),
|
|
1752
|
+
landmarkCount: dv.getUint32(20, true),
|
|
1753
|
+
landmarks: this.mod.HEAPF32.slice(lmStart, bsStart),
|
|
1754
|
+
blendshapes: this.mod.HEAPF32.slice(bsStart, bsStart + GOSS_FACE_BLENDSHAPE_COUNT),
|
|
1755
|
+
};
|
|
1756
|
+
}
|
|
1757
|
+
/// The tracked head pose as a 4x4 column-major matrix (16 floats), or null when
|
|
1758
|
+
/// no face is tracked, for anchoring 3D content to the head.
|
|
1759
|
+
facePose() {
|
|
1760
|
+
const ptr = this.scratch(64);
|
|
1761
|
+
if (this.mod.ccall("goss_session_face_pose", "number", ["number", "number"], [this.handle, ptr]) !== 0)
|
|
1762
|
+
return null;
|
|
1763
|
+
return this.mod.HEAPF32.slice(ptr >> 2, (ptr >> 2) + 16);
|
|
1764
|
+
}
|
|
1765
|
+
/// Reads the in-engine hand tracker's result, or null when no hand is tracked;
|
|
1766
|
+
/// hands past handCount are omitted.
|
|
1767
|
+
handResult() {
|
|
1768
|
+
const ptr = this.scratch(HAND_RESULT_BYTES);
|
|
1769
|
+
if (this.mod.ccall("goss_session_hand_result", "number", ["number", "number"], [this.handle, ptr]) !== 0)
|
|
1770
|
+
return null;
|
|
1771
|
+
const dv = new DataView(this.mod.HEAPU8.buffer, ptr, HAND_RESULT_BYTES);
|
|
1772
|
+
const count = dv.getUint32(16, true);
|
|
1773
|
+
const hands = [];
|
|
1774
|
+
for (let i = 0; i < count && i < GOSS_HAND_MAX; i += 1) {
|
|
1775
|
+
const base = ptr + 24 + i * HAND_ONE_BYTES;
|
|
1776
|
+
const lm = (base + 16) >> 2;
|
|
1777
|
+
hands.push({
|
|
1778
|
+
presence: dv.getFloat32(24 + i * HAND_ONE_BYTES, true),
|
|
1779
|
+
handedness: dv.getFloat32(24 + i * HAND_ONE_BYTES + 4, true),
|
|
1780
|
+
gesture: dv.getUint32(24 + i * HAND_ONE_BYTES + 8, true),
|
|
1781
|
+
gestureScore: dv.getFloat32(24 + i * HAND_ONE_BYTES + 12, true),
|
|
1782
|
+
landmarks: this.mod.HEAPF32.slice(lm, lm + GOSS_HAND_LANDMARK_COUNT * 3),
|
|
1783
|
+
});
|
|
1784
|
+
}
|
|
1785
|
+
return { frameSerial: dv.getBigUint64(0, true), timestampUs: dv.getBigInt64(8, true), hands };
|
|
1786
|
+
}
|
|
1787
|
+
/// Reads the in-engine pose tracker's single-body result, or null when no body
|
|
1788
|
+
/// is tracked; bodyResultAt visits every body on the multi-person path.
|
|
1789
|
+
poseResult() {
|
|
1790
|
+
const ptr = this.scratch(POSE_RESULT_BYTES);
|
|
1791
|
+
if (this.mod.ccall("goss_session_pose_result", "number", ["number", "number"], [this.handle, ptr]) !== 0)
|
|
1792
|
+
return null;
|
|
1793
|
+
const dv = new DataView(this.mod.HEAPU8.buffer, ptr, POSE_RESULT_BYTES);
|
|
1794
|
+
const lmStart = (ptr + 24) >> 2;
|
|
1795
|
+
const visStart = lmStart + GOSS_POSE_LANDMARK_COUNT * 3;
|
|
1796
|
+
const presStart = visStart + GOSS_POSE_LANDMARK_COUNT;
|
|
1797
|
+
return {
|
|
1798
|
+
frameSerial: dv.getBigUint64(0, true),
|
|
1799
|
+
timestampUs: dv.getBigInt64(8, true),
|
|
1800
|
+
presence: dv.getFloat32(16, true),
|
|
1801
|
+
landmarkCount: dv.getUint32(20, true),
|
|
1802
|
+
landmarks: this.mod.HEAPF32.slice(lmStart, visStart),
|
|
1803
|
+
visibilities: this.mod.HEAPF32.slice(visStart, presStart),
|
|
1804
|
+
presences: this.mod.HEAPF32.slice(presStart, presStart + GOSS_POSE_LANDMARK_COUNT),
|
|
1805
|
+
};
|
|
1806
|
+
}
|
|
1807
|
+
/// Samples a reference photo's makeup color per face part, so a tint.pass
|
|
1808
|
+
/// with a reference source paints the live face in that color. rgba is width
|
|
1809
|
+
/// by height RGBA8; landmarks is the reference face's 478 x, y, z points. An
|
|
1810
|
+
/// empty landmarks array clears the reference.
|
|
1811
|
+
setMakeupReference(rgba, width, height, landmarks) {
|
|
1812
|
+
if (landmarks.length === 0) {
|
|
1813
|
+
this.mod.ccall("goss_session_set_makeup_reference", "number", ["number", "number", "number", "number", "number", "number"], [this.handle, 0, 0, 0, 0, 0]);
|
|
1814
|
+
return;
|
|
1815
|
+
}
|
|
1816
|
+
const rptr = this.mod.ccall("goss_alloc", "number", ["number"], [rgba.length]);
|
|
1817
|
+
this.mod.HEAPU8.set(rgba, rptr);
|
|
1818
|
+
const lbytes = landmarks.length * 4;
|
|
1819
|
+
const lptr = this.mod.ccall("goss_alloc", "number", ["number"], [lbytes]);
|
|
1820
|
+
this.mod.HEAPF32.set(landmarks, lptr >> 2);
|
|
1821
|
+
this.mod.ccall("goss_session_set_makeup_reference", "number", ["number", "number", "number", "number", "number", "number"], [this.handle, rptr, width, height, lptr, landmarks.length / 3]);
|
|
1822
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [rptr, rgba.length]);
|
|
1823
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [lptr, lbytes]);
|
|
1824
|
+
}
|
|
1825
|
+
/// The number of bodies the last submitBodies kept, zero to GOSS_BODY_MAX.
|
|
1826
|
+
bodyCount() {
|
|
1827
|
+
const ptr = this.scratch(4);
|
|
1828
|
+
this.mod.ccall("goss_session_body_count", "number", ["number", "number"], [this.handle, ptr]);
|
|
1829
|
+
return this.mod.HEAP32[ptr >> 2];
|
|
1830
|
+
}
|
|
1831
|
+
/// Reads the index-th submitted body, or null once index reaches bodyCount,
|
|
1832
|
+
/// so a caller loops zero to bodyCount to visit every body.
|
|
1833
|
+
bodyResultAt(index) {
|
|
1834
|
+
const ptr = this.scratch(POSE_RESULT_BYTES);
|
|
1835
|
+
const status = this.mod.ccall("goss_session_body_result_at", "number", ["number", "number", "number"], [this.handle, index, ptr]);
|
|
1836
|
+
if (status !== 0)
|
|
1837
|
+
return null;
|
|
1838
|
+
const dv = new DataView(this.mod.HEAPU8.buffer, ptr, POSE_RESULT_BYTES);
|
|
1839
|
+
const lmStart = (ptr + 24) >> 2;
|
|
1840
|
+
const visStart = lmStart + GOSS_POSE_LANDMARK_COUNT * 3;
|
|
1841
|
+
const presStart = visStart + GOSS_POSE_LANDMARK_COUNT;
|
|
1842
|
+
const out = {
|
|
1843
|
+
frameSerial: dv.getBigUint64(0, true),
|
|
1844
|
+
timestampUs: dv.getBigInt64(8, true),
|
|
1845
|
+
presence: dv.getFloat32(16, true),
|
|
1846
|
+
landmarkCount: dv.getUint32(20, true),
|
|
1847
|
+
landmarks: this.mod.HEAPF32.slice(lmStart, visStart),
|
|
1848
|
+
visibilities: this.mod.HEAPF32.slice(visStart, presStart),
|
|
1849
|
+
presences: this.mod.HEAPF32.slice(presStart, presStart + GOSS_POSE_LANDMARK_COUNT),
|
|
1850
|
+
};
|
|
1851
|
+
return out;
|
|
1852
|
+
}
|
|
1853
|
+
/// The tracked point (x, y in frame pixels, z in the same scale) of a named
|
|
1854
|
+
/// face region, or null until a face is tracked.
|
|
1855
|
+
faceRegion(region) {
|
|
1856
|
+
const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [12]);
|
|
1857
|
+
const status = this.mod.ccall("goss_session_face_region", "number", ["number", "number", "number"], [this.handle, region, ptr]);
|
|
1858
|
+
if (status !== 0) {
|
|
1859
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [ptr, 12]);
|
|
1860
|
+
return null;
|
|
1861
|
+
}
|
|
1862
|
+
const w = ptr >> 2;
|
|
1863
|
+
const out = [this.mod.HEAPF32[w], this.mod.HEAPF32[w + 1], this.mod.HEAPF32[w + 2]];
|
|
1864
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [ptr, 12]);
|
|
1865
|
+
return out;
|
|
1866
|
+
}
|
|
1867
|
+
/// The tracked point (x, y in frame pixels, z in the same scale) of a named
|
|
1868
|
+
/// body skeleton joint, or null until a body is tracked.
|
|
1869
|
+
bodyJoint(joint) {
|
|
1870
|
+
const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [12]);
|
|
1871
|
+
const status = this.mod.ccall("goss_session_body_joint", "number", ["number", "number", "number"], [this.handle, joint, ptr]);
|
|
1872
|
+
if (status !== 0) {
|
|
1873
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [ptr, 12]);
|
|
1874
|
+
return null;
|
|
1875
|
+
}
|
|
1876
|
+
const w = ptr >> 2;
|
|
1877
|
+
const out = [this.mod.HEAPF32[w], this.mod.HEAPF32[w + 1], this.mod.HEAPF32[w + 2]];
|
|
1878
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [ptr, 12]);
|
|
1879
|
+
return out;
|
|
1880
|
+
}
|
|
1881
|
+
/// The tracked point (x, y in frame pixels, z in the same scale) of a named
|
|
1882
|
+
/// joint on the handIndex-th tracked hand, or null until that hand is tracked.
|
|
1883
|
+
handJoint(joint, handIndex = 0) {
|
|
1884
|
+
const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [12]);
|
|
1885
|
+
const status = this.mod.ccall("goss_session_hand_joint", "number", ["number", "number", "number", "number"], [this.handle, handIndex, joint, ptr]);
|
|
1886
|
+
if (status !== 0) {
|
|
1887
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [ptr, 12]);
|
|
1888
|
+
return null;
|
|
1889
|
+
}
|
|
1890
|
+
const w = ptr >> 2;
|
|
1891
|
+
const out = [this.mod.HEAPF32[w], this.mod.HEAPF32[w + 1], this.mod.HEAPF32[w + 2]];
|
|
1892
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [ptr, 12]);
|
|
1893
|
+
return out;
|
|
1894
|
+
}
|
|
1895
|
+
/// Runs the in-engine segmenter on the camera frames: model is any square RGB
|
|
1896
|
+
/// segmenter (portrait or a multi-class scene net, up to 32 classes), read for
|
|
1897
|
+
/// its own tensor dimensions; threads is the worker count. A build with no
|
|
1898
|
+
/// inference stack returns unsupported and setSegmentationMask is used instead.
|
|
1899
|
+
enableSegmentation(model, threads) {
|
|
1900
|
+
const modelPtr = model.length > 0 ? this.mod.ccall("goss_alloc", "number", ["number"], [model.length]) : 0;
|
|
1901
|
+
if (modelPtr)
|
|
1902
|
+
this.mod.HEAPU8.set(model, modelPtr);
|
|
1903
|
+
this.mod.ccall("goss_session_enable_segmentation", "number", ["number", "number", "number", "number"], [this.handle, modelPtr, model.length, threads]);
|
|
1904
|
+
if (modelPtr)
|
|
1905
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [modelPtr, model.length]);
|
|
1906
|
+
}
|
|
1907
|
+
/// Tears the in-engine segmenter down; the subject and class channels go empty.
|
|
1908
|
+
disableSegmentation() {
|
|
1909
|
+
this.mod.ccall("goss_session_disable_segmentation", "number", ["number"], [this.handle]);
|
|
1910
|
+
}
|
|
1911
|
+
enableTracker(sym, task, threads) {
|
|
1912
|
+
const ptr = task.length > 0 ? this.mod.ccall("goss_alloc", "number", ["number"], [task.length]) : 0;
|
|
1913
|
+
if (ptr)
|
|
1914
|
+
this.mod.HEAPU8.set(task, ptr);
|
|
1915
|
+
this.mod.ccall(sym, "number", ["number", "number", "number", "number"], [this.handle, ptr, task.length, threads]);
|
|
1916
|
+
if (ptr)
|
|
1917
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [ptr, task.length]);
|
|
1918
|
+
}
|
|
1919
|
+
/// Enables the in-engine face tracker: task is a mediapipe face-landmarker
|
|
1920
|
+
/// .task, threads the worker count. Web usually feeds faces through the
|
|
1921
|
+
/// producer path (submitFaces); a build with no inference stack returns
|
|
1922
|
+
/// unsupported.
|
|
1923
|
+
enableFaceTracking(task, threads) {
|
|
1924
|
+
this.enableTracker("goss_session_enable_face_tracking", task, threads);
|
|
1925
|
+
}
|
|
1926
|
+
/// Tears the in-engine face tracker down.
|
|
1927
|
+
disableFaceTracking() {
|
|
1928
|
+
this.mod.ccall("goss_session_disable_face_tracking", null, ["number"], [this.handle]);
|
|
1929
|
+
}
|
|
1930
|
+
/// Enables the in-engine hand tracker from a hand-landmarker .task.
|
|
1931
|
+
enableHandTracking(task, threads) {
|
|
1932
|
+
this.enableTracker("goss_session_enable_hand_tracking", task, threads);
|
|
1933
|
+
}
|
|
1934
|
+
/// Tears the in-engine hand tracker down.
|
|
1935
|
+
disableHandTracking() {
|
|
1936
|
+
this.mod.ccall("goss_session_disable_hand_tracking", null, ["number"], [this.handle]);
|
|
1937
|
+
}
|
|
1938
|
+
/// Enables the in-engine pose tracker from a pose-landmarker .task.
|
|
1939
|
+
enablePoseTracking(task, threads) {
|
|
1940
|
+
this.enableTracker("goss_session_enable_pose_tracking", task, threads);
|
|
1941
|
+
}
|
|
1942
|
+
/// Tears the in-engine pose tracker down.
|
|
1943
|
+
disablePoseTracking() {
|
|
1944
|
+
this.mod.ccall("goss_session_disable_pose_tracking", null, ["number"], [this.handle]);
|
|
1945
|
+
}
|
|
1946
|
+
/// Turns the in-engine beauty pass on. resourcePath points at the gpupixel
|
|
1947
|
+
/// resource directory the native build ships; on web it is unused, so an
|
|
1948
|
+
/// empty string is fine. A build without the beauty stack leaves it off.
|
|
1949
|
+
enableBeauty(resourcePath = "") {
|
|
1950
|
+
const bytes = new TextEncoder().encode(resourcePath);
|
|
1951
|
+
const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [bytes.length + 1]);
|
|
1952
|
+
this.mod.HEAPU8.set(bytes, ptr);
|
|
1953
|
+
this.mod.HEAPU8[ptr + bytes.length] = 0;
|
|
1954
|
+
this.mod.ccall("goss_session_enable_beauty", "number", ["number", "number"], [this.handle, ptr]);
|
|
1955
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [ptr, bytes.length + 1]);
|
|
1956
|
+
}
|
|
1957
|
+
/// Turns the in-engine beauty pass off.
|
|
1958
|
+
disableBeauty() {
|
|
1959
|
+
this.mod.ccall("goss_session_disable_beauty", null, ["number"], [this.handle]);
|
|
1960
|
+
}
|
|
1961
|
+
/// Beautifies one RGBA frame on the CPU and returns a new RGBA buffer the same
|
|
1962
|
+
/// size; the smoothing follows the beauty parameters set on the session. A
|
|
1963
|
+
/// build without the beauty stack returns the input unchanged.
|
|
1964
|
+
beautifyFrame(rgba, width, height) {
|
|
1965
|
+
const inPtr = this.mod.ccall("goss_alloc", "number", ["number"], [rgba.length]);
|
|
1966
|
+
const outPtr = this.mod.ccall("goss_alloc", "number", ["number"], [rgba.length]);
|
|
1967
|
+
this.mod.HEAPU8.set(rgba, inPtr);
|
|
1968
|
+
this.mod.ccall("goss_session_beautify_frame", "number", ["number", "number", "number", "number", "number"], [this.handle, inPtr, width, height, outPtr]);
|
|
1969
|
+
const out = this.mod.HEAPU8.slice(outPtr, outPtr + rgba.length);
|
|
1970
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [inPtr, rgba.length]);
|
|
1971
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [outPtr, rgba.length]);
|
|
1972
|
+
return out;
|
|
1973
|
+
}
|
|
1974
|
+
/// Feeds a segmentation mask (GOSS_SEGMENTATION_MASK_SIDE squared floats,
|
|
1975
|
+
/// from a GossSegmenter) into the session as the subject texture the blend
|
|
1976
|
+
/// and mask channels sample. Null clears it (no subject this frame).
|
|
1977
|
+
setSegmentationMask(mask) {
|
|
1978
|
+
const count = GOSS_SEGMENTATION_MASK_SIDE * GOSS_SEGMENTATION_MASK_SIDE;
|
|
1979
|
+
if (!mask || mask.length < count) {
|
|
1980
|
+
this.mod.ccall("goss_session_set_segmentation_mask", "number", ["number", "number", "number"], [this.handle, 0, 0]);
|
|
1981
|
+
return;
|
|
1982
|
+
}
|
|
1983
|
+
this.mod.HEAPF32.set(mask.subarray(0, count), this.segmentationMaskPtr >> 2);
|
|
1984
|
+
this.mod.ccall("goss_session_set_segmentation_mask", "number", ["number", "number", "number"], [this.handle, this.segmentationMaskPtr, count]);
|
|
1985
|
+
}
|
|
1986
|
+
/// The class channels the active lens samples, as a bitmask over
|
|
1987
|
+
/// GOSS_SEGMENTATION_CHANNELS. Upload exactly these with setSegmentationClassMask
|
|
1988
|
+
/// each frame; zero means only the subject mask is wanted.
|
|
1989
|
+
segmentationChannels() {
|
|
1990
|
+
return this.mod.ccall("goss_session_segmentation_channels", "number", ["number"], [this.handle]);
|
|
1991
|
+
}
|
|
1992
|
+
/// Feeds one class channel's mask (from a GossSegmenter's classMask) as the
|
|
1993
|
+
/// texture that channel's passes sample. channel indexes
|
|
1994
|
+
/// GOSS_SEGMENTATION_CHANNELS; channel 0 (person) goes through
|
|
1995
|
+
/// setSegmentationMask, which clears the classes, so upload these after.
|
|
1996
|
+
setSegmentationClassMask(channel, mask) {
|
|
1997
|
+
const count = GOSS_SEGMENTATION_MASK_SIDE * GOSS_SEGMENTATION_MASK_SIDE;
|
|
1998
|
+
if (!mask || mask.length < count) {
|
|
1999
|
+
this.mod.ccall("goss_session_set_segmentation_class_mask", "number", ["number", "number", "number", "number"], [this.handle, channel, 0, 0]);
|
|
2000
|
+
return;
|
|
2001
|
+
}
|
|
2002
|
+
this.mod.HEAPF32.set(mask.subarray(0, count), this.segmentationMaskPtr >> 2);
|
|
2003
|
+
this.mod.ccall("goss_session_set_segmentation_class_mask", "number", ["number", "number", "number", "number"], [this.handle, channel, this.segmentationMaskPtr, count]);
|
|
2004
|
+
}
|
|
2005
|
+
/// Uploads one of whiten's four lookup textures directly - slot 0
|
|
2006
|
+
/// gray, 1 origin, 2 skin, 3 custom. loadWhitenLuts is the sugar most
|
|
2007
|
+
/// callers want; this is the raw upload it calls internally.
|
|
2008
|
+
setBeautyLut(slot, rgba, width, height) {
|
|
2009
|
+
const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [rgba.length]);
|
|
2010
|
+
this.mod.HEAPU8.set(rgba, ptr);
|
|
2011
|
+
this.mod.ccall("goss_session_set_beauty_lut", "number", ["number", "number", "number", "number", "number"], [this.handle, slot, ptr, width, height]);
|
|
2012
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [ptr, rgba.length]);
|
|
2013
|
+
}
|
|
2014
|
+
/// Uploads lipstick's or blush's own source image directly.
|
|
2015
|
+
/// loadMakeupTextures is the sugar most callers want; this is the raw
|
|
2016
|
+
/// upload it calls internally.
|
|
2017
|
+
setBeautyMakeupTexture(effect, rgba, width, height) {
|
|
2018
|
+
const ptr = this.mod.ccall("goss_alloc", "number", ["number"], [rgba.length]);
|
|
2019
|
+
this.mod.HEAPU8.set(rgba, ptr);
|
|
2020
|
+
this.mod.ccall("goss_session_set_beauty_makeup_texture", "number", ["number", "number", "number", "number", "number"], [this.handle, effect, ptr, width, height]);
|
|
2021
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [ptr, rgba.length]);
|
|
2022
|
+
}
|
|
2023
|
+
/// Fetches the four whiten lookup textures (gray/origin/skin/custom),
|
|
2024
|
+
/// relative to lutBaseUrl. Safe to call once after construction;
|
|
2025
|
+
/// setWhiten stays a no-op until this resolves.
|
|
2026
|
+
async loadWhitenLuts(lutBaseUrl) {
|
|
2027
|
+
const names = ["lookup_gray", "lookup_origin", "lookup_skin", "lookup_light"];
|
|
2028
|
+
const images = await Promise.all(names.map((name) => fetch(new URL(`${name}.png`, lutBaseUrl)).then((r) => r.blob()).then(decodeImageRgba)));
|
|
2029
|
+
images.forEach((image, slot) => {
|
|
2030
|
+
this.setBeautyLut(slot, image.data, image.width, image.height);
|
|
2031
|
+
this.whitenLutsLoaded += 1;
|
|
2032
|
+
});
|
|
2033
|
+
}
|
|
2034
|
+
/// Fetches mouth.png/blusher.png, relative to baseUrl. Safe to call
|
|
2035
|
+
/// once after construction; setLipstick/setBlush stay a no-op until
|
|
2036
|
+
/// this resolves.
|
|
2037
|
+
async loadMakeupTextures(baseUrl) {
|
|
2038
|
+
const [mouth, blusher] = await Promise.all(["mouth.png", "blusher.png"].map((name) => fetch(new URL(name, baseUrl)).then((r) => r.blob()).then(decodeImageRgba)));
|
|
2039
|
+
for (const [effect, image] of [
|
|
2040
|
+
[4 /* GossBeautyEffect.Lipstick */, mouth],
|
|
2041
|
+
[5 /* GossBeautyEffect.Blush */, blusher],
|
|
2042
|
+
]) {
|
|
2043
|
+
this.setBeautyMakeupTexture(effect, image.data, image.width, image.height);
|
|
2044
|
+
}
|
|
2045
|
+
this.lipstickTextureLoaded = true;
|
|
2046
|
+
this.blushTextureLoaded = true;
|
|
2047
|
+
}
|
|
2048
|
+
ensureFramePixels(byteLength) {
|
|
2049
|
+
if (this.framePixelsCapacity >= byteLength)
|
|
2050
|
+
return;
|
|
2051
|
+
if (this.framePixelsPtr !== 0)
|
|
2052
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [this.framePixelsPtr, this.framePixelsCapacity]);
|
|
2053
|
+
this.framePixelsPtr = this.mod.ccall("goss_alloc", "number", ["number"], [byteLength]);
|
|
2054
|
+
this.framePixelsCapacity = byteLength;
|
|
2055
|
+
}
|
|
2056
|
+
/// rotationDegrees omitted means the setVideoFlip state decides (a
|
|
2057
|
+
/// flipped source is a 180-degree turn); timestampUs omitted means
|
|
2058
|
+
/// now.
|
|
2059
|
+
submitFrameRgbaCopy(rgba, stride, width, height, pixelFormat = 4 /* GossPixelFormat.Rgba8 */, rotationDegrees, mirrored = false, timestampUs) {
|
|
2060
|
+
this.frameWidth = width;
|
|
2061
|
+
this.frameHeight = height;
|
|
2062
|
+
const byteLength = stride * height;
|
|
2063
|
+
this.ensureFramePixels(byteLength);
|
|
2064
|
+
this.mod.HEAPU8.set(rgba.subarray(0, byteLength), this.framePixelsPtr);
|
|
2065
|
+
const rotationQuarters = ((rotationDegrees ?? (this.videoFlipped ? 180 : 0)) / 90) & 3;
|
|
2066
|
+
const flags = (mirrored ? FRAME_FLAG_MIRROR : 0) | (rotationQuarters << FRAME_ROTATION_SHIFT);
|
|
2067
|
+
this.mod.setValue(this.frameDescPtr, width, "i32");
|
|
2068
|
+
this.mod.setValue(this.frameDescPtr + 4, height, "i32");
|
|
2069
|
+
this.mod.setValue(this.frameDescPtr + 8, pixelFormat, "i32");
|
|
2070
|
+
this.mod.setValue(this.frameDescPtr + 12, 0, "i32");
|
|
2071
|
+
this.mod.setValue(this.frameDescPtr + 16, 0, "i32");
|
|
2072
|
+
this.mod.setValue(this.frameDescPtr + 20, flags, "i32");
|
|
2073
|
+
const stampUs = timestampUs ?? Math.round(performance.now() * 1000);
|
|
2074
|
+
this.mod.setValue(this.frameDescPtr + 24, stampUs >>> 0, "i32");
|
|
2075
|
+
this.mod.setValue(this.frameDescPtr + 28, Math.floor(stampUs / 4294967296), "i32");
|
|
2076
|
+
this.mod.ccall("goss_session_submit_frame_rgba_copy", "number", ["number", "number", "number", "number"], [this.handle, this.frameDescPtr, this.framePixelsPtr, stride]);
|
|
2077
|
+
}
|
|
2078
|
+
/// The web selfie path: runs each selfie-source splat.cloud once over one
|
|
2079
|
+
/// RGBA8 still (row major), so a photoreal avatar is generated from one photo
|
|
2080
|
+
/// and then held off the live camera. Reuses the per-session frame staging.
|
|
2081
|
+
submitAvatarSourceRgba(rgba, width, height) {
|
|
2082
|
+
const byteLength = width * 4 * height;
|
|
2083
|
+
this.ensureFramePixels(byteLength);
|
|
2084
|
+
this.mod.HEAPU8.set(rgba.subarray(0, byteLength), this.framePixelsPtr);
|
|
2085
|
+
this.mod.ccall("goss_session_submit_avatar_source_rgba", "number", ["number", "number", "number"], [this.handle, this.framePixelsPtr, width, height]);
|
|
2086
|
+
}
|
|
2087
|
+
/// Feeds the platform's world understanding into the session: camera
|
|
2088
|
+
/// pose and projection (column-major float16 arrays), tracked planes,
|
|
2089
|
+
/// anchors, and the light estimate. Drives the world.tracking_state
|
|
2090
|
+
/// trigger and world-anchored lens content.
|
|
2091
|
+
submitWorld(state, planes = [], anchors = [], light) {
|
|
2092
|
+
const stateBytes = 144;
|
|
2093
|
+
const planeBytes = 88;
|
|
2094
|
+
const anchorBytes = 72;
|
|
2095
|
+
const lightBytes = 8;
|
|
2096
|
+
const total = stateBytes + planes.length * planeBytes + anchors.length * anchorBytes + lightBytes;
|
|
2097
|
+
this.ensureWorldScratch(total);
|
|
2098
|
+
const base = this.worldScratchPtr;
|
|
2099
|
+
const heap = this.mod.HEAPU8;
|
|
2100
|
+
const view = new DataView(heap.buffer, base, total);
|
|
2101
|
+
view.setUint32(0, state.trackingState, true);
|
|
2102
|
+
for (let i = 0; i < 16; i++)
|
|
2103
|
+
view.setFloat32(4 + i * 4, state.worldFromCamera[i], true);
|
|
2104
|
+
for (let i = 0; i < 16; i++)
|
|
2105
|
+
view.setFloat32(68 + i * 4, state.projection[i], true);
|
|
2106
|
+
view.setBigInt64(136, BigInt(Math.round(state.timestampUs)), true);
|
|
2107
|
+
let at = stateBytes;
|
|
2108
|
+
const planesPtr = planes.length > 0 ? base + at : 0;
|
|
2109
|
+
for (const plane of planes) {
|
|
2110
|
+
view.setBigUint64(at, BigInt(plane.id), true);
|
|
2111
|
+
for (let i = 0; i < 16; i++)
|
|
2112
|
+
view.setFloat32(at + 8 + i * 4, plane.pose[i], true);
|
|
2113
|
+
view.setFloat32(at + 72, plane.extentX, true);
|
|
2114
|
+
view.setFloat32(at + 76, plane.extentZ, true);
|
|
2115
|
+
view.setUint32(at + 80, plane.classification, true);
|
|
2116
|
+
at += planeBytes;
|
|
2117
|
+
}
|
|
2118
|
+
const anchorsPtr = anchors.length > 0 ? base + at : 0;
|
|
2119
|
+
for (const anchorInput of anchors) {
|
|
2120
|
+
view.setBigUint64(at, BigInt(anchorInput.id), true);
|
|
2121
|
+
for (let i = 0; i < 16; i++)
|
|
2122
|
+
view.setFloat32(at + 8 + i * 4, anchorInput.pose[i], true);
|
|
2123
|
+
at += anchorBytes;
|
|
2124
|
+
}
|
|
2125
|
+
let lightPtr = 0;
|
|
2126
|
+
if (light) {
|
|
2127
|
+
lightPtr = base + at;
|
|
2128
|
+
view.setFloat32(at, light.ambientIntensity, true);
|
|
2129
|
+
view.setFloat32(at + 4, light.colorTemperatureKelvin, true);
|
|
2130
|
+
}
|
|
2131
|
+
this.mod.ccall("goss_session_submit_world", "number", ["number", "number", "number", "number", "number", "number", "number"], [this.handle, base, planesPtr, planes.length, anchorsPtr, anchors.length, lightPtr]);
|
|
2132
|
+
}
|
|
2133
|
+
ensureWorldScratch(byteLength) {
|
|
2134
|
+
if (this.worldScratchLen >= byteLength)
|
|
2135
|
+
return;
|
|
2136
|
+
if (this.worldScratchPtr !== 0)
|
|
2137
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [this.worldScratchPtr, this.worldScratchLen]);
|
|
2138
|
+
this.worldScratchPtr = this.mod.ccall("goss_alloc", "number", ["number"], [byteLength]);
|
|
2139
|
+
this.worldScratchLen = byteLength;
|
|
2140
|
+
}
|
|
2141
|
+
/// Reports one finished frame: measured whole-pipeline time plus
|
|
2142
|
+
/// thermal pressure (nominal by default - no browser API surfaces
|
|
2143
|
+
/// device thermal state). Returns the degradation level in effect
|
|
2144
|
+
/// for the next frame.
|
|
2145
|
+
reportFrame(frameTimeUs, thermal = 0 /* GossThermal.Nominal */) {
|
|
2146
|
+
return this.mod.ccall("goss_session_report_frame", "number", ["number", "number", "number"], [this.handle, frameTimeUs, thermal]);
|
|
2147
|
+
}
|
|
2148
|
+
degradeLevel() {
|
|
2149
|
+
return this.mod.ccall("goss_session_degrade_level", "number", ["number"], [this.handle]);
|
|
2150
|
+
}
|
|
2151
|
+
destroy() {
|
|
2152
|
+
// frameDescPtr is allocated at construction and only ever nulled here,
|
|
2153
|
+
// so it doubles as the guard that makes a second destroy() a no-op.
|
|
2154
|
+
if (this.frameDescPtr === 0)
|
|
2155
|
+
return;
|
|
2156
|
+
this.mod.ccall("goss_session_destroy", null, ["number"], [this.handle]);
|
|
2157
|
+
// goss_session_destroy owns only the native session; every scratch
|
|
2158
|
+
// buffer the session allocated is our heap and freed here.
|
|
2159
|
+
const free = (ptr, size) => {
|
|
2160
|
+
if (ptr !== 0 && size !== 0)
|
|
2161
|
+
this.mod.ccall("goss_free", null, ["number", "number"], [ptr, size]);
|
|
2162
|
+
};
|
|
2163
|
+
free(this.frameDescPtr, 32);
|
|
2164
|
+
free(this.landmarksPtr, GOSS_FACE_LANDMARK_COUNT * 3 * 4);
|
|
2165
|
+
free(this.signalsPtr, LENS_SIGNALS_BYTES);
|
|
2166
|
+
free(this.segmentationMaskPtr, GOSS_SEGMENTATION_MASK_SIDE * GOSS_SEGMENTATION_MASK_SIDE * 4);
|
|
2167
|
+
free(this.framePixelsPtr, this.framePixelsCapacity);
|
|
2168
|
+
free(this.worldScratchPtr, this.worldScratchLen);
|
|
2169
|
+
free(this.scratchPtr, this.scratchCapacity);
|
|
2170
|
+
this.frameDescPtr = 0;
|
|
2171
|
+
this.landmarksPtr = 0;
|
|
2172
|
+
this.signalsPtr = 0;
|
|
2173
|
+
this.segmentationMaskPtr = 0;
|
|
2174
|
+
this.framePixelsPtr = 0;
|
|
2175
|
+
this.framePixelsCapacity = 0;
|
|
2176
|
+
this.worldScratchPtr = 0;
|
|
2177
|
+
this.worldScratchLen = 0;
|
|
2178
|
+
this.scratchPtr = 0;
|
|
2179
|
+
this.scratchCapacity = 0;
|
|
2180
|
+
}
|
|
2181
|
+
}
|
|
2182
|
+
/// The SDK-facing orchestrator: capture loop, video element, DOM
|
|
2183
|
+
/// events. Composes Gosslens/GossEngine/GossSession rather than being one of
|
|
2184
|
+
/// them - the same relationship CameraController/PreviewViewController
|
|
2185
|
+
/// have to GossEngine/GossSession on iOS, not a fourth ABI-shaped type.
|
|
2186
|
+
export class GossPreviewSession {
|
|
2187
|
+
gosslens;
|
|
2188
|
+
engine;
|
|
2189
|
+
session;
|
|
2190
|
+
events;
|
|
2191
|
+
video = document.createElement("video");
|
|
2192
|
+
state = "idle";
|
|
2193
|
+
stream = null;
|
|
2194
|
+
raf = 0;
|
|
2195
|
+
lastTick = 0;
|
|
2196
|
+
fpsWindowStart = 0;
|
|
2197
|
+
fpsWindowFrames = 0;
|
|
2198
|
+
renderedFrames = 0;
|
|
2199
|
+
cameraFrames = 0;
|
|
2200
|
+
lastVideoTime = -1;
|
|
2201
|
+
scratchCanvas = document.createElement("canvas");
|
|
2202
|
+
scratchCtx;
|
|
2203
|
+
constructor(gosslens, engine, session, events) {
|
|
2204
|
+
this.gosslens = gosslens;
|
|
2205
|
+
this.engine = engine;
|
|
2206
|
+
this.session = session;
|
|
2207
|
+
this.events = events;
|
|
2208
|
+
this.scratchCtx = this.scratchCanvas.getContext("2d", { willReadFrequently: true });
|
|
2209
|
+
}
|
|
2210
|
+
static async create(canvas, wasmJsUrl, events = {}) {
|
|
2211
|
+
const gosslens = await Gosslens.load(canvas, wasmJsUrl);
|
|
2212
|
+
const engine = GossEngine.create(gosslens);
|
|
2213
|
+
await engine.initRenderer(canvas);
|
|
2214
|
+
const session = GossSession.create(engine);
|
|
2215
|
+
return new GossPreviewSession(gosslens, engine, session, events);
|
|
2216
|
+
}
|
|
2217
|
+
abiVersion() {
|
|
2218
|
+
return this.gosslens.abiVersion();
|
|
2219
|
+
}
|
|
2220
|
+
setState(state) {
|
|
2221
|
+
this.state = state;
|
|
2222
|
+
this.events.onState?.(state);
|
|
2223
|
+
}
|
|
2224
|
+
setWhiten(amount) {
|
|
2225
|
+
this.session.setWhiten(amount);
|
|
2226
|
+
}
|
|
2227
|
+
setSmooth(amount) {
|
|
2228
|
+
this.session.setSmooth(amount);
|
|
2229
|
+
}
|
|
2230
|
+
setThinFace(amount) {
|
|
2231
|
+
this.session.setThinFace(amount);
|
|
2232
|
+
}
|
|
2233
|
+
setBigEye(amount) {
|
|
2234
|
+
this.session.setBigEye(amount);
|
|
2235
|
+
}
|
|
2236
|
+
setLipstick(amount) {
|
|
2237
|
+
this.session.setLipstick(amount);
|
|
2238
|
+
}
|
|
2239
|
+
setBlush(amount) {
|
|
2240
|
+
this.session.setBlush(amount);
|
|
2241
|
+
}
|
|
2242
|
+
activateLens(manifestJson) {
|
|
2243
|
+
this.session.activateLens(manifestJson);
|
|
2244
|
+
}
|
|
2245
|
+
deactivateLens() {
|
|
2246
|
+
this.session.deactivateLens();
|
|
2247
|
+
}
|
|
2248
|
+
tickLens(dtUs, signals = {}) {
|
|
2249
|
+
this.session.tickLens(dtUs, signals);
|
|
2250
|
+
}
|
|
2251
|
+
setVideoFlip(enabled) {
|
|
2252
|
+
this.session.setVideoFlip(enabled);
|
|
2253
|
+
}
|
|
2254
|
+
isVideoFlipped() {
|
|
2255
|
+
return this.session.isVideoFlipped();
|
|
2256
|
+
}
|
|
2257
|
+
setFaceLandmarks(landmarks, sourceWidth, sourceHeight) {
|
|
2258
|
+
this.session.setFaceLandmarks(landmarks, sourceWidth, sourceHeight);
|
|
2259
|
+
}
|
|
2260
|
+
loadWhitenLuts(lutBaseUrl) {
|
|
2261
|
+
return this.session.loadWhitenLuts(lutBaseUrl);
|
|
2262
|
+
}
|
|
2263
|
+
loadMakeupTextures(baseUrl) {
|
|
2264
|
+
return this.session.loadMakeupTextures(baseUrl);
|
|
2265
|
+
}
|
|
2266
|
+
/// Uploads a still image directly into the frame the engine renders,
|
|
2267
|
+
/// bypassing the video element - freezeCamera() first stops tick()
|
|
2268
|
+
/// from re-submitting over it. Test/demo tooling only: skin-smoothing's
|
|
2269
|
+
/// content-adaptive blend needs a real face to prove, not a fake one.
|
|
2270
|
+
async loadStillFrame(url) {
|
|
2271
|
+
const image = await decodeImageRgba(await (await fetch(url)).blob(), {
|
|
2272
|
+
maxWidth: this.scratchCanvas.width,
|
|
2273
|
+
maxHeight: this.scratchCanvas.height,
|
|
2274
|
+
});
|
|
2275
|
+
// Not mirrored: a loaded test photo isn't a front camera, and
|
|
2276
|
+
// setLandmarksFromStill tracks this same unmirrored image - mirroring
|
|
2277
|
+
// only the background here would leave the tracked landmarks
|
|
2278
|
+
// pointing at the wrong side of the now-mirrored face.
|
|
2279
|
+
this.session.submitFrameRgbaCopy(image.data, image.width * 4, image.width, image.height);
|
|
2280
|
+
}
|
|
2281
|
+
async start() {
|
|
2282
|
+
try {
|
|
2283
|
+
this.stream = await navigator.mediaDevices.getUserMedia({
|
|
2284
|
+
video: { width: { ideal: 1280 }, height: { ideal: 720 } },
|
|
2285
|
+
audio: false,
|
|
2286
|
+
});
|
|
2287
|
+
}
|
|
2288
|
+
catch (err) {
|
|
2289
|
+
this.setState(err instanceof DOMException && err.name === "NotAllowedError" ? "denied" : "failed");
|
|
2290
|
+
return;
|
|
2291
|
+
}
|
|
2292
|
+
const track = this.stream.getVideoTracks()[0];
|
|
2293
|
+
track.addEventListener("mute", () => this.setState("interrupted"));
|
|
2294
|
+
track.addEventListener("unmute", () => this.setState("running"));
|
|
2295
|
+
track.addEventListener("ended", () => this.setState("failed"));
|
|
2296
|
+
this.video.srcObject = this.stream;
|
|
2297
|
+
this.video.muted = true;
|
|
2298
|
+
this.video.playsInline = true;
|
|
2299
|
+
await this.video.play();
|
|
2300
|
+
this.setState("running");
|
|
2301
|
+
this.fpsWindowStart = performance.now();
|
|
2302
|
+
this.lastTick = performance.now();
|
|
2303
|
+
this.tick();
|
|
2304
|
+
}
|
|
2305
|
+
stop() {
|
|
2306
|
+
cancelAnimationFrame(this.raf);
|
|
2307
|
+
this.stream?.getTracks().forEach((track) => track.stop());
|
|
2308
|
+
this.stream = null;
|
|
2309
|
+
this.setState("idle");
|
|
2310
|
+
}
|
|
2311
|
+
tick = () => {
|
|
2312
|
+
this.raf = requestAnimationFrame(this.tick);
|
|
2313
|
+
if (this.engine.isCaptureInFlight)
|
|
2314
|
+
return;
|
|
2315
|
+
const now = performance.now();
|
|
2316
|
+
const frameTimeUs = Math.max(0, Math.round((now - this.lastTick) * 1000));
|
|
2317
|
+
this.lastTick = now;
|
|
2318
|
+
this.session.reportFrame(frameTimeUs);
|
|
2319
|
+
if (this.video.readyState >= 2 && this.video.currentTime !== this.lastVideoTime) {
|
|
2320
|
+
this.lastVideoTime = this.video.currentTime;
|
|
2321
|
+
this.cameraFrames += 1;
|
|
2322
|
+
const width = this.video.videoWidth;
|
|
2323
|
+
const height = this.video.videoHeight;
|
|
2324
|
+
this.scratchCanvas.width = width;
|
|
2325
|
+
this.scratchCanvas.height = height;
|
|
2326
|
+
this.scratchCtx.drawImage(this.video, 0, 0, width, height);
|
|
2327
|
+
const pixels = this.scratchCtx.getImageData(0, 0, width, height);
|
|
2328
|
+
// Not mirrored here - the demo page's own CSS mirrors the canvas
|
|
2329
|
+
// for display, so the engine keeps working in the camera's real,
|
|
2330
|
+
// unmirrored coordinate space (matching tracking, which analyzes
|
|
2331
|
+
// this same unmirrored buffer).
|
|
2332
|
+
this.session.submitFrameRgbaCopy(pixels.data, width * 4, width, height);
|
|
2333
|
+
}
|
|
2334
|
+
const status = this.engine.renderFrame(this.session);
|
|
2335
|
+
if (status === GOSS_OK) {
|
|
2336
|
+
this.renderedFrames += 1;
|
|
2337
|
+
this.fpsWindowFrames += 1;
|
|
2338
|
+
}
|
|
2339
|
+
if (now - this.fpsWindowStart >= 1000) {
|
|
2340
|
+
const fps = (this.fpsWindowFrames * 1000) / (now - this.fpsWindowStart);
|
|
2341
|
+
this.events.onFps?.(fps, this.renderedFrames, this.cameraFrames);
|
|
2342
|
+
this.fpsWindowStart = now;
|
|
2343
|
+
this.fpsWindowFrames = 0;
|
|
2344
|
+
}
|
|
2345
|
+
};
|
|
2346
|
+
degradeLevel() {
|
|
2347
|
+
return this.session.degradeLevel();
|
|
2348
|
+
}
|
|
2349
|
+
captureFrame() {
|
|
2350
|
+
return this.engine.captureFrame(this.session);
|
|
2351
|
+
}
|
|
2352
|
+
readCenterPixel() {
|
|
2353
|
+
return this.engine.readCenterPixel(this.session);
|
|
2354
|
+
}
|
|
2355
|
+
readFrameSum() {
|
|
2356
|
+
return this.engine.readFrameSum(this.session);
|
|
2357
|
+
}
|
|
2358
|
+
}
|
|
2359
|
+
export { GossWebXRWorldSource } from "./world";
|
|
2360
|
+
//# sourceMappingURL=index.js.map
|