@myned-ai/gsplat-flame-avatar-renderer 1.0.5 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +130 -32
- package/dist/gsplat-flame-avatar-renderer.cjs.js +3478 -722
- package/dist/gsplat-flame-avatar-renderer.cjs.min.js +2 -0
- package/dist/gsplat-flame-avatar-renderer.cjs.min.js.map +1 -0
- package/dist/gsplat-flame-avatar-renderer.esm.js +3439 -724
- package/dist/gsplat-flame-avatar-renderer.esm.min.js +2 -0
- package/dist/gsplat-flame-avatar-renderer.esm.min.js.map +1 -0
- package/package.json +10 -6
- package/src/core/SplatMesh.js +53 -46
- package/src/core/Viewer.js +47 -196
- package/src/errors/ApplicationError.js +185 -0
- package/src/errors/index.js +17 -0
- package/src/flame/FlameAnimator.js +282 -57
- package/src/loaders/PlyLoader.js +302 -44
- package/src/materials/SplatMaterial.js +13 -10
- package/src/materials/SplatMaterial3D.js +72 -27
- package/src/renderer/AnimationManager.js +8 -5
- package/src/renderer/GaussianSplatRenderer.js +668 -217
- package/src/utils/BlobUrlManager.js +294 -0
- package/src/utils/EventEmitter.js +349 -0
- package/src/utils/LoaderUtils.js +2 -1
- package/src/utils/Logger.js +171 -0
- package/src/utils/ObjectPool.js +248 -0
- package/src/utils/RenderLoop.js +306 -0
- package/src/utils/Util.js +59 -18
- package/src/utils/ValidationUtils.js +331 -0
- package/src/utils/index.js +10 -1
- package/dist/gsplat-flame-avatar-renderer.cjs.js.map +0 -1
- package/dist/gsplat-flame-avatar-renderer.esm.js.map +0 -1
package/package.json
CHANGED
|
@@ -1,16 +1,20 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@myned-ai/gsplat-flame-avatar-renderer",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.6",
|
|
4
4
|
"description": "FLAME-enabled Gaussian Splatting library for animated avatars with ARKit blendshape support",
|
|
5
|
-
"main": "dist/gsplat-flame-avatar-renderer.cjs.js",
|
|
6
|
-
"module": "dist/gsplat-flame-avatar-renderer.esm.js",
|
|
5
|
+
"main": "dist/gsplat-flame-avatar-renderer.cjs.min.js",
|
|
6
|
+
"module": "dist/gsplat-flame-avatar-renderer.esm.min.js",
|
|
7
7
|
"types": "dist/index.d.ts",
|
|
8
8
|
"type": "module",
|
|
9
9
|
"exports": {
|
|
10
10
|
".": {
|
|
11
|
-
"import": "./dist/gsplat-flame-avatar-renderer.esm.js",
|
|
12
|
-
"require": "./dist/gsplat-flame-avatar-renderer.cjs.js",
|
|
13
|
-
"types": "./dist/index.d.ts"
|
|
11
|
+
"import": "./dist/gsplat-flame-avatar-renderer.esm.min.js",
|
|
12
|
+
"require": "./dist/gsplat-flame-avatar-renderer.cjs.min.js",
|
|
13
|
+
"types": "./dist/index.d.ts",
|
|
14
|
+
"development": {
|
|
15
|
+
"import": "./dist/gsplat-flame-avatar-renderer.esm.js",
|
|
16
|
+
"require": "./dist/gsplat-flame-avatar-renderer.cjs.js"
|
|
17
|
+
}
|
|
14
18
|
}
|
|
15
19
|
},
|
|
16
20
|
"scripts": {
|
package/src/core/SplatMesh.js
CHANGED
|
@@ -22,6 +22,9 @@ import { SplatTree } from './SplatTree.js';
|
|
|
22
22
|
import { SplatMaterial3D } from '../materials/SplatMaterial3D.js';
|
|
23
23
|
import { SplatMaterial2D } from '../materials/SplatMaterial2D.js';
|
|
24
24
|
import { getSphericalHarmonicsComponentCountForDegree, uintEncodedFloat, rgbaArrayToInteger, clamp } from '../utils/Util.js';
|
|
25
|
+
import { getLogger } from '../utils/Logger.js';
|
|
26
|
+
|
|
27
|
+
const logger = getLogger('SplatMesh');
|
|
25
28
|
|
|
26
29
|
// Dummy geometry and material for initial Mesh construction
|
|
27
30
|
const dummyGeometry = new BufferGeometry();
|
|
@@ -91,7 +94,7 @@ function WebGLExtensions$1(gl) {
|
|
|
91
94
|
get: function(name) {
|
|
92
95
|
const extension = getExtension(name);
|
|
93
96
|
if (extension === null) {
|
|
94
|
-
|
|
97
|
+
logger.warn('THREE.WebGLRenderer: ' + name + ' extension not supported.');
|
|
95
98
|
}
|
|
96
99
|
return extension;
|
|
97
100
|
}
|
|
@@ -138,7 +141,7 @@ function WebGLCapabilities$1(gl, extensions, parameters) {
|
|
|
138
141
|
const maxPrecision = getMaxPrecision(precision);
|
|
139
142
|
|
|
140
143
|
if (maxPrecision !== precision) {
|
|
141
|
-
|
|
144
|
+
logger.warn('THREE.WebGLRenderer:', precision, 'not supported, using', maxPrecision, 'instead.');
|
|
142
145
|
precision = maxPrecision;
|
|
143
146
|
}
|
|
144
147
|
|
|
@@ -396,7 +399,6 @@ export class SplatMesh extends Mesh {
|
|
|
396
399
|
this.bonesNum = null;
|
|
397
400
|
this.bonesWeight = null;
|
|
398
401
|
this.gaussianSplatCount = null;
|
|
399
|
-
this.useFlameModel = true;
|
|
400
402
|
|
|
401
403
|
this.morphTargetDictionary = null;
|
|
402
404
|
this.distancesTransformFeedback = {
|
|
@@ -416,6 +418,9 @@ export class SplatMesh extends Mesh {
|
|
|
416
418
|
this.globalSplatIndexToLocalSplatIndexMap = [];
|
|
417
419
|
this.globalSplatIndexToSceneIndexMap = [];
|
|
418
420
|
|
|
421
|
+
// Optional iris occlusion configuration from avatar ZIP
|
|
422
|
+
this.irisOcclusionConfig = null;
|
|
423
|
+
|
|
419
424
|
this.lastBuildSplatCount = 0;
|
|
420
425
|
this.lastBuildScenes = [];
|
|
421
426
|
this.lastBuildMaxSplatCount = 0;
|
|
@@ -531,7 +536,7 @@ export class SplatMesh extends Mesh {
|
|
|
531
536
|
}, onSplatTreeIndexesUpload, onSplatTreeConstruction)
|
|
532
537
|
.then(() => {
|
|
533
538
|
const buildTime = performance.now() - buildStartTime;
|
|
534
|
-
if (this.logLevel >= LogLevel.Info)
|
|
539
|
+
if (this.logLevel >= LogLevel.Info) logger.info('SplatTree build: ' + buildTime + ' ms');
|
|
535
540
|
if (this.disposed) {
|
|
536
541
|
resolve();
|
|
537
542
|
} else {
|
|
@@ -554,11 +559,11 @@ export class SplatMesh extends Mesh {
|
|
|
554
559
|
}
|
|
555
560
|
});
|
|
556
561
|
if (this.logLevel >= LogLevel.Info) {
|
|
557
|
-
|
|
558
|
-
|
|
562
|
+
logger.info(`SplatTree leaves: ${this.splatTree.countLeaves()}`);
|
|
563
|
+
logger.info(`SplatTree leaves with splats:${leavesWithVertices}`);
|
|
559
564
|
avgSplatCount = avgSplatCount / nodeCount;
|
|
560
|
-
|
|
561
|
-
|
|
565
|
+
logger.info(`Avg splat count per node: ${avgSplatCount}`);
|
|
566
|
+
logger.info(`Total splat count: ${this.getSplatCount()}`);
|
|
562
567
|
}
|
|
563
568
|
resolve();
|
|
564
569
|
}
|
|
@@ -655,7 +660,7 @@ export class SplatMesh extends Mesh {
|
|
|
655
660
|
if (this.splatRenderMode === SplatRenderMode.ThreeD) {
|
|
656
661
|
this.material = SplatMaterial3D.build(this.dynamicMode, this.enableOptionalEffects, this.antialiased,
|
|
657
662
|
this.maxScreenSpaceSplatSize, this.splatScale, this.pointCloudModeEnabled,
|
|
658
|
-
this.minSphericalHarmonicsDegree, this.kernel2DSize, this.
|
|
663
|
+
this.minSphericalHarmonicsDegree, this.kernel2DSize, this.irisOcclusionConfig);
|
|
659
664
|
} else {
|
|
660
665
|
this.material = SplatMaterial2D.build(this.dynamicMode, this.enableOptionalEffects,
|
|
661
666
|
this.splatScale, this.pointCloudModeEnabled, this.minSphericalHarmonicsDegree);
|
|
@@ -1188,17 +1193,18 @@ export class SplatMesh extends Mesh {
|
|
|
1188
1193
|
};
|
|
1189
1194
|
this.material.uniforms.sceneCount.value = this.scenes.length;
|
|
1190
1195
|
|
|
1191
|
-
|
|
1192
|
-
this.
|
|
1196
|
+
// FLAME-specific setup (only if flameModel has required data)
|
|
1197
|
+
if (this.flameModel && this.flameModel.geometry && this.flameModel.geometry.morphAttributes && this.flameModel.skeleton) {
|
|
1198
|
+
this.expressionBSNum = this.flameModel.geometry.morphAttributes.position.length;
|
|
1199
|
+
this.material.uniforms.bsCount.value = this.expressionBSNum;
|
|
1193
1200
|
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1201
|
+
this.flameModel.skeleton.bones.forEach((bone, index) => {
|
|
1202
|
+
if (bone.name == 'head')
|
|
1203
|
+
this.material.uniforms.headBoneIndex.value = index;
|
|
1204
|
+
});
|
|
1198
1205
|
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
if(this.useFlameModel) {
|
|
1206
|
+
this.buildModelTexture(this.flameModel);
|
|
1207
|
+
this.buildBoneMatrixTexture();
|
|
1202
1208
|
this.buildBoneWeightTexture(this.flameModel);
|
|
1203
1209
|
}
|
|
1204
1210
|
}
|
|
@@ -1206,21 +1212,25 @@ export class SplatMesh extends Mesh {
|
|
|
1206
1212
|
buildBoneMatrixTexture() {
|
|
1207
1213
|
if (!this.bsWeight)
|
|
1208
1214
|
return
|
|
1215
|
+
|
|
1216
|
+
// Guard: Requires FLAME model with morphTargetDictionary
|
|
1217
|
+
if (!this.flameModel || !this.flameModel.morphTargetDictionary) {
|
|
1218
|
+
return;
|
|
1219
|
+
}
|
|
1220
|
+
|
|
1209
1221
|
//this.bonesNum + this.expressionBSNum / 4 = 30, so 32
|
|
1210
1222
|
const boneTextureSize = new Vector2(4, 32);
|
|
1211
1223
|
let boneMatrixTextureData = new Float32Array(this.bonesMatrix);
|
|
1212
1224
|
let boneMatrixTextureDataInt = new Uint32Array(boneTextureSize.x * boneTextureSize.y * 4);
|
|
1213
1225
|
this.morphTargetDictionary = this.flameModel.morphTargetDictionary;
|
|
1214
1226
|
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
this.material.uniforms.bindMatrixInverse.value = this.flameModel.bindMatrixInverse;
|
|
1223
|
-
}
|
|
1227
|
+
for (let c = 0; c < this.bonesNum * 16; c++) {
|
|
1228
|
+
boneMatrixTextureDataInt[c] = uintEncodedFloat(boneMatrixTextureData[c]);
|
|
1229
|
+
}
|
|
1230
|
+
if (this.flameModel && this.flameModel.skeleton) {
|
|
1231
|
+
this.material.uniforms.boneTexture0.value = this.flameModel.skeleton.boneTexture;
|
|
1232
|
+
this.material.uniforms.bindMatrix.value = this.flameModel.bindMatrix;
|
|
1233
|
+
this.material.uniforms.bindMatrixInverse.value = this.flameModel.bindMatrixInverse;
|
|
1224
1234
|
}
|
|
1225
1235
|
for (const key in this.bsWeight) {
|
|
1226
1236
|
if (Object.hasOwn(this.bsWeight, key)) {
|
|
@@ -1252,17 +1262,10 @@ export class SplatMesh extends Mesh {
|
|
|
1252
1262
|
this.splatDataTextures.baseData['boneMatrix'] = boneMatrixTextureDataInt;
|
|
1253
1263
|
}
|
|
1254
1264
|
|
|
1255
|
-
updateBoneMatrixTexture(
|
|
1265
|
+
updateBoneMatrixTexture() {
|
|
1256
1266
|
if (!this.bsWeight || !this.morphTargetDictionary)
|
|
1257
1267
|
return
|
|
1258
1268
|
|
|
1259
|
-
if(updateFlameBoneMatrix == true) {
|
|
1260
|
-
let boneMatrixTextureData = new Float32Array(this.bonesMatrix);
|
|
1261
|
-
for (let c = 0; c < this.bonesNum * 16; c++) {
|
|
1262
|
-
this.splatDataTextures.baseData['boneMatrix'][c] = uintEncodedFloat(boneMatrixTextureData[c]);
|
|
1263
|
-
}
|
|
1264
|
-
}
|
|
1265
|
-
|
|
1266
1269
|
for (const key in this.bsWeight) {
|
|
1267
1270
|
if (Object.hasOwn(this.bsWeight, key)) {
|
|
1268
1271
|
const value = this.bsWeight[key];
|
|
@@ -1289,6 +1292,11 @@ export class SplatMesh extends Mesh {
|
|
|
1289
1292
|
}
|
|
1290
1293
|
|
|
1291
1294
|
buildBoneWeightTexture(flameModel) {
|
|
1295
|
+
// Guard: bonesWeight is required for FLAME bone weight texture building
|
|
1296
|
+
if (!this.bonesWeight) {
|
|
1297
|
+
return;
|
|
1298
|
+
}
|
|
1299
|
+
|
|
1292
1300
|
let shapedMesh = flameModel.geometry.attributes.position.array;
|
|
1293
1301
|
|
|
1294
1302
|
let pointNum = shapedMesh.length / 3;
|
|
@@ -1328,6 +1336,11 @@ export class SplatMesh extends Mesh {
|
|
|
1328
1336
|
|
|
1329
1337
|
|
|
1330
1338
|
buildModelTexture(flameModel) {
|
|
1339
|
+
// Guard: Requires FLAME model with morph attributes
|
|
1340
|
+
if (!flameModel || !flameModel.geometry || !flameModel.geometry.morphAttributes || !flameModel.morphTargetDictionary) {
|
|
1341
|
+
return;
|
|
1342
|
+
}
|
|
1343
|
+
|
|
1331
1344
|
const flameModelTexSize = new Vector2(4096, 2048);
|
|
1332
1345
|
|
|
1333
1346
|
var shapedMesh = flameModel.geometry.attributes.position.array;
|
|
@@ -1337,9 +1350,6 @@ export class SplatMesh extends Mesh {
|
|
|
1337
1350
|
let bsLength = flameModel.geometry.morphAttributes.position.length;
|
|
1338
1351
|
|
|
1339
1352
|
const morphTargetNames = Object.keys(flameModel.morphTargetDictionary);
|
|
1340
|
-
// if (this.useFlameModel == false) {
|
|
1341
|
-
// morphTargetNames.sort();
|
|
1342
|
-
// }
|
|
1343
1353
|
morphTargetNames.forEach((name, newIndex) => {
|
|
1344
1354
|
const originalIndex = flameModel.morphTargetDictionary[name];
|
|
1345
1355
|
var bsMesh = flameModel.geometry.morphAttributes.position[originalIndex];
|
|
@@ -1377,7 +1387,7 @@ export class SplatMesh extends Mesh {
|
|
|
1377
1387
|
this.splatDataTextures.baseData['flameModelPos'] = flameModelData;
|
|
1378
1388
|
}
|
|
1379
1389
|
|
|
1380
|
-
updateTetureAfterBSAndSkeleton(fromSplat, toSplat
|
|
1390
|
+
updateTetureAfterBSAndSkeleton(fromSplat, toSplat) {
|
|
1381
1391
|
const sceneTransform = new Matrix4();
|
|
1382
1392
|
|
|
1383
1393
|
this.getSceneTransform(0, sceneTransform);
|
|
@@ -1398,7 +1408,7 @@ export class SplatMesh extends Mesh {
|
|
|
1398
1408
|
fromSplat, toSplat);
|
|
1399
1409
|
}
|
|
1400
1410
|
|
|
1401
|
-
this.updateBoneMatrixTexture(
|
|
1411
|
+
this.updateBoneMatrixTexture();
|
|
1402
1412
|
}
|
|
1403
1413
|
|
|
1404
1414
|
updateBaseDataFromSplatBuffers(fromSplat, toSplat) {
|
|
@@ -1946,7 +1956,7 @@ export class SplatMesh extends Mesh {
|
|
|
1946
1956
|
const createShader = (gl, type, source) => {
|
|
1947
1957
|
const shader = gl.createShader(type);
|
|
1948
1958
|
if (!shader) {
|
|
1949
|
-
|
|
1959
|
+
logger.error('Fatal error: gl could not create a shader object.');
|
|
1950
1960
|
return null;
|
|
1951
1961
|
}
|
|
1952
1962
|
|
|
@@ -1959,7 +1969,7 @@ export class SplatMesh extends Mesh {
|
|
|
1959
1969
|
if (type === gl.VERTEX_SHADER) typeName = 'vertex shader';
|
|
1960
1970
|
else if (type === gl.FRAGMENT_SHADER) typeName = 'fragement shader';
|
|
1961
1971
|
const errors = gl.getShaderInfoLog(shader);
|
|
1962
|
-
|
|
1972
|
+
logger.error('Failed to compile ' + typeName + ' with these errors:' + errors);
|
|
1963
1973
|
gl.deleteShader(shader);
|
|
1964
1974
|
return null;
|
|
1965
1975
|
}
|
|
@@ -2046,7 +2056,7 @@ export class SplatMesh extends Mesh {
|
|
|
2046
2056
|
const linked = gl.getProgramParameter(program, gl.LINK_STATUS);
|
|
2047
2057
|
if (!linked) {
|
|
2048
2058
|
const error = gl.getProgramInfoLog(program);
|
|
2049
|
-
|
|
2059
|
+
logger.error('Fatal error: Failed to link program: ' + error);
|
|
2050
2060
|
gl.deleteProgram(program);
|
|
2051
2061
|
gl.deleteShader(fragmentShader);
|
|
2052
2062
|
gl.deleteShader(vertexShader);
|
|
@@ -2225,7 +2235,6 @@ export class SplatMesh extends Mesh {
|
|
|
2225
2235
|
return (modelViewProjMatrix, outComputedDistances) => {
|
|
2226
2236
|
if (!this.renderer) return;
|
|
2227
2237
|
|
|
2228
|
-
// console.time("gpu_compute_distances");
|
|
2229
2238
|
const gl = this.renderer.getContext();
|
|
2230
2239
|
|
|
2231
2240
|
const currentVao = gl.getParameter(gl.VERTEX_ARRAY_BINDING);
|
|
@@ -2316,8 +2325,6 @@ export class SplatMesh extends Mesh {
|
|
|
2316
2325
|
|
|
2317
2326
|
if (currentVao) gl.bindVertexArray(currentVao);
|
|
2318
2327
|
|
|
2319
|
-
// console.timeEnd("gpu_compute_distances");
|
|
2320
|
-
|
|
2321
2328
|
resolve();
|
|
2322
2329
|
}
|
|
2323
2330
|
}
|
package/src/core/Viewer.js
CHANGED
|
@@ -12,12 +12,12 @@
|
|
|
12
12
|
*/
|
|
13
13
|
|
|
14
14
|
import { Bone, DynamicDrawUsage, InstancedBufferAttribute, MathUtils, Matrix4, OrthographicCamera, PerspectiveCamera, Quaternion, Scene, Skeleton, Vector2, Vector3, WebGLRenderer } from 'three';
|
|
15
|
-
import {
|
|
16
|
-
getCurrentTime,
|
|
17
|
-
clamp,
|
|
18
|
-
delayedExecute,
|
|
19
|
-
isIOS,
|
|
20
|
-
getIOSSemever,
|
|
15
|
+
import {
|
|
16
|
+
getCurrentTime,
|
|
17
|
+
clamp,
|
|
18
|
+
delayedExecute,
|
|
19
|
+
isIOS,
|
|
20
|
+
getIOSSemever,
|
|
21
21
|
Semver,
|
|
22
22
|
fetchWithProgress,
|
|
23
23
|
nativePromiseWithExtractedComponents,
|
|
@@ -25,6 +25,9 @@ import {
|
|
|
25
25
|
disposeAllMeshes,
|
|
26
26
|
AbortedPromiseError
|
|
27
27
|
} from '../utils/Util.js';
|
|
28
|
+
import { getLogger } from '../utils/Logger.js';
|
|
29
|
+
|
|
30
|
+
const logger = getLogger('Viewer');
|
|
28
31
|
import { RenderMode } from '../enums/RenderMode.js';
|
|
29
32
|
import { LogLevel } from '../enums/LogLevel.js';
|
|
30
33
|
import { SplatRenderMode } from '../enums/SplatRenderMode.js';
|
|
@@ -415,15 +418,8 @@ export class Viewer {
|
|
|
415
418
|
|
|
416
419
|
this.lastTime = 0;
|
|
417
420
|
this.gaussianSplatCount = 0;
|
|
418
|
-
this.totalFrames = 0;
|
|
419
|
-
this.flame_params = null;
|
|
420
|
-
this.bone_tree = null;
|
|
421
|
-
this.lbs_weight_80k = null;
|
|
421
|
+
this.totalFrames = 0;
|
|
422
422
|
this.frame = 0;
|
|
423
|
-
|
|
424
|
-
this.useFlame = true;
|
|
425
|
-
this.bones = null;
|
|
426
|
-
this.skeleton = null;
|
|
427
423
|
this. avatarMesh = null;
|
|
428
424
|
this.skinModel = null;
|
|
429
425
|
this.boneRoot = null;
|
|
@@ -449,6 +445,9 @@ export class Viewer {
|
|
|
449
445
|
this.sceneFadeInRateMultiplier,
|
|
450
446
|
this.kernel2DSize
|
|
451
447
|
);
|
|
448
|
+
|
|
449
|
+
// Pass iris occlusion config if loaded
|
|
450
|
+
this.splatMesh.irisOcclusionConfig = this.irisOcclusionConfig;
|
|
452
451
|
this.splatMesh.frustumCulled = false;
|
|
453
452
|
if (this.onSplatMeshChangedCallback) this.onSplatMeshChangedCallback();
|
|
454
453
|
}
|
|
@@ -613,8 +612,6 @@ export class Viewer {
|
|
|
613
612
|
false
|
|
614
613
|
);
|
|
615
614
|
this.keyDownListener = this.onKeyDown.bind(this);
|
|
616
|
-
// 暂时去掉键盘事件的监听
|
|
617
|
-
// window.addEventListener('keydown', this.keyDownListener, false)
|
|
618
615
|
}
|
|
619
616
|
}
|
|
620
617
|
|
|
@@ -979,7 +976,7 @@ export class Viewer {
|
|
|
979
976
|
this.splatMesh.scenes &&
|
|
980
977
|
this.splatMesh.scenes.length > 0
|
|
981
978
|
) {
|
|
982
|
-
|
|
979
|
+
logger.warn(
|
|
983
980
|
'addSplatScene(): "progressiveLoad" option ignore because there are multiple splat scenes'
|
|
984
981
|
);
|
|
985
982
|
options.progressiveLoad = false;
|
|
@@ -1035,7 +1032,7 @@ export class Viewer {
|
|
|
1035
1032
|
}
|
|
1036
1033
|
}
|
|
1037
1034
|
} else if (loaderStatus === LoaderStatus.Processing) {
|
|
1038
|
-
|
|
1035
|
+
logger.debug('loaderStatus === LoaderStatus.Processing');
|
|
1039
1036
|
this.loadingSpinner.setMessageForTask(
|
|
1040
1037
|
loadingUITaskId,
|
|
1041
1038
|
'Processing splats...'
|
|
@@ -1280,7 +1277,7 @@ export class Viewer {
|
|
|
1280
1277
|
this.removeSplatSceneDownloadPromise(splatSceneDownloadPromise);
|
|
1281
1278
|
})
|
|
1282
1279
|
.catch((e) => {
|
|
1283
|
-
|
|
1280
|
+
logger.error('Viewer::addSplatScene actual error:', e);
|
|
1284
1281
|
this.clearSplatSceneDownloadAndBuildPromise();
|
|
1285
1282
|
this.removeSplatSceneDownloadPromise(splatSceneDownloadPromise);
|
|
1286
1283
|
const error =
|
|
@@ -1691,7 +1688,6 @@ export class Viewer {
|
|
|
1691
1688
|
0,
|
|
1692
1689
|
e.data.splatRenderCount
|
|
1693
1690
|
);
|
|
1694
|
-
// console.log(sortedIndexes);
|
|
1695
1691
|
this.splatMesh.updateRenderIndexes(
|
|
1696
1692
|
sortedIndexes,
|
|
1697
1693
|
e.data.splatRenderCount
|
|
@@ -1714,7 +1710,7 @@ export class Viewer {
|
|
|
1714
1710
|
this.sortRunning = false;
|
|
1715
1711
|
} else if (e.data.sortSetupPhase1Complete) {
|
|
1716
1712
|
if (this.logLevel >= LogLevel.Info)
|
|
1717
|
-
|
|
1713
|
+
logger.info('Sorting web worker WASM setup complete.');
|
|
1718
1714
|
if (this.sharedMemoryForWorkers) {
|
|
1719
1715
|
this.sortWorkerSortedIndexes = new Uint32Array(
|
|
1720
1716
|
e.data.sortedIndexesBuffer,
|
|
@@ -1750,17 +1746,17 @@ export class Viewer {
|
|
|
1750
1746
|
this.sortWorker.maxSplatCount = maxSplatCount;
|
|
1751
1747
|
|
|
1752
1748
|
if (this.logLevel >= LogLevel.Info) {
|
|
1753
|
-
|
|
1749
|
+
logger.info('Sorting web worker ready.');
|
|
1754
1750
|
const splatDataTextures = this.splatMesh.getSplatDataTextures();
|
|
1755
1751
|
const covariancesTextureSize = splatDataTextures.covariances.size;
|
|
1756
1752
|
const centersColorsTextureSize = splatDataTextures.centerColors.size;
|
|
1757
|
-
|
|
1753
|
+
logger.info(
|
|
1758
1754
|
'Covariances texture size: ' +
|
|
1759
1755
|
covariancesTextureSize.x +
|
|
1760
1756
|
' x ' +
|
|
1761
1757
|
covariancesTextureSize.y
|
|
1762
1758
|
);
|
|
1763
|
-
|
|
1759
|
+
logger.info(
|
|
1764
1760
|
'Centers/colors texture size: ' +
|
|
1765
1761
|
centersColorsTextureSize.x +
|
|
1766
1762
|
' x ' +
|
|
@@ -2695,113 +2691,14 @@ export class Viewer {
|
|
|
2695
2691
|
return navigator.userAgent.includes('Mobi')
|
|
2696
2692
|
}
|
|
2697
2693
|
|
|
2698
|
-
createBonesFromJson(bonesJson) {
|
|
2699
|
-
const bones = [];
|
|
2700
|
-
|
|
2701
|
-
function createBoneRecursive(jsonBone, parent = null) {
|
|
2702
|
-
const bone = new Bone();
|
|
2703
|
-
bone.name = jsonBone.name;
|
|
2704
|
-
if (parent) {
|
|
2705
|
-
parent.add(bone);
|
|
2706
|
-
}
|
|
2707
|
-
bone.position.set(...jsonBone.position);
|
|
2708
|
-
bones.push(bone);
|
|
2709
|
-
|
|
2710
|
-
if (jsonBone.children) {
|
|
2711
|
-
jsonBone.children.forEach((childJsonBone) =>
|
|
2712
|
-
createBoneRecursive(childJsonBone, bone)
|
|
2713
|
-
);
|
|
2714
|
-
}
|
|
2715
|
-
return bone
|
|
2716
|
-
}
|
|
2717
|
-
|
|
2718
|
-
bonesJson.forEach((boneJson) => createBoneRecursive(boneJson));
|
|
2719
|
-
|
|
2720
|
-
return bones
|
|
2721
|
-
}
|
|
2722
|
-
|
|
2723
2694
|
updateMorphTarget(inputMesh) {
|
|
2724
2695
|
this.avatarMesh = inputMesh;
|
|
2725
2696
|
this.splatMesh.flameModel = inputMesh;
|
|
2726
|
-
this.splatMesh.useFlameModel = this.useFlame;
|
|
2727
|
-
if(this.useFlame == true) {
|
|
2728
|
-
// const skinData = {
|
|
2729
|
-
// bones: [
|
|
2730
|
-
// {
|
|
2731
|
-
// "name": "root",
|
|
2732
|
-
// "position": [-1.7149e-04, -1.4252e-01, -8.2541e-02],
|
|
2733
|
-
// "children": [
|
|
2734
|
-
// {
|
|
2735
|
-
// "name": "neck",
|
|
2736
|
-
// "position": [-5.6988e-05, -1.6069e-02, -5.7859e-02],
|
|
2737
|
-
|
|
2738
|
-
// "children": [
|
|
2739
|
-
// {
|
|
2740
|
-
// "name": "jaw",
|
|
2741
|
-
// "position": [7.4429e-04, -8.7249e-03, -5.3760e-02]
|
|
2742
|
-
|
|
2743
|
-
// },
|
|
2744
|
-
// {
|
|
2745
|
-
// "name": "leftEye",
|
|
2746
|
-
// "position": [ 3.0240e-02, 2.3092e-02, 2.2900e-02]
|
|
2747
|
-
// },
|
|
2748
|
-
// {
|
|
2749
|
-
// "name": "rightEye",
|
|
2750
|
-
// "position": [-3.0296e-02, 2.3675e-02, 2.1837e-02]
|
|
2751
|
-
// }
|
|
2752
|
-
// ]
|
|
2753
|
-
// }
|
|
2754
|
-
// ]
|
|
2755
|
-
// }
|
|
2756
|
-
// ]
|
|
2757
|
-
// };
|
|
2758
|
-
|
|
2759
|
-
// this.bones = this.createBonesFromJson(skinData.bones);
|
|
2760
|
-
this.bones = this.createBonesFromJson(this.bone_tree["bones"]);
|
|
2761
|
-
|
|
2762
|
-
const bonesPosReserve = [new Vector3(this.bones[0].position.x, this.bones[0].position.y, this.bones[0].position.z),
|
|
2763
|
-
new Vector3(this.bones[1].position.x, this.bones[1].position.y, this.bones[1].position.z),
|
|
2764
|
-
new Vector3(this.bones[2].position.x, this.bones[2].position.y, this.bones[2].position.z),
|
|
2765
|
-
new Vector3(this.bones[3].position.x, this.bones[3].position.y, this.bones[3].position.z),
|
|
2766
|
-
new Vector3(this.bones[4].position.x, this.bones[4].position.y, this.bones[4].position.z)
|
|
2767
|
-
];
|
|
2768
|
-
this.bones[1].position.copy(new Vector3(bonesPosReserve[1].x - bonesPosReserve[0].x, bonesPosReserve[1].y - bonesPosReserve[0].y, bonesPosReserve[1].z - bonesPosReserve[0].z));
|
|
2769
|
-
this.bones[2].position.copy(new Vector3(bonesPosReserve[2].x - bonesPosReserve[1].x, bonesPosReserve[2].y - bonesPosReserve[1].y, bonesPosReserve[2].z - bonesPosReserve[1].z));
|
|
2770
|
-
this.bones[3].position.copy(new Vector3(bonesPosReserve[3].x - bonesPosReserve[1].x, bonesPosReserve[3].y - bonesPosReserve[1].y, bonesPosReserve[3].z - bonesPosReserve[1].z));
|
|
2771
|
-
this.bones[4].position.copy(new Vector3(bonesPosReserve[4].x - bonesPosReserve[1].x, bonesPosReserve[4].y - bonesPosReserve[1].y, bonesPosReserve[4].z - bonesPosReserve[1].z));
|
|
2772
|
-
|
|
2773
|
-
this.bones[0].updateMatrixWorld(true);
|
|
2774
|
-
const boneInverses = [this.bones[0].matrixWorld.clone().invert(),
|
|
2775
|
-
this.bones[1].matrixWorld.clone().invert(),
|
|
2776
|
-
this.bones[2].matrixWorld.clone().invert(),
|
|
2777
|
-
this.bones[3].matrixWorld.clone().invert(),
|
|
2778
|
-
this.bones[4].matrixWorld.clone().invert()];
|
|
2779
|
-
|
|
2780
|
-
this.skeleton = new Skeleton(this.bones, boneInverses);
|
|
2781
|
-
}
|
|
2782
2697
|
|
|
2783
2698
|
this.runMorphUpdate();
|
|
2784
2699
|
this.splatMesh.gaussianSplatCount = this.gaussianSplatCount;
|
|
2785
2700
|
}
|
|
2786
2701
|
|
|
2787
|
-
updatedBoneMatrices(boneNum){
|
|
2788
|
-
let updatedBoneMatrices = [];
|
|
2789
|
-
for (let j = 0; j < boneNum; j++) {
|
|
2790
|
-
let boneMatrix;
|
|
2791
|
-
boneMatrix = this.skeleton.bones[j].matrixWorld.clone().multiply(this.skeleton.boneInverses[j].clone());
|
|
2792
|
-
|
|
2793
|
-
function addMatrixToArray(matrix, array) {
|
|
2794
|
-
let elements = matrix.elements;
|
|
2795
|
-
for (let i = 0; i < elements.length; i++) {
|
|
2796
|
-
array.push(elements[i]);
|
|
2797
|
-
}
|
|
2798
|
-
}
|
|
2799
|
-
|
|
2800
|
-
addMatrixToArray(boneMatrix, updatedBoneMatrices);
|
|
2801
|
-
}
|
|
2802
|
-
let bonesMatrix = new Float32Array(updatedBoneMatrices);
|
|
2803
|
-
return bonesMatrix;
|
|
2804
|
-
}
|
|
2805
2702
|
runMorphUpdate() {
|
|
2806
2703
|
this.gaussianSplatCount = this.avatarMesh.geometry.attributes.position.count;
|
|
2807
2704
|
var morphedMesh = new Float32Array(
|
|
@@ -2809,87 +2706,41 @@ export class Viewer {
|
|
|
2809
2706
|
);
|
|
2810
2707
|
const numBones = 5;
|
|
2811
2708
|
this.splatMesh.bonesNum = numBones;
|
|
2812
|
-
if (this.useFlame == false)
|
|
2813
|
-
{
|
|
2814
|
-
this.skinModel.skeleton.update();
|
|
2815
|
-
this.boneRoot.updateMatrixWorld(true);
|
|
2816
|
-
if (this.splatMesh.geometry.getAttribute('splatIndex') && this.setSkinAttibutes === false) {
|
|
2817
2709
|
|
|
2818
|
-
|
|
2819
|
-
|
|
2710
|
+
this.skinModel.skeleton.update();
|
|
2711
|
+
this.boneRoot.updateMatrixWorld(true);
|
|
2712
|
+
if (this.splatMesh.geometry.getAttribute('splatIndex') && this.setSkinAttibutes === false) {
|
|
2820
2713
|
|
|
2821
|
-
|
|
2822
|
-
|
|
2714
|
+
this.setSkinAttibutes = true;
|
|
2715
|
+
const geometry = this.splatMesh.geometry;
|
|
2823
2716
|
|
|
2824
|
-
|
|
2825
|
-
|
|
2826
|
-
|
|
2827
|
-
|
|
2828
|
-
|
|
2829
|
-
|
|
2830
|
-
|
|
2831
|
-
|
|
2832
|
-
|
|
2833
|
-
|
|
2834
|
-
|
|
2835
|
-
|
|
2836
|
-
|
|
2837
|
-
|
|
2838
|
-
|
|
2839
|
-
|
|
2840
|
-
|
|
2841
|
-
|
|
2842
|
-
|
|
2843
|
-
|
|
2717
|
+
const skinIndexSource = this.skinModel.geometry.attributes.skinIndex;
|
|
2718
|
+
const skinWeightSource = this.skinModel.geometry.attributes.skinWeight;
|
|
2719
|
+
|
|
2720
|
+
const newSkinIndex = new InstancedBufferAttribute(
|
|
2721
|
+
new skinIndexSource.array.constructor(skinIndexSource.array),
|
|
2722
|
+
4,
|
|
2723
|
+
skinIndexSource.normalized,
|
|
2724
|
+
1
|
|
2725
|
+
);
|
|
2726
|
+
|
|
2727
|
+
const newSkinWeight = new InstancedBufferAttribute(
|
|
2728
|
+
new skinWeightSource.array.constructor(skinWeightSource.array),
|
|
2729
|
+
4,
|
|
2730
|
+
skinWeightSource.normalized,
|
|
2731
|
+
1
|
|
2732
|
+
);
|
|
2733
|
+
newSkinIndex.setUsage(DynamicDrawUsage);
|
|
2734
|
+
newSkinWeight.setUsage(DynamicDrawUsage);
|
|
2735
|
+
geometry.setAttribute('skinIndex', newSkinIndex);
|
|
2736
|
+
geometry.setAttribute('skinWeight', newSkinWeight);
|
|
2844
2737
|
}
|
|
2845
|
-
|
|
2738
|
+
|
|
2846
2739
|
this.splatMesh.morphedMesh = morphedMesh;
|
|
2847
2740
|
|
|
2848
2741
|
let splatNum = this.splatMesh.morphedMesh.length / 3;
|
|
2849
2742
|
if (this.splatMesh.splatDataTextures['flameModel'] != undefined) {
|
|
2850
|
-
this.splatMesh.updateTetureAfterBSAndSkeleton(0, splatNum - 1,
|
|
2743
|
+
this.splatMesh.updateTetureAfterBSAndSkeleton(0, splatNum - 1, false);
|
|
2851
2744
|
}
|
|
2852
2745
|
}
|
|
2853
|
-
|
|
2854
|
-
updateFlameBones(){
|
|
2855
|
-
this.splatMesh.bsWeight = this.flame_params['expr'][this.frame];
|
|
2856
|
-
|
|
2857
|
-
function setBoneRotationAndMatrix(bone, angles, isQuat = false) {
|
|
2858
|
-
let quaternion;
|
|
2859
|
-
if(isQuat == true) {
|
|
2860
|
-
quaternion = new Quaternion(angles[0], angles[1], angles[2], angles[3]);
|
|
2861
|
-
} else {
|
|
2862
|
-
const value = new Vector3(angles[0], angles[1], angles[2]);
|
|
2863
|
-
const angleInRadians = value.length();
|
|
2864
|
-
const axis = value.normalize();
|
|
2865
|
-
quaternion = new Quaternion().setFromAxisAngle(axis, angleInRadians);
|
|
2866
|
-
}
|
|
2867
|
-
bone.quaternion.copy(quaternion);
|
|
2868
|
-
bone.updateMatrixWorld(true);
|
|
2869
|
-
}
|
|
2870
|
-
|
|
2871
|
-
let angle = this.flame_params['rotation'][this.frame];
|
|
2872
|
-
|
|
2873
|
-
setBoneRotationAndMatrix(this.skeleton.bones[0], angle);
|
|
2874
|
-
|
|
2875
|
-
angle = this.flame_params['neck_pose'][this.frame];
|
|
2876
|
-
setBoneRotationAndMatrix(this.skeleton.bones[1], angle);
|
|
2877
|
-
|
|
2878
|
-
angle = this.flame_params['jaw_pose'][this.frame];
|
|
2879
|
-
setBoneRotationAndMatrix(this.skeleton.bones[2], angle);
|
|
2880
|
-
|
|
2881
|
-
angle = this.flame_params['eyes_pose'][this.frame];
|
|
2882
|
-
setBoneRotationAndMatrix(this.skeleton.bones[3], angle);
|
|
2883
|
-
|
|
2884
|
-
setBoneRotationAndMatrix(this.skeleton.bones[4], [angle[3], angle[4], angle[5]]);
|
|
2885
|
-
|
|
2886
|
-
// update skeleton
|
|
2887
|
-
this.skeleton.update();
|
|
2888
|
-
|
|
2889
|
-
const numBones = 5;
|
|
2890
|
-
const bonesMatrix = this.updatedBoneMatrices(numBones);
|
|
2891
|
-
this.splatMesh.bonesMatrix = bonesMatrix;
|
|
2892
|
-
this.splatMesh.bonesNum = numBones;
|
|
2893
|
-
this.splatMesh.bonesWeight = this.lbs_weight_80k;
|
|
2894
|
-
}
|
|
2895
2746
|
}
|