@myned-ai/gsplat-flame-avatar-renderer 1.0.2 → 1.0.4

package/src/core/index.js

@@ -0,0 +1,13 @@
+/**
+ * gsplat-flame-avatar - Core Module
+ * Core rendering engine components.
+ * 
+ * Based on @mkkellogg/gaussian-splats-3d (MIT License)
+ * Extended with FLAME avatar support.
+ */
+
+export { SplatGeometry } from './SplatGeometry.js';
+export { SplatMesh } from './SplatMesh.js';
+export { SplatScene } from './SplatScene.js';
+export { SplatTree } from './SplatTree.js';
+export { Viewer } from './Viewer.js';

package/src/enums/EngineConstants.js

@@ -0,0 +1,58 @@
+/**
+ * EngineConstants
+ * 
+ * Derived from @mkkellogg/gaussian-splats-3d (MIT License)
+ * https://github.com/mkkellogg/GaussianSplats3D
+ * 
+ * Core constants for Gaussian Splat rendering.
+ */
+
+export class Constants {
+    static DefaultSplatSortDistanceMapPrecision = 16;
+    static MemoryPageSize = 65536;
+    static BytesPerFloat = 4;
+    static BytesPerInt = 4;
+    static MaxScenes = 32;
+    static ProgressiveLoadSectionSize = 262144;
+    static ProgressiveLoadSectionDelayDuration = 15;
+    static SphericalHarmonics8BitCompressionRange = 3;
+}
+
+export const DefaultSphericalHarmonics8BitCompressionRange = Constants.SphericalHarmonics8BitCompressionRange;
+
+// SplatMesh constants
+export const COVARIANCES_ELEMENTS_PER_SPLAT = 6;
+export const CENTER_COLORS_ELEMENTS_PER_SPLAT = 4;
+
+export const COVARIANCES_ELEMENTS_PER_TEXEL_STORED = 4;
+export const COVARIANCES_ELEMENTS_PER_TEXEL_ALLOCATED = 4;
+export const COVARIANCES_ELEMENTS_PER_TEXEL_COMPRESSED_STORED = 6;
+export const COVARIANCES_ELEMENTS_PER_TEXEL_COMPRESSED_ALLOCATED = 8;
+export const SCALES_ROTATIONS_ELEMENTS_PER_TEXEL = 4;
+export const CENTER_COLORS_ELEMENTS_PER_TEXEL = 4;
+export const SCENE_INDEXES_ELEMENTS_PER_TEXEL = 1;
+
+export const SCENE_FADEIN_RATE_FAST = 0.012;
+export const SCENE_FADEIN_RATE_GRADUAL = 0.003;
+
+// Viewer constants
+export const THREE_CAMERA_FOV = 50;
+export const MINIMUM_DISTANCE_TO_NEW_FOCAL_POINT = 0.5;
+export const CONSECUTIVE_RENDERED_FRAMES_FOR_FPS_CALCULATION = 60;
+export const MIN_SPLAT_COUNT_TO_SHOW_SPLAT_TREE_LOADING_SPINNER = 1500000;
+export const FOCUS_MARKER_FADE_IN_SPEED = 0.4;
+export const FOCUS_MARKER_FADE_OUT_SPEED = 0.12;
+
+// Internal load types for loaders
+export const InternalLoadType = {
+    DirectToSplatBuffer: 0,
+    DirectToSplatArray: 1,
+    DownloadBeforeProcessing: 2
+};
+
+// Loader status
+export const LoaderStatus = {
+    'Downloading': 0,
+    'Processing': 1,
+    'Done': 2
+};

package/src/enums/LogLevel.js

@@ -0,0 +1,13 @@
+/**
+ * Log level enumeration for controlling verbosity
+ * 
+ * Derived from @mkkellogg/gaussian-splats-3d (MIT License)
+ * https://github.com/mkkellogg/GaussianSplats3D
+ */
+export const LogLevel = {
+    None: 0,
+    Error: 1,
+    Warning: 2,
+    Info: 3,
+    Debug: 4
+};

package/src/enums/RenderMode.js

@@ -0,0 +1,11 @@
+/**
+ * Render mode enumeration
+ * 
+ * Derived from @mkkellogg/gaussian-splats-3d (MIT License)
+ * https://github.com/mkkellogg/GaussianSplats3D
+ */
+export const RenderMode = {
+    Always: 0,
+    OnChange: 1,
+    Never: 2
+};

package/src/enums/SceneFormat.js

@@ -0,0 +1,21 @@
+/**
+ * Scene format enumeration for supported file types
+ * 
+ * Derived from @mkkellogg/gaussian-splats-3d (MIT License)
+ * https://github.com/mkkellogg/GaussianSplats3D
+ * 
+ * Simplified for FLAME avatar usage - only PLY format is supported.
+ */
+export const SceneFormat = {
+    'Ply': 0
+};
+
+/**
+ * Determine scene format from file path
+ * @param {string} path - File path
+ * @returns {number|null} Scene format or null if unknown
+ */
+export const sceneFormatFromPath = (path) => {
+    if (path.endsWith('.ply')) return SceneFormat.Ply;
+    return null;
+};

package/src/enums/SceneRevealMode.js

@@ -0,0 +1,11 @@
+/**
+ * Scene reveal mode enumeration
+ * 
+ * Derived from @mkkellogg/gaussian-splats-3d (MIT License)
+ * https://github.com/mkkellogg/GaussianSplats3D
+ */
+export const SceneRevealMode = {
+    Default: 0,
+    Gradual: 1,
+    Instant: 2
+};

package/src/enums/SplatRenderMode.js

@@ -0,0 +1,10 @@
+/**
+ * Splat render mode enumeration
+ * 
+ * Derived from @mkkellogg/gaussian-splats-3d (MIT License)
+ * https://github.com/mkkellogg/GaussianSplats3D
+ */
+export const SplatRenderMode = {
+    ThreeD: 0,
+    TwoD: 1
+};

package/src/enums/index.js

@@ -0,0 +1,13 @@
+/**
+ * gsplat-flame-avatar - Enums Module
+ * Constants and enumerations for rendering configuration.
+ * 
+ * Derived from @mkkellogg/gaussian-splats-3d (MIT License)
+ */
+
+export * from './LogLevel.js';
+export * from './RenderMode.js';
+export * from './SceneFormat.js';
+export * from './SceneRevealMode.js';
+export * from './SplatRenderMode.js';
+export * from './EngineConstants.js';

package/src/flame/FlameAnimator.js

@@ -0,0 +1,271 @@
+/**
+ * FlameAnimator
+ * 
+ * Derived from gaussian-splat-renderer-for-lam
+ * 
+ * Handles FLAME model animation and skeleton updates:
+ * - Bone rotation from FLAME parameters
+ * - Blendshape weight updates
+ * - Skeleton synchronization with gaussian splat mesh
+ */
+
+import {
+    Vector3,
+    Quaternion,
+    Matrix4,
+    Bone,
+    Skeleton
+} from 'three';
+
+import { FlameTextureManager } from './FlameTextureManager.js';
+
+/**
+ * FlameAnimator - Manages FLAME parametric head model animation
+ */
+export class FlameAnimator {
+    constructor() {
+        this.skeleton = null;
+        this.bones = null;
+        this.flameParams = null;
+        this.lbsWeight = null;
+        this.frame = 0;
+        this.totalFrames = 0;
+        this.useFlame = true;
+        this.avatarMesh = null;
+        this.gaussianSplatCount = 0;
+    }
+
+    /**
+     * Initialize with FLAME parameters and LBS weights
+     * @param {object} flameParams - FLAME animation parameters
+     * @param {object} boneTree - Skeleton hierarchy data
+     * @param {Array} lbsWeight - LBS weights for each vertex
+     * @param {THREE.SkinnedMesh} avatarMesh - The FLAME avatar mesh
+     */
+    initialize(flameParams, boneTree, lbsWeight, avatarMesh) {
+        this.flameParams = flameParams;
+        this.lbsWeight = lbsWeight;
+        this.avatarMesh = avatarMesh;
+        
+        if (flameParams && flameParams.rotation) {
+            this.totalFrames = flameParams.rotation.length;
+        }
+
+        this.gaussianSplatCount = avatarMesh.geometry.attributes.position.count;
+        
+        // Build skeleton from bone tree
+        this.buildSkeleton(boneTree);
+    }
+
+    /**
+     * Build skeleton from bone tree data
+     * @param {object} boneTree - Skeleton hierarchy definition
+     */
+    buildSkeleton(boneTree) {
+        if (!boneTree) return;
+
+        this.bones = [];
+        const boneInverses = [];
+
+        // Create bones from tree structure
+        const createBone = (boneData, parentBone = null) => {
+            const bone = new Bone();
+            bone.name = boneData.name || `bone_${this.bones.length}`;
+
+            if (boneData.position) {
+                bone.position.fromArray(boneData.position);
+            }
+            if (boneData.rotation) {
+                bone.rotation.fromArray(boneData.rotation);
+            }
+            if (boneData.scale) {
+                bone.scale.fromArray(boneData.scale);
+            }
+
+            if (parentBone) {
+                parentBone.add(bone);
+            }
+
+            this.bones.push(bone);
+
+            // Create inverse bind matrix
+            const inverseMatrix = new Matrix4();
+            if (boneData.inverseBindMatrix) {
+                inverseMatrix.fromArray(boneData.inverseBindMatrix);
+            } else {
+                bone.updateMatrixWorld(true);
+                inverseMatrix.copy(bone.matrixWorld).invert();
+            }
+            boneInverses.push(inverseMatrix);
+
+            // Process children
+            if (boneData.children) {
+                boneData.children.forEach(child => createBone(child, bone));
+            }
+        };
+
+        // If boneTree is array, create bones directly
+        if (Array.isArray(boneTree)) {
+            boneTree.forEach((boneData, index) => {
+                const bone = new Bone();
+                bone.name = boneData.name || `bone_${index}`;
+                
+                if (boneData.position) {
+                    bone.position.fromArray(boneData.position);
+                }
+                
+                this.bones.push(bone);
+                
+                const inverseMatrix = new Matrix4();
+                if (boneData.inverseBindMatrix) {
+                    inverseMatrix.fromArray(boneData.inverseBindMatrix);
+                }
+                boneInverses.push(inverseMatrix);
+            });
+
+            // Set up hierarchy (FLAME: root -> neck -> jaw, eyes)
+            if (this.bones.length >= 5) {
+                this.bones[0].add(this.bones[1]); // root -> neck
+                this.bones[1].add(this.bones[2]); // neck -> jaw
+                this.bones[1].add(this.bones[3]); // neck -> leftEye
+                this.bones[1].add(this.bones[4]); // neck -> rightEye
+            }
+        } else if (boneTree.root) {
+            createBone(boneTree.root);
+        }
+
+        // Create skeleton
+        this.skeleton = new Skeleton(this.bones, boneInverses);
+    }
+
+    /**
+     * Set bone rotation from axis-angle or quaternion
+     * @param {THREE.Bone} bone - Target bone
+     * @param {Array} angles - Rotation values
+     * @param {boolean} isQuat - Whether angles are quaternion
+     */
+    setBoneRotation(bone, angles, isQuat = false) {
+        let quaternion;
+        
+        if (isQuat) {
+            quaternion = new Quaternion(angles[0], angles[1], angles[2], angles[3]);
+        } else {
+            // Axis-angle representation
+            const value = new Vector3(angles[0], angles[1], angles[2]);
+            const angleInRadians = value.length();
+            const axis = value.normalize();
+            quaternion = new Quaternion().setFromAxisAngle(axis, angleInRadians);
+        }
+        
+        bone.quaternion.copy(quaternion);
+        bone.updateMatrixWorld(true);
+    }
+
+    /**
+     * Update FLAME bones from current frame parameters
+     */
+    updateFlameBones() {
+        if (!this.flameParams || !this.skeleton) return {};
+
+        const frame = this.frame;
+        const bsWeight = this.flameParams['expr'][frame];
+
+        // Root bone rotation
+        const rootAngle = this.flameParams['rotation'][frame];
+        this.setBoneRotation(this.skeleton.bones[0], rootAngle);
+
+        // Neck rotation
+        const neckAngle = this.flameParams['neck_pose'][frame];
+        this.setBoneRotation(this.skeleton.bones[1], neckAngle);
+
+        // Jaw rotation
+        const jawAngle = this.flameParams['jaw_pose'][frame];
+        this.setBoneRotation(this.skeleton.bones[2], jawAngle);
+
+        // Eyes rotation (combined in eyes_pose array)
+        const eyesAngle = this.flameParams['eyes_pose'][frame];
+        this.setBoneRotation(this.skeleton.bones[3], eyesAngle);
+        this.setBoneRotation(this.skeleton.bones[4], [eyesAngle[3], eyesAngle[4], eyesAngle[5]]);
+
+        // Update skeleton matrices
+        this.skeleton.update();
+
+        // Get updated bone matrices
+        const numBones = 5;
+        const bonesMatrix = FlameTextureManager.getUpdatedBoneMatrices(this.skeleton, numBones);
+
+        return {
+            bsWeight,
+            bonesMatrix,
+            bonesNum: numBones,
+            bonesWeight: this.lbsWeight
+        };
+    }
+
+    /**
+     * Run morph update - main animation loop function
+     * @param {SplatMesh} splatMesh - The gaussian splat mesh to update
+     */
+    runMorphUpdate(splatMesh) {
+        const morphedMesh = new Float32Array(
+            this.avatarMesh.geometry.attributes.position.array
+        );
+
+        splatMesh.gaussianSplatCount = this.gaussianSplatCount;
+        splatMesh.bonesNum = 5;
+
+        if (this.useFlame) {
+            const updateData = this.updateFlameBones();
+            splatMesh.bsWeight = updateData.bsWeight;
+            splatMesh.bonesMatrix = updateData.bonesMatrix;
+            splatMesh.bonesNum = updateData.bonesNum;
+            splatMesh.bonesWeight = updateData.bonesWeight;
+        }
+
+        splatMesh.morphedMesh = morphedMesh;
+
+        // Update textures
+        const splatNum = splatMesh.morphedMesh.length / 3;
+        if (splatMesh.splatDataTextures && splatMesh.splatDataTextures['flameModel']) {
+            splatMesh.updateTextureAfterBSAndSkeleton(0, splatNum - 1, this.useFlame);
+        }
+    }
+
+    /**
+     * Set current animation frame
+     * @param {number} frame - Frame number
+     */
+    setFrame(frame) {
+        this.frame = frame % this.totalFrames;
+    }
+
+    /**
+     * Advance to next frame
+     */
+    nextFrame() {
+        this.frame = (this.frame + 1) % this.totalFrames;
+    }
+
+    /**
+     * Get skeleton for external use
+     */
+    getSkeleton() {
+        return this.skeleton;
+    }
+
+    /**
+     * Get current frame number
+     */
+    getFrame() {
+        return this.frame;
+    }
+
+    /**
+     * Get total frame count
+     */
+    getTotalFrames() {
+        return this.totalFrames;
+    }
+}
+
+export default FlameAnimator;

package/src/flame/FlameConstants.js

@@ -0,0 +1,21 @@
+/**
+ * FlameConstants
+ * 
+ * Derived from gaussian-splat-renderer-for-lam
+ * FLAME-specific constants for parametric head model.
+ * 
+ * Note: General engine constants (MaxScenes, etc.) are in enums/EngineConstants.js
+ */
+
+export const Constants = {
+    // FLAME model constants
+    FlameBonesCount: 5,  // root, neck, jaw, leftEye, rightEye
+    DefaultBlendshapeCount: 52,  // ARKit blendshapes
+    
+    // Texture sizes for FLAME data
+    FlameModelTextureSize: { width: 4096, height: 2048 },
+    BoneTextureSize: { width: 4, height: 32 },
+    BoneWeightTextureSize: { width: 512, height: 512 }
+};
+
+export default Constants;