@myned-ai/gsplat-flame-avatar-renderer 1.0.1 → 1.0.4

package/src/materials/SplatMaterial2D.js

@@ -0,0 +1,358 @@
+/**
+ * SplatMaterial2D
+ * 
+ * Based on @mkkellogg/gaussian-splats-3d (MIT License)
+ * https://github.com/mkkellogg/GaussianSplats3D
+ * 
+ * Modified for FLAME avatar support:
+ * - Extended vertex shader for FLAME integration
+ */
+
+import { DoubleSide, NormalBlending, ShaderMaterial, Vector2 } from 'three';
+import { SplatMaterial } from './SplatMaterial.js';
+
+export class SplatMaterial2D {
+
+    /**
+     * Build the Three.js material that is used to render the splats.
+     * @param {number} dynamicMode If true, it means the scene geometry represented by this splat mesh is not stationary or
+     *                             that the splat count might change
+     * @param {boolean} enableOptionalEffects When true, allows for usage of extra properties and attributes in the shader for effects
+     *                                        such as opacity adjustment. Default is false for performance reasons.
+     * @param {number} splatScale Value by which all splats are scaled in screen-space (default is 1.0)
+     * @param {number} pointCloudModeEnabled Render all splats as screen-space circles
+     * @param {number} maxSphericalHarmonicsDegree Degree of spherical harmonics to utilize in rendering splats
+     * @return {THREE.ShaderMaterial}
+     */
+    static build(dynamicMode = false, enableOptionalEffects = false, splatScale = 1.0,
+                 pointCloudModeEnabled = false, maxSphericalHarmonicsDegree = 0) {
+
+        const customVertexVars = `
+            uniform vec2 scaleRotationsTextureSize;
+            uniform highp sampler2D scaleRotationsTexture;
+            varying mat3 vT;
+            varying vec2 vQuadCenter;
+            varying vec2 vFragCoord;
+        `;
+
+        let vertexShaderSource = SplatMaterial.buildVertexShaderBase(dynamicMode, enableOptionalEffects,
+                                                                     maxSphericalHarmonicsDegree, customVertexVars);
+        vertexShaderSource += SplatMaterial2D.buildVertexShaderProjection();
+        const fragmentShaderSource = SplatMaterial2D.buildFragmentShader();
+
+        const uniforms = SplatMaterial.getUniforms(dynamicMode, enableOptionalEffects,
+                                                   maxSphericalHarmonicsDegree, splatScale, pointCloudModeEnabled);
+
+        uniforms['scaleRotationsTexture'] = {
+            'type': 't',
+            'value': null
+        };
+        uniforms['scaleRotationsTextureSize'] = {
+            'type': 'v2',
+            'value': new Vector2(1024, 1024)
+        };
+
+        const material = new ShaderMaterial({
+            uniforms: uniforms,
+            vertexShader: vertexShaderSource,
+            fragmentShader: fragmentShaderSource,
+            transparent: true,
+            alphaTest: 1.0,
+            blending: NormalBlending,
+            depthTest: true,
+            depthWrite: false,
+            side: DoubleSide
+        });
+
+        return material;
+    }
+
+    static buildVertexShaderProjection() {
+
+        // Original CUDA code for calculating splat-to-screen transformation, for reference
+        /*
+            glm::mat3 R = quat_to_rotmat(rot);
+            glm::mat3 S = scale_to_mat(scale, mod);
+            glm::mat3 L = R * S;
+
+            // center of Gaussians in the camera coordinate
+            glm::mat3x4 splat2world = glm::mat3x4(
+                glm::vec4(L[0], 0.0),
+                glm::vec4(L[1], 0.0),
+                glm::vec4(p_orig.x, p_orig.y, p_orig.z, 1)
+            );
+
+            glm::mat4 world2ndc = glm::mat4(
+                projmatrix[0], projmatrix[4], projmatrix[8], projmatrix[12],
+                projmatrix[1], projmatrix[5], projmatrix[9], projmatrix[13],
+                projmatrix[2], projmatrix[6], projmatrix[10], projmatrix[14],
+                projmatrix[3], projmatrix[7], projmatrix[11], projmatrix[15]
+            );
+
+            glm::mat3x4 ndc2pix = glm::mat3x4(
+                glm::vec4(float(W) / 2.0, 0.0, 0.0, float(W-1) / 2.0),
+                glm::vec4(0.0, float(H) / 2.0, 0.0, float(H-1) / 2.0),
+                glm::vec4(0.0, 0.0, 0.0, 1.0)
+            );
+
+            T = glm::transpose(splat2world) * world2ndc * ndc2pix;
+            normal = transformVec4x3({L[2].x, L[2].y, L[2].z}, viewmatrix);
+        */
+
+        // Compute a 2D-to-2D mapping matrix from a tangent plane into a image plane
+        // given a 2D gaussian parameters. T = WH (from the paper: https://arxiv.org/pdf/2403.17888)
+        let vertexShaderSource = `
+
+            vec4 scaleRotationA = texture(scaleRotationsTexture, getDataUVF(nearestEvenIndex, 1.5,
+                                                                            oddOffset, scaleRotationsTextureSize));
+            vec4 scaleRotationB = texture(scaleRotationsTexture, getDataUVF(nearestEvenIndex, 1.5,
+                                                                            oddOffset + uint(1), scaleRotationsTextureSize));
+
+            vec3 scaleRotation123 = vec3(scaleRotationA.rgb) * (1.0 - fOddOffset) +
+                                    vec3(scaleRotationA.ba, scaleRotationB.r) * fOddOffset;
+            vec3 scaleRotation456 = vec3(scaleRotationA.a, scaleRotationB.rg) * (1.0 - fOddOffset) +
+                                    vec3(scaleRotationB.gba) * fOddOffset;
+
+            float missingW = sqrt(1.0 - scaleRotation456.x * scaleRotation456.x - scaleRotation456.y *
+                                    scaleRotation456.y - scaleRotation456.z * scaleRotation456.z);
+            mat3 R = quaternionToRotationMatrix(scaleRotation456.r, scaleRotation456.g, scaleRotation456.b, missingW);
+            mat3 S = mat3(scaleRotation123.r, 0.0, 0.0,
+                            0.0, scaleRotation123.g, 0.0,
+                            0.0, 0.0, scaleRotation123.b);
+            
+            mat3 L = R * S;
+
+            mat3x4 splat2World = mat3x4(vec4(L[0], 0.0),
+                                        vec4(L[1], 0.0),
+                                        vec4(splatCenter.x, splatCenter.y, splatCenter.z, 1.0));
+
+            mat4 world2ndc = transpose(projectionMatrix * transformModelViewMatrix);
+
+            mat3x4 ndc2pix = mat3x4(vec4(viewport.x / 2.0, 0.0, 0.0, (viewport.x - 1.0) / 2.0),
+                                    vec4(0.0, viewport.y / 2.0, 0.0, (viewport.y - 1.0) / 2.0),
+                                    vec4(0.0, 0.0, 0.0, 1.0));
+
+            mat3 T = transpose(splat2World) * world2ndc * ndc2pix;
+            vec3 normal = vec3(viewMatrix * vec4(L[0][2], L[1][2], L[2][2], 0.0));
+        `;
+
+        // Original CUDA code for projection to 2D, for reference
+        /*
+            float3 T0 = {T[0][0], T[0][1], T[0][2]};
+            float3 T1 = {T[1][0], T[1][1], T[1][2]};
+            float3 T3 = {T[2][0], T[2][1], T[2][2]};
+
+            // Compute AABB
+            float3 temp_point = {1.0f, 1.0f, -1.0f};
+            float distance = sumf3(T3 * T3 * temp_point);
+            float3 f = (1 / distance) * temp_point;
+            if (distance == 0.0) return false;
+
+            point_image = {
+                sumf3(f * T0 * T3),
+                sumf3(f * T1 * T3)
+            };
+
+            float2 temp = {
+                sumf3(f * T0 * T0),
+                sumf3(f * T1 * T1)
+            };
+            float2 half_extend = point_image * point_image - temp;
+            extent = sqrtf2(maxf2(1e-4, half_extend));
+            return true;
+        */
+
+        // Computing the bounding box of the 2D Gaussian and its center
+        // The center of the bounding box is used to create a low pass filter.
+        // This code is based off the reference implementation and creates an AABB aligned
+        // with the screen for the quad to be rendered.
+        const referenceQuadGeneration = `
+            vec3 T0 = vec3(T[0][0], T[0][1], T[0][2]);
+            vec3 T1 = vec3(T[1][0], T[1][1], T[1][2]);
+            vec3 T3 = vec3(T[2][0], T[2][1], T[2][2]);
+
+            vec3 tempPoint = vec3(1.0, 1.0, -1.0);
+            float distance = (T3.x * T3.x * tempPoint.x) + (T3.y * T3.y * tempPoint.y) + (T3.z * T3.z * tempPoint.z);
+            vec3 f = (1.0 / distance) * tempPoint;
+            if (abs(distance) < 0.00001) return;
+
+            float pointImageX = (T0.x * T3.x * f.x) + (T0.y * T3.y * f.y) + (T0.z * T3.z * f.z);
+            float pointImageY = (T1.x * T3.x * f.x) + (T1.y * T3.y * f.y) + (T1.z * T3.z * f.z);
+            vec2 pointImage = vec2(pointImageX, pointImageY);
+
+            float tempX = (T0.x * T0.x * f.x) + (T0.y * T0.y * f.y) + (T0.z * T0.z * f.z);
+            float tempY = (T1.x * T1.x * f.x) + (T1.y * T1.y * f.y) + (T1.z * T1.z * f.z);
+            vec2 temp = vec2(tempX, tempY);
+
+            vec2 halfExtend = pointImage * pointImage - temp;
+            vec2 extent = sqrt(max(vec2(0.0001), halfExtend));
+            float radius = max(extent.x, extent.y);
+
+            vec2 ndcOffset = ((position.xy * radius * 3.0) * basisViewport * 2.0);
+
+            vec4 quadPos = vec4(ndcCenter.xy + ndcOffset, ndcCenter.z, 1.0);
+            gl_Position = quadPos;
+
+            vT = T;
+            vQuadCenter = pointImage;
+            vFragCoord = (quadPos.xy * 0.5 + 0.5) * viewport;
+        `;
+
+        const useRefImplementation = false;
+        if (useRefImplementation) {
+            vertexShaderSource += referenceQuadGeneration;
+        } else {
+            // Create a quad that is aligned with the eigen vectors of the projected gaussian for rendering.
+            // This is a different approach than the reference implementation, similar to how the rendering of
+            // 3D gaussians in this viewer differs from the reference implementation. If the quad is too small
+            // (smaller than a pixel), then revert to the reference implementation.
+            vertexShaderSource += `
+
+                mat4 splat2World4 = mat4(vec4(L[0], 0.0),
+                                        vec4(L[1], 0.0),
+                                        vec4(L[2], 0.0),
+                                        vec4(splatCenter.x, splatCenter.y, splatCenter.z, 1.0));
+
+                mat4 Tt = transpose(transpose(splat2World4) * world2ndc);
+
+                vec4 tempPoint1 = Tt * vec4(1.0, 0.0, 0.0, 1.0);
+                tempPoint1 /= tempPoint1.w;
+
+                vec4 tempPoint2 = Tt * vec4(0.0, 1.0, 0.0, 1.0);
+                tempPoint2 /= tempPoint2.w;
+
+                vec4 center = Tt * vec4(0.0, 0.0, 0.0, 1.0);
+                center /= center.w;
+
+                vec2 basisVector1 = tempPoint1.xy - center.xy;
+                vec2 basisVector2 = tempPoint2.xy - center.xy;
+
+                vec2 basisVector1Screen = basisVector1 * 0.5 * viewport;
+                vec2 basisVector2Screen = basisVector2 * 0.5 * viewport;
+
+                const float minPix = 1.;
+                if (length(basisVector1Screen) < minPix || length(basisVector2Screen) < minPix) {
+                    ${referenceQuadGeneration}
+                } else {
+                    vec2 ndcOffset = vec2(position.x * basisVector1 + position.y * basisVector2) * 3.0 * inverseFocalAdjustment;
+                    vec4 quadPos = vec4(ndcCenter.xy + ndcOffset, ndcCenter.z, 1.0);
+                    gl_Position = quadPos;
+
+                    vT = T;
+                    vQuadCenter = center.xy;
+                    vFragCoord = (quadPos.xy * 0.5 + 0.5) * viewport;
+                }
+            `;
+        }
+
+        vertexShaderSource += SplatMaterial.getVertexShaderFadeIn();
+        vertexShaderSource += `}`;
+
+        return vertexShaderSource;
+    }
+
+    static buildFragmentShader() {
+
+        // Original CUDA code for splat intersection, for reference
+        /*
+            const float2 xy = collected_xy[j];
+            const float3 Tu = collected_Tu[j];
+            const float3 Tv = collected_Tv[j];
+            const float3 Tw = collected_Tw[j];
+            float3 k = pix.x * Tw - Tu;
+            float3 l = pix.y * Tw - Tv;
+            float3 p = cross(k, l);
+            if (p.z == 0.0) continue;
+            float2 s = {p.x / p.z, p.y / p.z};
+            float rho3d = (s.x * s.x + s.y * s.y);
+            float2 d = {xy.x - pixf.x, xy.y - pixf.y};
+            float rho2d = FilterInvSquare * (d.x * d.x + d.y * d.y);
+
+            // compute intersection and depth
+            float rho = min(rho3d, rho2d);
+            float depth = (rho3d <= rho2d) ? (s.x * Tw.x + s.y * Tw.y) + Tw.z : Tw.z;
+            if (depth < near_n) continue;
+            float4 nor_o = collected_normal_opacity[j];
+            float normal[3] = {nor_o.x, nor_o.y, nor_o.z};
+            float opa = nor_o.w;
+
+            float power = -0.5f * rho;
+            if (power > 0.0f)
+                continue;
+
+            // Eq. (2) from 3D Gaussian splatting paper.
+            // Obtain alpha by multiplying with Gaussian opacity
+            // and its exponential falloff from mean.
+            // Avoid numerical instabilities (see paper appendix).
+            float alpha = min(0.99f, opa * exp(power));
+            if (alpha < 1.0f / 255.0f)
+                continue;
+            float test_T = T * (1 - alpha);
+            if (test_T < 0.0001f)
+            {
+                done = true;
+                continue;
+            }
+
+            float w = alpha * T;
+        */
+        let fragmentShaderSource = `
+            precision highp float;
+            #include <common>
+
+            uniform vec3 debugColor;
+
+            varying vec4 vColor;
+            varying vec2 vUv;
+            varying vec2 vPosition;
+            varying mat3 vT;
+            varying vec2 vQuadCenter;
+            varying vec2 vFragCoord;
+
+            void main () {
+
+                const float FilterInvSquare = 2.0;
+                const float near_n = 0.2;
+                const float T = 1.0;
+
+                vec2 xy = vQuadCenter;
+                vec3 Tu = vT[0];
+                vec3 Tv = vT[1];
+                vec3 Tw = vT[2];
+                vec3 k = vFragCoord.x * Tw - Tu;
+                vec3 l = vFragCoord.y * Tw - Tv;
+                vec3 p = cross(k, l);
+                if (p.z == 0.0) discard;
+                vec2 s = vec2(p.x / p.z, p.y / p.z);
+                float rho3d = (s.x * s.x + s.y * s.y); 
+                vec2 d = vec2(xy.x - vFragCoord.x, xy.y - vFragCoord.y);
+                float rho2d = FilterInvSquare * (d.x * d.x + d.y * d.y); 
+
+                // compute intersection and depth
+                float rho = min(rho3d, rho2d);
+                float depth = (rho3d <= rho2d) ? (s.x * Tw.x + s.y * Tw.y) + Tw.z : Tw.z; 
+                if (depth < near_n) discard;
+                //  vec4 nor_o = collected_normal_opacity[j];
+                //  float normal[3] = {nor_o.x, nor_o.y, nor_o.z};
+                float opa = vColor.a;
+
+                float power = -0.5f * rho;
+                if (power > 0.0f) discard;
+
+                // Eq. (2) from 3D Gaussian splatting paper.
+                // Obtain alpha by multiplying with Gaussian opacity
+                // and its exponential falloff from mean.
+                // Avoid numerical instabilities (see paper appendix). 
+                float alpha = min(0.99f, opa * exp(power));
+                if (alpha < 1.0f / 255.0f) discard;
+                float test_T = T * (1.0 - alpha);
+                if (test_T < 0.0001)discard;
+
+                float w = alpha * T;
+                gl_FragColor = vec4(vColor.rgb, w);
+            }
+        `;
+
+        return fragmentShaderSource;
+    }
+}

package/src/materials/SplatMaterial3D.js

@@ -0,0 +1,278 @@
+/**
+ * SplatMaterial3D
+ * 
+ * Based on @mkkellogg/gaussian-splats-3d (MIT License)
+ * https://github.com/mkkellogg/GaussianSplats3D
+ * 
+ * Modified for FLAME avatar support:
+ * - Extended vertex shader for FLAME integration
+ */
+
+import { DoubleSide, NormalBlending, ShaderMaterial, Vector2 } from 'three';
+import { SplatMaterial } from './SplatMaterial.js';
+
+export class SplatMaterial3D {
+
+    /**
+     * Build the Three.js material that is used to render the splats.
+     * @param {number} dynamicMode If true, it means the scene geometry represented by this splat mesh is not stationary or
+     *                             that the splat count might change
+     * @param {boolean} enableOptionalEffects When true, allows for usage of extra properties and attributes in the shader for effects
+     *                                        such as opacity adjustment. Default is false for performance reasons.
+     * @param {boolean} antialiased If true, calculate compensation factor to deal with gaussians being rendered at a significantly
+     *                              different resolution than that of their training
+     * @param {number} maxScreenSpaceSplatSize The maximum clip space splat size
+     * @param {number} splatScale Value by which all splats are scaled in screen-space (default is 1.0)
+     * @param {number} pointCloudModeEnabled Render all splats as screen-space circles
+     * @param {number} maxSphericalHarmonicsDegree Degree of spherical harmonics to utilize in rendering splats
+     * @return {THREE.ShaderMaterial}
+     */
+    static build(dynamicMode = false, enableOptionalEffects = false, antialiased = false, maxScreenSpaceSplatSize = 2048,
+                 splatScale = 1.0, pointCloudModeEnabled = false, maxSphericalHarmonicsDegree = 0, kernel2DSize = 0.3, useFlame = true) {
+
+        const customVertexVars = `
+            uniform vec2 covariancesTextureSize;
+            uniform highp sampler2D covariancesTexture;
+            uniform highp usampler2D covariancesTextureHalfFloat;
+            uniform int covariancesAreHalfFloat;
+
+            void fromCovarianceHalfFloatV4(uvec4 val, out vec4 first, out vec4 second) {
+                vec2 r = unpackHalf2x16(val.r);
+                vec2 g = unpackHalf2x16(val.g);
+                vec2 b = unpackHalf2x16(val.b);
+
+                first = vec4(r.x, r.y, g.x, g.y);
+                second = vec4(b.x, b.y, 0.0, 0.0);
+            }
+        `;
+
+        let vertexShaderSource = SplatMaterial.buildVertexShaderBase(dynamicMode, enableOptionalEffects,
+                                                                     maxSphericalHarmonicsDegree, customVertexVars, useFlame);
+        vertexShaderSource += SplatMaterial3D.buildVertexShaderProjection(antialiased, enableOptionalEffects,
+                                                                          maxScreenSpaceSplatSize, kernel2DSize);
+        const fragmentShaderSource = SplatMaterial3D.buildFragmentShader();
+
+        const uniforms = SplatMaterial.getUniforms(dynamicMode, enableOptionalEffects,
+                                                   maxSphericalHarmonicsDegree, splatScale, pointCloudModeEnabled);
+
+        uniforms['covariancesTextureSize'] = {
+            'type': 'v2',
+            'value': new Vector2(1024, 1024)
+        };
+        uniforms['covariancesTexture'] = {
+            'type': 't',
+            'value': null
+        };
+        uniforms['covariancesTextureHalfFloat'] = {
+            'type': 't',
+            'value': null
+        };
+        uniforms['covariancesAreHalfFloat'] = {
+            'type': 'i',
+            'value': 0
+        };
+
+        const material = new ShaderMaterial({
+            uniforms: uniforms,
+            vertexShader: vertexShaderSource,
+            fragmentShader: fragmentShaderSource,
+            transparent: true,
+            alphaTest: 1.0,
+            blending: NormalBlending,
+            depthTest: true,
+            depthWrite: false,
+            side: DoubleSide
+        });
+
+        return material;
+    }
+
+    static buildVertexShaderProjection(antialiased, enableOptionalEffects, maxScreenSpaceSplatSize, kernel2DSize) {
+        let vertexShaderSource = `
+
+            vec4 sampledCovarianceA;
+            vec4 sampledCovarianceB;
+            vec3 cov3D_M11_M12_M13;
+            vec3 cov3D_M22_M23_M33;
+            if (covariancesAreHalfFloat == 0) {
+                sampledCovarianceA = texture(covariancesTexture, getDataUVF(nearestEvenIndex, 1.5, oddOffset,
+                                                                            covariancesTextureSize));
+                sampledCovarianceB = texture(covariancesTexture, getDataUVF(nearestEvenIndex, 1.5, oddOffset + uint(1),
+                                                                            covariancesTextureSize));
+
+                cov3D_M11_M12_M13 = vec3(sampledCovarianceA.rgb) * (1.0 - fOddOffset) +
+                                    vec3(sampledCovarianceA.ba, sampledCovarianceB.r) * fOddOffset;
+                cov3D_M22_M23_M33 = vec3(sampledCovarianceA.a, sampledCovarianceB.rg) * (1.0 - fOddOffset) +
+                                    vec3(sampledCovarianceB.gba) * fOddOffset;
+            } else {
+                uvec4 sampledCovarianceU = texture(covariancesTextureHalfFloat, getDataUV(1, 0, covariancesTextureSize));
+                fromCovarianceHalfFloatV4(sampledCovarianceU, sampledCovarianceA, sampledCovarianceB);
+                cov3D_M11_M12_M13 = sampledCovarianceA.rgb;
+                cov3D_M22_M23_M33 = vec3(sampledCovarianceA.a, sampledCovarianceB.rg);
+            }
+        
+            // Construct the 3D covariance matrix
+            mat3 Vrk = mat3(
+                cov3D_M11_M12_M13.x, cov3D_M11_M12_M13.y, cov3D_M11_M12_M13.z,
+                cov3D_M11_M12_M13.y, cov3D_M22_M23_M33.x, cov3D_M22_M23_M33.y,
+                cov3D_M11_M12_M13.z, cov3D_M22_M23_M33.y, cov3D_M22_M23_M33.z
+            );
+
+            mat3 J;
+            if (orthographicMode == 1) {
+                // Since the projection is linear, we don't need an approximation
+                J = transpose(mat3(orthoZoom, 0.0, 0.0,
+                                0.0, orthoZoom, 0.0,
+                                0.0, 0.0, 0.0));
+            } else {
+                // Construct the Jacobian of the affine approximation of the projection matrix. It will be used to transform the
+                // 3D covariance matrix instead of using the actual projection matrix because that transformation would
+                // require a non-linear component (perspective division) which would yield a non-gaussian result.
+                float s = 1.0 / (viewCenter.z * viewCenter.z);
+                J = mat3(
+                    focal.x / viewCenter.z, 0., -(focal.x * viewCenter.x) * s,
+                    0., focal.y / viewCenter.z, -(focal.y * viewCenter.y) * s,
+                    0., 0., 0.
+                );
+            }
+
+            // Concatenate the projection approximation with the model-view transformation
+            mat3 W = transpose(mat3(transformModelViewMatrix));
+            mat3 T = W * J;
+
+            // Transform the 3D covariance matrix (Vrk) to compute the 2D covariance matrix
+            mat3 cov2Dm = transpose(T) * Vrk * T;
+            `;
+
+        if (antialiased) {
+            vertexShaderSource += `
+                float detOrig = cov2Dm[0][0] * cov2Dm[1][1] - cov2Dm[0][1] * cov2Dm[0][1];
+                cov2Dm[0][0] += ${kernel2DSize};
+                cov2Dm[1][1] += ${kernel2DSize};
+                float detBlur = cov2Dm[0][0] * cov2Dm[1][1] - cov2Dm[0][1] * cov2Dm[0][1];
+                vColor.a *= sqrt(max(detOrig / detBlur, 0.0));
+                if (vColor.a < minAlpha) return;
+            `;
+        } else {
+            vertexShaderSource += `
+                cov2Dm[0][0] += ${kernel2DSize};
+                cov2Dm[1][1] += ${kernel2DSize};
+            `;
+        }
+
+        vertexShaderSource += `
+
+            // We are interested in the upper-left 2x2 portion of the projected 3D covariance matrix because
+            // we only care about the X and Y values. We want the X-diagonal, cov2Dm[0][0],
+            // the Y-diagonal, cov2Dm[1][1], and the correlation between the two cov2Dm[0][1]. We don't
+            // need cov2Dm[1][0] because it is a symetric matrix.
+            vec3 cov2Dv = vec3(cov2Dm[0][0], cov2Dm[0][1], cov2Dm[1][1]);
+
+            // We now need to solve for the eigen-values and eigen vectors of the 2D covariance matrix
+            // so that we can determine the 2D basis for the splat. This is done using the method described
+            // here: https://people.math.harvard.edu/~knill/teaching/math21b2004/exhibits/2dmatrices/index.html
+            // After calculating the eigen-values and eigen-vectors, we calculate the basis for rendering the splat
+            // by normalizing the eigen-vectors and then multiplying them by (sqrt(8) * sqrt(eigen-value)), which is
+            // equal to scaling them by sqrt(8) standard deviations.
+            //
+            // This is a different approach than in the original work at INRIA. In that work they compute the
+            // max extents of the projected splat in screen space to form a screen-space aligned bounding rectangle
+            // which forms the geometry that is actually rasterized. The dimensions of that bounding box are 3.0
+            // times the square root of the maximum eigen-value, or 3 standard deviations. They then use the inverse
+            // 2D covariance matrix (called 'conic') in the CUDA rendering thread to determine fragment opacity by
+            // calculating the full gaussian: exp(-0.5 * (X - mean) * conic * (X - mean)) * splat opacity
+            float a = cov2Dv.x;
+            float d = cov2Dv.z;
+            float b = cov2Dv.y;
+            float D = a * d - b * b;
+            float trace = a + d;
+            float traceOver2 = 0.5 * trace;
+            float term2 = sqrt(max(0.1f, traceOver2 * traceOver2 - D));
+            float eigenValue1 = traceOver2 + term2;
+            float eigenValue2 = traceOver2 - term2;
+
+            if (pointCloudModeEnabled == 1) {
+                eigenValue1 = eigenValue2 = 0.2;
+            }
+
+            if (eigenValue2 <= 0.0) return;
+
+            vec2 eigenVector1 = normalize(vec2(b, eigenValue1 - a));
+            // since the eigen vectors are orthogonal, we derive the second one from the first
+            vec2 eigenVector2 = vec2(eigenVector1.y, -eigenVector1.x);
+
+            // We use sqrt(8) standard deviations instead of 3 to eliminate more of the splat with a very low opacity.
+            vec2 basisVector1 = eigenVector1 * splatScale * min(sqrt8 * sqrt(eigenValue1), ${parseInt(maxScreenSpaceSplatSize)}.0);
+            vec2 basisVector2 = eigenVector2 * splatScale * min(sqrt8 * sqrt(eigenValue2), ${parseInt(maxScreenSpaceSplatSize)}.0);
+            `;
+
+        if (enableOptionalEffects) {
+            vertexShaderSource += `
+                vColor.a *= splatOpacityFromScene;
+            `;
+        }
+
+        vertexShaderSource += `
+            vec2 ndcOffset = vec2(vPosition.x * basisVector1 + vPosition.y * basisVector2) *
+                             basisViewport * 2.0 * inverseFocalAdjustment;
+
+            vec4 quadPos = vec4(ndcCenter.xy + ndcOffset, ndcCenter.z, 1.0);
+            gl_Position = quadPos;
+
+            // Scale the position data we send to the fragment shader
+            vPosition *= sqrt8;
+        `;
+
+        vertexShaderSource += SplatMaterial.getVertexShaderFadeIn();
+        vertexShaderSource += `}`;
+
+        return vertexShaderSource;
+    }
+
+    static buildFragmentShader() {
+        let fragmentShaderSource = `
+            precision highp float;
+            #include <common>
+ 
+            uniform vec3 debugColor;
+
+            varying vec4 vColor;
+            varying vec2 vUv;
+            varying vec2 vPosition;
+            varying vec2 vSplatIndex;
+
+        `;
+
+        fragmentShaderSource += `
+            void main () {
+                // Compute the positional squared distance from the center of the splat to the current fragment.
+                float A = dot(vPosition, vPosition);
+                // Since the positional data in vPosition has been scaled by sqrt(8), the squared result will be
+                // scaled by a factor of 8. If the squared result is larger than 8, it means it is outside the ellipse
+                // defined by the rectangle formed by vPosition. It also means it's farther
+                // away than sqrt(8) standard deviations from the mean.
+
+                // if(vSplatIndex.x > 20000.0) discard;
+                // if (A > 8.0) discard;
+                vec3 color = vColor.rgb;
+
+                // Since the rendered splat is scaled by sqrt(8), the inverse covariance matrix that is part of
+                // the gaussian formula becomes the identity matrix. We're then left with (X - mean) * (X - mean),
+                // and since 'mean' is zero, we have X * X, which is the same as A:
+                float opacity = exp( -0.5*A) * vColor.a;
+                if(opacity < 1.0 / 255.0)
+                    discard;
+
+                // uint a = uint(255);
+                // vec3 c = vec3(vSplatIndex.x / 256.0 / 256.0, float(uint(vSplatIndex.x / 256.0 )% a) / 256.0, float(uint(vSplatIndex.x)% a) / 256.0);
+                // gl_FragColor = vec4(c, 1.0);
+                gl_FragColor = vec4(color, opacity);
+
+
+            }
+        `;
+
+        return fragmentShaderSource;
+    }
+
+}

package/src/materials/index.js

@@ -0,0 +1,11 @@
+/**
+ * gsplat-flame-avatar - Materials Module
+ * WebGL shaders and materials for Gaussian Splat rendering.
+ * 
+ * Based on @mkkellogg/gaussian-splats-3d (MIT License)
+ * Extended with FLAME skinning shaders.
+ */
+
+export { SplatMaterial } from './SplatMaterial.js';
+export { SplatMaterial2D } from './SplatMaterial2D.js';
+export { SplatMaterial3D } from './SplatMaterial3D.js';

package/src/raycaster/Hit.js

@@ -0,0 +1,37 @@
+/**
+ * Hit
+ * 
+ * Derived from @mkkellogg/gaussian-splats-3d (MIT License)
+ * https://github.com/mkkellogg/GaussianSplats3D
+ * 
+ * This file is functionally identical to the original.
+ */
+
+import * as THREE from 'three';
+
+export class Hit {
+
+    constructor() {
+        this.origin = new THREE.Vector3();
+        this.normal = new THREE.Vector3();
+        this.distance = 0;
+        this.splatIndex = 0;
+    }
+
+    set(origin, normal, distance, splatIndex) {
+        this.origin.copy(origin);
+        this.normal.copy(normal);
+        this.distance = distance;
+        this.splatIndex = splatIndex;
+    }
+
+    clone() {
+        const hitClone = new Hit();
+        hitClone.origin.copy(this.origin);
+        hitClone.normal.copy(this.normal);
+        hitClone.distance = this.distance;
+        hitClone.splatIndex = this.splatIndex;
+        return hitClone;
+    }
+
+}