@sage-rsc/talking-head-react 1.1.7 → 1.1.9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sage-rsc/talking-head-react",
-  "version": "1.1.7",
+  "version": "1.1.9",
   "description": "A reusable React component for 3D talking avatars with lip-sync and text-to-speech",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

package/src/lib/talkinghead.mjs

@@ -5471,6 +5471,279 @@ class TalkingHead {
       if ( fbx && fbx.animations && fbx.animations[ndx] ) {
         let anim = fbx.animations[ndx];
 
+        // Get available bone names from avatar skeleton for mapping
+        const availableBones = new Set();
+        if (this.armature) {
+          this.armature.traverse((child) => {
+            if (child.isBone || child.type === 'Bone') {
+              availableBones.add(child.name);
+            }
+          });
+        }
+
+        // Map bone names from FBX to avatar skeleton
+        const boneNameMap = new Map();
+        const mapBoneName = (fbxBoneName) => {
+          // Direct match
+          if (availableBones.has(fbxBoneName)) {
+            return fbxBoneName;
+          }
+          
+          // Remove common prefixes (mixamorig, CC_Base_, etc.)
+          let normalized = fbxBoneName
+            .replace(/^mixamorig/i, '')
+            .replace(/^CC_Base_/i, '')
+            .replace(/^RPM_/i, '');
+            
+          if (availableBones.has(normalized)) {
+            return normalized;
+          }
+          
+          // Pattern-based matching for Ready Player Me / Mixamo
+          const lowerNormalized = normalized.toLowerCase();
+          
+          // Arm bones - pattern matching
+          if (lowerNormalized.includes('left') && lowerNormalized.includes('arm')) {
+            if (lowerNormalized.includes('fore') || lowerNormalized.includes('lower')) {
+              if (availableBones.has('LeftForeArm')) return 'LeftForeArm';
+              if (availableBones.has('LeftForearm')) return 'LeftForearm';
+            } else if (!lowerNormalized.includes('fore') && !lowerNormalized.includes('hand')) {
+              if (availableBones.has('LeftArm')) return 'LeftArm';
+            }
+          }
+          
+          if (lowerNormalized.includes('right') && lowerNormalized.includes('arm')) {
+            if (lowerNormalized.includes('fore') || lowerNormalized.includes('lower')) {
+              if (availableBones.has('RightForeArm')) return 'RightForeArm';
+              if (availableBones.has('RightForearm')) return 'RightForearm';
+            } else if (!lowerNormalized.includes('fore') && !lowerNormalized.includes('hand')) {
+              if (availableBones.has('RightArm')) return 'RightArm';
+            }
+          }
+          
+          // Hand bones
+          if (lowerNormalized.includes('left') && lowerNormalized.includes('hand') && 
+              !lowerNormalized.includes('index') && !lowerNormalized.includes('thumb') &&
+              !lowerNormalized.includes('middle') && !lowerNormalized.includes('ring') &&
+              !lowerNormalized.includes('pinky')) {
+            if (availableBones.has('LeftHand')) return 'LeftHand';
+          }
+          
+          if (lowerNormalized.includes('right') && lowerNormalized.includes('hand') && 
+              !lowerNormalized.includes('index') && !lowerNormalized.includes('thumb') &&
+              !lowerNormalized.includes('middle') && !lowerNormalized.includes('ring') &&
+              !lowerNormalized.includes('pinky')) {
+            if (availableBones.has('RightHand')) return 'RightHand';
+          }
+          
+          // Shoulder bones
+          if (lowerNormalized.includes('left') && (lowerNormalized.includes('shoulder') || lowerNormalized.includes('clavicle'))) {
+            if (availableBones.has('LeftShoulder')) return 'LeftShoulder';
+          }
+          
+          if (lowerNormalized.includes('right') && (lowerNormalized.includes('shoulder') || lowerNormalized.includes('clavicle'))) {
+            if (availableBones.has('RightShoulder')) return 'RightShoulder';
+          }
+          
+          // Common bone name mappings
+          const mappings = {
+            // Arm bones - exact matches
+            'LeftArm': 'LeftArm',
+            'leftArm': 'LeftArm',
+            'LEFTARM': 'LeftArm',
+            'RightArm': 'RightArm',
+            'rightArm': 'RightArm',
+            'RIGHTARM': 'RightArm',
+            'LeftForeArm': 'LeftForeArm',
+            'leftForeArm': 'LeftForeArm',
+            'leftForearm': 'LeftForeArm',
+            'LeftForearm': 'LeftForeArm',
+            'RightForeArm': 'RightForeArm',
+            'rightForeArm': 'RightForeArm',
+            'rightForearm': 'RightForeArm',
+            'RightForearm': 'RightForeArm',
+            'LeftHand': 'LeftHand',
+            'leftHand': 'LeftHand',
+            'RightHand': 'RightHand',
+            'rightHand': 'RightHand',
+            'LeftShoulder': 'LeftShoulder',
+            'leftShoulder': 'LeftShoulder',
+            'RightShoulder': 'RightShoulder',
+            'rightShoulder': 'RightShoulder',
+            // Spine
+            'Spine': 'Spine1',
+            'spine': 'Spine1',
+            'Spine1': 'Spine1',
+            'Spine2': 'Spine2',
+            // Head/Neck
+            'Head': 'Head',
+            'head': 'Head',
+            'Neck': 'Neck',
+            'neck': 'Neck',
+            // Hips
+            'Hips': 'Hips',
+            'hips': 'Hips',
+            'Root': 'Hips',
+            'root': 'Hips',
+          };
+          
+          if (mappings[normalized]) {
+            const mapped = mappings[normalized];
+            if (availableBones.has(mapped)) {
+              return mapped;
+            }
+          }
+          
+          // Try case-insensitive match
+          for (const boneName of availableBones) {
+            if (boneName.toLowerCase() === lowerNormalized) {
+              return boneName;
+            }
+          }
+          
+          // Try partial match (contains)
+          for (const boneName of availableBones) {
+            const boneLower = boneName.toLowerCase();
+            // Match if normalized contains key parts of bone name
+            if ((lowerNormalized.includes('left') && boneLower.includes('left')) ||
+                (lowerNormalized.includes('right') && boneLower.includes('right'))) {
+              if ((lowerNormalized.includes('arm') && boneLower.includes('arm') && !boneLower.includes('fore')) ||
+                  (lowerNormalized.includes('forearm') && boneLower.includes('forearm')) ||
+                  (lowerNormalized.includes('hand') && boneLower.includes('hand') && !boneLower.includes('index') && !boneLower.includes('thumb')) ||
+                  (lowerNormalized.includes('shoulder') && boneLower.includes('shoulder'))) {
+                return boneName;
+              }
+            }
+          }
+          
+          return null; // No mapping found
+        };
+
+        // Debug: Log FBX bone names and avatar bone names for comparison
+        const fbxBoneNames = new Set();
+        anim.tracks.forEach(track => {
+          const trackParts = track.name.split('.');
+          fbxBoneNames.add(trackParts[0]);
+        });
+        
+        console.log('=== Ready Player Me Animation Bone Analysis ===');
+        console.log('FBX bone names:', Array.from(fbxBoneNames).sort().join(', '));
+        console.log('Avatar skeleton bone names:', Array.from(availableBones).sort().join(', '));
+        
+        // Check for arm bones specifically
+        const fbxArmBones = Array.from(fbxBoneNames).filter(b => 
+          b.toLowerCase().includes('arm') || 
+          b.toLowerCase().includes('hand') || 
+          b.toLowerCase().includes('shoulder')
+        );
+        const avatarArmBones = Array.from(availableBones).filter(b => 
+          b.includes('Arm') || 
+          b.includes('Hand') || 
+          b.includes('Shoulder')
+        );
+        console.log('FBX arm/hand/shoulder bones:', fbxArmBones.sort().join(', '));
+        console.log('Avatar arm/hand/shoulder bones:', avatarArmBones.sort().join(', '));
+
+        // Filter and map animation tracks
+        const mappedTracks = [];
+        const unmappedBones = new Set();
+        anim.tracks.forEach(track => {
+          // Remove mixamorig prefix first
+          let trackName = track.name.replaceAll('mixamorig', '');
+          const trackParts = trackName.split('.');
+          const fbxBoneName = trackParts[0];
+          const property = trackParts[1];
+          
+          // Map bone name to avatar skeleton
+          const mappedBoneName = mapBoneName(fbxBoneName);
+          
+          if (mappedBoneName && property) {
+            // Create new track with mapped bone name
+            const newTrackName = `${mappedBoneName}.${property}`;
+            const newTrack = track.clone();
+            newTrack.name = newTrackName;
+            
+            // Fix rotations for Ready Player Me arm bones (coordinate system correction)
+            const isArmBone = mappedBoneName.includes('Arm') || mappedBoneName.includes('Hand') || mappedBoneName.includes('Shoulder');
+            const isLeftArm = mappedBoneName.includes('Left');
+            
+            if (isArmBone && (property === 'quaternion' || property === 'rotation')) {
+              if (property === 'quaternion' && newTrack.values && newTrack.values.length >= 4) {
+                // Adjust quaternion rotations for Ready Player Me coordinate system
+                const numKeyframes = newTrack.times.length;
+                for (let i = 0; i < numKeyframes; i++) {
+                  const baseIdx = i * 4;
+                  if (baseIdx + 3 < newTrack.values.length) {
+                    let x = newTrack.values[baseIdx];
+                    let y = newTrack.values[baseIdx + 1];
+                    let z = newTrack.values[baseIdx + 2];
+                    let w = newTrack.values[baseIdx + 3];
+                    
+                    // For Ready Player Me, we may need to flip certain axes
+                    // Try rotating 180 degrees around Y axis for left arm
+                    if (isLeftArm && mappedBoneName.includes('ForeArm')) {
+                      // Left forearm might need Y-axis flip
+                      const flipY = new THREE.Quaternion().setFromAxisAngle(new THREE.Vector3(0, 1, 0), Math.PI);
+                      const q = new THREE.Quaternion(x, y, z, w);
+                      q.multiply(flipY);
+                      x = q.x;
+                      y = q.y;
+                      z = q.z;
+                      w = q.w;
+                    }
+                    
+                    newTrack.values[baseIdx] = x;
+                    newTrack.values[baseIdx + 1] = y;
+                    newTrack.values[baseIdx + 2] = z;
+                    newTrack.values[baseIdx + 3] = w;
+                  }
+                }
+              }
+            }
+            
+            mappedTracks.push(newTrack);
+            
+            // Store mapping for logging
+            if (fbxBoneName !== mappedBoneName) {
+              boneNameMap.set(fbxBoneName, mappedBoneName);
+            }
+          } else {
+            unmappedBones.add(fbxBoneName);
+            // Log unmapped bones (especially arm bones)
+            if (fbxBoneName.toLowerCase().includes('arm') || 
+                fbxBoneName.toLowerCase().includes('hand') || 
+                fbxBoneName.toLowerCase().includes('shoulder')) {
+              console.warn(`⚠️ Arm bone "${fbxBoneName}" could not be mapped to avatar skeleton`);
+            }
+          }
+        });
+        
+        if (unmappedBones.size > 0) {
+          console.warn(`⚠️ ${unmappedBones.size} bone(s) could not be mapped:`, Array.from(unmappedBones).sort().join(', '));
+        }
+
+        // Use mapped tracks if we have any, otherwise use original
+        if (mappedTracks.length > 0) {
+          anim = new THREE.AnimationClip(anim.name, anim.duration, mappedTracks);
+          console.log(`✓ Created animation with ${mappedTracks.length} mapped tracks (from ${anim.tracks.length} original tracks)`);
+          if (boneNameMap.size > 0) {
+            console.log(`✓ Mapped ${boneNameMap.size} bone(s):`, 
+              Array.from(boneNameMap.entries()).map(([from, to]) => `${from}→${to}`).join(', '));
+          }
+          
+          // Check if arm bones were mapped
+          const mappedArmBones = Array.from(boneNameMap.values()).filter(b => 
+            b.includes('Arm') || b.includes('Hand') || b.includes('Shoulder')
+          );
+          if (mappedArmBones.length > 0) {
+            console.log(`✓ Arm bones mapped: ${mappedArmBones.join(', ')}`);
+          } else {
+            console.warn('⚠️ No arm bones were mapped! This may cause arm rigging issues.');
+          }
+        } else {
+          console.error('❌ No tracks could be mapped! Animation may not work correctly.');
+        }
+
         // Rename and scale Mixamo tracks, create a pose
         const props = {};
         anim.tracks.forEach( t => {