@sage-rsc/talking-head-react 1.1.8 → 1.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sage-rsc/talking-head-react",
-  "version": "1.1.8",
+  "version": "1.2.0",
   "description": "A reusable React component for 3D talking avatars with lip-sync and text-to-speech",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

package/src/lib/talkinghead.mjs

@@ -5489,15 +5489,65 @@ class TalkingHead {
             return fbxBoneName;
           }
           
-          // Remove mixamorig prefix if present
-          let normalized = fbxBoneName.replace(/^mixamorig/i, '');
+          // Remove common prefixes (mixamorig, CC_Base_, etc.)
+          let normalized = fbxBoneName
+            .replace(/^mixamorig/i, '')
+            .replace(/^CC_Base_/i, '')
+            .replace(/^RPM_/i, '');
+            
           if (availableBones.has(normalized)) {
             return normalized;
           }
           
-          // Common bone name mappings for Ready Player Me / Mixamo
+          // Pattern-based matching for Ready Player Me / Mixamo
+          const lowerNormalized = normalized.toLowerCase();
+          
+          // Arm bones - pattern matching
+          if (lowerNormalized.includes('left') && lowerNormalized.includes('arm')) {
+            if (lowerNormalized.includes('fore') || lowerNormalized.includes('lower')) {
+              if (availableBones.has('LeftForeArm')) return 'LeftForeArm';
+              if (availableBones.has('LeftForearm')) return 'LeftForearm';
+            } else if (!lowerNormalized.includes('fore') && !lowerNormalized.includes('hand')) {
+              if (availableBones.has('LeftArm')) return 'LeftArm';
+            }
+          }
+          
+          if (lowerNormalized.includes('right') && lowerNormalized.includes('arm')) {
+            if (lowerNormalized.includes('fore') || lowerNormalized.includes('lower')) {
+              if (availableBones.has('RightForeArm')) return 'RightForeArm';
+              if (availableBones.has('RightForearm')) return 'RightForearm';
+            } else if (!lowerNormalized.includes('fore') && !lowerNormalized.includes('hand')) {
+              if (availableBones.has('RightArm')) return 'RightArm';
+            }
+          }
+          
+          // Hand bones
+          if (lowerNormalized.includes('left') && lowerNormalized.includes('hand') && 
+              !lowerNormalized.includes('index') && !lowerNormalized.includes('thumb') &&
+              !lowerNormalized.includes('middle') && !lowerNormalized.includes('ring') &&
+              !lowerNormalized.includes('pinky')) {
+            if (availableBones.has('LeftHand')) return 'LeftHand';
+          }
+          
+          if (lowerNormalized.includes('right') && lowerNormalized.includes('hand') && 
+              !lowerNormalized.includes('index') && !lowerNormalized.includes('thumb') &&
+              !lowerNormalized.includes('middle') && !lowerNormalized.includes('ring') &&
+              !lowerNormalized.includes('pinky')) {
+            if (availableBones.has('RightHand')) return 'RightHand';
+          }
+          
+          // Shoulder bones
+          if (lowerNormalized.includes('left') && (lowerNormalized.includes('shoulder') || lowerNormalized.includes('clavicle'))) {
+            if (availableBones.has('LeftShoulder')) return 'LeftShoulder';
+          }
+          
+          if (lowerNormalized.includes('right') && (lowerNormalized.includes('shoulder') || lowerNormalized.includes('clavicle'))) {
+            if (availableBones.has('RightShoulder')) return 'RightShoulder';
+          }
+          
+          // Common bone name mappings
           const mappings = {
-            // Arm bones - handle common variations
+            // Arm bones - exact matches
             'LeftArm': 'LeftArm',
             'leftArm': 'LeftArm',
             'LEFTARM': 'LeftArm',
@@ -5546,11 +5596,26 @@ class TalkingHead {
           
           // Try case-insensitive match
           for (const boneName of availableBones) {
-            if (boneName.toLowerCase() === normalized.toLowerCase()) {
+            if (boneName.toLowerCase() === lowerNormalized) {
               return boneName;
             }
           }
           
+          // Try partial match (contains)
+          for (const boneName of availableBones) {
+            const boneLower = boneName.toLowerCase();
+            // Match if normalized contains key parts of bone name
+            if ((lowerNormalized.includes('left') && boneLower.includes('left')) ||
+                (lowerNormalized.includes('right') && boneLower.includes('right'))) {
+              if ((lowerNormalized.includes('arm') && boneLower.includes('arm') && !boneLower.includes('fore')) ||
+                  (lowerNormalized.includes('forearm') && boneLower.includes('forearm')) ||
+                  (lowerNormalized.includes('hand') && boneLower.includes('hand') && !boneLower.includes('index') && !boneLower.includes('thumb')) ||
+                  (lowerNormalized.includes('shoulder') && boneLower.includes('shoulder'))) {
+                return boneName;
+              }
+            }
+          }
+          
           return null; // No mapping found
         };
 
@@ -5561,8 +5626,23 @@ class TalkingHead {
           fbxBoneNames.add(trackParts[0]);
         });
         
-        console.log('Ready Player Me FBX bone names:', Array.from(fbxBoneNames).sort().join(', '));
+        console.log('=== Ready Player Me Animation Bone Analysis ===');
+        console.log('FBX bone names:', Array.from(fbxBoneNames).sort().join(', '));
         console.log('Avatar skeleton bone names:', Array.from(availableBones).sort().join(', '));
+        
+        // Check for arm bones specifically
+        const fbxArmBones = Array.from(fbxBoneNames).filter(b => 
+          b.toLowerCase().includes('arm') || 
+          b.toLowerCase().includes('hand') || 
+          b.toLowerCase().includes('shoulder')
+        );
+        const avatarArmBones = Array.from(availableBones).filter(b => 
+          b.includes('Arm') || 
+          b.includes('Hand') || 
+          b.includes('Shoulder')
+        );
+        console.log('FBX arm/hand/shoulder bones:', fbxArmBones.sort().join(', '));
+        console.log('Avatar arm/hand/shoulder bones:', avatarArmBones.sort().join(', '));
 
         // Filter and map animation tracks
         const mappedTracks = [];
@@ -5582,6 +5662,54 @@ class TalkingHead {
             const newTrackName = `${mappedBoneName}.${property}`;
             const newTrack = track.clone();
             newTrack.name = newTrackName;
+            
+            // Fix rotations for Ready Player Me arm bones (coordinate system correction)
+            // Ready Player Me uses a different coordinate system, so we need to adjust rotations
+            const isArmBone = mappedBoneName.includes('Arm') || mappedBoneName.includes('Hand') || mappedBoneName.includes('Shoulder');
+            const isLeftSide = mappedBoneName.includes('Left');
+            const isRightSide = mappedBoneName.includes('Right');
+            const isForearm = mappedBoneName.includes('ForeArm') || mappedBoneName.includes('Forearm');
+            const isHand = mappedBoneName.includes('Hand') && !mappedBoneName.includes('Index') && 
+                          !mappedBoneName.includes('Thumb') && !mappedBoneName.includes('Middle') &&
+                          !mappedBoneName.includes('Ring') && !mappedBoneName.includes('Pinky');
+            
+            if (isArmBone && (property === 'quaternion' || property === 'rotation')) {
+              if (property === 'quaternion' && newTrack.values && newTrack.values.length >= 4) {
+                // Adjust quaternion rotations for Ready Player Me coordinate system
+                const numKeyframes = newTrack.times.length;
+                for (let i = 0; i < numKeyframes; i++) {
+                  const baseIdx = i * 4;
+                  if (baseIdx + 3 < newTrack.values.length) {
+                    let x = newTrack.values[baseIdx];
+                    let y = newTrack.values[baseIdx + 1];
+                    let z = newTrack.values[baseIdx + 2];
+                    let w = newTrack.values[baseIdx + 3];
+                    
+                    const q = new THREE.Quaternion(x, y, z, w);
+                    
+                    // For Ready Player Me, hands may need coordinate system correction
+                    // Hands folding behind suggests we need to rotate around Y axis
+                    if (isHand) {
+                      // Rotate 180 degrees around Y axis to fix hands folding behind
+                      const flipY = new THREE.Quaternion().setFromAxisAngle(new THREE.Vector3(0, 1, 0), Math.PI);
+                      // Apply rotation: q_result = q_original * q_flip (post-multiply)
+                      q.multiply(flipY);
+                    } else if (isForearm) {
+                      // For forearms, try a smaller correction - rotate 90 degrees around Z axis
+                      // This might help with the twisting
+                      const adjustZ = new THREE.Quaternion().setFromAxisAngle(new THREE.Vector3(0, 0, 1), Math.PI / 2);
+                      q.multiply(adjustZ);
+                    }
+                    
+                    newTrack.values[baseIdx] = q.x;
+                    newTrack.values[baseIdx + 1] = q.y;
+                    newTrack.values[baseIdx + 2] = q.z;
+                    newTrack.values[baseIdx + 3] = q.w;
+                  }
+                }
+              }
+            }
+            
             mappedTracks.push(newTrack);
             
             // Store mapping for logging
@@ -5606,10 +5734,23 @@ class TalkingHead {
         // Use mapped tracks if we have any, otherwise use original
         if (mappedTracks.length > 0) {
           anim = new THREE.AnimationClip(anim.name, anim.duration, mappedTracks);
+          console.log(`✓ Created animation with ${mappedTracks.length} mapped tracks (from ${anim.tracks.length} original tracks)`);
           if (boneNameMap.size > 0) {
-            console.log(`Mapped ${boneNameMap.size} bone(s) for Ready Player Me animation:`, 
-              Array.from(boneNameMap.entries()).slice(0, 5).map(([from, to]) => `${from}→${to}`).join(', '));
+            console.log(`✓ Mapped ${boneNameMap.size} bone(s):`, 
+              Array.from(boneNameMap.entries()).map(([from, to]) => `${from}→${to}`).join(', '));
           }
+          
+          // Check if arm bones were mapped
+          const mappedArmBones = Array.from(boneNameMap.values()).filter(b => 
+            b.includes('Arm') || b.includes('Hand') || b.includes('Shoulder')
+          );
+          if (mappedArmBones.length > 0) {
+            console.log(`✓ Arm bones mapped: ${mappedArmBones.join(', ')}`);
+          } else {
+            console.warn('⚠️ No arm bones were mapped! This may cause arm rigging issues.');
+          }
+        } else {
+          console.error('❌ No tracks could be mapped! Animation may not work correctly.');
         }
 
         // Rename and scale Mixamo tracks, create a pose