@sage-rsc/talking-head-react 1.1.3 → 1.1.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sage-rsc/talking-head-react",
-  "version": "1.1.3",
+  "version": "1.1.6",
   "description": "A reusable React component for 3D talking avatars with lip-sync and text-to-speech",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

package/src/lib/talkinghead.mjs

@@ -5407,18 +5407,33 @@ class TalkingHead {
       'Spine01': 'Spine1',
       'Spine02': 'Spine2',
       'Spine03': 'Spine2',
+      'Spine1': 'Spine1',
+      'Spine2': 'Spine2',
+      'Spine': 'Spine1',
+      
+      // Head and neck
+      'Head': 'Head',
+      'Neck': 'Neck',
+      'Neck1': 'Neck',
+      'Neck2': 'Neck',
       
       // Left arm mapping
       'L_Upperarm': 'LeftArm',
       'L_Forearm': 'LeftForeArm',
       'L_Hand': 'LeftHand',
       'L_Shoulder': 'LeftShoulder',
+      'L_Clavicle': 'LeftShoulder',
+      'L_UpperArm': 'LeftArm',
+      'L_ForeArm': 'LeftForeArm',
       'L_Index1': 'LeftHandIndex1',
       'L_Index2': 'LeftHandIndex2',
       'L_Index3': 'LeftHandIndex3',
       'L_Middle1': 'LeftHandMiddle1',
       'L_Middle2': 'LeftHandMiddle2',
       'L_Middle3': 'LeftHandMiddle3',
+      'L_Mid1': 'LeftHandMiddle1',
+      'L_Mid2': 'LeftHandMiddle2',
+      'L_Mid3': 'LeftHandMiddle3',
       'L_Ring1': 'LeftHandRing1',
       'L_Ring2': 'LeftHandRing2',
       'L_Ring3': 'LeftHandRing3',
@@ -5434,12 +5449,18 @@ class TalkingHead {
       'R_Forearm': 'RightForeArm',
       'R_Hand': 'RightHand',
       'R_Shoulder': 'RightShoulder',
+      'R_Clavicle': 'RightShoulder',
+      'R_UpperArm': 'RightArm',
+      'R_ForeArm': 'RightForeArm',
       'R_Index1': 'RightHandIndex1',
       'R_Index2': 'RightHandIndex2',
       'R_Index3': 'RightHandIndex3',
       'R_Middle1': 'RightHandMiddle1',
       'R_Middle2': 'RightHandMiddle2',
       'R_Middle3': 'RightHandMiddle3',
+      'R_Mid1': 'RightHandMiddle1',
+      'R_Mid2': 'RightHandMiddle2',
+      'R_Mid3': 'RightHandMiddle3',
       'R_Ring1': 'RightHandRing1',
       'R_Ring2': 'RightHandRing2',
       'R_Ring3': 'RightHandRing3',
@@ -5454,9 +5475,18 @@ class TalkingHead {
       'L_Thigh': 'LeftUpLeg',
       'L_Calf': 'LeftLeg',
       'L_Foot': 'LeftFoot',
+      'L_UpLeg': 'LeftUpLeg',
+      'L_Leg': 'LeftLeg',
       'R_Thigh': 'RightUpLeg',
       'R_Calf': 'RightLeg',
       'R_Foot': 'RightFoot',
+      'R_UpLeg': 'RightUpLeg',
+      'R_Leg': 'RightLeg',
+      
+      // Root/Hips
+      'Hips': 'Hips',
+      'Root': 'Hips',
+      'Pelvis': 'Hips',
     };
 
     // Try mapping
@@ -5504,8 +5534,8 @@ class TalkingHead {
       }
     }
     
-    // Pattern: R_Middle1/2/3 -> RightHandMiddle1/2/3
-    const middleMatch = lowerNormalized.match(/^[rl]_middle(\d+)$/);
+    // Pattern: R_Middle1/2/3 or R_Mid1/2/3 -> RightHandMiddle1/2/3
+    const middleMatch = lowerNormalized.match(/^[rl]_(?:middle|mid)(\d+)$/);
     if (middleMatch) {
       const digit = middleMatch[1];
       const side = lowerNormalized.startsWith('r') ? 'Right' : 'Left';
@@ -5535,8 +5565,12 @@ class TalkingHead {
       }
     }
     
-    // Pattern: R_UpperarmTwist01/02 -> ignore (twist bones)
-    if (lowerNormalized.includes('upperarmtwist') || lowerNormalized.includes('forearmtwist')) {
+    // Pattern: R_UpperarmTwist01/02 -> ignore (twist bones and other extra bones)
+    if (lowerNormalized.includes('upperarmtwist') || 
+        lowerNormalized.includes('forearmtwist') ||
+        lowerNormalized.includes('ribstwist') ||
+        lowerNormalized.includes('breast') ||
+        lowerNormalized.includes('twist')) {
       return null;
     }
     
@@ -5557,6 +5591,49 @@ class TalkingHead {
         return candidate;
       }
     }
+    
+    // Pattern: R_Clavicle -> RightShoulder
+    if (lowerNormalized.match(/^[rl]_clavicle/)) {
+      const side = lowerNormalized.startsWith('r') ? 'Right' : 'Left';
+      const candidate = `${side}Shoulder`;
+      if (availableBones.has(candidate)) {
+        return candidate;
+      }
+    }
+    
+    // Pattern: R_Shoulder -> RightShoulder
+    if (lowerNormalized.match(/^[rl]_shoulder/)) {
+      const side = lowerNormalized.startsWith('r') ? 'Right' : 'Left';
+      const candidate = `${side}Shoulder`;
+      if (availableBones.has(candidate)) {
+        return candidate;
+      }
+    }
+    
+    // Pattern: Spine01/02/03 -> Spine1/Spine2
+    if (lowerNormalized.match(/^spine0?(\d+)$/)) {
+      const spineNum = lowerNormalized.match(/^spine0?(\d+)$/)[1];
+      if (spineNum === '1') {
+        if (availableBones.has('Spine1')) return 'Spine1';
+      } else if (spineNum === '2' || spineNum === '3') {
+        if (availableBones.has('Spine2')) return 'Spine2';
+      }
+    }
+    
+    // Pattern: Neck1/2 -> Neck
+    if (lowerNormalized.match(/^neck\d*$/)) {
+      if (availableBones.has('Neck')) return 'Neck';
+    }
+    
+    // Pattern: Head -> Head
+    if (lowerNormalized === 'head') {
+      if (availableBones.has('Head')) return 'Head';
+    }
+    
+    // Pattern: Hips/Pelvis/Root -> Hips
+    if (lowerNormalized.match(/^(hips|pelvis|root)$/)) {
+      if (availableBones.has('Hips')) return 'Hips';
+    }
 
     // Try case-insensitive exact match
     for (const boneName of availableBones) {
@@ -5564,6 +5641,18 @@ class TalkingHead {
         return boneName;
       }
     }
+    
+    // Try partial match (contains)
+    for (const boneName of availableBones) {
+      const boneLower = boneName.toLowerCase();
+      if (boneLower.includes(lowerNormalized) || lowerNormalized.includes(boneLower)) {
+        // Prefer exact matches or very close matches
+        if (boneLower === lowerNormalized || 
+            boneLower.replace(/[^a-z0-9]/g, '') === lowerNormalized.replace(/[^a-z0-9]/g, '')) {
+          return boneName;
+        }
+      }
+    }
 
     return null; // No mapping found
   }
@@ -5579,14 +5668,25 @@ class TalkingHead {
     const validTracks = [];
     const missingBones = new Set();
     const mappedBones = new Map(); // Track mappings for logging
+    const fbxBoneNames = new Set(); // Track all unique FBX bone names
     
     // Debug: Log available bones (first time only)
     if (!this._loggedAvailableBones) {
-      console.log('Available avatar bones:', Array.from(availableBones).sort().slice(0, 50).join(', '), 
-        availableBones.size > 50 ? `... (${availableBones.size} total)` : '');
+      console.log('Available avatar bones:', Array.from(availableBones).sort().join(', '));
       this._loggedAvailableBones = true;
     }
     
+    // First pass: collect all unique FBX bone names
+    clip.tracks.forEach(track => {
+      const trackNameParts = track.name.split('.');
+      const fbxBoneName = trackNameParts[0];
+      fbxBoneNames.add(fbxBoneName);
+    });
+    
+    console.log(`\n=== FBX Animation "${clip.name}" Bone Analysis ===`);
+    console.log('FBX bone names in animation:', Array.from(fbxBoneNames).sort().join(', '));
+    console.log(`Total FBX bones: ${fbxBoneNames.size}, Avatar bones: ${availableBones.size}\n`);
+    
     clip.tracks.forEach(track => {
       // Extract bone name from track name (e.g., "CC_Base_R_Index3.position" -> "CC_Base_R_Index3")
       const trackNameParts = track.name.split('.');
@@ -5604,6 +5704,72 @@ class TalkingHead {
         const newTrack = track.clone();
         newTrack.name = newTrackName;
         
+        // Fix rotations for arm/hand bones that might be inverted
+        // If hands are folding behind instead of in front, we need to adjust rotations
+        const isArmBone = mappedBoneName.includes('Arm') || mappedBoneName.includes('Hand') || mappedBoneName.includes('Shoulder');
+        const isForearmBone = mappedBoneName.includes('ForeArm');
+        
+        if (isArmBone && (property === 'quaternion' || property === 'rotation')) {
+          // For quaternion tracks, we might need to adjust the rotation
+          // Check if this is a quaternion track
+          if (property === 'quaternion' && newTrack.values && newTrack.values.length >= 4) {
+            // Quaternion format: [x, y, z, w] per keyframe
+            // For arm bones, we might need to invert Y or Z rotation
+            // Adjust quaternion values to fix hand position
+            const numKeyframes = newTrack.times.length;
+            for (let i = 0; i < numKeyframes; i++) {
+              const baseIdx = i * 4;
+              if (baseIdx + 3 < newTrack.values.length) {
+                // Get quaternion values
+                let x = newTrack.values[baseIdx];
+                let y = newTrack.values[baseIdx + 1];
+                let z = newTrack.values[baseIdx + 2];
+                let w = newTrack.values[baseIdx + 3];
+                
+                // For arms, adjust rotation to flip hands from back to front
+                // This is a common fix for FBX animations with different coordinate systems
+                if (isForearmBone || mappedBoneName.includes('Hand')) {
+                  // Rotate 180 degrees around X axis to flip hands from behind to in front
+                  // Create a 180-degree rotation around X axis
+                  const flipAngle = Math.PI;
+                  const flipX = Math.cos(flipAngle / 2); // w component
+                  const flipY = Math.sin(flipAngle / 2); // x component (axis X = 1,0,0)
+                  
+                  // Multiply quaternions: q_result = q_flip * q_original
+                  // For rotation around X axis: q_flip = (sin(θ/2), 0, 0, cos(θ/2))
+                  const qw = flipX * w - flipY * x;
+                  const qx = flipX * x + flipY * w;
+                  const qy = flipX * y - flipY * z;
+                  const qz = flipX * z + flipY * y;
+                  
+                  x = qx;
+                  y = qy;
+                  z = qz;
+                  w = qw;
+                }
+                
+                newTrack.values[baseIdx] = x;
+                newTrack.values[baseIdx + 1] = y;
+                newTrack.values[baseIdx + 2] = z;
+                newTrack.values[baseIdx + 3] = w;
+              }
+            }
+          } else if (property === 'rotation' && newTrack.values && newTrack.values.length >= 3) {
+            // Euler rotation format: [x, y, z] per keyframe
+            const numKeyframes = newTrack.times.length;
+            for (let i = 0; i < numKeyframes; i++) {
+              const baseIdx = i * 3;
+              if (baseIdx + 2 < newTrack.values.length) {
+                // For arm bones, adjust Y rotation to flip hands
+                if (isForearmBone || mappedBoneName.includes('Hand')) {
+                  // Add 180 degrees (PI radians) to Y rotation
+                  newTrack.values[baseIdx + 1] += Math.PI;
+                }
+              }
+            }
+          }
+        }
+        
         validTracks.push(newTrack);
         
         // Track mappings for logging
@@ -5615,22 +5781,64 @@ class TalkingHead {
       }
     });
     
-    // Log results
+    // Log results with detailed mapping information
+    console.log(`\n=== Mapping Results for "${clip.name}" ===`);
+    
+    // Show all mapped bones
     if (mappedBones.size > 0) {
-      console.info(`FBX animation "${clip.name}": Mapped ${mappedBones.size} bone(s) to avatar skeleton:`, 
-        Array.from(mappedBones.entries()).slice(0, 5).map(([from, to]) => `${from} → ${to}`).join(', '),
-        mappedBones.size > 5 ? '...' : '');
+      console.info(`✓ Successfully mapped ${mappedBones.size} bone(s):`);
+      Array.from(mappedBones.entries()).forEach(([from, to]) => {
+        console.log(`  ${from} → ${to}`);
+      });
     }
     
+    // Show all missing bones with suggestions
     if (missingBones.size > 0) {
-      console.warn(`FBX animation "${clip.name}" contains tracks for ${missingBones.size} bone(s) that couldn't be mapped:`, 
-        Array.from(missingBones).slice(0, 10).join(', '), 
-        missingBones.size > 10 ? '...' : '');
+      console.warn(`\n✗ Could not map ${missingBones.size} bone(s):`);
+      Array.from(missingBones).sort().forEach(boneName => {
+        console.log(`  - ${boneName}`);
+        // Try to suggest a potential mapping
+        const normalized = boneName.replace(/^(CC_Base_|mixamorig)/i, '').replace(/^[RL]_/, (match) => match.toLowerCase());
+        const suggestions = [];
+        for (const avatarBone of availableBones) {
+          const avatarLower = avatarBone.toLowerCase();
+          const normalizedLower = normalized.toLowerCase();
+          if (avatarLower.includes(normalizedLower) || normalizedLower.includes(avatarLower)) {
+            suggestions.push(avatarBone);
+          }
+        }
+        if (suggestions.length > 0) {
+          console.log(`    → Possible matches: ${suggestions.slice(0, 3).join(', ')}`);
+        }
+      });
     }
     
-    if (validTracks.length > 0) {
-      console.info(`Filtered ${clip.tracks.length} tracks down to ${validTracks.length} valid tracks (${mappedBones.size} mapped)`);
-    } else {
+    // Show bones that matched directly (no mapping needed)
+    const directMatches = new Set();
+    fbxBoneNames.forEach(fbxBone => {
+      if (availableBones.has(fbxBone) && !mappedBones.has(fbxBone)) {
+        directMatches.add(fbxBone);
+      }
+    });
+    if (directMatches.size > 0) {
+      console.info(`\n✓ Direct matches (no mapping needed): ${directMatches.size} bone(s)`);
+      Array.from(directMatches).sort().slice(0, 10).forEach(bone => {
+        console.log(`  - ${bone}`);
+      });
+      if (directMatches.size > 10) {
+        console.log(`  ... and ${directMatches.size - 10} more`);
+      }
+    }
+    
+    console.log(`\n=== Summary ===`);
+    console.log(`Total FBX tracks: ${clip.tracks.length}`);
+    console.log(`Valid tracks: ${validTracks.length}`);
+    console.log(`Mapped bones: ${mappedBones.size}`);
+    console.log(`Direct matches: ${directMatches.size}`);
+    console.log(`Missing bones: ${missingBones.size}`);
+    console.log(`========================================\n`);
+    
+    if (validTracks.length === 0) {
       console.error(`No valid tracks found for animation "${clip.name}". All bones are missing or couldn't be mapped.`);
     }