@sage-rsc/talking-head-react 1.8.8 → 1.9.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sage-rsc/talking-head-react",
-  "version": "1.8.8",
+  "version": "1.9.1",
   "description": "A reusable React component for 3D talking avatars with lip-sync and text-to-speech",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",

package/src/components/SimpleTalkingAvatar.jsx

@@ -27,6 +27,7 @@ import { loadAnimationsFromManifest, autoLoadAnimationsFromFolders } from '../ut
  * @param {Function} props.onError - Callback for errors
  * @param {Function} props.onSpeechStart - Callback when avatar starts speaking
  * @param {Function} props.onSpeechEnd - Callback when avatar finishes speaking
+ * @param {Function} props.onSubtitle - Callback for closed captions/subtitles (receives current text being spoken)
  * @param {string} props.className - Additional CSS classes
  * @param {Object} props.style - Additional inline styles
  * @param {Object} props.animations - Object mapping animation names to FBX file paths, or animation groups
@@ -56,11 +57,12 @@ const SimpleTalkingAvatar = forwardRef(({
   onError = () => {},
   onSpeechStart = () => {},
   onSpeechEnd = () => {},
+  onSubtitle = null,
   className = "",
   style = {},
   animations = {},
-  autoAnimationGroup = null, // e.g., "talking" - will randomly select from this group when speaking
-  autoIdleGroup = null, // e.g., "idle" - will randomly select from this group when idle
+  autoAnimationGroup = null, // e.g., "talking" - will select from this group in order when speaking
+  autoIdleGroup = null, // e.g., "idle" - will select from this group in order when idle
   autoSpeak = false
 }, ref) => {
   const containerRef = useRef(null);
@@ -85,12 +87,19 @@ const SimpleTalkingAvatar = forwardRef(({
   const currentSentenceIndexRef = useRef(0); // Track which sentence is currently playing
   const pausedAudioDataRef = useRef(null); // Store trimmed audio buffer when paused
   const restartIdleAnimationsRef = useRef(null); // Ref to restartIdleAnimations function to avoid circular dependency
+  const onSubtitleRef = useRef(onSubtitle); // Store onSubtitle in ref to avoid stale closures
+  const currentSubtitleTextRef = useRef(''); // Track current text being spoken for closed captions
   
   // Keep ref in sync with state
   useEffect(() => {
     isPausedRef.current = isPaused;
   }, [isPaused]);
 
+  // Keep onSubtitle ref in sync with prop
+  useEffect(() => {
+    onSubtitleRef.current = onSubtitle;
+  }, [onSubtitle]);
+
   // Helper function to normalize avatar body to gender key
   const getGenderKey = useCallback((body) => {
     // Normalize avatarBody - handle 'M'/'F', 'male'/'female', or undefined
@@ -466,26 +475,16 @@ const SimpleTalkingAvatar = forwardRef(({
     return [];
   }, [loadedAnimations, avatarBody, getGenderKey]);
   
-  // Helper function to shuffle array (Fisher-Yates algorithm)
-  const shuffleArray = useCallback((array) => {
-    const shuffled = [...array];
-    for (let i = shuffled.length - 1; i > 0; i--) {
-      const j = Math.floor(Math.random() * (i + 1));
-      [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
-    }
-    return shuffled;
-  }, []);
-  
-  // Helper function to get next animation from queue (or create new shuffled queue)
+  // Helper function to get next animation from queue (or create new queue in order)
   const getNextAnimation = useCallback((groupName) => {
-    // If queue is empty, create a new shuffled queue
+    // If queue is empty, create a new queue in order
     if (animationQueueRef.current.length === 0) {
       const allAnimations = getAllAnimationsFromGroup(groupName);
       if (allAnimations.length === 0) {
         return null;
       }
-      // Shuffle and create new queue
-      animationQueueRef.current = shuffleArray(allAnimations);
+      // Create queue in order (no shuffling)
+      animationQueueRef.current = [...allAnimations];
       playedAnimationsRef.current = []; // Reset played list
     }
     
@@ -496,7 +495,7 @@ const SimpleTalkingAvatar = forwardRef(({
     }
     
     return nextAnimation;
-  }, [getAllAnimationsFromGroup, shuffleArray]);
+  }, [getAllAnimationsFromGroup]);
   
   // Helper function to extract animation name from path
   const getAnimationName = useCallback((animationPath) => {
@@ -629,7 +628,7 @@ const SimpleTalkingAvatar = forwardRef(({
         return null;
       }
     } else {
-      // No more animations in queue, but check if we should reshuffle and continue
+      // No more animations in queue, but check if we should reset queue and continue
       const checkStillSpeaking = () => {
         if (!talkingHeadRef.current) return false;
         
@@ -655,7 +654,7 @@ const SimpleTalkingAvatar = forwardRef(({
       const isStillSpeaking = checkStillSpeaking();
       
       if (isStillSpeaking) {
-        // Queue is empty but still speaking, reshuffle and continue playing animations IMMEDIATELY
+        // Queue is empty but still speaking, reset queue and continue playing animations IMMEDIATELY
         animationQueueRef.current = [];
         playedAnimationsRef.current = [];
         // Continue immediately - no delay for continuous playback
@@ -779,9 +778,65 @@ const SimpleTalkingAvatar = forwardRef(({
     const sentences = textToSpeak.split(/[.!?]+/).filter(s => s.trim().length > 0);
     originalSentencesRef.current = sentences;
     currentSentenceIndexRef.current = 0; // Reset sentence tracking
+    
+    // Reset subtitle text
+    currentSubtitleTextRef.current = '';
+    
+    // Create subtitle callback that extracts text from sentences/words as they're spoken
+    // Use a ref to store the latest onSubtitle callback to avoid stale closures
+    const subtitleCallback = (subtitleData) => {
+      // Extract text from subtitle data
+      // Can be called with:
+      // 1. Sentence-level: { text: "sentence", subtitles: ["sentence"] }
+      // 2. Word-level: { vs: { subtitles: [' word'] } }
+      let subtitleText = '';
+      
+      if (subtitleData) {
+        // Check for text property first (sentence-level, from startSpeaking)
+        if (subtitleData.text) {
+          subtitleText = subtitleData.text;
+        } 
+        // Check for vs.subtitles (word-level, from animation queue)
+        else if (subtitleData.vs && subtitleData.vs.subtitles) {
+          if (Array.isArray(subtitleData.vs.subtitles)) {
+            subtitleText = subtitleData.vs.subtitles.join('');
+          } else if (typeof subtitleData.vs.subtitles === 'string') {
+            subtitleText = subtitleData.vs.subtitles;
+          }
+        }
+        // Check for direct subtitles array (fallback)
+        else if (subtitleData.subtitles) {
+          if (Array.isArray(subtitleData.subtitles)) {
+            subtitleText = subtitleData.subtitles.join('');
+          } else if (typeof subtitleData.subtitles === 'string') {
+            subtitleText = subtitleData.subtitles;
+          }
+        }
+      }
+      
+      // Update current subtitle text (accumulate words/sentences)
+      if (subtitleText.trim()) {
+        // Append the new text (words already include spaces)
+        currentSubtitleTextRef.current += subtitleText;
+        
+        // Call onSubtitle callback if provided (use ref to avoid stale closures)
+        const currentCallback = onSubtitleRef.current;
+        if (currentCallback && typeof currentCallback === 'function') {
+          try {
+            // Use setTimeout to defer the callback and avoid calling setState during render
+            setTimeout(() => {
+              currentCallback(currentSubtitleTextRef.current.trim());
+            }, 0);
+          } catch (err) {
+            console.warn('Error in onSubtitle callback:', err);
+          }
+        }
+      }
+    };
 
     const speakOptions = {
-      lipsyncLang: options.lipsyncLang || 'en'
+      lipsyncLang: options.lipsyncLang || 'en',
+      onSubtitles: onSubtitle ? subtitleCallback : null
     };
     
     // Set up polling mechanism to detect when speech ends
@@ -888,6 +943,9 @@ const SimpleTalkingAvatar = forwardRef(({
               animationQueueRef.current = [];
               playedAnimationsRef.current = [];
               
+              // Clear subtitle text when speech ends
+              currentSubtitleTextRef.current = '';
+              
               // CRITICAL: Stop current animation to return to idle pose
               if (currentTalkingHead) {
                 currentTalkingHead.stopAnimation();
@@ -1238,6 +1296,9 @@ const SimpleTalkingAvatar = forwardRef(({
                   animationQueueRef.current = [];
                   playedAnimationsRef.current = [];
                   
+                  // Clear subtitle text when speech ends
+                  currentSubtitleTextRef.current = '';
+                  
                   // CRITICAL: Stop current animation to return to idle pose
                   if (currentTalkingHead) {
                     currentTalkingHead.stopAnimation();
@@ -1336,6 +1397,9 @@ const SimpleTalkingAvatar = forwardRef(({
       animationQueueRef.current = [];
       playedAnimationsRef.current = [];
       
+      // Clear subtitle text when speech is stopped
+      currentSubtitleTextRef.current = '';
+      
       setIsPaused(false);
       isPausedRef.current = false;
       pausedAudioDataRef.current = null; // Clear any paused audio data
@@ -1367,6 +1431,7 @@ const SimpleTalkingAvatar = forwardRef(({
     stopSpeaking,
     resumeAudioContext,
     isPaused: () => isPaused,
+    getCurrentSubtitle: () => currentSubtitleTextRef.current, // Get current subtitle text
     setMood: (mood) => talkingHeadRef.current?.setMood(mood),
     setBodyMovement: (movement) => {
       if (talkingHeadRef.current) {

package/src/lib/talkinghead.mjs

@@ -4538,6 +4538,25 @@ class TalkingHead {
         // Look at the camera
         this.lookAtCamera(500);
 
+        // Extract text for subtitle callback
+        let sentenceText = '';
+        if (Array.isArray(line.text)) {
+          // Array format: [{word: "..."}, {word: "..."}]
+          sentenceText = line.text.map(x => x.word).join(' ');
+        } else if (typeof line.text === 'string') {
+          sentenceText = line.text;
+        }
+        
+        // Call subtitle callback with sentence text if available
+        if (sentenceText && this.onSubtitles && typeof this.onSubtitles === 'function') {
+          try {
+            // Call with the sentence text
+            this.onSubtitles({ text: sentenceText, subtitles: [sentenceText] });
+          } catch (err) {
+            console.warn('Error in subtitle callback:', err);
+          }
+        }
+
         // Spoken text
         try {
           // Check which TTS service to use