@telnyx/voice-agent-tester 0.4.3 → 0.4.4

package/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## [0.4.4](https://github.com/team-telnyx/voice-agent-tester/compare/v0.4.3...v0.4.4) (2026-03-11)
+
+### Features
+
+* fix speechend race condition, add --retries flag ([#21](https://github.com/team-telnyx/voice-agent-tester/issues/21)) ([09e3b65](https://github.com/team-telnyx/voice-agent-tester/commit/09e3b6578face6c407d058991ab5495d9463e544))
+
+### Chores
+
+* release v0.4.3 ([#20](https://github.com/team-telnyx/voice-agent-tester/issues/20)) ([bdeb87b](https://github.com/team-telnyx/voice-agent-tester/commit/bdeb87bed502919a9fed9950e69242b1c2aefcfc))
+
 ## [0.4.3](https://github.com/team-telnyx/voice-agent-tester/compare/v0.4.2...v0.4.3) (2026-03-11)
 
 ### Features

package/javascript/audio_input_hooks.js

@@ -62,20 +62,24 @@ function createControlledMediaStream() {
 }
 
 // Replace getUserMedia to return our controlled stream
-const originalGetUserMedia = navigator.mediaDevices.getUserMedia.bind(navigator.mediaDevices);
-navigator.mediaDevices.getUserMedia = function (constraints) {
-  console.log("🎤 Intercepted getUserMedia call with constraints:", constraints);
-
-  // If audio is requested, return our controlled stream
-  if (constraints && constraints.audio) {
-    console.log("🎤 Returning controlled MediaStream instead of real microphone");
-    const controlledStream = createControlledMediaStream();
-    return Promise.resolve(controlledStream);
-  }
+if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) {
+  const originalGetUserMedia = navigator.mediaDevices.getUserMedia.bind(navigator.mediaDevices);
+  navigator.mediaDevices.getUserMedia = function (constraints) {
+    console.log("🎤 Intercepted getUserMedia call with constraints:", constraints);
+
+    // If audio is requested, return our controlled stream
+    if (constraints && constraints.audio) {
+      console.log("🎤 Returning controlled MediaStream instead of real microphone");
+      const controlledStream = createControlledMediaStream();
+      return Promise.resolve(controlledStream);
+    }
 
-  // For video-only or other requests, use original implementation
-  return originalGetUserMedia(constraints);
-};
+    // For video-only or other requests, use original implementation
+    return originalGetUserMedia(constraints);
+  };
+} else {
+  console.warn("🎤 navigator.mediaDevices.getUserMedia not available, skipping microphone intercept");
+}
 
 // Expose __speak method to be called from voice-agent-tester.js
 window.__speak = function (textOrUrl) {
@@ -152,6 +156,24 @@ function playAudioInMediaStream(url) {
   const audio = new Audio(url);
   audio.crossOrigin = 'anonymous'; // Enable CORS if needed
 
+  // Keep a strong reference so the element is not garbage collected
+  currentSpeakAudio = audio;
+
+  let speechEndFired = false;
+  let safetyTimeoutId = null;
+
+  function fireSpeechEnd(reason) {
+    if (speechEndFired) return;
+    speechEndFired = true;
+    if (safetyTimeoutId) clearTimeout(safetyTimeoutId);
+    console.log(`🎤 Audio playback ended (${reason})`);
+    if (typeof __publishEvent === 'function') {
+      __publishEvent('speechend', { url: url, reason: reason });
+    }
+    // Release reference
+    if (currentSpeakAudio === audio) currentSpeakAudio = null;
+  }
+
   // Set up audio routing through all MediaStreams
   audio.addEventListener('canplaythrough', function () {
     console.log(`🎤 Audio ready to play, routing to ${mediaStreams.length} MediaStreams`);
@@ -181,7 +203,33 @@ function playAudioInMediaStream(url) {
       }
 
       // Play the audio
-      audio.play();
+      audio.play().then(() => {
+        // Set up safety timeout based on audio duration
+        // audio.duration should be available after canplaythrough
+        const duration = audio.duration;
+        if (duration && isFinite(duration)) {
+          const safetyMs = Math.max((duration * 1000) + 5000, 15000);
+          console.log(`🎤 Audio duration: ${duration.toFixed(1)}s, safety timeout: ${(safetyMs / 1000).toFixed(1)}s`);
+          safetyTimeoutId = setTimeout(() => {
+            if (!speechEndFired) {
+              console.warn(`🎤 Safety timeout: speechend not fired after ${(safetyMs / 1000).toFixed(1)}s (audio paused=${audio.paused}, ended=${audio.ended}, currentTime=${audio.currentTime.toFixed(1)})`);
+              fireSpeechEnd('safety_timeout');
+            }
+          }, safetyMs);
+        } else {
+          // Unknown duration — use 20s fallback
+          console.warn('🎤 Audio duration unknown, using 20s safety timeout');
+          safetyTimeoutId = setTimeout(() => {
+            if (!speechEndFired) {
+              console.warn('🎤 Safety timeout: speechend not fired after 20s');
+              fireSpeechEnd('safety_timeout');
+            }
+          }, 20000);
+        }
+      }).catch(error => {
+        console.error('Error playing audio:', error);
+        fireSpeechEnd('play_error');
+      });
     } catch (error) {
       console.error('Error setting up audio source:', error);
       if (typeof __publishEvent === 'function') {
@@ -190,11 +238,19 @@ function playAudioInMediaStream(url) {
     }
   });
 
-  // Handle audio end
+  // Handle audio end — primary path
   audio.addEventListener('ended', function () {
-    console.log('🎤 Audio playback ended');
-    if (typeof __publishEvent === 'function') {
-      __publishEvent('speechend', { url: url });
+    fireSpeechEnd('ended');
+  });
+
+  // Handle pause — if something pauses the audio externally
+  audio.addEventListener('pause', function () {
+    // Only treat as speechend if the audio is past 90% of its duration (near end)
+    // or if it was paused externally (not by us)
+    if (audio.ended || (audio.duration && audio.currentTime >= audio.duration * 0.9)) {
+      fireSpeechEnd('pause_near_end');
+    } else {
+      console.warn(`🎤 Audio paused at ${audio.currentTime.toFixed(1)}s / ${(audio.duration || 0).toFixed(1)}s`);
     }
   });
 
@@ -204,17 +260,31 @@ function playAudioInMediaStream(url) {
     if (typeof __publishEvent === 'function') {
       __publishEvent('speecherror', { error: 'Audio playback failed', url: url });
     }
+    fireSpeechEnd('error');
   });
 
   // Start loading the audio
   audio.load();
 }
 
+// Keep a reference to the current speak Audio element so it doesn't get GC'd
+let currentSpeakAudio = null;
+
 // Helper function to stop current audio and reset to silence
 function stopCurrentAudio() {
+  // Stop the speak audio element if playing
+  if (currentSpeakAudio) {
+    try {
+      currentSpeakAudio.pause();
+      currentSpeakAudio.currentTime = 0;
+    } catch (e) {
+      console.warn('Error stopping speak audio:', e);
+    }
+    currentSpeakAudio = null;
+  }
+
   currentPlaybackNodes.forEach((sourceNode, index) => {
     try {
-      sourceNode.stop();
       sourceNode.disconnect();
       console.log(`🎤 Stopped audio source ${index}`);
     } catch (e) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@telnyx/voice-agent-tester",
-  "version": "0.4.3",
+  "version": "0.4.4",
   "description": "A command-line tool to test voice agents using Puppeteer",
   "main": "src/index.js",
   "type": "module",

package/src/index.js

@@ -327,6 +327,11 @@ const argv = yargs(hideBin(process.argv))
     description: 'Volume level for audio input (0.0 to 1.0)',
     default: 1.0
   })
+  .option('retries', {
+    type: 'number',
+    description: 'Number of retries for failed test runs (0 = no retries)',
+    default: 0
+  })
   .help()
   .argv;
 
@@ -409,22 +414,49 @@ async function runBenchmark({ applications, scenarios, repeat, concurrency, argv
       audioVolume: argv.audioVolume
     });
 
-    try {
-      await tester.runScenario(targetUrl, app.steps, scenario.steps, app.name, scenario.name, repetition);
-      console.log(`✅ Completed successfully (Run ${runNumber}/${totalRuns})`);
-      return { success: true };
-    } catch (error) {
-      // Store only the first line for summary, but print full message here (with diagnostics)
-      const shortMessage = error.message.split('\n')[0];
-      const errorInfo = {
-        app: app.name,
-        scenario: scenario.name,
-        repetition,
-        error: shortMessage
-      };
-      // Print full diagnostics here (only place they appear)
-      console.error(`❌ Error (Run ${runNumber}/${totalRuns}):\n${error.message}`);
-      return { success: false, error: errorInfo };
+    const maxAttempts = (argv.retries || 0) + 1;
+
+    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+      // Create a fresh tester for each attempt (after first, original tester is closed)
+      const currentTester = attempt === 1 ? tester : new VoiceAgentTester({
+        verbose: argv.verbose,
+        headless: argv.headless,
+        assetsServerUrl: argv.assetsServer,
+        reportGenerator: reportGenerator,
+        record: argv.record,
+        debug: argv.debug,
+        audioUrl: argv.audioUrl,
+        audioVolume: argv.audioVolume
+      });
+
+      try {
+        await currentTester.runScenario(targetUrl, app.steps, scenario.steps, app.name, scenario.name, repetition);
+        console.log(`✅ Completed successfully (Run ${runNumber}/${totalRuns})`);
+        return { success: true };
+      } catch (error) {
+        const shortMessage = error.message.split('\n')[0];
+
+        if (attempt < maxAttempts) {
+          console.warn(`\n⚠️ Attempt ${attempt}/${maxAttempts} failed: ${shortMessage}`);
+          console.warn(`🔄 Retrying in 3s... (${maxAttempts - attempt} retries left)\n`);
+          await new Promise(r => setTimeout(r, 3000));
+          continue;
+        }
+
+        // Final attempt failed
+        const errorInfo = {
+          app: app.name,
+          scenario: scenario.name,
+          repetition,
+          error: shortMessage
+        };
+        // Print full diagnostics here (only place they appear)
+        console.error(`❌ Error (Run ${runNumber}/${totalRuns}):\n${error.message}`);
+        if (maxAttempts > 1) {
+          console.error(`   Failed after ${maxAttempts} attempts`);
+        }
+        return { success: false, error: errorInfo };
+      }
     }
   }
 

package/src/voice-agent-tester.js

@@ -238,6 +238,7 @@ export class VoiceAgentTester {
           } else {
             errorMessage += '\n  (Could not collect browser diagnostics)';
           }
+
         }
 
         reject(new Error(errorMessage));
@@ -363,6 +364,7 @@ export class VoiceAgentTester {
         console.error(error.stack);
       }
     });
+
   }
 
   async close() {
@@ -866,10 +868,40 @@ export class VoiceAgentTester {
 
     // Wait for speech to complete by listening for speechend event
     try {
-      await this.waitForAudioEvent('speechend');
+      // Use a shorter timeout for speechend (15s) since we have safety fallback in browser
+      await this.waitForAudioEvent('speechend', 15000);
     } catch (error) {
-      console.error('Timeout waiting for speech to complete:', error.message);
-      throw error;
+      // speechend timeout is recoverable — the audio likely finished but the event was lost
+      // (e.g., agent started responding and disrupted the audio element)
+      if (this.debug) {
+        // Check the state of the speak audio in the browser
+        const speakState = await this.page.evaluate(() => {
+          const info = {
+            currentSpeakAudio: null,
+            audioContextState: null,
+          };
+          try {
+            if (window.currentSpeakAudio) {
+              info.currentSpeakAudio = {
+                paused: window.currentSpeakAudio.paused,
+                ended: window.currentSpeakAudio.ended,
+                currentTime: window.currentSpeakAudio.currentTime,
+                duration: window.currentSpeakAudio.duration,
+                readyState: window.currentSpeakAudio.readyState,
+              };
+            }
+            if (window.globalAudioContext) {
+              info.audioContextState = window.globalAudioContext.state;
+            }
+          } catch (e) { /* ignore */ }
+          return info;
+        }).catch(() => null);
+
+        console.warn(`\t⚠️ speechend timeout (recovered) — speak audio state:`, JSON.stringify(speakState));
+      } else {
+        console.warn(`\t⚠️ speechend timeout — continuing (audio likely finished)`);
+      }
+      // Don't throw — treat speechend timeout as recoverable
     }
   }
 

package/tests/integration.test.js

@@ -44,8 +44,9 @@ describe('Integration Tests', () => {
               this.text = text;
             };
 
-            // Mock __speak function that will be called by the tester
-            // This needs to be in the page itself since evaluateOnNewDocument runs before navigation
+            // Mock __speak and __waitForMediaStream functions
+            // These override the injected audio hooks since inline scripts run after evaluateOnNewDocument
+            window.__waitForMediaStream = () => Promise.resolve();
             window.__speak = (text) => {
               document.getElementById('speech-output').textContent = text;
               // Signal speech end after a small delay to allow waitForAudioEvent to be set up
@@ -75,7 +76,7 @@ describe('Integration Tests', () => {
 
     // The scenario should complete without throwing errors
     expect(true).toBe(true);
-  });
+  }, 15000);
 
   test('should handle scenario with wait step', async () => {
     const testPageContent = `