@telnyx/voice-agent-tester 0.4.3 → 0.4.5

package/javascript/audio_output_hooks.js

@@ -526,10 +526,98 @@ class AudioElementMonitor {
 
       console.log(`Started monitoring programmatic audio element: ${elementId}`);
     } catch (error) {
-      console.error(`Failed to monitor programmatic audio element ${elementId}:`, error);
+      console.error(`Failed to monitor programmatic audio element ${elementId} via analyser:`, error.message);
+      console.log(`Falling back to event-based monitoring for ${elementId}`);
+      this.monitorViaEvents(audioElement, elementId);
     }
   }
 
+  /**
+   * Fallback monitoring using audio element events (timeupdate/playing/pause).
+   * Used when AudioContext-based monitoring fails (e.g., when the audio element
+   * is already connected to another AudioContext via MediaStreamDestination).
+   */
+  monitorViaEvents(audioElement, elementId) {
+    const monitorData = {
+      element: audioElement,
+      source: null,
+      analyser: null,
+      dataArray: null,
+      isPlaying: false,
+      lastAudioTime: 0,
+      silenceThreshold: 10,
+      checkInterval: null,
+      isProgrammatic: true,
+      eventBased: true
+    };
+
+    this.monitoredElements.set(elementId, monitorData);
+
+    // Use timeupdate to detect audio activity — fires ~4x/sec during playback
+    let lastTimeUpdate = 0;
+    let silenceTimeoutId = null;
+    const SILENCE_DELAY = 1500; // ms of no timeupdate before declaring silence
+
+    const resetSilenceTimer = () => {
+      if (silenceTimeoutId) clearTimeout(silenceTimeoutId);
+      silenceTimeoutId = setTimeout(() => {
+        if (monitorData.isPlaying) {
+          monitorData.isPlaying = false;
+          this.dispatchAudioEvent('audiostop', elementId, audioElement);
+          if (typeof window.__publishEvent === 'function') {
+            window.__publishEvent('audiostop', { elementId, timestamp: Date.now() });
+          }
+          console.log(`Audio stopped (event-based): ${elementId}`);
+        }
+      }, SILENCE_DELAY);
+    };
+
+    audioElement.addEventListener('timeupdate', () => {
+      const now = Date.now();
+      // timeupdate fires even when seeking; only count if currentTime advances
+      if (audioElement.currentTime > 0 && now - lastTimeUpdate > 50) {
+        lastTimeUpdate = now;
+        monitorData.lastAudioTime = now;
+
+        if (!monitorData.isPlaying) {
+          monitorData.isPlaying = true;
+          this.dispatchAudioEvent('audiostart', elementId, audioElement);
+          if (typeof window.__publishEvent === 'function') {
+            window.__publishEvent('audiostart', { elementId, timestamp: Date.now() });
+          }
+          console.log(`Audio started (event-based): ${elementId}`);
+        }
+        resetSilenceTimer();
+      }
+    });
+
+    audioElement.addEventListener('pause', () => {
+      if (monitorData.isPlaying) {
+        monitorData.isPlaying = false;
+        if (silenceTimeoutId) clearTimeout(silenceTimeoutId);
+        this.dispatchAudioEvent('audiostop', elementId, audioElement);
+        if (typeof window.__publishEvent === 'function') {
+          window.__publishEvent('audiostop', { elementId, timestamp: Date.now() });
+        }
+        console.log(`Audio stopped (event-based, pause): ${elementId}`);
+      }
+    });
+
+    audioElement.addEventListener('ended', () => {
+      if (monitorData.isPlaying) {
+        monitorData.isPlaying = false;
+        if (silenceTimeoutId) clearTimeout(silenceTimeoutId);
+        this.dispatchAudioEvent('audiostop', elementId, audioElement);
+        if (typeof window.__publishEvent === 'function') {
+          window.__publishEvent('audiostop', { elementId, timestamp: Date.now() });
+        }
+        console.log(`Audio stopped (event-based, ended): ${elementId}`);
+      }
+    });
+
+    console.log(`Started event-based monitoring for programmatic audio element: ${elementId}`);
+  }
+
   monitorAudioElement(audioElement, elementId) {
     if (!this.audioContext) {
       console.warn("AudioContext not available, cannot monitor audio");
@@ -564,7 +652,9 @@ class AudioElementMonitor {
 
       console.log(`Started monitoring audio element: ${elementId}`);
     } catch (error) {
-      console.error(`Failed to monitor audio element ${elementId}:`, error);
+      console.error(`Failed to monitor audio element ${elementId} via analyser:`, error.message);
+      console.log(`Falling back to event-based monitoring for ${elementId}`);
+      this.monitorViaEvents(audioElement, elementId);
     }
   }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@telnyx/voice-agent-tester",
-  "version": "0.4.3",
+  "version": "0.4.5",
   "description": "A command-line tool to test voice agents using Puppeteer",
   "main": "src/index.js",
   "type": "module",

package/src/index.js

@@ -109,14 +109,7 @@ function getCompareRequiredParams(argv) {
       }
       break;
     case 'elevenlabs':
-      if (!argv.branchId) {
-        missing.push({
-          key: 'branchId',
-          flag: '--branch-id',
-          description: 'ElevenLabs branch ID',
-          hint: 'In the ElevenLabs Dashboard, go to Agents, select your target agent, then click the dropdown next to Publish and select "Copy shareable link". This copies the demo link containing your branch ID.'
-        });
-      }
+      // branchId is optional — the talk-to URL works with just agent_id
       break;
     // retell and others: no extra params needed yet
   }
@@ -134,8 +127,22 @@ function getCompareTemplateParams(argv) {
   switch (argv.provider) {
     case 'vapi':
       return { shareKey: argv.shareKey };
+    default:
+      return {};
+  }
+}
+
+/**
+ * Get provider-specific extra query parameters to append to the comparison URL.
+ * Unlike template params, these are appended as-is (not substituted into {{...}} placeholders).
+ *
+ * @param {Object} argv - Parsed CLI arguments
+ * @returns {Object} Key-value pairs to append as query parameters
+ */
+function getCompareExtraQueryParams(argv) {
+  switch (argv.provider) {
     case 'elevenlabs':
-      return { branchId: argv.branchId };
+      return argv.branchId ? { branch_id: argv.branchId } : {};
     default:
       return {};
   }
@@ -295,7 +302,7 @@ const argv = yargs(hideBin(process.argv))
   })
   .option('branch-id', {
     type: 'string',
-    description: 'ElevenLabs branch ID for direct widget testing (required for comparison mode with --provider elevenlabs)'
+    description: 'ElevenLabs branch ID for direct widget testing (optional, appended to demo URL when provided)'
   })
   .option('assistant-id', {
     type: 'string',
@@ -327,6 +334,11 @@ const argv = yargs(hideBin(process.argv))
     description: 'Volume level for audio input (0.0 to 1.0)',
     default: 1.0
   })
+  .option('retries', {
+    type: 'number',
+    description: 'Number of retries for failed test runs (0 = no retries)',
+    default: 0
+  })
   .help()
   .argv;
 
@@ -409,22 +421,49 @@ async function runBenchmark({ applications, scenarios, repeat, concurrency, argv
       audioVolume: argv.audioVolume
     });
 
-    try {
-      await tester.runScenario(targetUrl, app.steps, scenario.steps, app.name, scenario.name, repetition);
-      console.log(`✅ Completed successfully (Run ${runNumber}/${totalRuns})`);
-      return { success: true };
-    } catch (error) {
-      // Store only the first line for summary, but print full message here (with diagnostics)
-      const shortMessage = error.message.split('\n')[0];
-      const errorInfo = {
-        app: app.name,
-        scenario: scenario.name,
-        repetition,
-        error: shortMessage
-      };
-      // Print full diagnostics here (only place they appear)
-      console.error(`❌ Error (Run ${runNumber}/${totalRuns}):\n${error.message}`);
-      return { success: false, error: errorInfo };
+    const maxAttempts = (argv.retries || 0) + 1;
+
+    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+      // Create a fresh tester for each attempt (after first, original tester is closed)
+      const currentTester = attempt === 1 ? tester : new VoiceAgentTester({
+        verbose: argv.verbose,
+        headless: argv.headless,
+        assetsServerUrl: argv.assetsServer,
+        reportGenerator: reportGenerator,
+        record: argv.record,
+        debug: argv.debug,
+        audioUrl: argv.audioUrl,
+        audioVolume: argv.audioVolume
+      });
+
+      try {
+        await currentTester.runScenario(targetUrl, app.steps, scenario.steps, app.name, scenario.name, repetition);
+        console.log(`✅ Completed successfully (Run ${runNumber}/${totalRuns})`);
+        return { success: true };
+      } catch (error) {
+        const shortMessage = error.message.split('\n')[0];
+
+        if (attempt < maxAttempts) {
+          console.warn(`\n⚠️ Attempt ${attempt}/${maxAttempts} failed: ${shortMessage}`);
+          console.warn(`🔄 Retrying in 3s... (${maxAttempts - attempt} retries left)\n`);
+          await new Promise(r => setTimeout(r, 3000));
+          continue;
+        }
+
+        // Final attempt failed
+        const errorInfo = {
+          app: app.name,
+          scenario: scenario.name,
+          repetition,
+          error: shortMessage
+        };
+        // Print full diagnostics here (only place they appear)
+        console.error(`❌ Error (Run ${runNumber}/${totalRuns}):\n${error.message}`);
+        if (maxAttempts > 1) {
+          console.error(`   Failed after ${maxAttempts} attempts`);
+        }
+        return { success: false, error: errorInfo };
+      }
     }
   }
 
@@ -678,7 +717,19 @@ async function main() {
         throw new Error(`Provider application config not found: ${providerAppPath}\nPlease create applications/${argv.provider}.yaml for direct provider benchmarking.`);
       }
 
-      const providerApplications = [loadApplicationConfig(providerAppPath, providerParams)];
+      const providerApp = loadApplicationConfig(providerAppPath, providerParams);
+
+      // Append optional extra query parameters (e.g. branch_id for ElevenLabs)
+      const extraQueryParams = getCompareExtraQueryParams(argv);
+      if (providerApp.url && Object.keys(extraQueryParams).length > 0) {
+        const url = new URL(providerApp.url);
+        for (const [key, value] of Object.entries(extraQueryParams)) {
+          url.searchParams.set(key, value);
+        }
+        providerApp.url = url.toString();
+      }
+
+      const providerApplications = [providerApp];
       
       const providerResults = await runBenchmark({
         applications: providerApplications,
@@ -730,7 +781,7 @@ async function main() {
       telnyxReportGenerator.generateMetricsSummary();
 
       // Generate comparison report
-      ReportGenerator.generateComparisonSummary(providerReportGenerator, telnyxReportGenerator, argv.provider);
+      ReportGenerator.generateComparisonSummary(providerReportGenerator, telnyxReportGenerator, argv.provider, { debug: argv.debug });
 
       // Generate CSV reports if requested
       if (argv.report) {

package/src/report.js

@@ -28,7 +28,7 @@ export class ReportGenerator {
     });
   }
 
-  recordStepMetric(appName, scenarioName, repetition, stepIndex, action, name, value) {
+  recordStepMetric(appName, scenarioName, repetition, stepIndex, action, name, value, scenarioStepIndex = null) {
     const key = this._getRunKey(appName, scenarioName, repetition);
     const run = this.runs.get(key);
 
@@ -52,6 +52,26 @@ export class ReportGenerator {
     if (!this.stepColumns.get(stepIndex).has(name)) {
       this.stepColumns.get(stepIndex).set(name, `step_${stepIndex + 1}_${action}_${name}`);
     }
+
+    // Track scenario step index for cross-provider comparison alignment
+    if (scenarioStepIndex !== null) {
+      if (!this.scenarioStepMap) {
+        this.scenarioStepMap = new Map();
+      }
+      // Map absolute stepIndex -> scenarioStepIndex (1-based)
+      this.scenarioStepMap.set(stepIndex, scenarioStepIndex);
+
+      // Track scenario-based column names for comparison display
+      if (!this.scenarioStepColumns) {
+        this.scenarioStepColumns = new Map();
+      }
+      if (!this.scenarioStepColumns.has(scenarioStepIndex)) {
+        this.scenarioStepColumns.set(scenarioStepIndex, new Map());
+      }
+      if (!this.scenarioStepColumns.get(scenarioStepIndex).has(name)) {
+        this.scenarioStepColumns.get(scenarioStepIndex).set(name, `scenario_step_${scenarioStepIndex}_${action}_${name}`);
+      }
+    }
   }
 
   endRun(appName, scenarioName, repetition, success = true) {
@@ -285,119 +305,178 @@ export class ReportGenerator {
     return result;
   }
 
+  /**
+   * Get aggregated metrics keyed by scenario step index for cross-provider comparison.
+   * Returns a Map of scenarioStepIndex -> { metricName -> { avg, min, max, p50, columnName } }
+   */
+  getAggregatedMetricsByScenarioStep() {
+    const result = new Map();
+
+    // Build reverse map: absolute stepIndex -> scenarioStepIndex
+    const scenarioStepMap = this.scenarioStepMap || new Map();
+
+    // Collect values grouped by scenarioStepIndex
+    const grouped = new Map(); // scenarioStepIndex -> metricName -> values[]
+
+    this.allRunsData.forEach(run => {
+      run.stepMetrics.forEach((metrics, stepIndex) => {
+        const scenarioIdx = scenarioStepMap.get(stepIndex);
+        if (scenarioIdx == null) return; // skip steps without scenario mapping
+
+        if (!grouped.has(scenarioIdx)) {
+          grouped.set(scenarioIdx, new Map());
+        }
+
+        metrics.forEach((value, metricName) => {
+          if (!grouped.get(scenarioIdx).has(metricName)) {
+            grouped.get(scenarioIdx).set(metricName, []);
+          }
+          grouped.get(scenarioIdx).get(metricName).push(value);
+        });
+      });
+    });
+
+    grouped.forEach((metricMap, scenarioIdx) => {
+      const stepResult = new Map();
+
+      metricMap.forEach((values, metricName) => {
+        if (values.length > 0) {
+          const sum = values.reduce((a, b) => a + b, 0);
+          const avg = sum / values.length;
+          const min = Math.min(...values);
+          const max = Math.max(...values);
+
+          const sortedValues = [...values].sort((a, b) => a - b);
+          let p50;
+          if (sortedValues.length % 2 === 0) {
+            p50 = (sortedValues[sortedValues.length / 2 - 1] + sortedValues[sortedValues.length / 2]) / 2;
+          } else {
+            p50 = sortedValues[Math.floor(sortedValues.length / 2)];
+          }
+
+          const columnName = this.scenarioStepColumns?.get(scenarioIdx)?.get(metricName) ||
+                            `scenario_step_${scenarioIdx}_${metricName}`;
+
+          stepResult.set(metricName, { avg, min, max, p50, columnName });
+        }
+      });
+
+      result.set(scenarioIdx, stepResult);
+    });
+
+    return result;
+  }
+
   /**
    * Generate a comparison summary between two providers.
+   * Aligns metrics by scenario step index so that identical scenario steps
+   * are compared regardless of different application setup steps.
    * @param {ReportGenerator} providerReport - Report from the provider benchmark
    * @param {ReportGenerator} telnyxReport - Report from the Telnyx benchmark
    * @param {string} providerName - Name of the external provider
    */
-  static generateComparisonSummary(providerReport, telnyxReport, providerName) {
+  static generateComparisonSummary(providerReport, telnyxReport, providerName, { debug = false } = {}) {
     console.log('\n' + '='.repeat(80));
-    console.log('📊 COMPARISON SUMMARY: ' + providerName.toUpperCase() + ' vs TELNYX');
+    console.log('📊 COMPARISON: ' + providerName.toUpperCase() + ' vs TELNYX');
     console.log('='.repeat(80));
 
-    const providerMetrics = providerReport.getAggregatedMetrics();
-    const telnyxMetrics = telnyxReport.getAggregatedMetrics();
+    // Use scenario-step-aligned metrics for comparison
+    const providerMetrics = providerReport.getAggregatedMetricsByScenarioStep();
+    const telnyxMetrics = telnyxReport.getAggregatedMetricsByScenarioStep();
 
-    // Find all unique step indices from both reports
-    const allStepIndices = new Set([
+    // Find matched scenario steps (present in both providers)
+    const allScenarioSteps = new Set([
       ...providerMetrics.keys(),
       ...telnyxMetrics.keys()
     ]);
-    const sortedIndices = Array.from(allStepIndices).sort((a, b) => a - b);
+    const sortedIndices = Array.from(allScenarioSteps).sort((a, b) => a - b);
+
+    // Collect matched latencies
+    const providerLatencies = [];
+    const telnyxLatencies = [];
+    const perResponse = []; // for debug output
 
-    if (sortedIndices.length === 0) {
-      console.log('No metrics available for comparison.');
+    sortedIndices.forEach(scenarioStep => {
+      const providerElapsed = providerMetrics.get(scenarioStep)?.get('elapsed_time');
+      const telnyxElapsed = telnyxMetrics.get(scenarioStep)?.get('elapsed_time');
+
+      if (providerElapsed && telnyxElapsed) {
+        providerLatencies.push(providerElapsed.avg);
+        telnyxLatencies.push(telnyxElapsed.avg);
+        perResponse.push({
+          providerAvg: providerElapsed.avg,
+          telnyxAvg: telnyxElapsed.avg,
+          columnName: providerElapsed.columnName || telnyxElapsed.columnName
+        });
+      }
+    });
+
+    if (providerLatencies.length === 0) {
+      console.log('\n⚠️  No comparable metrics found between providers.');
+      console.log('='.repeat(80));
       return;
     }
 
-    // Compare elapsed_time metrics (primary latency indicator)
-    console.log('\n📈 Latency Comparison (elapsed_time):');
-    console.log('-'.repeat(80));
-    console.log(
-      'Step'.padEnd(40) + 
-      providerName.padEnd(12) + 
-      'Telnyx'.padEnd(12) + 
-      'Delta'.padEnd(12) + 
-      'Winner'
-    );
-    console.log('-'.repeat(80));
-
-    sortedIndices.forEach(stepIndex => {
-      const providerStep = providerMetrics.get(stepIndex);
-      const telnyxStep = telnyxMetrics.get(stepIndex);
-
-      const providerElapsed = providerStep?.get('elapsed_time');
-      const telnyxElapsed = telnyxStep?.get('elapsed_time');
-
-      if (providerElapsed || telnyxElapsed) {
-        const columnName = providerElapsed?.columnName || telnyxElapsed?.columnName || `step_${stepIndex + 1}`;
-        const shortName = columnName.length > 38 ? columnName.substring(0, 35) + '...' : columnName;
-        
-        const providerAvg = providerElapsed ? Math.round(providerElapsed.avg) : '-';
-        const telnyxAvg = telnyxElapsed ? Math.round(telnyxElapsed.avg) : '-';
-        
-        let delta = '-';
-        let winner = '-';
-        
-        if (providerElapsed && telnyxElapsed) {
-          const diff = telnyxElapsed.avg - providerElapsed.avg;
-          const pct = ((diff / providerElapsed.avg) * 100).toFixed(1);
-          delta = diff > 0 ? `+${Math.round(diff)}ms` : `${Math.round(diff)}ms`;
-          
-          if (Math.abs(diff) < 50) {
-            winner = '≈ Tie';
-          } else if (diff < 0) {
-            winner = '🏆 Telnyx';
-          } else {
-            winner = `🏆 ${providerName}`;
-          }
-          delta += ` (${pct}%)`;
+    // Debug: show per-response breakdown
+    if (debug && perResponse.length > 0) {
+      console.log('\n📈 Per-response breakdown:');
+      console.log('-'.repeat(80));
+      console.log(
+        'Response'.padEnd(40) +
+        providerName.padEnd(12) +
+        'Telnyx'.padEnd(12) +
+        'Delta'.padEnd(16) +
+        'Winner'
+      );
+      console.log('-'.repeat(80));
+
+      perResponse.forEach((r, i) => {
+        const action = (r.columnName || '').replace(/^scenario_step_\d+_/, '');
+        const label = `#${i + 1} (${action})`;
+        const shortLabel = label.length > 38 ? label.substring(0, 35) + '...' : label;
+
+        const diff = r.telnyxAvg - r.providerAvg;
+        const pct = ((diff / r.providerAvg) * 100).toFixed(1);
+        const delta = `${diff > 0 ? '+' : ''}${Math.round(diff)}ms (${pct}%)`;
+
+        let winner;
+        if (Math.abs(diff) < 50) {
+          winner = '≈ Tie';
+        } else if (diff < 0) {
+          winner = '🏆 Telnyx';
+        } else {
+          winner = `🏆 ${providerName}`;
         }
 
         console.log(
-          shortName.padEnd(40) + 
-          `${providerAvg}ms`.padEnd(12) + 
-          `${telnyxAvg}ms`.padEnd(12) + 
-          delta.padEnd(12) + 
+          shortLabel.padEnd(40) +
+          `${Math.round(r.providerAvg)}ms`.padEnd(12) +
+          `${Math.round(r.telnyxAvg)}ms`.padEnd(12) +
+          delta.padEnd(16) +
           winner
         );
-      }
-    });
-
-    console.log('-'.repeat(80));
+      });
 
-    // Summary
-    let providerTotal = 0, telnyxTotal = 0, comparableSteps = 0;
-    sortedIndices.forEach(stepIndex => {
-      const providerStep = providerMetrics.get(stepIndex);
-      const telnyxStep = telnyxMetrics.get(stepIndex);
-      const providerElapsed = providerStep?.get('elapsed_time');
-      const telnyxElapsed = telnyxStep?.get('elapsed_time');
-      
-      if (providerElapsed && telnyxElapsed) {
-        providerTotal += providerElapsed.avg;
-        telnyxTotal += telnyxElapsed.avg;
-        comparableSteps++;
-      }
-    });
+      console.log('-'.repeat(80));
+    }
 
-    if (comparableSteps > 0) {
-      const totalDiff = telnyxTotal - providerTotal;
-      const totalPct = ((totalDiff / providerTotal) * 100).toFixed(1);
-      
-      console.log('\n📊 Overall Summary:');
-      console.log(`   ${providerName} total latency: ${Math.round(providerTotal)}ms`);
-      console.log(`   Telnyx total latency: ${Math.round(telnyxTotal)}ms`);
-      console.log(`   Difference: ${totalDiff > 0 ? '+' : ''}${Math.round(totalDiff)}ms (${totalPct}%)`);
-      
-      if (Math.abs(totalDiff) < 100) {
-        console.log('\n   🤝 Result: Both providers perform similarly');
-      } else if (totalDiff < 0) {
-        console.log('\n   🏆 Result: Telnyx is faster overall');
-      } else {
-        console.log(`\n   🏆 Result: ${providerName} is faster overall`);
-      }
+    // One headline number: average response latency
+    const providerAvg = providerLatencies.reduce((a, b) => a + b, 0) / providerLatencies.length;
+    const telnyxAvg = telnyxLatencies.reduce((a, b) => a + b, 0) / telnyxLatencies.length;
+    const diff = telnyxAvg - providerAvg;
+    const pct = ((diff / providerAvg) * 100).toFixed(1);
+
+    console.log(`\n   Average response latency (${providerLatencies.length} matched responses):\n`);
+    console.log(`   ${providerName.padEnd(16)} ${Math.round(providerAvg)}ms`);
+    console.log(`   ${'Telnyx'.padEnd(16)} ${Math.round(telnyxAvg)}ms`);
+    console.log(`   ${'Difference'.padEnd(16)} ${diff > 0 ? '+' : ''}${Math.round(diff)}ms (${pct}%)`);
+
+    if (Math.abs(diff) < 50) {
+      console.log('\n   🤝 Result: Both providers perform similarly');
+    } else if (diff < 0) {
+      console.log(`\n   🏆 Telnyx is ${Math.abs(pct)}% faster`);
+    } else {
+      console.log(`\n   🏆 ${providerName} is ${Math.abs(pct)}% faster`);
     }
 
     console.log('\n' + '='.repeat(80));