@telnyx/voice-agent-tester 0.4.0 → 0.4.3

package/.github/workflows/draft-release.yml

@@ -46,8 +46,8 @@ jobs:
 
       - name: Setup Git user
         run: |
-          git config user.name TelnyxIntegrations
-          git config user.email integrations@telnyx.com
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
 
       - name: Use Node.js 20.x
         uses: actions/setup-node@v4
@@ -64,9 +64,44 @@ jobs:
         env:
           CI: true
 
-      - name: Create draft release
+      - name: Determine next version
+        id: version
         run: |
-          npx release-it --ci --github.draft --no-npm.publish${{ env.INCREMENT_ARG }}${{ env.PRERELEASE_ARGS }}
+          NEXT=$(npx release-it --ci --release-version${{ env.INCREMENT_ARG }}${{ env.PRERELEASE_ARGS }} 2>/dev/null)
+          echo "next=$NEXT" >> "$GITHUB_OUTPUT"
+          echo "branch=release/v$NEXT" >> "$GITHUB_OUTPUT"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create release branch
+        run: |
+          git checkout -b "${{ steps.version.outputs.branch }}"
+
+      - name: Create draft release on branch
+        run: |
+          npx release-it --ci --github.draft --no-npm.publish --no-git.push --no-git.requireUpstream${{ env.INCREMENT_ARG }}${{ env.PRERELEASE_ARGS }}
         env:
           NPM_TOKEN: ${{ secrets.NPM_CI_TOKEN }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Push release branch
+        run: |
+          git push origin "${{ steps.version.outputs.branch }}"
+
+      - name: Create pull request
+        run: |
+          gh pr create \
+            --title "chore: release v${{ steps.version.outputs.next }}" \
+            --body "## Release v${{ steps.version.outputs.next }}
+
+          Automated release PR created by the draft-release workflow.
+
+          - Version bump in \`package.json\`
+          - Updated \`CHANGELOG.md\`
+          - Draft GitHub release created
+
+          **After merging**, publish the release from the [releases page](https://github.com/${{ github.repository }}/releases)." \
+            --base "${{ env.TARGET_REF }}" \
+            --head "${{ steps.version.outputs.branch }}"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

package/.github/workflows/publish-release.yml

@@ -19,8 +19,8 @@ jobs:
 
       - name: Setup Git user
         run: |
-          git config user.name TelnyxIntegrations
-          git config user.email integrations@telnyx.com
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
 
       - name: Use Node.js 20.x
         uses: actions/setup-node@v4

package/CHANGELOG.md

@@ -1,5 +1,37 @@
 # Changelog
 
+## [0.4.3](https://github.com/team-telnyx/voice-agent-tester/compare/v0.4.2...v0.4.3) (2026-03-11)
+
+### Features
+
+* add click_with_retry action and fix audio event race conditions ([#19](https://github.com/team-telnyx/voice-agent-tester/issues/19)) ([#19](https://github.com/team-telnyx/voice-agent-tester/issues/19)) ([13e2009](https://github.com/team-telnyx/voice-agent-tester/commit/13e2009a94b4e2f7e05972f01a47c9b31758bf58))
+
+### Chores
+
+* release v0.4.2 ([#18](https://github.com/team-telnyx/voice-agent-tester/issues/18)) ([1cf64ef](https://github.com/team-telnyx/voice-agent-tester/commit/1cf64ef563e813c2f06b2b655bfcc414637594cb))
+
+## [0.4.2](https://github.com/team-telnyx/voice-agent-tester/compare/v0.4.1...v0.4.2) (2026-02-23)
+
+### Features
+
+* add dashboard hints for Vapi and ElevenLabs comparison mode params ([#16](https://github.com/team-telnyx/voice-agent-tester/issues/16)) ([7fda40b](https://github.com/team-telnyx/voice-agent-tester/commit/7fda40b6971a968dde1fc1c3466662227a3bc77e))
+
+### Chores
+
+* improve event logs and comparison mode docs ([#17](https://github.com/team-telnyx/voice-agent-tester/issues/17)) ([24a9683](https://github.com/team-telnyx/voice-agent-tester/commit/24a968337a0b4a6c2d6baddd0aa507d5a87c9488))
+
+## [0.4.1](https://github.com/team-telnyx/voice-agent-tester/compare/v0.4.0...v0.4.1) (2026-02-18)
+
+### Features
+
+* require provider-specific params for comparison mode ([#10](https://github.com/team-telnyx/voice-agent-tester/issues/10)) ([db9eb27](https://github.com/team-telnyx/voice-agent-tester/commit/db9eb273c139374a9f6358126113cab92f8f5b32))
+* use Qwen/Qwen3-235B-A22B as model for imported assistants ([#11](https://github.com/team-telnyx/voice-agent-tester/issues/11)) ([3c4ed0a](https://github.com/team-telnyx/voice-agent-tester/commit/3c4ed0a14498833544f1797426b234585adcb49b))
+
+### Bug Fixes
+
+* add --no-git.requireUpstream to release-it in draft workflow ([#14](https://github.com/team-telnyx/voice-agent-tester/issues/14)) ([9553e65](https://github.com/team-telnyx/voice-agent-tester/commit/9553e65bdc6f0094853895da6b806befc5a898f6))
+* use triggering user as git author and create PR for releases ([#13](https://github.com/team-telnyx/voice-agent-tester/issues/13)) ([8ebecba](https://github.com/team-telnyx/voice-agent-tester/commit/8ebecba1839985949e46bec457f327711f89138d))
+
 ## [0.4.0](https://github.com/team-telnyx/voice-agent-tester/compare/v0.3.0...v0.4.0) (2026-01-26)
 
 ### Features

package/README.md

@@ -31,6 +31,8 @@ voice-agent-tester -a applications/telnyx.yaml -s scenarios/appointment.yaml --a
 | `--provider` | | Import from provider (`vapi`, `elevenlabs`, `retell`) |
 | `--provider-api-key` | | External provider API key (required with `--provider`) |
 | `--provider-import-id` | | Provider assistant ID to import (required with `--provider`) |
+| `--share-key` | | Vapi share key for comparison mode (prompted if missing) |
+| `--branch-id` | | ElevenLabs branch ID for comparison mode (prompted if missing) |
 | `--compare` | `true` | Run both provider direct and Telnyx import benchmarks |
 | `--no-compare` | | Disable comparison (run only Telnyx import) |
 | `-d, --debug` | `false` | Enable detailed timeout diagnostics |
@@ -190,20 +192,45 @@ When importing from an external provider, the tool automatically runs both bench
 1. **Provider Direct** - Benchmarks the assistant on the original provider's widget
 2. **Telnyx Import** - Benchmarks the same assistant after importing to Telnyx
 
+### Provider-Specific Keys
+
+Comparison mode requires a provider-specific key to load the provider's direct widget. If not passed via CLI, the tool will prompt you with instructions on how to find it.
+
+| Provider | Flag | How to find it |
+|----------|------|----------------|
+| Vapi | `--share-key` | In the Vapi Dashboard, select your assistant, then click the link icon (🔗) next to the assistant ID at the top. This copies the demo link containing your share key. |
+| ElevenLabs | `--branch-id` | In the ElevenLabs Dashboard, go to Agents, select your target agent, then click the dropdown next to Publish and select "Copy shareable link". This copies the demo link containing your branch ID. |
+
 ### Import and Compare (Default)
 
+**Vapi:**
+
 ```bash
 npx @telnyx/voice-agent-tester@latest \
   -a applications/telnyx.yaml \
   -s scenarios/appointment.yaml \
   --provider vapi \
+  --share-key <VAPI_SHARE_KEY> \
   --api-key <TELNYX_KEY> \
   --provider-api-key <VAPI_KEY> \
   --provider-import-id <VAPI_ASSISTANT_ID>
 ```
 
+**ElevenLabs:**
+
+```bash
+npx @telnyx/voice-agent-tester@latest \
+  -a applications/telnyx.yaml \
+  -s scenarios/appointment.yaml \
+  --provider elevenlabs \
+  --branch-id <ELEVENLABS_BRANCH_ID> \
+  --api-key <TELNYX_KEY> \
+  --provider-api-key <ELEVENLABS_KEY> \
+  --provider-import-id <ELEVENLABS_AGENT_ID>
+```
+
 This will:
-- Run Phase 1: VAPI direct benchmark
+- Run Phase 1: Provider direct benchmark
 - Run Phase 2: Telnyx import benchmark
 - Generate a side-by-side latency comparison report
 

package/applications/telnyx.yaml

@@ -4,7 +4,8 @@ steps:
     selector: "telnyx-ai-agent"
   - action: sleep
     time: 3000
-  - action: click
+  - action: click_with_retry
     selector: "telnyx-ai-agent >>> button"
-  - action: sleep
-    time: 4000
+    retries: 5
+    checkDelay: 4000
+    retryDelay: 5000

package/applications/vapi.yaml

@@ -13,7 +13,3 @@ steps:
     time: 2000
   - action: speak
     text: "Hello, what can you do?"
-  - action: wait_for_voice
-    metrics: elapsed_time
-  - action: wait_for_silence
-    metrics: elapsed_time

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@telnyx/voice-agent-tester",
-  "version": "0.4.0",
+  "version": "0.4.3",
   "description": "A command-line tool to test voice agents using Puppeteer",
   "main": "src/index.js",
   "type": "module",

package/src/index.js

@@ -87,6 +87,60 @@ function substituteUrlParams(url, params) {
   return result;
 }
 
+/**
+ * Get the list of missing provider-specific parameters required for comparison mode.
+ * Each provider has its own set of required params for the direct widget benchmark.
+ *
+ * @param {Object} argv - Parsed CLI arguments
+ * @returns {Array<{key: string, flag: string, description: string}>} Missing params
+ */
+function getCompareRequiredParams(argv) {
+  const missing = [];
+
+  switch (argv.provider) {
+    case 'vapi':
+      if (!argv.shareKey) {
+        missing.push({
+          key: 'shareKey',
+          flag: '--share-key',
+          description: 'Vapi share key',
+          hint: 'In the Vapi Dashboard, select your assistant, then click the link icon (🔗) next to the assistant ID at the top. This copies the demo link containing your share key.'
+        });
+      }
+      break;
+    case 'elevenlabs':
+      if (!argv.branchId) {
+        missing.push({
+          key: 'branchId',
+          flag: '--branch-id',
+          description: 'ElevenLabs branch ID',
+          hint: 'In the ElevenLabs Dashboard, go to Agents, select your target agent, then click the dropdown next to Publish and select "Copy shareable link". This copies the demo link containing your branch ID.'
+        });
+      }
+      break;
+    // retell and others: no extra params needed yet
+  }
+
+  return missing;
+}
+
+/**
+ * Get provider-specific template parameters for comparison mode URL/HTML substitution.
+ *
+ * @param {Object} argv - Parsed CLI arguments
+ * @returns {Object} Template params to merge into provider params
+ */
+function getCompareTemplateParams(argv) {
+  switch (argv.provider) {
+    case 'vapi':
+      return { shareKey: argv.shareKey };
+    case 'elevenlabs':
+      return { branchId: argv.branchId };
+    default:
+      return {};
+  }
+}
+
 // Helper function to load and validate application config
 function loadApplicationConfig(configPath, params = {}) {
   const configFile = fs.readFileSync(configPath, 'utf8');
@@ -235,6 +289,14 @@ const argv = yargs(hideBin(process.argv))
     type: 'string',
     description: 'Provider assistant/agent ID to import (required with --provider)'
   })
+  .option('share-key', {
+    type: 'string',
+    description: 'Vapi share key for direct widget testing (required for comparison mode with --provider vapi)'
+  })
+  .option('branch-id', {
+    type: 'string',
+    description: 'ElevenLabs branch ID for direct widget testing (required for comparison mode with --provider elevenlabs)'
+  })
   .option('assistant-id', {
     type: 'string',
     description: 'Assistant/agent ID for direct benchmarking (works with all providers)'
@@ -456,8 +518,8 @@ async function main() {
     // Parse URL parameters for template substitution
     const params = parseParams(argv.params);
     
-    // Determine if we should run comparison benchmark
-    const shouldCompare = argv.provider && argv.compare && !argv.noCompare;
+    // Determine if we should run comparison benchmark (may be updated later if public key is missing)
+    let shouldCompare = argv.provider && argv.compare && !argv.noCompare;
     
     // Store credentials for potential comparison run
     let telnyxApiKey = argv.apiKey;
@@ -492,6 +554,32 @@ async function main() {
         }
       }
 
+      // Require provider-specific params when comparison mode is enabled
+      if (shouldCompare) {
+        const missingParams = getCompareRequiredParams(argv);
+        if (missingParams.length > 0) {
+          for (const param of missingParams) {
+            console.log(`\n🔑 ${param.description} is required for comparison mode`);
+            if (param.hint) {
+              console.log(`   ${param.hint}`);
+            }
+            const inputVal = await promptUserInput(`Enter ${param.description} (or press Enter to skip comparison): `);
+            if (inputVal) {
+              argv[param.key] = inputVal;
+            } else {
+              console.warn(`⚠️  Missing ${param.flag}. Disabling comparison mode (--no-compare).`);
+              console.warn(`   To run comparison benchmarks, pass ${param.flag} <value>\n`);
+              argv.compare = false;
+              argv.noCompare = true;
+              break;
+            }
+          }
+        }
+      }
+
+      // Re-evaluate shouldCompare after potential public key prompt
+      shouldCompare = argv.provider && argv.compare && !argv.noCompare;
+
       const importResult = await importAssistantsFromProvider({
         provider: argv.provider,
         providerApiKey: providerApiKey,
@@ -583,7 +671,7 @@ async function main() {
 
       // Phase 1: Provider Direct Benchmark
       // Load provider-specific application config with provider assistant ID
-      const providerParams = { ...params, assistantId: providerImportId };
+      const providerParams = { ...params, assistantId: providerImportId, ...getCompareTemplateParams(argv) };
       const providerAppPath = path.resolve(__packageDir, 'applications', `${argv.provider}.yaml`);
       
       if (!fs.existsSync(providerAppPath)) {

package/src/provider-import.js

@@ -200,7 +200,7 @@ async function configureImportedAssistant({ assistantId, assistantName, telnyxAp
         },
         body: JSON.stringify({
           name: newName,
-          model: 'Qwen/Qwen3-235B-A22',
+          model: 'Qwen/Qwen3-235B-A22B',
           telephony_settings: {
             supports_unauthenticated_web_calls: true
           },

package/src/voice-agent-tester.js

@@ -330,7 +330,8 @@ export class VoiceAgentTester {
     await this.page.exposeFunction('__publishEvent', (eventType, data) => {
       const event = { eventType, data, timestamp: Date.now() };
 
-      console.log(`\t📢 Event received: ${eventType}`);
+      const elementSuffix = data && data.elementId ? ` (audio element: ${data.elementId})` : '';
+      console.log(`\t📢 ${eventType}${elementSuffix}`);
 
       // Check if there are any pending promises waiting for this event type
       const pendingPromises = this.pendingPromises.get(eventType);
@@ -534,6 +535,9 @@ export class VoiceAgentTester {
         case 'screenshot':
           handlerResult = await this.handleScreenshot(step);
           break;
+        case 'click_with_retry':
+          handlerResult = await this.handleClickWithRetry(step);
+          break;
 
         default:
           console.log(`Unknown action: ${action}`);
@@ -576,10 +580,173 @@ export class VoiceAgentTester {
     await this.page.click(selector);
   }
 
+  async handleClickWithRetry(step) {
+    const selector = step.selector;
+    if (!selector) {
+      throw new Error('No selector specified for click_with_retry action');
+    }
+
+    const maxRetries = step.retries || 2;
+    const retryDelay = step.retryDelay || 3000;
+    const checkDelay = step.checkDelay || 4000;
+
+    for (let attempt = 1; attempt <= maxRetries; attempt++) {
+      let clicked = false;
+      try {
+        await this.page.waitForSelector(selector, { timeout: attempt === 1 ? 30000 : 5000 });
+        await this.page.click(selector);
+        clicked = true;
+      } catch {
+        // Selector not found — will check for widget config errors below
+      }
+
+      if (!clicked) {
+        // Check if the widget is showing a configuration error
+        const widgetState = await this._getWidgetErrorState(selector);
+
+        if (widgetState.isConfigError) {
+          // Widget is showing "unauthenticated web calls" or similar config error.
+          // This means the API config hasn't propagated to the widget yet.
+          if (attempt < maxRetries) {
+            console.log(`\t⚠️ Click attempt ${attempt}/${maxRetries}: widget not ready — "${widgetState.errorText}"`);
+            console.log(`\t⏳ Waiting for configuration to propagate (reloading in ${retryDelay}ms)...`);
+            await this.sleep(retryDelay);
+            await this.page.reload({ waitUntil: 'networkidle0', timeout: 30000 });
+            await this.sleep(2000); // extra time after reload
+            continue;
+          }
+          throw new Error(
+            `Widget configuration not ready after ${maxRetries} attempts: "${widgetState.errorText}"\n` +
+            `The "Supports Unauthenticated Web Calls" setting may not have propagated yet.\n` +
+            `Try running again in a few seconds, or verify the setting in the Telnyx portal.`
+          );
+        }
+
+        // Not a config error — genuinely missing selector
+        if (attempt < maxRetries) {
+          console.log(`\t⚠️ Click attempt ${attempt}/${maxRetries}: selector not found, retrying in ${retryDelay}ms...`);
+          await this.sleep(retryDelay);
+          continue;
+        }
+        throw new Error(`Selector "${selector}" not found after ${maxRetries} attempts`);
+      }
+
+      console.log(`\t🖱️ Click attempt ${attempt}/${maxRetries}`);
+
+      // Wait for connection to establish
+      await this.sleep(checkDelay);
+
+      // Check if audio elements are monitored or WebRTC connections exist
+      const status = await this._checkConnectionStatus();
+
+      if (status.isConnected) {
+        console.log(`\t✅ Connection established (monitored: ${status.monitoredElements}, rtc: ${status.rtcConnections})`);
+        return;
+      }
+
+      if (attempt < maxRetries) {
+        console.log(`\t⚠️ No connection detected (monitored: ${status.monitoredElements}, rtc: ${status.rtcConnections}), retrying in ${retryDelay}ms...`);
+        await this.sleep(retryDelay);
+      } else {
+        console.log(`\t⚠️ No connection detected after ${maxRetries} attempts, proceeding anyway`);
+      }
+    }
+  }
+
+  /**
+   * Check if a widget is showing a configuration error (e.g., "unauthenticated web calls" not enabled).
+   * Inspects the shadow DOM for error indicators.
+   */
+  async _getWidgetErrorState(selector) {
+    const parts = selector.split('>>>').map(s => s.trim());
+    const hostSelector = parts[0];
+
+    return await this.page.evaluate((host) => {
+      const el = document.querySelector(host);
+      if (!el || !el.shadowRoot) return { isConfigError: false };
+
+      const text = el.shadowRoot.textContent || '';
+
+      // Check for known configuration error messages
+      const configErrors = [
+        'unauthenticated web calls',
+        'support unauthenticated',
+        'not configured',
+        'configuration required'
+      ];
+
+      const lowerText = text.toLowerCase();
+      for (const pattern of configErrors) {
+        if (lowerText.includes(pattern)) {
+          // Extract a readable error message
+          const errorText = text.trim().replace(/\s+/g, ' ').substring(0, 200);
+          return { isConfigError: true, errorText };
+        }
+      }
+
+      return { isConfigError: false };
+    }, hostSelector);
+  }
+
+  async _checkConnectionStatus() {
+    const status = await this.page.evaluate(() => {
+      const info = { monitoredElements: 0, hasActiveConnection: false };
+
+      if (window.audioMonitor && window.audioMonitor.monitoredElements) {
+        info.monitoredElements = window.audioMonitor.monitoredElements.size;
+      }
+
+      document.querySelectorAll('audio').forEach(el => {
+        if (el.srcObject) info.hasActiveConnection = true;
+      });
+
+      return info;
+    });
+
+    let rtcConnections = 0;
+    try {
+      const rtpStats = await this.page.evaluate(async () => {
+        if (typeof window.__getRtpStats === 'function') {
+          return await window.__getRtpStats();
+        }
+        return null;
+      });
+      if (rtpStats) rtcConnections = rtpStats.connectionCount || 0;
+    } catch {
+      // Ignore RTP stats errors
+    }
+
+    return {
+      monitoredElements: status.monitoredElements,
+      rtcConnections,
+      isConnected: status.monitoredElements > 0 || status.hasActiveConnection || rtcConnections > 0
+    };
+  }
+
   async handleWaitForVoice() {
     if (this.debug) {
       console.log('\t⏳ Waiting for audio to start (AI agent response)...');
     }
+
+    // Check if audio is already playing before waiting for a new event.
+    // This handles the case where audiostart fired before we started listening
+    // (e.g., during click_with_retry or between steps).
+    const alreadyPlaying = await this.page.evaluate(() => {
+      if (window.audioMonitor && window.audioMonitor.monitoredElements) {
+        for (const [, data] of window.audioMonitor.monitoredElements) {
+          if (data.isPlaying) return true;
+        }
+      }
+      return false;
+    });
+
+    if (alreadyPlaying) {
+      if (this.debug) {
+        console.log('\t✅ Audio already playing');
+      }
+      return;
+    }
+
     await this.waitForAudioEvent('audiostart');
     if (this.debug) {
       console.log('\t✅ Audio detected');
@@ -590,6 +757,27 @@ export class VoiceAgentTester {
     if (this.debug) {
       console.log('\t⏳ Waiting for audio to stop (silence)...');
     }
+
+    // Check if all monitored elements are already silent.
+    // This handles the case where audiostop fired before we started listening.
+    const allSilent = await this.page.evaluate(() => {
+      if (window.audioMonitor && window.audioMonitor.monitoredElements) {
+        if (window.audioMonitor.monitoredElements.size === 0) return false; // no elements yet
+        for (const [, data] of window.audioMonitor.monitoredElements) {
+          if (data.isPlaying) return false;
+        }
+        return true; // all elements exist and are silent
+      }
+      return false;
+    });
+
+    if (allSilent) {
+      if (this.debug) {
+        console.log('\t✅ Already silent');
+      }
+      return;
+    }
+
     await this.waitForAudioEvent('audiostop');
     if (this.debug) {
       console.log('\t✅ Silence detected');