@telnyx/voice-agent-tester 0.2.0

package/src/provider-import.js

@@ -0,0 +1,315 @@
+/**
+ * External Provider to Telnyx Assistant Import Module
+ *
+ * Imports assistants from external providers (vapi, elevenlabs, retell)
+ * into Telnyx using the Telnyx AI Assistants Import API.
+ * 
+ * Features:
+ * - Auto-creates integration secrets from provider API keys
+ * - Enables unauthenticated web calls for imported assistants
+ */
+
+const TELNYX_BASE_URL = 'https://api.telnyx.com/v2';
+const TELNYX_SECRETS_ENDPOINT = `${TELNYX_BASE_URL}/integration_secrets`;
+const TELNYX_IMPORT_ENDPOINT = `${TELNYX_BASE_URL}/ai/assistants/import`;
+const TELNYX_ASSISTANTS_ENDPOINT = `${TELNYX_BASE_URL}/ai/assistants`;
+
+// Supported providers
+const SUPPORTED_PROVIDERS = ['vapi', 'elevenlabs', 'retell'];
+
+// Default widget settings for benchmarking
+const DEFAULT_WIDGET_SETTINGS = {
+  theme: 'dark',
+  audio_visualizer_config: {
+    color: 'verdant',
+    preset: 'roundBars'
+  },
+  start_call_text: '',
+  default_state: 'expanded',
+  position: 'fixed',
+  view_history_url: null,
+  report_issue_url: null,
+  give_feedback_url: null,
+  agent_thinking_text: '',
+  speak_to_interrupt_text: '',
+  logo_icon_url: null
+};
+
+/**
+ * Create an integration secret in Telnyx from a provider's API key.
+ *
+ * @param {Object} options
+ * @param {string} options.identifier - Unique identifier for the secret
+ * @param {string} options.token - The API key/token to store
+ * @param {string} options.telnyxApiKey - Telnyx API key for authentication
+ * @returns {Promise<{id: string, identifier: string}>}
+ */
+async function createIntegrationSecret({ identifier, token, telnyxApiKey }) {
+  console.log(`🔐 Creating integration secret: ${identifier}`);
+
+  const response = await fetch(TELNYX_SECRETS_ENDPOINT, {
+    method: 'POST',
+    headers: {
+      'Authorization': `Bearer ${telnyxApiKey}`,
+      'Content-Type': 'application/json'
+    },
+    body: JSON.stringify({
+      identifier: identifier,
+      type: 'bearer',
+      token: token
+    })
+  });
+
+  if (!response.ok) {
+    const errorText = await response.text();
+    throw new Error(`Failed to create integration secret: ${response.status} - ${errorText}`);
+  }
+
+  const data = await response.json();
+  console.log(`✅ Integration secret created: ${data.data.identifier}`);
+  
+  return {
+    id: data.data.id,
+    identifier: data.data.identifier
+  };
+}
+
+/**
+ * Get assistant details from Telnyx API.
+ *
+ * @param {Object} options
+ * @param {string} options.assistantId - The Telnyx assistant ID
+ * @param {string} options.telnyxApiKey - Telnyx API key
+ * @returns {Promise<Object>} - Assistant details
+ */
+export async function getAssistant({ assistantId, telnyxApiKey }) {
+  const response = await fetch(`${TELNYX_ASSISTANTS_ENDPOINT}/${assistantId}`, {
+    method: 'GET',
+    headers: {
+      'Authorization': `Bearer ${telnyxApiKey}`,
+      'Content-Type': 'application/json'
+    }
+  });
+
+  if (!response.ok) {
+    const errorText = await response.text();
+    throw new Error(`Failed to get assistant: ${response.status} ${errorText}`);
+  }
+
+  const data = await response.json();
+  
+  // API returns data at root level for single assistant GET
+  if (!data.id) {
+    throw new Error(`Assistant not found: ${assistantId}`);
+  }
+  
+  return data;
+}
+
+/**
+ * Enable unauthenticated web calls for a Telnyx assistant.
+ *
+ * @param {Object} options
+ * @param {string} options.assistantId - The Telnyx assistant ID
+ * @param {string} options.telnyxApiKey - Telnyx API key
+ * @param {Object} options.assistant - Optional existing assistant data to preserve settings
+ * @returns {Promise<boolean>} - true if successful
+ */
+export async function enableWebCalls({ assistantId, telnyxApiKey, assistant }) {
+  console.log(`🔧 Enabling unauthenticated web calls for assistant ${assistantId}...`);
+  
+  // Preserve existing telephony_settings and just enable web calls
+  const telephonySettings = {
+    ...(assistant?.telephony_settings || {}),
+    supports_unauthenticated_web_calls: true
+  };
+
+  // Build request body preserving widget_settings if they exist
+  const requestBody = {
+    telephony_settings: telephonySettings
+  };
+
+  // Use existing widget_settings or set defaults
+  if (assistant?.widget_settings && Object.keys(assistant.widget_settings).length > 0) {
+    requestBody.widget_settings = assistant.widget_settings;
+  } else {
+    requestBody.widget_settings = DEFAULT_WIDGET_SETTINGS;
+  }
+  
+  const response = await fetch(`${TELNYX_ASSISTANTS_ENDPOINT}/${assistantId}`, {
+    method: 'POST',
+    headers: {
+      'Authorization': `Bearer ${telnyxApiKey}`,
+      'Content-Type': 'application/json'
+    },
+    body: JSON.stringify(requestBody)
+  });
+
+  if (!response.ok) {
+    const errorText = await response.text();
+    throw new Error(`Failed to enable web calls: ${response.status} ${errorText}`);
+  }
+
+  console.log(`✅ Unauthenticated web calls enabled`);
+  return true;
+}
+
+/**
+ * Configure an imported assistant with web calls enabled, widget settings, and timestamped name.
+ * Returns true if successful, false if failed (with warning).
+ *
+ * @param {Object} options
+ * @param {string} options.assistantId - The assistant ID
+ * @param {string} options.assistantName - The original assistant name
+ * @param {string} options.telnyxApiKey - Telnyx API key for authentication
+ * @returns {Promise<boolean>}
+ */
+async function configureImportedAssistant({ assistantId, assistantName, telnyxApiKey, provider }) {
+  // Generate UTC timestamp suffix
+  const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
+  const providerLabel = provider ? `_${provider}` : '';
+  const newName = `${assistantName || 'Imported'}${providerLabel}_${timestamp}`;
+  
+  console.log(`🔧 Configuring assistant: ${assistantId}`);
+  console.log(`   📝 Renaming to: ${newName}`);
+
+  try {
+    const response = await fetch(`${TELNYX_ASSISTANTS_ENDPOINT}/${assistantId}`, {
+      method: 'POST',
+      headers: {
+        'Authorization': `Bearer ${telnyxApiKey}`,
+        'Content-Type': 'application/json'
+      },
+      body: JSON.stringify({
+        name: newName,
+        model: 'Qwen/Qwen3-235B-A22',
+        telephony_settings: {
+          supports_unauthenticated_web_calls: true
+        },
+        widget_settings: DEFAULT_WIDGET_SETTINGS
+      })
+    });
+
+    if (!response.ok) {
+      const errorText = await response.text();
+      console.warn(`⚠️  Could not configure assistant ${assistantId}: ${response.status}`);
+      console.warn(`   This may require manual configuration in the Telnyx portal.`);
+      return false;
+    }
+
+    console.log(`✅ Assistant configured: ${newName}`);
+    return true;
+  } catch (error) {
+    console.warn(`⚠️  Error configuring assistant ${assistantId}: ${error.message}`);
+    return false;
+  }
+}
+
+/**
+ * Import assistants from an external provider into Telnyx.
+ * 
+ * This function:
+ * 1. Creates an integration secret from the provider's private API key
+ * 2. Imports assistants from the provider (optionally filtered by ID)
+ * 3. Enables unauthenticated web calls for each imported assistant
+ *
+ * @param {Object} options - Import options
+ * @param {string} options.provider - External provider name (vapi, elevenlabs, retell)
+ * @param {string} options.providerApiKey - The provider's private API key
+ * @param {string} options.telnyxApiKey - Telnyx API key for authentication
+ * @param {string} [options.assistantId] - Optional: specific assistant ID to import
+ * @returns {Promise<{assistants: Array<{id: string, name: string}>, assistantId: string}>}
+ */
+export async function importAssistantsFromProvider({ provider, providerApiKey, telnyxApiKey, assistantId }) {
+  // Validate provider
+  if (!SUPPORTED_PROVIDERS.includes(provider)) {
+    throw new Error(`Unsupported provider: ${provider}. Supported providers: ${SUPPORTED_PROVIDERS.join(', ')}`);
+  }
+
+  console.log(`\n🔄 Importing assistants from ${provider} into Telnyx...`);
+
+  try {
+    // Step 1: Create integration secret from provider API key
+    const secretIdentifier = `${provider}_import_${Date.now()}`;
+    const secret = await createIntegrationSecret({
+      identifier: secretIdentifier,
+      token: providerApiKey,
+      telnyxApiKey
+    });
+
+    // Step 2: Import assistant using the secret reference
+    console.log(`📥 Importing assistant ${assistantId} using secret: ${secret.identifier}`);
+    
+    // Build import request body with specific assistant ID
+    const importBody = {
+      provider: provider,
+      api_key_ref: secret.identifier,
+      import_ids: [assistantId]
+    };
+    
+    const importResponse = await fetch(TELNYX_IMPORT_ENDPOINT, {
+      method: 'POST',
+      headers: {
+        'Authorization': `Bearer ${telnyxApiKey}`,
+        'Content-Type': 'application/json'
+      },
+      body: JSON.stringify(importBody)
+    });
+
+    if (!importResponse.ok) {
+      const errorText = await importResponse.text();
+      throw new Error(`Telnyx import API failed with status ${importResponse.status}: ${errorText}`);
+    }
+
+    const importData = await importResponse.json();
+    const assistants = importData.data || [];
+
+    if (assistants.length === 0) {
+      throw new Error(`No assistant was imported for ID "${assistantId}"`);
+    }
+
+    // Validate that we got the correct assistant
+    const importedAssistant = assistants[0];
+    const importId = importedAssistant.import_metadata?.import_id;
+    
+    if (importId !== assistantId) {
+      throw new Error(`Import mismatch: requested "${assistantId}" but got "${importId}"`);
+    }
+
+    // Check if this is a previously imported assistant (re-using existing Telnyx assistant)
+    const importedAt = importedAssistant.import_metadata?.imported_at;
+    const isReused = importedAt && (Date.now() - new Date(importedAt).getTime() > 60000); // More than 1 minute ago
+    
+    if (isReused) {
+      console.log(`♻️  Re-using previously imported assistant: ${importedAssistant.name} (${importedAssistant.id})`);
+      console.log(`   Originally imported at: ${importedAt}`);
+    } else {
+      console.log(`✅ Successfully imported: ${importedAssistant.name} (${importedAssistant.id})`);
+      
+      // Only configure newly imported assistants (rename with timestamp, enable web calls, set widget settings)
+      console.log(`\n🔧 Configuring imported assistant...`);
+      
+      await configureImportedAssistant({
+        assistantId: importedAssistant.id,
+        assistantName: importedAssistant.name,
+        telnyxApiKey,
+        provider
+      });
+    }
+
+    return {
+      assistants: [{ 
+        id: importedAssistant.id, 
+        name: importedAssistant.name,
+        import_id: importId 
+      }],
+      assistantId: importedAssistant.id
+    };
+  } catch (error) {
+    console.error(`❌ Failed to import assistants from ${provider}:`, error.message);
+    throw error;
+  }
+}
+
+// Export supported providers for CLI validation
+export { SUPPORTED_PROVIDERS };

package/src/report.js

@@ -0,0 +1,228 @@
+import fs from 'fs';
+import path from 'path';
+
+export class ReportGenerator {
+  constructor(filePath) {
+    this.filePath = path.resolve(filePath);
+    this.allRunsData = [];
+    // Map: "app|scenario|repetition" -> { metadata, stepMetrics, startTime }
+    this.runs = new Map();
+    this.stepColumns = new Map(); // Map of stepIndex -> Map of metricName -> column name
+  }
+
+  // Create a unique key for each run
+  _getRunKey(appName, scenarioName, repetition) {
+    return `${appName}|${scenarioName}|${repetition}`;
+  }
+
+  beginRun(appName, scenarioName, repetition) {
+    const key = this._getRunKey(appName, scenarioName, repetition);
+    this.runs.set(key, {
+      metadata: {
+        app: appName,
+        scenario: scenarioName,
+        repetition: repetition,
+        startTime: Date.now()
+      },
+      stepMetrics: new Map() // Map of stepIndex -> Map of metricName -> value
+    });
+  }
+
+  recordStepMetric(appName, scenarioName, repetition, stepIndex, action, name, value) {
+    const key = this._getRunKey(appName, scenarioName, repetition);
+    const run = this.runs.get(key);
+
+    if (!run) {
+      console.warn(`Warning: Attempting to record metric for non-existent run: ${key}`);
+      return;
+    }
+
+    // Initialize step metrics map if it doesn't exist
+    if (!run.stepMetrics.has(stepIndex)) {
+      run.stepMetrics.set(stepIndex, new Map());
+    }
+
+    // Record the metric value
+    run.stepMetrics.get(stepIndex).set(name, value);
+
+    // Track column names based on step index, action, and metric name
+    if (!this.stepColumns.has(stepIndex)) {
+      this.stepColumns.set(stepIndex, new Map());
+    }
+    if (!this.stepColumns.get(stepIndex).has(name)) {
+      this.stepColumns.get(stepIndex).set(name, `step_${stepIndex + 1}_${action}_${name}`);
+    }
+  }
+
+  endRun(appName, scenarioName, repetition, success = true) {
+    const key = this._getRunKey(appName, scenarioName, repetition);
+    const run = this.runs.get(key);
+
+    if (!run) {
+      console.warn(`Warning: Attempting to end non-existent run: ${key}`);
+      return;
+    }
+
+    // Calculate duration
+    const duration = Date.now() - run.metadata.startTime;
+
+    // Deep copy the nested Map structure
+    const runCopy = new Map();
+    run.stepMetrics.forEach((metricsMap, stepIndex) => {
+      runCopy.set(stepIndex, new Map(metricsMap));
+    });
+
+    this.allRunsData.push({
+      metadata: {
+        app: run.metadata.app,
+        scenario: run.metadata.scenario,
+        repetition: run.metadata.repetition,
+        success: success ? 1 : 0,
+        duration: duration
+      },
+      stepMetrics: runCopy
+    });
+
+    // Remove the run from active runs map
+    this.runs.delete(key);
+  }
+
+  generateCSV() {
+    if (this.allRunsData.length === 0) {
+      console.warn('No step times recorded for report generation');
+      return;
+    }
+
+    // Collect all step indices and their metrics
+    const allStepMetrics = new Map(); // Map of stepIndex -> Set of metricNames
+    this.allRunsData.forEach(run => {
+      run.stepMetrics.forEach((metrics, stepIndex) => {
+        if (!allStepMetrics.has(stepIndex)) {
+          allStepMetrics.set(stepIndex, new Set());
+        }
+        metrics.forEach((_, metricName) => {
+          allStepMetrics.get(stepIndex).add(metricName);
+        });
+      });
+    });
+
+    // Sort step indices
+    const sortedStepIndices = Array.from(allStepMetrics.keys()).sort((a, b) => a - b);
+
+    // Build column headers - start with app, scenario, repetition, success, and duration
+    const headers = ['app', 'scenario', 'repetition', 'success', 'duration'];
+    sortedStepIndices.forEach(stepIndex => {
+      const metricNames = Array.from(allStepMetrics.get(stepIndex)).sort();
+      metricNames.forEach(metricName => {
+        const columnName = this.stepColumns.get(stepIndex)?.get(metricName) ||
+                          `step_${stepIndex + 1}_${metricName}`;
+        headers.push(columnName);
+      });
+    });
+
+    // Create CSV data rows
+    const dataRows = this.allRunsData.map(run => {
+      // Start with metadata columns
+      const row = [
+        run.metadata.app,
+        run.metadata.scenario,
+        run.metadata.repetition,
+        run.metadata.success,
+        run.metadata.duration
+      ];
+
+      // Add step metrics
+      sortedStepIndices.forEach(stepIndex => {
+        const stepMetrics = run.stepMetrics.get(stepIndex) || new Map();
+        const metricNames = Array.from(allStepMetrics.get(stepIndex)).sort();
+        metricNames.forEach(metricName => {
+          const value = stepMetrics.get(metricName);
+          row.push(value !== undefined ? value : '');
+        });
+      });
+      return row.join(', ');
+    });
+
+    const csvContent = `${headers.join(', ')}\n${dataRows.join('\n')}\n`;
+
+    // Write to file
+    fs.writeFileSync(this.filePath, csvContent, 'utf8');
+    console.log(`Report generated: ${this.filePath}`);
+  }
+
+  generateMetricsSummary() {
+    if (this.allRunsData.length === 0) {
+      return;
+    }
+
+    console.log('\n=== METRICS SUMMARY ===');
+
+    // Collect all step indices and their metrics
+    const allStepMetrics = new Map(); // Map of stepIndex -> Set of metricNames
+    this.allRunsData.forEach(run => {
+      run.stepMetrics.forEach((metrics, stepIndex) => {
+        if (!allStepMetrics.has(stepIndex)) {
+          allStepMetrics.set(stepIndex, new Set());
+        }
+        metrics.forEach((_, metricName) => {
+          allStepMetrics.get(stepIndex).add(metricName);
+        });
+      });
+    });
+
+    const sortedStepIndices = Array.from(allStepMetrics.keys()).sort((a, b) => a - b);
+
+    if (sortedStepIndices.length === 0) {
+      console.log('No metrics collected during test runs.');
+      return;
+    }
+
+    sortedStepIndices.forEach(stepIndex => {
+      const metricNames = Array.from(allStepMetrics.get(stepIndex)).sort();
+
+      metricNames.forEach(metricName => {
+        const columnName = this.stepColumns.get(stepIndex)?.get(metricName) ||
+                          `step_${stepIndex + 1}_${metricName}`;
+        const values = [];
+
+        this.allRunsData.forEach(run => {
+          const stepMetrics = run.stepMetrics.get(stepIndex);
+          if (stepMetrics && stepMetrics.has(metricName)) {
+            values.push(stepMetrics.get(metricName));
+          }
+        });
+
+        if (values.length > 0) {
+          const sum = values.reduce((a, b) => a + b, 0);
+          const average = sum / values.length;
+          const min = Math.min(...values);
+          const max = Math.max(...values);
+
+          // Calculate p50 (median)
+          const sortedValues = [...values].sort((a, b) => a - b);
+          let p50;
+          if (sortedValues.length % 2 === 0) {
+            // Even number of samples: average of two middle values
+            const mid1 = sortedValues[sortedValues.length / 2 - 1];
+            const mid2 = sortedValues[sortedValues.length / 2];
+            p50 = (mid1 + mid2) / 2;
+          } else {
+            // Odd number of samples: middle value
+            p50 = sortedValues[Math.floor(sortedValues.length / 2)];
+          }
+
+          // Only add "ms" unit for elapsed_time metrics
+          const unit = metricName === 'elapsed_time' ? 'ms' : '';
+          const formatValue = (val) => unit ? `${Math.round(val)}${unit}` : val.toFixed(2);
+
+          console.log(`${columnName}:`);
+          console.log(`  Average: ${formatValue(average)}`);
+          console.log(`  Min: ${formatValue(min)}`);
+          console.log(`  Max: ${formatValue(max)}`);
+          console.log(`  p50: ${formatValue(p50)}`);
+          console.log('');
+        }
+      });
+    });
+  }
+}

package/src/server.js

@@ -0,0 +1,31 @@
+#!/usr/bin/env node
+
+import express from 'express';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const app = express();
+const PORT = process.env.HTTP_PORT || process.env.PORT || 3333;
+
+// Serve static assets from the assets folder
+app.use('/assets', express.static(path.join(__dirname, '..', 'assets')));
+
+// Serve JavaScript files
+app.use('/javascript', express.static(path.join(__dirname, '..', 'javascript')));
+
+// Basic health check endpoint
+app.get('/health', (req, res) => {
+  res.json({ status: 'OK', timestamp: new Date().toISOString() });
+});
+
+export function createServer() {
+  const server = app.listen(PORT, () => {
+    console.log(`Server running on http://localhost:${PORT}`);
+  });
+  return server;
+}
+
+export default app;