@telnyx/ai-agent-lib 0.2.2 → 0.2.4

package/README.md

@@ -149,6 +149,7 @@ function VoiceChat() {
 | `versionId` | `string` | ❌ | `"main"` | Agent version to use |
 | `environment` | `"production" \| "development"` | ❌ | `"production"` | Telnyx environment |
 | `debug` | `boolean` | ❌ | `false` | Enable debug logging |
+| `vad` | `VADOptions` | ❌ | See below | Voice Activity Detection configuration |
 
 ### Debug Logging
 
@@ -339,21 +340,81 @@ agent.on('conversation.update', (notification) => {
 The library automatically measures round-trip latency using client-side Voice Activity Detection (VAD). This provides accurate timing from when the user stops speaking until the agent's response audio begins.
 
 **How it works:**
-1. **Local VAD (User's microphone)**: Monitors the user's audio stream. After detecting 1 second of silence following speech, the library records `thinkingStartedAt` timestamp and transitions to "thinking" state.
-2. **Remote VAD (Agent's audio)**: Monitors the agent's audio stream. When audio volume crosses the threshold, the library calculates `latencyMs` as the time elapsed since `thinkingStartedAt` and transitions to "speaking" state.
+1. **Local VAD (User's microphone)**: Monitors the user's audio stream. After detecting silence following speech, the library records `thinkingStartedAt` timestamp and transitions to "thinking" state.
+2. **Remote VAD (Agent's audio)**: Monitors the agent's audio stream. When audio volume crosses the threshold, the library calculates `userPerceivedLatencyMs` as the time elapsed since user went silent and transitions to "speaking" state.
+3. **Greeting Latency**: For the first agent speech (greeting), the library calculates `greetingLatencyMs` from when the audio stream monitoring started.
 
-**Configuration constants:**
-- Volume threshold: 10 (frequency data average)
-- Silence duration: 1000ms (time of silence before triggering "thinking" state)
-- Check interval: 20ms (polling frequency for local audio)
+**VAD Configuration:**
+
+The VAD behavior can be customized using the `vad` option:
+
+```typescript
+// React
+<TelnyxAIAgentProvider
+  agentId="your-agent-id"
+  vad={{
+    volumeThreshold: 10,      // 0-255, audio level to detect speech
+    silenceDurationMs: 500,   // ms of silence before "thinking" state
+    minSpeechDurationMs: 100, // min ms of speech to count as real (filters noise)
+    maxLatencyMs: 15000,      // ignore latency above this (optional, filters stale)
+  }}
+>
+
+// Direct usage
+const agent = new TelnyxAIAgent({
+  agentId: 'your-agent-id',
+  vad: {
+    volumeThreshold: 10,
+    silenceDurationMs: 500,
+    minSpeechDurationMs: 100,
+    maxLatencyMs: 15000,
+  },
+});
+```
+
+**VAD Options:**
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `volumeThreshold` | `number` | `10` | Audio level (0-255) to detect speech |
+| `silenceDurationMs` | `number` | `500` | Silence duration before triggering "thinking" state |
+| `minSpeechDurationMs` | `number` | `100` | Minimum speech duration to count as real user speech (filters brief noise) |
+| `maxLatencyMs` | `number` | `undefined` | Maximum latency to report (values above are ignored as stale) |
+
+**Tuning for different scenarios:**
+
+```typescript
+// Fast-paced conversation (aggressive turn detection)
+vad: {
+  silenceDurationMs: 500,
+  minSpeechDurationMs: 80,
+}
+
+// Thoughtful conversation (tolerant of pauses)
+vad: {
+  silenceDurationMs: 1500,
+  minSpeechDurationMs: 150,
+}
+
+// Noisy environment
+vad: {
+  volumeThreshold: 20,
+  minSpeechDurationMs: 200,
+}
+```
+
+**Note:** Silence-based VAD has inherent tradeoffs. Lower `silenceDurationMs` values detect turn-endings faster but may cut off natural pauses ("I need to... think about that"). Higher values are more tolerant but add latency. For production use cases requiring precise turn detection, consider integrating server-side semantic endpointing.
 
 ```tsx
 const agentState = useAgentState();
 
 // Access latency when agent starts speaking
 useEffect(() => {
-  if (agentState.latencyMs !== undefined) {
-    console.log(`Response latency: ${agentState.latencyMs}ms`);
+  if (agentState.greetingLatencyMs !== undefined) {
+    console.log(`Greeting latency: ${agentState.greetingLatencyMs}ms`);
+  }
+  if (agentState.userPerceivedLatencyMs !== undefined) {
+    console.log(`Response latency: ${agentState.userPerceivedLatencyMs}ms`);
   }
   if (agentState.thinkingStartedAt) {
     console.log(`Started thinking at: ${agentState.thinkingStartedAt}`);
@@ -446,9 +507,12 @@ type TranscriptItem = {
 // Agent state with optional latency information
 type AgentStateData = {
   state: "speaking" | "listening" | "thinking";
-  // Round-trip latency in ms from when user stopped speaking until agent response began.
+  // Latency in ms from when user stopped speaking until agent response began.
   // Only present when state is "speaking"
-  latencyMs?: number;
+  userPerceivedLatencyMs?: number;
+  // Latency in ms for the initial agent greeting (first speech).
+  // Only present on first "speaking" state
+  greetingLatencyMs?: number;
   // UTC timestamp (ISO 8601) when user stopped speaking and thinking state began.
   // Only present when state is "thinking"
   thinkingStartedAt?: string;
@@ -501,8 +565,11 @@ agent.on('conversation.agent.state', (data) => {
       break;
     case 'speaking':
       // Show speaking indicator (e.g., animated waveform)
-      if (data.latencyMs !== undefined) {
-        console.log(`Response latency: ${data.latencyMs}ms`);
+      if (data.greetingLatencyMs !== undefined) {
+        console.log(`Greeting latency: ${data.greetingLatencyMs}ms`);
+      }
+      if (data.userPerceivedLatencyMs !== undefined) {
+        console.log(`Response latency: ${data.userPerceivedLatencyMs}ms`);
         // Track latency for analytics or display to user
       }
       break;
@@ -585,8 +652,8 @@ function ConversationMonitor() {
   // Subscribe to events for additional handling beyond the built-in hooks
   useEffect(() => {
     const handleAgentState = (data: AgentStateData) => {
-      if (data.latencyMs !== undefined) {
-        setLatencyHistory(prev => [...prev, data.latencyMs!]);
+      if (data.userPerceivedLatencyMs !== undefined) {
+        setLatencyHistory(prev => [...prev, data.userPerceivedLatencyMs!]);
       }
     };
 

package/dist/audio-stream-monitor.d.ts

@@ -1,3 +1,4 @@
+import type { VADOptions } from "./types";
 export declare class AudioStreamMonitor {
     private remoteIntervalId;
     private localIntervalId;
@@ -13,7 +14,15 @@ export declare class AudioStreamMonitor {
     private lastState;
     private userIsSpeaking;
     private lastUserAudioTime;
-    constructor();
+    private userSpeechStartTime;
+    private userSilenceStartTime;
+    private isFirstAgentSpeech;
+    private monitorStartTime;
+    private volumeThreshold;
+    private silenceDurationMs;
+    private minSpeechDurationMs;
+    private maxLatencyMs;
+    constructor(options?: VADOptions);
     private updateAgentState;
     /**
      * Set the remote audio stream (agent's voice) to monitor for speech detection

package/dist/client.d.ts

@@ -1,12 +1,16 @@
 import { Call } from "@telnyx/webrtc";
 import EventEmitter from "eventemitter3";
-import type { AIAgentEvents, TranscriptItem } from "./types";
+import type { AIAgentEvents, TranscriptItem, VADOptions } from "./types";
 export type TelnyxAIAgentConstructorParams = {
     agentId: string;
     versionId?: string;
     environment?: "production" | "development";
     debug?: boolean;
     trickleIce?: boolean;
+    /**
+     * Voice Activity Detection options for controlling speech detection and latency measurement.
+     */
+    vad?: VADOptions;
 };
 export declare class TelnyxAIAgent extends EventEmitter<AIAgentEvents> {
     private telnyxRTC;