@telnyx/ai-agent-lib 0.1.10 → 0.2.1

package/README.md

@@ -11,6 +11,7 @@ A TypeScript/React library for building AI-powered voice conversation applicatio
 - 🔄 **State Management** - Automatic state synchronization using Jotai
 - 📱 **Connection Management** - Robust connection handling with error recovery
 - 🎚️ **Agent State Tracking** - Monitor agent states (listening, speaking, thinking)
+- ⏱️ **Latency Measurement** - Automatic round-trip latency tracking using client-side Voice Activity Detection (VAD)
 
 ## Installation
 
@@ -149,6 +150,22 @@ function VoiceChat() {
 | `environment` | `"production" \| "development"` | ❌ | `"production"` | Telnyx environment |
 | `debug` | `boolean` | ❌ | `false` | Enable debug logging |
 
+### Debug Logging
+
+When `debug: true` is set, the library outputs detailed logs to the console using a timestamped format. This is useful for troubleshooting connection issues, audio stream problems, or understanding the internal state transitions.
+
+```tsx
+<TelnyxAIAgentProvider agentId="your-agent-id" debug={true}>
+  <App />
+</TelnyxAIAgentProvider>
+```
+
+Debug logs include:
+- Audio stream monitoring events (local and remote)
+- Agent state transitions with timing information
+- AudioContext state changes
+- Volume threshold detection
+
 ### Hooks
 
 #### `useClient()`
@@ -160,6 +177,7 @@ Returns the `TelnyxAIAgent` instance for direct API access.
 - `startConversation(options?)` - Start a new conversation with optional caller metadata and headers
 - `endConversation()` - End the current conversation
 - `sendConversationMessage(message: string)` - Send a text message during an active conversation
+- `setRemoteStream(stream: MediaStream)` - Manually set the remote audio stream for monitoring (useful when `call.remoteStream` is not available)
 - `transcript` - Get current transcript array
 
 **`startConversation` Options:**
@@ -316,6 +334,33 @@ agent.on('conversation.update', (notification) => {
 - The agent will receive and process text messages just like spoken input
 - Text messages may appear in the transcript depending on the agent configuration
 
+### Latency Measurement
+
+The library automatically measures round-trip latency using client-side Voice Activity Detection (VAD). This provides accurate timing from when the user stops speaking until the agent's response audio begins.
+
+**How it works:**
+1. **Local VAD (User's microphone)**: Monitors the user's audio stream. After detecting 1 second of silence following speech, the library records `thinkingStartedAt` timestamp and transitions to "thinking" state.
+2. **Remote VAD (Agent's audio)**: Monitors the agent's audio stream. When audio volume crosses the threshold, the library calculates `latencyMs` as the time elapsed since `thinkingStartedAt` and transitions to "speaking" state.
+
+**Configuration constants:**
+- Volume threshold: 10 (frequency data average)
+- Silence duration: 1000ms (time of silence before triggering "thinking" state)
+- Check interval: 20ms (polling frequency for local audio)
+
+```tsx
+const agentState = useAgentState();
+
+// Access latency when agent starts speaking
+useEffect(() => {
+  if (agentState.latencyMs !== undefined) {
+    console.log(`Response latency: ${agentState.latencyMs}ms`);
+  }
+  if (agentState.thinkingStartedAt) {
+    console.log(`Started thinking at: ${agentState.thinkingStartedAt}`);
+  }
+}, [agentState]);
+```
+
 ### Custom Audio Handling
 
 The library automatically handles audio stream monitoring and agent state detection based on audio levels. The audio stream is available through the conversation object:
@@ -325,6 +370,28 @@ const conversation = useConversation();
 const audioStream = conversation?.call?.remoteStream;
 ```
 
+If `call.remoteStream` is not available, you can manually provide the stream using `setRemoteStream`:
+
+```tsx
+const client = useClient();
+const conversation = useConversation();
+
+useEffect(() => {
+  const call = conversation?.call;
+  if (call?.state === 'active') {
+    // Get stream from peer connection if remoteStream is not available
+    const peerConnection = call.peer?.instance;
+    const receivers = peerConnection?.getReceivers?.();
+    const audioReceiver = receivers?.find(r => r.track?.kind === 'audio');
+
+    if (audioReceiver?.track) {
+      const stream = new MediaStream([audioReceiver.track]);
+      client.setRemoteStream(stream);
+    }
+  }
+}, [conversation, client]);
+```
+
 ### Error Handling
 
 ```tsx
@@ -348,6 +415,244 @@ useEffect(() => {
 
 The library uses Jotai for state management, which automatically handles state updates across components. All state is ephemeral and resets when the provider unmounts.
 
+## Events Reference
+
+The `TelnyxAIAgent` class extends `EventEmitter` and provides a comprehensive set of events for monitoring connection status, conversation state, and agent behavior. Events can be subscribed to using `on()`, `once()`, or `addListener()` and unsubscribed using `off()` or `removeListener()`.
+
+### Event Types
+
+| Event | Payload | Description |
+|-------|---------|-------------|
+| `agent.connected` | - | Emitted when successfully connected to the Telnyx platform |
+| `agent.disconnected` | - | Emitted when disconnected from the Telnyx platform |
+| `agent.error` | `Error` | Emitted when any operational error occurs |
+| `transcript.item` | `TranscriptItem` | Emitted when a new transcript item is received |
+| `conversation.update` | `INotification` | Emitted when conversation state changes |
+| `conversation.agent.state` | `AgentStateData` | Emitted when agent state changes (listening/speaking/thinking) |
+| `agent.audio.mute` | `boolean` | Emitted when agent audio is muted or unmuted |
+
+### Data Types
+
+```typescript
+// Transcript item representing a message in the conversation
+type TranscriptItem = {
+  id: string;
+  role: "user" | "assistant";
+  content: string;
+  timestamp: Date;
+  attachments?: Array<{ type: "image"; url: string }>;
+};
+
+// Agent state with optional latency information
+type AgentStateData = {
+  state: "speaking" | "listening" | "thinking";
+  // Round-trip latency in ms from when user stopped speaking until agent response began.
+  // Only present when state is "speaking"
+  latencyMs?: number;
+  // UTC timestamp (ISO 8601) when user stopped speaking and thinking state began.
+  // Only present when state is "thinking"
+  thinkingStartedAt?: string;
+};
+```
+
+### Usage Examples
+
+#### Connection Events
+
+Monitor connection lifecycle to handle reconnection logic or update UI:
+
+```typescript
+const agent = new TelnyxAIAgent({ agentId: 'your-agent-id' });
+
+agent.on('agent.connected', () => {
+  console.log('Connected to Telnyx platform');
+  // Enable UI controls, start conversation, etc.
+});
+
+agent.on('agent.disconnected', () => {
+  console.log('Disconnected from Telnyx platform');
+  // Disable UI controls, show reconnection prompt, etc.
+});
+
+agent.on('agent.error', (error) => {
+  console.error('Agent error:', error.message);
+  // Handle error: show notification, attempt reconnection, etc.
+});
+```
+
+#### Agent State Events
+
+Track whether the agent is listening, thinking, or speaking. This is useful for visual feedback like animated indicators. The library uses client-side Voice Activity Detection (VAD) to detect when the user stops speaking (after 1 second of silence) and when the agent starts responding, providing accurate round-trip latency measurements.
+
+```typescript
+agent.on('conversation.agent.state', (data) => {
+  console.log(`Agent state: ${data.state}`);
+
+  switch (data.state) {
+    case 'listening':
+      // Show listening indicator (e.g., pulsing microphone)
+      break;
+    case 'thinking':
+      // Show thinking indicator (e.g., loading spinner)
+      // thinkingStartedAt contains the UTC timestamp when user stopped speaking
+      if (data.thinkingStartedAt) {
+        console.log(`Thinking started at: ${data.thinkingStartedAt}`);
+      }
+      break;
+    case 'speaking':
+      // Show speaking indicator (e.g., animated waveform)
+      if (data.latencyMs !== undefined) {
+        console.log(`Response latency: ${data.latencyMs}ms`);
+        // Track latency for analytics or display to user
+      }
+      break;
+  }
+});
+```
+
+#### Transcript Events
+
+Build a real-time chat interface by listening to transcript updates:
+
+```typescript
+const conversationHistory: TranscriptItem[] = [];
+
+agent.on('transcript.item', (item) => {
+  conversationHistory.push(item);
+
+  // Display the new message
+  console.log(`[${item.timestamp.toLocaleTimeString()}] ${item.role}: ${item.content}`);
+
+  // Handle attachments if present
+  if (item.attachments?.length) {
+    item.attachments.forEach((attachment) => {
+      if (attachment.type === 'image') {
+        console.log(`Image attachment: ${attachment.url}`);
+      }
+    });
+  }
+});
+```
+
+#### Conversation Update Events
+
+Monitor call state changes to know when to enable/disable features:
+
+```typescript
+agent.on('conversation.update', (notification) => {
+  const call = notification.call;
+
+  if (!call) return;
+
+  console.log(`Call state: ${call.state}`);
+
+  switch (call.state) {
+    case 'new':
+      console.log('Call initiated');
+      break;
+    case 'trying':
+      console.log('Connecting...');
+      break;
+    case 'ringing':
+      console.log('Ringing...');
+      break;
+    case 'active':
+      console.log('Call is active - voice and text messaging enabled');
+      // Enable send message button, show active call UI
+      break;
+    case 'hangup':
+    case 'destroy':
+      console.log('Call ended');
+      // Clean up UI, show call summary
+      break;
+  }
+});
+```
+
+#### React Hook Pattern
+
+In React applications, use the hooks with `useEffect` to manage event subscriptions:
+
+```tsx
+import { useClient, useAgentState, useConnectionState } from '@telnyx/ai-agent-lib';
+
+function ConversationMonitor() {
+  const client = useClient();
+  const agentState = useAgentState();
+  const connectionState = useConnectionState();
+  const [latencyHistory, setLatencyHistory] = useState<number[]>([]);
+
+  // Subscribe to events for additional handling beyond the built-in hooks
+  useEffect(() => {
+    const handleAgentState = (data: AgentStateData) => {
+      if (data.latencyMs !== undefined) {
+        setLatencyHistory(prev => [...prev, data.latencyMs!]);
+      }
+    };
+
+    const handleError = (error: Error) => {
+      // Custom error handling (e.g., send to error tracking service)
+      console.error('Agent error:', error);
+    };
+
+    client.on('conversation.agent.state', handleAgentState);
+    client.on('agent.error', handleError);
+
+    return () => {
+      client.off('conversation.agent.state', handleAgentState);
+      client.off('agent.error', handleError);
+    };
+  }, [client]);
+
+  const averageLatency = latencyHistory.length > 0
+    ? Math.round(latencyHistory.reduce((a, b) => a + b, 0) / latencyHistory.length)
+    : null;
+
+  return (
+    <div>
+      <p>Connection: {connectionState}</p>
+      <p>Agent: {agentState.state}</p>
+      {averageLatency && <p>Avg Response Time: {averageLatency}ms</p>}
+    </div>
+  );
+}
+```
+
+#### One-Time Event Listeners
+
+Use `once()` for events you only need to handle one time:
+
+```typescript
+// Wait for connection before starting conversation
+agent.once('agent.connected', () => {
+  agent.startConversation({ callerName: 'User' });
+});
+
+await agent.connect();
+```
+
+#### Removing Event Listeners
+
+Clean up listeners when no longer needed:
+
+```typescript
+const handleTranscript = (item: TranscriptItem) => {
+  console.log(`${item.role}: ${item.content}`);
+};
+
+// Add listener
+agent.on('transcript.item', handleTranscript);
+
+// Remove specific listener
+agent.off('transcript.item', handleTranscript);
+
+// Remove all listeners for an event
+agent.removeAllListeners('transcript.item');
+
+// Remove all listeners for all events
+agent.removeAllListeners();
+```
+
 ## TypeScript Support
 
 This library is built with TypeScript and provides full type definitions. All hooks and components are fully typed for the best development experience.
@@ -361,6 +666,7 @@ Check out the [example](https://github.com/team-telnyx/telnyx-ai-agent-lib-examp
 - `@telnyx/webrtc` - Telnyx WebRTC SDK
 - `eventemitter3` - Event handling
 - `jotai` - State management
+- `loglevel` - Lightweight logging with level control
 
 ## License
 
@@ -372,4 +678,4 @@ For support, please contact Telnyx support or check the [Telnyx documentation](h
 
 ## Contributing
 
-This library is maintained by Telnyx. For bug reports or feature requests, please contact Telnyx support.elnyx AI Agent Library
+This library is maintained by Telnyx. For bug reports or feature requests, please contact Telnyx support.

package/dist/audio-stream-monitor.d.ts

@@ -1,11 +1,41 @@
 export declare class AudioStreamMonitor {
-    private animationFrameId;
-    private stream;
-    private audioContext;
-    private source;
-    private analyser;
+    private remoteIntervalId;
+    private localIntervalId;
+    private remoteStream;
+    private localStream;
+    private remoteAudioContext;
+    private remoteSource;
+    private remoteAnalyser;
+    private localAudioContext;
+    private localSource;
+    private localAnalyser;
+    private thinkingStartTime;
+    private lastState;
+    private userIsSpeaking;
+    private lastUserAudioTime;
+    constructor();
     private updateAgentState;
+    /**
+     * Set the remote audio stream (agent's voice) to monitor for speech detection
+     */
+    setRemoteStream(stream: MediaStream): void;
+    /**
+     * Set the local audio stream (user's microphone) to monitor for VAD
+     */
+    setLocalStream(stream: MediaStream): void;
     setMonitoredAudioStream(stream: MediaStream): void;
+    private stopRemoteMonitor;
+    private stopLocalMonitor;
     stopAudioStreamMonitor(): void;
-    startAudioStreamMonitor(): void;
+    /**
+     * Monitor remote stream (agent's audio) for speech detection
+     * Detects when agent starts speaking to calculate latency
+     */
+    private startRemoteMonitor;
+    /**
+     * Monitor local stream (user's microphone) for VAD
+     * Detects when user stops speaking (1000ms of silence) to start latency measurement
+     */
+    private startLocalMonitor;
+    destroy(): void;
 }

package/dist/client.d.ts

@@ -89,6 +89,14 @@ export declare class TelnyxAIAgent extends EventEmitter<AIAgentEvents> {
      * @returns Promise that resolves when the call is hung up, or undefined if there is no active call
      */
     endConversation(): void | undefined;
+    /**
+     * Sets the remote audio stream for monitoring agent speech.
+     * Use this when call.remoteStream is not available and you need to provide
+     * the stream from the peer connection receiver.
+     *
+     * @param stream - The MediaStream containing the remote (agent) audio
+     */
+    setRemoteStream(stream: MediaStream): void;
     private onClientReady;
     private onClientOrSocketError;
     private onNotification;