@convai/web-sdk 0.2.3 → 0.3.0-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/README.md +529 -868
  2. package/dist/core/AudioManager.d.ts.map +1 -1
  3. package/dist/core/AudioManager.js +0 -5
  4. package/dist/core/AudioManager.js.map +1 -1
  5. package/dist/core/BlendshapeQueue.d.ts +128 -0
  6. package/dist/core/BlendshapeQueue.d.ts.map +1 -0
  7. package/dist/core/BlendshapeQueue.js +229 -0
  8. package/dist/core/BlendshapeQueue.js.map +1 -0
  9. package/dist/core/ConvaiClient.d.ts +19 -0
  10. package/dist/core/ConvaiClient.d.ts.map +1 -1
  11. package/dist/core/ConvaiClient.js +67 -29
  12. package/dist/core/ConvaiClient.js.map +1 -1
  13. package/dist/core/MessageHandler.d.ts +7 -0
  14. package/dist/core/MessageHandler.d.ts.map +1 -1
  15. package/dist/core/MessageHandler.js +35 -2
  16. package/dist/core/MessageHandler.js.map +1 -1
  17. package/dist/core/ScreenShareManager.d.ts.map +1 -1
  18. package/dist/core/ScreenShareManager.js +0 -3
  19. package/dist/core/ScreenShareManager.js.map +1 -1
  20. package/dist/core/VideoManager.d.ts.map +1 -1
  21. package/dist/core/VideoManager.js +0 -4
  22. package/dist/core/VideoManager.js.map +1 -1
  23. package/dist/core/index.d.ts +2 -0
  24. package/dist/core/index.d.ts.map +1 -1
  25. package/dist/core/index.js +2 -0
  26. package/dist/core/index.js.map +1 -1
  27. package/dist/core/types.d.ts +19 -0
  28. package/dist/core/types.d.ts.map +1 -1
  29. package/dist/index.d.ts +2 -0
  30. package/dist/index.d.ts.map +1 -1
  31. package/dist/index.js +2 -0
  32. package/dist/index.js.map +1 -1
  33. package/dist/lipsync-helpers/arkitBlendshapeHelpers.d.ts +80 -0
  34. package/dist/lipsync-helpers/arkitBlendshapeHelpers.d.ts.map +1 -0
  35. package/dist/lipsync-helpers/arkitBlendshapeHelpers.js +201 -0
  36. package/dist/lipsync-helpers/arkitBlendshapeHelpers.js.map +1 -0
  37. package/dist/lipsync-helpers/arkitOrder61.d.ts +121 -0
  38. package/dist/lipsync-helpers/arkitOrder61.d.ts.map +1 -0
  39. package/dist/lipsync-helpers/arkitOrder61.js +287 -0
  40. package/dist/lipsync-helpers/arkitOrder61.js.map +1 -0
  41. package/dist/lipsync-helpers/arkitPhonemeReference.d.ts +155 -0
  42. package/dist/lipsync-helpers/arkitPhonemeReference.d.ts.map +1 -0
  43. package/dist/lipsync-helpers/arkitPhonemeReference.js +362 -0
  44. package/dist/lipsync-helpers/arkitPhonemeReference.js.map +1 -0
  45. package/dist/lipsync-helpers/index.d.ts +10 -0
  46. package/dist/lipsync-helpers/index.d.ts.map +1 -0
  47. package/dist/lipsync-helpers/index.js +21 -0
  48. package/dist/lipsync-helpers/index.js.map +1 -0
  49. package/dist/lipsync-helpers/metahumanOrder251.d.ts +115 -0
  50. package/dist/lipsync-helpers/metahumanOrder251.d.ts.map +1 -0
  51. package/dist/lipsync-helpers/metahumanOrder251.js +432 -0
  52. package/dist/lipsync-helpers/metahumanOrder251.js.map +1 -0
  53. package/dist/lipsync-helpers/neurosyncBlendshapeMapper.d.ts +30 -0
  54. package/dist/lipsync-helpers/neurosyncBlendshapeMapper.d.ts.map +1 -0
  55. package/dist/lipsync-helpers/neurosyncBlendshapeMapper.js +315 -0
  56. package/dist/lipsync-helpers/neurosyncBlendshapeMapper.js.map +1 -0
  57. package/dist/react/components/ConvaiWidget.d.ts.map +1 -1
  58. package/dist/react/components/ConvaiWidget.js +43 -3
  59. package/dist/react/components/ConvaiWidget.js.map +1 -1
  60. package/dist/react/hooks/useCharacterInfo.d.ts.map +1 -1
  61. package/dist/react/hooks/useCharacterInfo.js +1 -1
  62. package/dist/react/hooks/useCharacterInfo.js.map +1 -1
  63. package/dist/react/hooks/useConvaiClient.d.ts.map +1 -1
  64. package/dist/react/hooks/useConvaiClient.js +5 -0
  65. package/dist/react/hooks/useConvaiClient.js.map +1 -1
  66. package/dist/utils/speakerManagement.d.ts.map +1 -1
  67. package/dist/utils/speakerManagement.js +0 -5
  68. package/dist/utils/speakerManagement.js.map +1 -1
  69. package/dist/vanilla/AudioRenderer.d.ts +5 -0
  70. package/dist/vanilla/AudioRenderer.d.ts.map +1 -1
  71. package/dist/vanilla/AudioRenderer.js +27 -18
  72. package/dist/vanilla/AudioRenderer.js.map +1 -1
  73. package/package.json +4 -4
package/README.md CHANGED
@@ -1,47 +1,6 @@
1
1
  # @convai/web-sdk
2
2
 
3
- [![npm version](https://badge.fury.io/js/%40convai%2Fweb-sdk.svg)](https://www.npmjs.com/package/@convai/web-sdk)
4
-
5
- JavaScript/TypeScript SDK for building AI voice assistants with real-time audio/video streaming. Drop-in widgets for **React** and **Vanilla JavaScript/TypeScript** with customizable UI components.
6
-
7
- ---
8
-
9
- ## 📑 Table of Contents
10
-
11
- - [Installation](#installation)
12
- - [Quick Start](#quick-start)
13
- - [React - ConvaiWidget](#react---convaiwidget)
14
- - [Vanilla JS/TS - ConvaiWidget](#vanilla-jsts---convaiwidget)
15
- - [Core Concepts](#core-concepts)
16
- - [React SDK](#react-sdk)
17
- - [useConvaiClient Hook](#useconvaiclient-hook)
18
- - [AudioRenderer Component](#audiorenderer-component)
19
- - [AudioContext](#audiocontext)
20
- - [React Exports Reference](#react-exports-reference)
21
- - [Vanilla SDK](#vanilla-sdk)
22
- - [ConvaiClient Class](#convaiclient-class)
23
- - [AudioRenderer Class](#audiorenderer-class)
24
- - [Vanilla Exports Reference](#vanilla-exports-reference)
25
- - [Video & Screen Share](#video--screen-share)
26
- - [Critical Requirements](#critical-requirements)
27
- - [Enabling Video](#enabling-video)
28
- - [Enabling Screen Share](#enabling-screen-share)
29
- - [Building Custom UIs](#building-custom-uis)
30
- - [Custom Chat Interface](#custom-chat-interface)
31
- - [Audio Visualizer](#audio-visualizer)
32
- - [Message Types](#message-types)
33
- - [API Reference](#api-reference)
34
- - [Configuration](#configuration)
35
- - [Connection Management](#connection-management)
36
- - [Messaging](#messaging)
37
- - [Audio Controls](#audio-controls)
38
- - [Video Controls](#video-controls)
39
- - [Screen Share Controls](#screen-share-controls)
40
- - [Getting Credentials](#getting-credentials)
41
- - [TypeScript Support](#typescript-support)
42
- - [Support](#support)
43
-
44
- ---
3
+ JavaScript/TypeScript SDK for Convai AI voice assistants. Build voice-powered AI interactions for web applications with real-time audio/video streaming. Supports both React and Vanilla JavaScript/TypeScript.
45
4
 
46
5
  ## Installation
47
6
 
@@ -49,25 +8,14 @@ JavaScript/TypeScript SDK for building AI voice assistants with real-time audio/
49
8
  npm install @convai/web-sdk
50
9
  ```
51
10
 
52
- **Peer Dependencies (React only):**
11
+ ## Basic Setup
53
12
 
54
- ```bash
55
- npm install react@^18.0.0 react-dom@^18.0.0
56
- ```
57
-
58
- ---
59
-
60
- ## Quick Start
61
-
62
- ### React - ConvaiWidget
63
-
64
- The `ConvaiWidget` is a complete, pre-built chat interface with voice/video capabilities.
13
+ ### React
65
14
 
66
15
  ```tsx
67
- import { useConvaiClient, ConvaiWidget } from "@convai/web-sdk/react";
16
+ import { useConvaiClient, ConvaiWidget } from "@convai/web-sdk";
68
17
 
69
18
  function App() {
70
- // Initialize the Convai client
71
19
  const convaiClient = useConvaiClient({
72
20
  apiKey: "your-api-key",
73
21
  characterId: "your-character-id",
@@ -77,9 +25,7 @@ function App() {
77
25
  }
78
26
  ```
79
27
 
80
- **That's it!** The widget auto-connects on first user interaction and handles all UI/audio for you.
81
-
82
- ### Vanilla JS/TS - ConvaiWidget
28
+ ### Vanilla TypeScript
83
29
 
84
30
  ```typescript
85
31
  import { ConvaiClient, createConvaiWidget } from "@convai/web-sdk/vanilla";
@@ -90,7 +36,7 @@ const client = new ConvaiClient({
90
36
  characterId: "your-character-id",
91
37
  });
92
38
 
93
- // Create and mount widget - auto-connects on first user click
39
+ // Create widget - auto-connects on first user click
94
40
  const widget = createConvaiWidget(document.body, {
95
41
  convaiClient: client,
96
42
  });
@@ -99,1025 +45,770 @@ const widget = createConvaiWidget(document.body, {
99
45
  widget.destroy();
100
46
  ```
101
47
 
102
- ---
48
+ ## Exports
103
49
 
104
- ## Core Concepts
50
+ ### React Exports (`@convai/web-sdk` or `@convai/web-sdk/react`)
105
51
 
106
- ### The Architecture
52
+ **Components:**
107
53
 
108
- ```
109
- ┌─────────────────────────────────────────────────┐
110
- │ ConvaiWidget (UI Layer) │
111
- │ ├─ Chat Interface │
112
- │ ├─ Voice Mode │
113
- │ └─ Video/Screen Share UI │
114
- └─────────────────────────────────────────────────┘
115
-
116
- ┌─────────────────────────────────────────────────┐
117
- │ ConvaiClient (Core Logic) │
118
- │ ├─ Connection Management │
119
- │ ├─ Message Handling │
120
- │ ├─ State Management │
121
- │ └─ Audio/Video Controls │
122
- └─────────────────────────────────────────────────┘
123
-
124
- ┌─────────────────────────────────────────────────┐
125
- │ WebRTC Room (Communication Layer) │
126
- │ ├─ Real-time Audio/Video Streaming │
127
- │ ├─ Track Management │
128
- │ └─ Network Communication │
129
- └─────────────────────────────────────────────────┘
130
-
131
- ┌─────────────────────────────────────────────────┐
132
- │ AudioRenderer (Critical for Playback) │
133
- │ ├─ Attaches audio tracks to DOM │
134
- │ ├─ Manages audio elements │
135
- │ └─ Enables bot voice playback │
136
- └─────────────────────────────────────────────────┘
137
- ```
54
+ - `ConvaiWidget` - Main chat widget component
138
55
 
139
- ### Key Principles
56
+ **Hooks:**
140
57
 
141
- 1. **ConvaiClient** - The brain. Manages connection, state, and communication with Convai servers.
142
- 2. **AudioRenderer** - **CRITICAL**: Without this, you won't hear the bot. It renders audio to the user's speakers.
143
- 3. **ConvaiWidget** - The complete UI. Uses both ConvaiClient and AudioRenderer internally.
144
- 4. **Connection Type** - Determines capabilities:
145
- - `"audio"` (default) - Audio only
146
- - `"video"` - Audio + Video + Screen Share
58
+ - `useConvaiClient(config?)` - Main client hook
59
+ - `useCharacterInfo(characterId, apiKey)` - Fetch character metadata
60
+ - `useLocalCameraTrack()` - Get local camera track
147
61
 
148
- ---
62
+ **Core Client:**
149
63
 
150
- ## React SDK
64
+ - `ConvaiClient` - Core client class
151
65
 
152
- ### useConvaiClient Hook
66
+ **Types:**
153
67
 
154
- **Purpose**: Returns a fully configured `ConvaiClient` instance with reactive state updates.
68
+ - `ConvaiConfig` - Configuration interface
69
+ - `ConvaiClientState` - Client state interface
70
+ - `ChatMessage` - Message interface
71
+ - `IConvaiClient` - Client interface
72
+ - `AudioControls` - Audio control interface
73
+ - `VideoControls` - Video control interface
74
+ - `ScreenShareControls` - Screen share control interface
155
75
 
156
- **When to Use**: Every React app using Convai needs this hook.
76
+ **Components:**
157
77
 
158
- **What It Does**:
78
+ - `AudioRenderer` - Audio playback component
79
+ - `AudioContext` - Audio context provider
159
80
 
160
- - Creates and manages a ConvaiClient instance
161
- - Provides reactive state (connection, messages, activity)
162
- - Handles connection lifecycle
163
- - Exposes audio/video/screen share controls
164
-
165
- ```tsx
166
- import { useConvaiClient } from "@convai/web-sdk/react";
81
+ ### Vanilla Exports (`@convai/web-sdk/vanilla`)
167
82
 
168
- function ChatbotWrapper() {
169
- const convaiClient = useConvaiClient({
170
- apiKey: "your-api-key",
171
- characterId: "your-character-id",
172
- enableVideo: false, // Default: audio only
173
- startWithAudioOn: false, // Mic starts muted
174
- });
83
+ **Functions:**
175
84
 
176
- // Access reactive state
177
- const { state, chatMessages, userTranscription, isBotReady } = convaiClient;
85
+ - `createConvaiWidget(container, options)` - Create widget instance
86
+ - `destroyConvaiWidget(widget)` - Destroy widget instance
178
87
 
179
- // Use controls
180
- const handleMute = () => convaiClient.audioControls.muteAudio();
181
- const handleSend = () =>
182
- convaiClient.sendUserTextMessage("Hello, character!");
88
+ **Classes:**
183
89
 
184
- return (
185
- <div>
186
- <p>Status: {state.agentState}</p>
187
- <p>Messages: {chatMessages.length}</p>
188
- <button onClick={handleMute}>Mute</button>
189
- <button onClick={handleSend}>Send</button>
190
- </div>
191
- );
192
- }
193
- ```
90
+ - `ConvaiClient` - Core client class
91
+ - `AudioRenderer` - Audio playback handler
194
92
 
195
- ### AudioRenderer Component
93
+ **Types:**
196
94
 
197
- **Purpose**: Renders remote audio tracks to the user's speakers.
95
+ - `VanillaWidget` - Widget instance interface
96
+ - `VanillaWidgetOptions` - Widget options interface
97
+ - `IConvaiClient` - Client interface
98
+ - `ConvaiConfig` - Configuration interface
99
+ - `ConvaiClientState` - Client state interface
100
+ - `ChatMessage` - Message interface
198
101
 
199
- **⚠️ CRITICAL**: Without `AudioRenderer`, you will NOT hear the bot's voice.
102
+ ### Core Exports (`@convai/web-sdk/core`)
200
103
 
201
- **When to Use**:
104
+ **Classes:**
202
105
 
203
- - Always when building custom UIs
204
- - Already included in `ConvaiWidget` (no need to add separately)
106
+ - `ConvaiClient` - Main client class
107
+ - `AudioManager` - Audio management
108
+ - `VideoManager` - Video management
109
+ - `ScreenShareManager` - Screen share management
110
+ - `MessageHandler` - Message handling
111
+ - `EventEmitter` - Event emitter base class
205
112
 
206
- **How It Works**:
113
+ **Types:**
207
114
 
208
- - Attaches to the WebRTC room
209
- - Automatically creates `<audio>` elements for remote participants (the bot)
210
- - Manages audio playback lifecycle
115
+ - All types from React/Vanilla exports
116
+ - `ConvaiClientType` - Type alias for ConvaiClient
211
117
 
212
- ```tsx
213
- import { useConvaiClient, AudioRenderer } from "@convai/web-sdk/react";
118
+ ## Props and Configuration
214
119
 
215
- function CustomChatUI() {
216
- const convaiClient = useConvaiClient({
217
- apiKey: "your-api-key",
218
- characterId: "your-character-id",
219
- });
220
-
221
- return (
222
- <div>
223
- {/* CRITICAL: This component renders bot audio to speakers */}
224
- <AudioRenderer />
225
-
226
- {/* Your custom UI */}
227
- <div>
228
- {convaiClient.chatMessages.map((msg) => (
229
- <div key={msg.id}>{msg.content}</div>
230
- ))}
231
- </div>
232
- </div>
233
- );
234
- }
235
- ```
236
-
237
- ### AudioContext
238
-
239
- **Purpose**: Provides the WebRTC Room to child components.
240
-
241
- **When to Use**: When building deeply nested custom UIs that need direct access to the audio room.
242
-
243
- **How It Works**: React Context that holds the active WebRTC room.
120
+ ### ConvaiWidget Props (React)
244
121
 
245
122
  ```tsx
246
- import { useConvaiClient, AudioRenderer, AudioContext } from "@convai/web-sdk/react";
247
- import { useContext } from "react";
248
-
249
- function ChatbotWrapper() {
250
- const convaiClient = useConvaiClient({
251
- /* config */
252
- });
253
-
254
- return (
255
- <AudioContext.Provider value={convaiClient.room}>
256
- <AudioRenderer />
257
- <ChildComponent />
258
- </AudioContext.Provider>
259
- );
260
- }
261
-
262
- function ChildComponent() {
263
- const room = useContext(AudioContext);
264
- // Access WebRTC room directly
265
- console.log("Room state:", room?.state);
266
- return <div>Child has access to Room</div>;
123
+ interface ConvaiWidgetProps {
124
+ /** Convai client instance (required) */
125
+ convaiClient: IConvaiClient & {
126
+ activity?: string;
127
+ isAudioMuted: boolean;
128
+ isVideoEnabled: boolean;
129
+ isScreenShareActive: boolean;
130
+ };
131
+ /** Show video toggle button in settings (default: true) */
132
+ showVideo?: boolean;
133
+ /** Show screen share toggle button in settings (default: true) */
134
+ showScreenShare?: boolean;
267
135
  }
268
136
  ```
269
137
 
270
- ### React Exports Reference
271
-
272
- ```tsx
273
- // Components
274
- import { ConvaiWidget } from "@convai/web-sdk/react";
275
-
276
- // Hooks
277
- import { useConvaiClient, useCharacterInfo } from "@convai/web-sdk/react";
278
-
279
- // Audio Rendering (Critical)
280
- import { AudioRenderer, AudioContext } from "@convai/web-sdk/react";
281
-
282
- // Core Client (for advanced usage)
283
- import { ConvaiClient } from "@convai/web-sdk/react";
284
-
285
- // Types
286
- import type {
287
- ConvaiConfig,
288
- ConvaiClientState,
289
- ChatMessage,
290
- IConvaiClient,
291
- AudioControls,
292
- VideoControls,
293
- ScreenShareControls,
294
- } from "@convai/web-sdk/react";
295
- ```
296
-
297
- ---
298
-
299
- ## Vanilla SDK
300
-
301
- ### ConvaiClient Class
302
-
303
- **Purpose**: Core client for managing Convai connections in vanilla JavaScript/TypeScript.
304
-
305
- **When to Use**: Any non-React application or when you need full control.
306
-
307
- **What It Provides**:
308
-
309
- - Connection management
310
- - Message handling
311
- - State management (via events)
312
- - Audio/video/screen share controls
313
-
314
- ```typescript
315
- import { ConvaiClient } from "@convai/web-sdk/vanilla";
316
-
317
- const client = new ConvaiClient({
318
- apiKey: "your-api-key",
319
- characterId: "your-character-id",
320
- });
321
-
322
- // Connect
323
- await client.connect();
324
-
325
- // Listen to events
326
- client.on("stateChange", (state) => {
327
- console.log("Agent state:", state.agentState);
328
- });
329
-
330
- client.on("message", (message) => {
331
- console.log("New message:", message.content);
332
- });
333
-
334
- // Send messages
335
- client.sendUserTextMessage("Hello!");
336
-
337
- // Control audio
338
- await client.audioControls.muteAudio();
339
- await client.audioControls.unmuteAudio();
340
-
341
- // Disconnect
342
- await client.disconnect();
343
- ```
344
-
345
- ### AudioRenderer Class
346
-
347
- **Purpose**: Manages audio playback for vanilla JavaScript/TypeScript applications.
348
-
349
- **⚠️ CRITICAL**: Without this, you will NOT hear the bot's voice.
350
-
351
- **When to Use**:
352
-
353
- - Always when building custom vanilla UIs
354
- - Already included in vanilla `ConvaiWidget` (no need to add separately)
355
-
356
- **How It Works**:
357
-
358
- - Attaches to the WebRTC room
359
- - Automatically creates hidden `<audio>` elements
360
- - Manages audio playback for remote participants (the bot)
138
+ ### createConvaiWidget Options (Vanilla)
361
139
 
362
140
  ```typescript
363
- import { ConvaiClient, AudioRenderer } from "@convai/web-sdk/vanilla";
364
-
365
- const client = new ConvaiClient({
366
- apiKey: "your-api-key",
367
- characterId: "your-character-id",
368
- });
369
-
370
- await client.connect();
371
-
372
- // CRITICAL: Create AudioRenderer to hear bot audio
373
- const audioRenderer = new AudioRenderer(client.room);
374
-
375
- // Your custom UI logic...
376
-
377
- // Cleanup
378
- audioRenderer.destroy();
379
- await client.disconnect();
141
+ interface VanillaWidgetOptions {
142
+ /** Convai client instance (required) */
143
+ convaiClient: IConvaiClient & {
144
+ activity?: string;
145
+ chatMessages: ChatMessage[];
146
+ };
147
+ /** Show video toggle button in settings (default: true) */
148
+ showVideo?: boolean;
149
+ /** Show screen share toggle button in settings (default: true) */
150
+ showScreenShare?: boolean;
151
+ }
380
152
  ```
381
153
 
382
- ### Vanilla Exports Reference
154
+ ### ConvaiConfig
383
155
 
384
156
  ```typescript
385
- // Widget
386
- import { createConvaiWidget, destroyConvaiWidget } from "@convai/web-sdk/vanilla";
387
-
388
- // Core Client
389
- import { ConvaiClient } from "@convai/web-sdk/vanilla";
390
-
391
- // Audio Rendering (Critical)
392
- import { AudioRenderer } from "@convai/web-sdk/vanilla";
393
-
394
- // Types
395
- import type {
396
- VanillaWidget,
397
- VanillaWidgetOptions,
398
- ConvaiConfig,
399
- ConvaiClientState,
400
- ChatMessage,
401
- IConvaiClient,
402
- } from "@convai/web-sdk/vanilla";
157
+ interface ConvaiConfig {
158
+ /** Your Convai API key from convai.com dashboard (required) */
159
+ apiKey: string;
160
+ /** The Character ID to connect to (required) */
161
+ characterId: string;
162
+ /**
163
+ * End user identifier for speaker management (optional).
164
+ * When provided: enables long-term memory and analytics
165
+ * When not provided: anonymous mode, no persistent memory
166
+ */
167
+ endUserId?: string;
168
+ /** Custom Convai API URL (optional, defaults to production endpoint) */
169
+ url?: string;
170
+ /**
171
+ * Enable video capability (default: false).
172
+ * If true, connection_type will be "video" (supports audio, video, and screenshare).
173
+ * If false, connection_type will be "audio" (audio only).
174
+ */
175
+ enableVideo?: boolean;
176
+ /**
177
+ * Start with video camera on when connecting (default: false).
178
+ * Only works if enableVideo is true.
179
+ */
180
+ startWithVideoOn?: boolean;
181
+ /**
182
+ * Start with microphone on when connecting (default: false).
183
+ * If false, microphone stays off until user enables it.
184
+ */
185
+ startWithAudioOn?: boolean;
186
+ /** Enable text-to-speech audio generation (default: true) */
187
+ ttsEnabled?: boolean;
188
+ }
403
189
  ```
404
190
 
405
- ---
406
-
407
- ## Video & Screen Share
191
+ ## Features
408
192
 
409
- ### Critical Requirements
193
+ ### Video Enabled Chat
410
194
 
411
- > ⚠️ **IMPORTANT**: Video and Screen Share features require **TWO** configuration changes:
195
+ To enable video capabilities, set `enableVideo: true` in your configuration. This enables audio, video, and screen sharing.
412
196
 
413
- #### 1. Set `enableVideo: true` in Client Configuration
414
-
415
- This sets the connection type to `"video"` which enables video capabilities.
416
-
417
- #### 2. Set `showVideo` and/or `showScreenShare` in Widget Props
418
-
419
- This shows the UI controls for video/screen share.
420
-
421
- **Without both, video features will NOT work.**
422
-
423
- ---
424
-
425
- ### Enabling Video
426
-
427
- #### React
197
+ **React:**
428
198
 
429
199
  ```tsx
430
- import { useConvaiClient, ConvaiWidget } from "@convai/web-sdk/react";
200
+ import { useConvaiClient, ConvaiWidget } from "@convai/web-sdk";
431
201
 
432
202
  function App() {
433
- // ✅ STEP 1: Enable video in client config
434
203
  const convaiClient = useConvaiClient({
435
204
  apiKey: "your-api-key",
436
205
  characterId: "your-character-id",
437
- enableVideo: true, // ← REQUIRED for video
206
+ enableVideo: true,
438
207
  startWithVideoOn: false, // Camera off by default
439
208
  });
440
209
 
441
210
  return (
442
211
  <ConvaiWidget
443
212
  convaiClient={convaiClient}
444
- showVideo={true} // ← STEP 2: Show video controls
445
- showScreenShare={false} // Optional: hide screen share
213
+ showVideo={true}
214
+ showScreenShare={true}
446
215
  />
447
216
  );
448
217
  }
449
218
  ```
450
219
 
451
- #### Vanilla
220
+ **Vanilla:**
452
221
 
453
222
  ```typescript
454
223
  import { ConvaiClient, createConvaiWidget } from "@convai/web-sdk/vanilla";
455
224
 
456
- // ✅ STEP 1: Enable video in client config
457
225
  const client = new ConvaiClient({
458
226
  apiKey: "your-api-key",
459
227
  characterId: "your-character-id",
460
- enableVideo: true, // ← REQUIRED for video
228
+ enableVideo: true,
461
229
  startWithVideoOn: false,
462
230
  });
463
231
 
464
232
  const widget = createConvaiWidget(document.body, {
465
233
  convaiClient: client,
466
- showVideo: true, // ← STEP 2: Show video controls
467
- showScreenShare: false,
234
+ showVideo: true,
235
+ showScreenShare: true,
468
236
  });
469
237
  ```
470
238
 
471
- #### Manual Video Control
239
+ **Manual Video Controls:**
472
240
 
473
241
  ```typescript
474
- // Enable camera
242
+ // Enable video camera
475
243
  await convaiClient.videoControls.enableVideo();
476
244
 
477
- // Disable camera
245
+ // Disable video camera
478
246
  await convaiClient.videoControls.disableVideo();
479
247
 
480
- // Toggle camera
248
+ // Toggle video
481
249
  await convaiClient.videoControls.toggleVideo();
482
250
 
483
- // Check state
484
- console.log(convaiClient.isVideoEnabled);
251
+ // Check video state
252
+ const isVideoEnabled = convaiClient.videoControls.isVideoEnabled;
485
253
 
486
254
  // Set video quality
487
255
  await convaiClient.videoControls.setVideoQuality("high"); // 'low' | 'medium' | 'high'
488
256
 
489
- // Get available cameras
257
+ // Get available video devices
490
258
  const devices = await convaiClient.videoControls.getVideoDevices();
491
259
 
492
- // Switch camera
260
+ // Set specific video device
493
261
  await convaiClient.videoControls.setVideoDevice(deviceId);
494
262
  ```
495
263
 
496
- ---
264
+ **Screen Sharing:**
497
265
 
498
- ### Enabling Screen Share
266
+ ```typescript
267
+ // Enable screen share
268
+ await convaiClient.screenShareControls.enableScreenShare();
499
269
 
500
- Screen sharing **requires** `enableVideo: true` (connection type must be `"video"`).
270
+ // Enable screen share with audio
271
+ await convaiClient.screenShareControls.enableScreenShareWithAudio();
501
272
 
502
- #### React
273
+ // Disable screen share
274
+ await convaiClient.screenShareControls.disableScreenShare();
503
275
 
504
- ```tsx
505
- import { useConvaiClient, ConvaiWidget } from "@convai/web-sdk/react";
276
+ // Toggle screen share
277
+ await convaiClient.screenShareControls.toggleScreenShare();
506
278
 
507
- function App() {
508
- // STEP 1: Enable video (required for screen share)
279
+ // Check screen share state
280
+ const isActive = convaiClient.screenShareControls.isScreenShareActive;
281
+ ```
282
+
283
+ **Video State Monitoring:**
284
+
285
+ ```typescript
286
+ // React
287
+ const { isVideoEnabled } = convaiClient;
288
+
289
+ // Core API (event-based)
290
+ convaiClient.videoControls.on("videoStateChange", (state) => {
291
+ console.log("Video enabled:", state.isVideoEnabled);
292
+ console.log("Video hidden:", state.isVideoHidden);
293
+ });
294
+ ```
295
+
296
+ ### Interruption
297
+
298
+ Interrupt the character's current response to allow the user to speak immediately.
299
+
300
+ **React:**
301
+
302
+ ```tsx
303
+ function ChatInterface() {
509
304
  const convaiClient = useConvaiClient({
510
- apiKey: "your-api-key",
511
- characterId: "your-character-id",
512
- enableVideo: true, // ← REQUIRED for screen share
305
+ /* config */
513
306
  });
514
307
 
515
- return (
516
- <ConvaiWidget
517
- convaiClient={convaiClient}
518
- showVideo={true} // Optional: show video controls
519
- showScreenShare={true} // ← STEP 2: Show screen share controls
520
- />
521
- );
308
+ const handleInterrupt = () => {
309
+ // Interrupt the bot's current response
310
+ convaiClient.sendInterruptMessage();
311
+ };
312
+
313
+ return <button onClick={handleInterrupt}>Interrupt</button>;
522
314
  }
523
315
  ```
524
316
 
525
- #### Vanilla
317
+ **Vanilla:**
526
318
 
527
319
  ```typescript
528
- import { ConvaiClient, createConvaiWidget } from "@convai/web-sdk/vanilla";
529
-
530
- // ✅ STEP 1: Enable video (required for screen share)
531
- const client = new ConvaiClient({
532
- apiKey: "your-api-key",
533
- characterId: "your-character-id",
534
- enableVideo: true, // ← REQUIRED for screen share
535
- });
320
+ const interruptButton = document.getElementById("interrupt-btn");
536
321
 
537
- const widget = createConvaiWidget(document.body, {
538
- convaiClient: client,
539
- showVideo: true,
540
- showScreenShare: true, // ← STEP 2: Show screen share controls
322
+ interruptButton.addEventListener("click", () => {
323
+ client.sendInterruptMessage();
541
324
  });
542
325
  ```
543
326
 
544
- #### Manual Screen Share Control
327
+ **Voice Mode Interruption Pattern:**
545
328
 
546
- ```typescript
547
- // Start screen share
548
- await convaiClient.screenShareControls.enableScreenShare();
329
+ When implementing voice mode, interrupt the bot when the user starts speaking:
549
330
 
550
- // Start screen share with audio
551
- await convaiClient.screenShareControls.enableScreenShareWithAudio();
331
+ ```typescript
332
+ // When user enters voice mode
333
+ const enterVoiceMode = async () => {
334
+ // Interrupt any ongoing bot response
335
+ convaiClient.sendInterruptMessage();
552
336
 
553
- // Stop screen share
554
- await convaiClient.screenShareControls.disableScreenShare();
337
+ // Unmute microphone
338
+ await convaiClient.audioControls.unmuteAudio();
339
+ };
555
340
 
556
- // Toggle screen share
557
- await convaiClient.screenShareControls.toggleScreenShare();
341
+ // When user exits voice mode
342
+ const exitVoiceMode = async () => {
343
+ // Interrupt any ongoing bot response
344
+ convaiClient.sendInterruptMessage();
558
345
 
559
- // Check state
560
- console.log(convaiClient.isScreenShareActive);
346
+ // Mute microphone
347
+ await convaiClient.audioControls.muteAudio();
348
+ };
561
349
  ```
562
350
 
563
- ---
564
-
565
- ## Building Custom UIs
351
+ ### User Microphone Mute/Unmute
566
352
 
567
- ### Custom Chat Interface
353
+ Control the user's microphone input.
568
354
 
569
- Use the `chatMessages` array from ConvaiClient to build your own chat UI.
570
-
571
- #### React Example
355
+ **React:**
572
356
 
573
357
  ```tsx
574
- import { useConvaiClient, AudioRenderer } from "@convai/web-sdk/react";
575
- import { useState } from "react";
576
-
577
- function CustomChatUI() {
358
+ function AudioControls() {
578
359
  const convaiClient = useConvaiClient({
579
- apiKey: "your-api-key",
580
- characterId: "your-character-id",
360
+ /* config */
581
361
  });
582
362
 
583
- const { chatMessages, state } = convaiClient;
584
- const [inputValue, setInputValue] = useState("");
363
+ const handleMute = async () => {
364
+ await convaiClient.audioControls.muteAudio();
365
+ };
585
366
 
586
- const handleSend = () => {
587
- if (inputValue.trim() && state.isConnected) {
588
- convaiClient.sendUserTextMessage(inputValue);
589
- setInputValue("");
590
- }
367
+ const handleUnmute = async () => {
368
+ await convaiClient.audioControls.unmuteAudio();
369
+ };
370
+
371
+ const handleToggle = async () => {
372
+ await convaiClient.audioControls.toggleAudio();
591
373
  };
592
374
 
593
375
  return (
594
376
  <div>
595
- {/* CRITICAL: AudioRenderer for bot voice */}
596
- <AudioRenderer />
597
-
598
- {/* Chat Messages */}
599
- <div className="chat-container">
600
- {chatMessages.map((msg) => {
601
- const isUser = msg.type.includes("user");
602
- const displayMessage =
603
- msg.type === "user-llm-text" || msg.type === "bot-llm-text";
604
-
605
- if (!displayMessage) return null;
606
-
607
- return (
608
- <div
609
- key={msg.id}
610
- className={isUser ? "user-message" : "bot-message"}
611
- >
612
- <span className="sender">
613
- {isUser ? "You" : "Character"}
614
- </span>
615
- <p>{msg.content}</p>
616
- <span className="timestamp">
617
- {new Date(msg.timestamp).toLocaleTimeString()}
618
- </span>
619
- </div>
620
- );
621
- })}
622
- </div>
623
-
624
- {/* Input */}
625
- <div className="input-container">
626
- <input
627
- type="text"
628
- value={inputValue}
629
- onChange={(e) => setInputValue(e.target.value)}
630
- onKeyPress={(e) => e.key === "Enter" && handleSend()}
631
- placeholder="Type a message..."
632
- disabled={!state.isConnected}
633
- />
634
- <button onClick={handleSend} disabled={!state.isConnected}>
635
- Send
636
- </button>
637
- </div>
638
-
639
- {/* Status Indicator */}
640
- <div className="status">
641
- {state.isConnecting && "Connecting..."}
642
- {state.isConnected && state.agentState}
643
- {!state.isConnected && "Disconnected"}
644
- </div>
377
+ <button onClick={handleMute}>Mute</button>
378
+ <button onClick={handleUnmute}>Unmute</button>
379
+ <button onClick={handleToggle}>Toggle</button>
380
+ <p>Muted: {convaiClient.audioControls.isAudioMuted ? "Yes" : "No"}</p>
645
381
  </div>
646
382
  );
647
383
  }
648
384
  ```
649
385
 
650
- #### Vanilla Example
386
+ **Vanilla:**
651
387
 
652
388
  ```typescript
653
- import { ConvaiClient, AudioRenderer } from "@convai/web-sdk/vanilla";
654
-
655
- const client = new ConvaiClient({
656
- apiKey: "your-api-key",
657
- characterId: "your-character-id",
658
- });
659
-
660
- await client.connect();
661
-
662
- // CRITICAL: Create AudioRenderer for bot voice
663
- const audioRenderer = new AudioRenderer(client.room);
389
+ // Mute microphone
390
+ await client.audioControls.muteAudio();
664
391
 
665
- const chatContainer = document.getElementById("chat-container");
666
- const inputElement = document.getElementById("message-input");
667
- const sendButton = document.getElementById("send-button");
392
+ // Unmute microphone
393
+ await client.audioControls.unmuteAudio();
668
394
 
669
- // Render messages
670
- client.on("messagesChange", (messages) => {
671
- chatContainer.innerHTML = "";
395
+ // Toggle mute state
396
+ await client.audioControls.toggleAudio();
672
397
 
673
- messages.forEach((msg) => {
674
- const isUser = msg.type.includes("user");
675
- const displayMessage =
676
- msg.type === "user-llm-text" || msg.type === "bot-llm-text";
398
+ // Check mute state
399
+ const isMuted = client.audioControls.isAudioMuted;
677
400
 
678
- if (!displayMessage) return;
401
+ // Enable audio (request permissions if needed)
402
+ await client.audioControls.enableAudio();
679
403
 
680
- const messageDiv = document.createElement("div");
681
- messageDiv.className = isUser ? "user-message" : "bot-message";
404
+ // Disable audio
405
+ await client.audioControls.disableAudio();
406
+ ```
682
407
 
683
- const sender = document.createElement("span");
684
- sender.textContent = isUser ? "You" : "Character";
685
- sender.className = "sender";
408
+ **Audio Device Management:**
686
409
 
687
- const content = document.createElement("p");
688
- content.textContent = msg.content;
410
+ ```typescript
411
+ // Get available audio devices
412
+ const devices = await convaiClient.audioControls.getAudioDevices();
689
413
 
690
- const timestamp = document.createElement("span");
691
- timestamp.textContent = new Date(msg.timestamp).toLocaleTimeString();
692
- timestamp.className = "timestamp";
414
+ // Set specific audio device
415
+ await convaiClient.audioControls.setAudioDevice(deviceId);
693
416
 
694
- messageDiv.appendChild(sender);
695
- messageDiv.appendChild(content);
696
- messageDiv.appendChild(timestamp);
697
- chatContainer.appendChild(messageDiv);
698
- });
417
+ // Monitor audio level
418
+ convaiClient.audioControls.startAudioLevelMonitoring();
699
419
 
700
- // Auto-scroll
701
- chatContainer.scrollTop = chatContainer.scrollHeight;
420
+ convaiClient.audioControls.on("audioLevelChange", (level) => {
421
+ console.log("Audio level:", level);
422
+ // level is a number between 0 and 1
702
423
  });
703
424
 
704
- // Send message
705
- sendButton.addEventListener("click", () => {
706
- const text = inputElement.value.trim();
707
- if (text && client.state.isConnected) {
708
- client.sendUserTextMessage(text);
709
- inputElement.value = "";
710
- }
711
- });
425
+ convaiClient.audioControls.stopAudioLevelMonitoring();
426
+ ```
712
427
 
713
- inputElement.addEventListener("keypress", (e) => {
714
- if (e.key === "Enter") {
715
- sendButton.click();
716
- }
717
- });
428
+ **Audio State Monitoring:**
718
429
 
719
- // Cleanup
720
- // audioRenderer.destroy();
721
- // await client.disconnect();
430
+ ```typescript
431
+ // React
432
+ const { isAudioMuted } = convaiClient;
433
+
434
+ // Core API (event-based)
435
+ convaiClient.audioControls.on("audioStateChange", (state) => {
436
+ console.log("Audio enabled:", state.isAudioEnabled);
437
+ console.log("Audio muted:", state.isAudioMuted);
438
+ console.log("Audio level:", state.audioLevel);
439
+ });
722
440
  ```
723
441
 
724
- ---
442
+ ### Character TTS Mute/Unmute
725
443
 
726
- ### Audio Visualizer
444
+ Control whether the character's responses are spoken aloud (text-to-speech).
727
445
 
728
- Create real-time audio visualizers using the WebRTC room's audio tracks.
729
-
730
- #### React Example
446
+ **React:**
731
447
 
732
448
  ```tsx
733
- import { useConvaiClient } from "@convai/web-sdk/react";
734
- import { useEffect, useRef, useState } from "react";
735
-
736
- function AudioVisualizer() {
449
+ function TTSControls() {
737
450
  const convaiClient = useConvaiClient({
738
- apiKey: "your-api-key",
739
- characterId: "your-character-id",
451
+ /* config */
740
452
  });
741
453
 
742
- const canvasRef = useRef<HTMLCanvasElement>(null);
743
- const [audioLevel, setAudioLevel] = useState(0);
744
-
745
- useEffect(() => {
746
- if (!convaiClient.room) return;
747
-
748
- let animationId: number;
749
- let analyzer: AnalyserNode | null = null;
750
- let dataArray: Uint8Array | null = null;
751
-
752
- const setupAnalyzer = async () => {
753
- const audioContext = new AudioContext();
754
-
755
- // Get remote participant (bot)
756
- const remoteParticipants = Array.from(
757
- convaiClient.room.remoteParticipants.values()
758
- );
759
-
760
- if (remoteParticipants.length === 0) return;
761
-
762
- const participant = remoteParticipants[0];
763
- const audioTracks = Array.from(
764
- participant.audioTrackPublications.values()
765
- );
766
-
767
- if (audioTracks.length === 0) return;
768
-
769
- const audioTrack = audioTracks[0].track;
770
- if (!audioTrack) return;
454
+ const handleToggleTTS = (enabled: boolean) => {
455
+ convaiClient.toggleTts(enabled);
456
+ };
771
457
 
772
- // Get MediaStream from track
773
- const mediaStream = new MediaStream([audioTrack.mediaStreamTrack]);
458
+ return (
459
+ <div>
460
+ <button onClick={() => handleToggleTTS(true)}>Enable TTS</button>
461
+ <button onClick={() => handleToggleTTS(false)}>Disable TTS</button>
462
+ </div>
463
+ );
464
+ }
465
+ ```
774
466
 
775
- // Create analyzer
776
- const source = audioContext.createMediaStreamSource(mediaStream);
777
- analyzer = audioContext.createAnalyser();
778
- analyzer.fftSize = 256;
467
+ **Vanilla:**
779
468
 
780
- source.connect(analyzer);
781
- dataArray = new Uint8Array(analyzer.frequencyBinCount);
469
+ ```typescript
470
+ // Enable text-to-speech (character will speak responses)
471
+ client.toggleTts(true);
782
472
 
783
- // Animate
784
- const animate = () => {
785
- if (!analyzer || !dataArray) return;
473
+ // Disable text-to-speech (character will only send text, no audio)
474
+ client.toggleTts(false);
475
+ ```
786
476
 
787
- analyzer.getByteFrequencyData(dataArray);
477
+ **Initial TTS Configuration:**
788
478
 
789
- // Calculate average volume
790
- const sum = dataArray.reduce((a, b) => a + b, 0);
791
- const average = sum / dataArray.length;
792
- const normalizedLevel = average / 255;
479
+ ```typescript
480
+ // Set TTS state during connection
481
+ const client = new ConvaiClient({
482
+ apiKey: "your-api-key",
483
+ characterId: "your-character-id",
484
+ ttsEnabled: true, // Enable TTS by default
485
+ });
793
486
 
794
- setAudioLevel(normalizedLevel);
487
+ // Or disable initially
488
+ const client = new ConvaiClient({
489
+ apiKey: "your-api-key",
490
+ characterId: "your-character-id",
491
+ ttsEnabled: false, // Disable TTS
492
+ });
493
+ ```
795
494
 
796
- // Draw visualization
797
- drawVisualizer(dataArray);
495
+ ### Voice Mode Implementation
798
496
 
799
- animationId = requestAnimationFrame(animate);
800
- };
497
+ Voice mode allows users to speak instead of typing. The widget automatically handles voice mode, but you can implement it manually.
801
498
 
802
- animate();
803
- };
499
+ **React - Manual Voice Mode:**
804
500
 
805
- const drawVisualizer = (dataArray: Uint8Array) => {
806
- const canvas = canvasRef.current;
807
- if (!canvas) return;
501
+ ```tsx
502
+ import { useConvaiClient } from "@convai/web-sdk";
503
+ import { useState, useEffect } from "react";
808
504
 
809
- const ctx = canvas.getContext("2d");
810
- if (!ctx) return;
505
+ function CustomChatInterface() {
506
+ const convaiClient = useConvaiClient({
507
+ /* config */
508
+ });
509
+ const [isVoiceMode, setIsVoiceMode] = useState(false);
811
510
 
812
- const width = canvas.width;
813
- const height = canvas.height;
511
+ const enterVoiceMode = async () => {
512
+ // Interrupt any ongoing bot response
513
+ convaiClient.sendInterruptMessage();
814
514
 
815
- ctx.clearRect(0, 0, width, height);
515
+ // Unmute microphone
516
+ await convaiClient.audioControls.unmuteAudio();
816
517
 
817
- const barWidth = (width / dataArray.length) * 2.5;
818
- let x = 0;
518
+ setIsVoiceMode(true);
519
+ };
819
520
 
820
- for (let i = 0; i < dataArray.length; i++) {
821
- const barHeight = (dataArray[i] / 255) * height;
521
+ const exitVoiceMode = async () => {
522
+ // Interrupt any ongoing bot response
523
+ convaiClient.sendInterruptMessage();
822
524
 
823
- ctx.fillStyle = `rgb(${barHeight + 100}, 50, 150)`;
824
- ctx.fillRect(x, height - barHeight, barWidth, barHeight);
525
+ // Mute microphone
526
+ await convaiClient.audioControls.muteAudio();
825
527
 
826
- x += barWidth + 1;
827
- }
828
- };
528
+ setIsVoiceMode(false);
529
+ };
829
530
 
830
- if (convaiClient.state.isConnected) {
831
- setupAnalyzer();
531
+ // Monitor user transcription for voice input
532
+ useEffect(() => {
533
+ const transcription = convaiClient.userTranscription;
534
+ if (transcription && isVoiceMode) {
535
+ // Display real-time transcription
536
+ console.log("User is saying:", transcription);
832
537
  }
833
-
834
- return () => {
835
- if (animationId) cancelAnimationFrame(animationId);
836
- };
837
- }, [convaiClient.room, convaiClient.state.isConnected]);
538
+ }, [convaiClient.userTranscription, isVoiceMode]);
838
539
 
839
540
  return (
840
541
  <div>
841
- <canvas
842
- ref={canvasRef}
843
- width={800}
844
- height={200}
845
- style={{ border: "1px solid #ccc" }}
846
- />
847
- <div>Audio Level: {(audioLevel * 100).toFixed(0)}%</div>
848
- <div>
849
- Bot is {convaiClient.state.isSpeaking ? "speaking" : "silent"}
850
- </div>
542
+ {isVoiceMode ? (
543
+ <div>
544
+ <p>Listening: {convaiClient.userTranscription}</p>
545
+ <button onClick={exitVoiceMode}>Stop Voice Mode</button>
546
+ </div>
547
+ ) : (
548
+ <button onClick={enterVoiceMode}>Start Voice Mode</button>
549
+ )}
851
550
  </div>
852
551
  );
853
552
  }
854
553
  ```
855
554
 
856
- #### Vanilla Example
555
+ **Vanilla - Manual Voice Mode:**
857
556
 
858
557
  ```typescript
859
- import { ConvaiClient, AudioRenderer } from "@convai/web-sdk/vanilla";
558
+ let isVoiceMode = false;
860
559
 
861
- const client = new ConvaiClient({
862
- apiKey: "your-api-key",
863
- characterId: "your-character-id",
864
- });
865
-
866
- await client.connect();
867
-
868
- // CRITICAL: AudioRenderer for playback
869
- const audioRenderer = new AudioRenderer(client.room);
870
-
871
- const canvas = document.getElementById("visualizer") as HTMLCanvasElement;
872
- const ctx = canvas.getContext("2d")!;
873
-
874
- let analyzer: AnalyserNode | null = null;
875
- let dataArray: Uint8Array | null = null;
876
- let animationId: number;
877
-
878
- // Setup analyzer
879
- const audioContext = new AudioContext();
560
+ const enterVoiceMode = async () => {
561
+ // Interrupt any ongoing bot response
562
+ client.sendInterruptMessage();
880
563
 
881
- const remoteParticipants = Array.from(client.room.remoteParticipants.values());
882
- const participant = remoteParticipants[0];
883
- const audioTracks = Array.from(participant.audioTrackPublications.values());
884
- const audioTrack = audioTracks[0].track;
564
+ // Unmute microphone
565
+ await client.audioControls.unmuteAudio();
885
566
 
886
- const mediaStream = new MediaStream([audioTrack.mediaStreamTrack]);
887
- const source = audioContext.createMediaStreamSource(mediaStream);
567
+ isVoiceMode = true;
568
+ updateUI();
569
+ };
888
570
 
889
- analyzer = audioContext.createAnalyser();
890
- analyzer.fftSize = 256;
891
- source.connect(analyzer);
571
+ const exitVoiceMode = async () => {
572
+ // Interrupt any ongoing bot response
573
+ client.sendInterruptMessage();
892
574
 
893
- dataArray = new Uint8Array(analyzer.frequencyBinCount);
575
+ // Mute microphone
576
+ await client.audioControls.muteAudio();
894
577
 
895
- // Animate
896
- function animate() {
897
- if (!analyzer || !dataArray) return;
578
+ isVoiceMode = false;
579
+ updateUI();
580
+ };
898
581
 
899
- analyzer.getByteFrequencyData(dataArray);
900
-
901
- // Clear canvas
902
- ctx.clearRect(0, 0, canvas.width, canvas.height);
903
-
904
- const barWidth = (canvas.width / dataArray.length) * 2.5;
905
- let x = 0;
906
-
907
- for (let i = 0; i < dataArray.length; i++) {
908
- const barHeight = (dataArray[i] / 255) * canvas.height;
582
+ // Monitor user transcription
583
+ client.on("userTranscriptionChange", (transcription) => {
584
+ if (isVoiceMode && transcription) {
585
+ // Display real-time transcription
586
+ document.getElementById("transcription").textContent = transcription;
587
+ }
588
+ });
909
589
 
910
- ctx.fillStyle = `rgb(${barHeight + 100}, 50, 150)`;
911
- ctx.fillRect(x, canvas.height - barHeight, barWidth, barHeight);
590
+ function updateUI() {
591
+ const voiceButton = document.getElementById("voice-btn");
592
+ const transcriptionDiv = document.getElementById("transcription");
912
593
 
913
- x += barWidth + 1;
594
+ if (isVoiceMode) {
595
+ voiceButton.textContent = "Stop Voice Mode";
596
+ transcriptionDiv.style.display = "block";
597
+ } else {
598
+ voiceButton.textContent = "Start Voice Mode";
599
+ transcriptionDiv.style.display = "none";
914
600
  }
915
-
916
- animationId = requestAnimationFrame(animate);
917
601
  }
918
-
919
- animate();
920
-
921
- // Cleanup
922
- // cancelAnimationFrame(animationId);
923
- // audioRenderer.destroy();
924
- // await client.disconnect();
925
602
  ```
926
603
 
927
- ---
928
-
929
- ### Message Types
930
-
931
- All messages from `convaiClient.chatMessages` have a `type` field:
604
+ **Voice Mode with State Monitoring:**
932
605
 
933
606
  ```typescript
934
- type ChatMessageType =
935
- | "user" // User's sent message (raw)
936
- | "user-transcription" // Real-time speech-to-text from user
937
- | "user-llm-text" // User text processed by LLM (final)
938
- | "convai" // Character's response (raw)
939
- | "bot-llm-text" // Character's LLM-generated text (final)
940
- | "bot-emotion" // Character's emotional state
941
- | "emotion" // Generic emotion
942
- | "behavior-tree" // Behavior tree response
943
- | "action" // Action execution
944
- | "interrupt-bot"; // Interrupt message
607
+ // Monitor agent state to handle voice mode transitions
608
+ convaiClient.on("stateChange", (state) => {
609
+ if (isVoiceMode) {
610
+ switch (state.agentState) {
611
+ case "listening":
612
+ // User can speak
613
+ console.log("Bot is listening");
614
+ break;
615
+ case "thinking":
616
+ // Bot is processing
617
+ console.log("Bot is thinking");
618
+ break;
619
+ case "speaking":
620
+ // Bot is responding
621
+ console.log("Bot is speaking");
622
+ // Optionally interrupt if user wants to speak
623
+ break;
624
+ }
625
+ }
626
+ });
945
627
  ```
946
628
 
947
- **For Chat UIs, filter to:**
629
+ ### Connection Management
630
+
631
+ **Connect:**
948
632
 
949
633
  ```typescript
950
- const displayMessages = chatMessages.filter(
951
- (msg) => msg.type === "user-llm-text" || msg.type === "bot-llm-text"
952
- );
953
- ```
634
+ // React - config passed to hook
635
+ const convaiClient = useConvaiClient({
636
+ apiKey: "your-api-key",
637
+ characterId: "your-character-id",
638
+ });
954
639
 
955
- ---
640
+ // Or connect manually
641
+ await convaiClient.connect({
642
+ apiKey: "your-api-key",
643
+ characterId: "your-character-id",
644
+ });
956
645
 
957
- ## API Reference
646
+ // Vanilla
647
+ const client = new ConvaiClient();
648
+ await client.connect({
649
+ apiKey: "your-api-key",
650
+ characterId: "your-character-id",
651
+ });
652
+ ```
958
653
 
959
- ### Configuration
654
+ **Disconnect:**
960
655
 
961
656
  ```typescript
962
- interface ConvaiConfig {
963
- /** Your Convai API key from convai.com dashboard (required) */
964
- apiKey: string;
965
-
966
- /** The Character ID to connect to (required) */
967
- characterId: string;
968
-
969
- /**
970
- * End user identifier for speaker management (optional).
971
- * When provided: enables long-term memory and analytics
972
- * When not provided: anonymous mode, no persistent memory
973
- */
974
- endUserId?: string;
657
+ await convaiClient.disconnect();
658
+ ```
975
659
 
976
- /** Custom Convai API URL (optional) */
977
- url?: string;
660
+ **Reconnect:**
978
661
 
979
- /**
980
- * Enable video capability (default: false).
981
- * If true, connection_type will be "video" (supports audio, video, screenshare).
982
- * If false, connection_type will be "audio" (audio only).
983
- * ⚠️ REQUIRED for video and screen share features.
984
- */
985
- enableVideo?: boolean;
662
+ ```typescript
663
+ await convaiClient.reconnect();
664
+ ```
986
665
 
987
- /**
988
- * Start with video camera on when connecting (default: false).
989
- * Only works if enableVideo is true.
990
- */
991
- startWithVideoOn?: boolean;
666
+ **Reset Session:**
992
667
 
993
- /**
994
- * Start with microphone on when connecting (default: false).
995
- * If false, microphone stays off until user enables it.
996
- */
997
- startWithAudioOn?: boolean;
998
-
999
- /**
1000
- * Enable text-to-speech audio generation (default: true).
1001
- */
1002
- ttsEnabled?: boolean;
1003
- }
668
+ ```typescript
669
+ // Clear conversation history and start new session
670
+ convaiClient.resetSession();
1004
671
  ```
1005
672
 
1006
- ### Connection Management
673
+ **Connection State:**
1007
674
 
1008
675
  ```typescript
1009
- // Connect
1010
- await convaiClient.connect({
1011
- apiKey: "your-api-key",
1012
- characterId: "your-character-id",
1013
- enableVideo: true,
676
+ // React
677
+ const { state } = convaiClient;
678
+ console.log("Connected:", state.isConnected);
679
+ console.log("Connecting:", state.isConnecting);
680
+ console.log("Agent state:", state.agentState); // 'disconnected' | 'connected' | 'listening' | 'thinking' | 'speaking'
681
+
682
+ // Core API (event-based)
683
+ convaiClient.on("stateChange", (state) => {
684
+ console.log("State changed:", state);
1014
685
  });
1015
686
 
1016
- // Disconnect
1017
- await convaiClient.disconnect();
1018
-
1019
- // Reconnect
1020
- await convaiClient.reconnect();
1021
-
1022
- // Reset session (clear conversation history)
1023
- convaiClient.resetSession();
687
+ convaiClient.on("connect", () => {
688
+ console.log("Connected");
689
+ });
1024
690
 
1025
- // Check connection state
1026
- console.log(convaiClient.state.isConnected);
1027
- console.log(convaiClient.state.isConnecting);
1028
- console.log(convaiClient.state.agentState); // 'disconnected' | 'connected' | 'listening' | 'thinking' | 'speaking'
1029
- console.log(convaiClient.isBotReady); // Bot ready to receive messages
691
+ convaiClient.on("disconnect", () => {
692
+ console.log("Disconnected");
693
+ });
1030
694
  ```
1031
695
 
1032
696
  ### Messaging
1033
697
 
698
+ **Send Text Message:**
699
+
1034
700
  ```typescript
1035
- // Send text message
1036
701
  convaiClient.sendUserTextMessage("Hello, how are you?");
702
+ ```
1037
703
 
1038
- // Send trigger message (invoke character action)
704
+ **Send Trigger Message:**
705
+
706
+ ```typescript
707
+ // Trigger specific character action
1039
708
  convaiClient.sendTriggerMessage("greet", "User entered the room");
1040
709
 
1041
- // Interrupt character's current response
1042
- convaiClient.sendInterruptMessage();
710
+ // Trigger without message
711
+ convaiClient.sendTriggerMessage("wave");
712
+ ```
1043
713
 
1044
- // Update context
1045
- convaiClient.updateTemplateKeys({ user_name: "John" });
1046
- convaiClient.updateDynamicInfo({ text: "User is browsing products" });
714
+ **Update Context:**
1047
715
 
1048
- // Access messages
1049
- console.log(convaiClient.chatMessages);
716
+ ```typescript
717
+ // Update template keys (e.g., user name, location)
718
+ convaiClient.updateTemplateKeys({
719
+ user_name: "John",
720
+ location: "New York",
721
+ });
1050
722
 
1051
- // Access real-time user transcription
1052
- console.log(convaiClient.userTranscription);
723
+ // Update dynamic information
724
+ convaiClient.updateDynamicInfo({
725
+ text: "User is currently browsing the products page",
726
+ });
1053
727
  ```
1054
728
 
1055
- ### Audio Controls
729
+ **Message History:**
1056
730
 
1057
731
  ```typescript
1058
- // Mute/unmute microphone
1059
- await convaiClient.audioControls.muteAudio();
1060
- await convaiClient.audioControls.unmuteAudio();
1061
- await convaiClient.audioControls.toggleAudio();
732
+ // React
733
+ const { chatMessages } = convaiClient;
1062
734
 
1063
- // Check mute state
1064
- console.log(convaiClient.isAudioMuted);
735
+ // Core API (event-based)
736
+ convaiClient.on("message", (message: ChatMessage) => {
737
+ console.log("New message:", message.content);
738
+ console.log("Message type:", message.type);
739
+ });
1065
740
 
1066
- // Get available microphones
1067
- const devices = await convaiClient.audioControls.getAudioDevices();
741
+ convaiClient.on("messagesChange", (messages: ChatMessage[]) => {
742
+ console.log("All messages:", messages);
743
+ });
744
+ ```
1068
745
 
1069
- // Set microphone
1070
- await convaiClient.audioControls.setAudioDevice(deviceId);
746
+ **Message Types:**
1071
747
 
1072
- // Monitor audio level
1073
- convaiClient.audioControls.startAudioLevelMonitoring();
1074
- convaiClient.audioControls.on("audioLevelChange", (level) => {
1075
- console.log("Audio level:", level); // 0 to 1
1076
- });
1077
- convaiClient.audioControls.stopAudioLevelMonitoring();
748
+ ```typescript
749
+ type ChatMessageType =
750
+ | "user" // User's sent message
751
+ | "convai" // Character's response
752
+ | "user-transcription" // Real-time speech-to-text from user
753
+ | "bot-llm-text" // Character's LLM-generated text
754
+ | "emotion" // Character's emotional state
755
+ | "behavior-tree" // Behavior tree response
756
+ | "action" // Action execution
757
+ | "bot-emotion" // Bot emotional response
758
+ | "user-llm-text" // User text processed by LLM
759
+ | "interrupt-bot"; // Interrupt message
1078
760
  ```
1079
761
 
1080
- ### Video Controls
762
+ ### State Monitoring
1081
763
 
1082
- **⚠️ Requires `enableVideo: true` in config.**
764
+ **Agent State:**
1083
765
 
1084
766
  ```typescript
1085
- // Enable/disable camera
1086
- await convaiClient.videoControls.enableVideo();
1087
- await convaiClient.videoControls.disableVideo();
1088
- await convaiClient.videoControls.toggleVideo();
767
+ // React
768
+ const { state } = convaiClient;
1089
769
 
1090
- // Check video state
1091
- console.log(convaiClient.isVideoEnabled);
770
+ // Check specific states
771
+ if (state.isListening) {
772
+ console.log("Bot is listening");
773
+ }
1092
774
 
1093
- // Set video quality
1094
- await convaiClient.videoControls.setVideoQuality("high"); // 'low' | 'medium' | 'high'
775
+ if (state.isThinking) {
776
+ console.log("Bot is thinking");
777
+ }
1095
778
 
1096
- // Get available cameras
1097
- const devices = await convaiClient.videoControls.getVideoDevices();
779
+ if (state.isSpeaking) {
780
+ console.log("Bot is speaking");
781
+ }
1098
782
 
1099
- // Switch camera
1100
- await convaiClient.videoControls.setVideoDevice(deviceId);
783
+ // Combined state
784
+ console.log(state.agentState); // 'disconnected' | 'connected' | 'listening' | 'thinking' | 'speaking'
1101
785
  ```
1102
786
 
1103
- ### Screen Share Controls
1104
-
1105
- **⚠️ Requires `enableVideo: true` in config.**
787
+ **User Transcription:**
1106
788
 
1107
789
  ```typescript
1108
- // Start/stop screen share
1109
- await convaiClient.screenShareControls.enableScreenShare();
1110
- await convaiClient.screenShareControls.enableScreenShareWithAudio();
1111
- await convaiClient.screenShareControls.disableScreenShare();
1112
- await convaiClient.screenShareControls.toggleScreenShare();
790
+ // React
791
+ const { userTranscription } = convaiClient;
1113
792
 
1114
- // Check screen share state
1115
- console.log(convaiClient.isScreenShareActive);
793
+ // Core API (event-based)
794
+ convaiClient.on("userTranscriptionChange", (transcription: string) => {
795
+ console.log("User is saying:", transcription);
796
+ });
1116
797
  ```
1117
798
 
1118
- ---
799
+ **Bot Ready State:**
800
+
801
+ ```typescript
802
+ // React
803
+ const { isBotReady } = convaiClient;
804
+
805
+ // Core API (event-based)
806
+ convaiClient.on("botReady", () => {
807
+ console.log("Bot is ready to receive messages");
808
+ });
809
+ ```
1119
810
 
1120
- ## Getting Credentials
811
+ ## Getting Convai Credentials
1121
812
 
1122
813
  1. Visit [convai.com](https://convai.com) and create an account
1123
814
  2. Navigate to your dashboard
@@ -1125,71 +816,41 @@ console.log(convaiClient.isScreenShareActive);
1125
816
  4. Copy your **API Key** from the dashboard
1126
817
  5. Copy your **Character ID** from the character details
1127
818
 
1128
- ---
1129
-
1130
- ## TypeScript Support
1131
-
1132
- All exports are fully typed:
1133
-
1134
- **React:**
819
+ ## Import Paths
1135
820
 
1136
821
  ```typescript
1137
- import type {
1138
- // Configuration
1139
- ConvaiConfig,
1140
-
1141
- // State
1142
- ConvaiClientState,
822
+ // Default: React version (backward compatible)
823
+ import { useConvaiClient, ConvaiWidget } from "@convai/web-sdk";
1143
824
 
1144
- // Messages
1145
- ChatMessage,
1146
- ChatMessageType,
825
+ // Explicit React import
826
+ import { useConvaiClient, ConvaiWidget } from "@convai/web-sdk/react";
1147
827
 
1148
- // Client
1149
- IConvaiClient,
1150
- ConvaiClient,
828
+ // Vanilla JS/TS
829
+ import { ConvaiClient, createConvaiWidget } from "@convai/web-sdk/vanilla";
1151
830
 
1152
- // Controls
1153
- AudioControls,
1154
- VideoControls,
1155
- ScreenShareControls,
1156
- } from "@convai/web-sdk/react";
831
+ // Core only (no UI, framework agnostic)
832
+ import { ConvaiClient } from "@convai/web-sdk/core";
1157
833
  ```
1158
834
 
1159
- **Vanilla:**
835
+ ## TypeScript Support
836
+
837
+ All exports are fully typed:
1160
838
 
1161
839
  ```typescript
1162
840
  import type {
1163
- // Configuration
841
+ ConvaiClient,
1164
842
  ConvaiConfig,
1165
-
1166
- // State
1167
843
  ConvaiClientState,
1168
-
1169
- // Messages
1170
844
  ChatMessage,
1171
- ChatMessageType,
1172
-
1173
- // Client
1174
- IConvaiClient,
1175
- ConvaiClient,
1176
-
1177
- // Controls
1178
845
  AudioControls,
1179
846
  VideoControls,
1180
847
  ScreenShareControls,
1181
-
1182
- // Widget
1183
- VanillaWidget,
1184
- VanillaWidgetOptions,
1185
- } from "@convai/web-sdk/vanilla";
848
+ IConvaiClient,
849
+ } from "@convai/web-sdk";
1186
850
  ```
1187
851
 
1188
- ---
1189
-
1190
852
  ## Support
1191
853
 
1192
- - **Documentation**: [API Reference](./API_REFERENCE.md)
1193
- - **Forum**: [Convai Forum](https://forum.convai.com)
1194
- - **Website**: [convai.com](https://convai.com)
1195
- - **Issues**: [GitHub Issues](https://github.com/convai/web-sdk/issues)
854
+ - [Convai Forum](https://forum.convai.com)
855
+ - [API Reference](./API_REFERENCE.md)
856
+ - [Convai Website](https://convai.com)