@convai/web-sdk 0.2.3-beta.1 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +868 -529
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -1,6 +1,47 @@
|
|
|
1
1
|
# @convai/web-sdk
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
[](https://www.npmjs.com/package/@convai/web-sdk)
|
|
4
|
+
|
|
5
|
+
JavaScript/TypeScript SDK for building AI voice assistants with real-time audio/video streaming. Drop-in widgets for **React** and **Vanilla JavaScript/TypeScript** with customizable UI components.
|
|
6
|
+
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
## 📑 Table of Contents
|
|
10
|
+
|
|
11
|
+
- [Installation](#installation)
|
|
12
|
+
- [Quick Start](#quick-start)
|
|
13
|
+
- [React - ConvaiWidget](#react---convaiwidget)
|
|
14
|
+
- [Vanilla JS/TS - ConvaiWidget](#vanilla-jsts---convaiwidget)
|
|
15
|
+
- [Core Concepts](#core-concepts)
|
|
16
|
+
- [React SDK](#react-sdk)
|
|
17
|
+
- [useConvaiClient Hook](#useconvaiclient-hook)
|
|
18
|
+
- [AudioRenderer Component](#audiorenderer-component)
|
|
19
|
+
- [AudioContext](#audiocontext)
|
|
20
|
+
- [React Exports Reference](#react-exports-reference)
|
|
21
|
+
- [Vanilla SDK](#vanilla-sdk)
|
|
22
|
+
- [ConvaiClient Class](#convaiclient-class)
|
|
23
|
+
- [AudioRenderer Class](#audiorenderer-class)
|
|
24
|
+
- [Vanilla Exports Reference](#vanilla-exports-reference)
|
|
25
|
+
- [Video & Screen Share](#video--screen-share)
|
|
26
|
+
- [Critical Requirements](#critical-requirements)
|
|
27
|
+
- [Enabling Video](#enabling-video)
|
|
28
|
+
- [Enabling Screen Share](#enabling-screen-share)
|
|
29
|
+
- [Building Custom UIs](#building-custom-uis)
|
|
30
|
+
- [Custom Chat Interface](#custom-chat-interface)
|
|
31
|
+
- [Audio Visualizer](#audio-visualizer)
|
|
32
|
+
- [Message Types](#message-types)
|
|
33
|
+
- [API Reference](#api-reference)
|
|
34
|
+
- [Configuration](#configuration)
|
|
35
|
+
- [Connection Management](#connection-management)
|
|
36
|
+
- [Messaging](#messaging)
|
|
37
|
+
- [Audio Controls](#audio-controls)
|
|
38
|
+
- [Video Controls](#video-controls)
|
|
39
|
+
- [Screen Share Controls](#screen-share-controls)
|
|
40
|
+
- [Getting Credentials](#getting-credentials)
|
|
41
|
+
- [TypeScript Support](#typescript-support)
|
|
42
|
+
- [Support](#support)
|
|
43
|
+
|
|
44
|
+
---
|
|
4
45
|
|
|
5
46
|
## Installation
|
|
6
47
|
|
|
@@ -8,14 +49,25 @@ JavaScript/TypeScript SDK for Convai AI voice assistants. Build voice-powered AI
|
|
|
8
49
|
npm install @convai/web-sdk
|
|
9
50
|
```
|
|
10
51
|
|
|
11
|
-
|
|
52
|
+
**Peer Dependencies (React only):**
|
|
12
53
|
|
|
13
|
-
|
|
54
|
+
```bash
|
|
55
|
+
npm install react@^18.0.0 react-dom@^18.0.0
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
---
|
|
59
|
+
|
|
60
|
+
## Quick Start
|
|
61
|
+
|
|
62
|
+
### React - ConvaiWidget
|
|
63
|
+
|
|
64
|
+
The `ConvaiWidget` is a complete, pre-built chat interface with voice/video capabilities.
|
|
14
65
|
|
|
15
66
|
```tsx
|
|
16
|
-
import { useConvaiClient, ConvaiWidget } from "@convai/web-sdk";
|
|
67
|
+
import { useConvaiClient, ConvaiWidget } from "@convai/web-sdk/react";
|
|
17
68
|
|
|
18
69
|
function App() {
|
|
70
|
+
// Initialize the Convai client
|
|
19
71
|
const convaiClient = useConvaiClient({
|
|
20
72
|
apiKey: "your-api-key",
|
|
21
73
|
characterId: "your-character-id",
|
|
@@ -25,7 +77,9 @@ function App() {
|
|
|
25
77
|
}
|
|
26
78
|
```
|
|
27
79
|
|
|
28
|
-
|
|
80
|
+
**That's it!** The widget auto-connects on first user interaction and handles all UI/audio for you.
|
|
81
|
+
|
|
82
|
+
### Vanilla JS/TS - ConvaiWidget
|
|
29
83
|
|
|
30
84
|
```typescript
|
|
31
85
|
import { ConvaiClient, createConvaiWidget } from "@convai/web-sdk/vanilla";
|
|
@@ -36,7 +90,7 @@ const client = new ConvaiClient({
|
|
|
36
90
|
characterId: "your-character-id",
|
|
37
91
|
});
|
|
38
92
|
|
|
39
|
-
// Create widget - auto-connects on first user click
|
|
93
|
+
// Create and mount widget - auto-connects on first user click
|
|
40
94
|
const widget = createConvaiWidget(document.body, {
|
|
41
95
|
convaiClient: client,
|
|
42
96
|
});
|
|
@@ -45,770 +99,1025 @@ const widget = createConvaiWidget(document.body, {
|
|
|
45
99
|
widget.destroy();
|
|
46
100
|
```
|
|
47
101
|
|
|
48
|
-
|
|
102
|
+
---
|
|
49
103
|
|
|
50
|
-
|
|
104
|
+
## Core Concepts
|
|
51
105
|
|
|
52
|
-
|
|
106
|
+
### The Architecture
|
|
53
107
|
|
|
54
|
-
|
|
108
|
+
```
|
|
109
|
+
┌─────────────────────────────────────────────────┐
|
|
110
|
+
│ ConvaiWidget (UI Layer) │
|
|
111
|
+
│ ├─ Chat Interface │
|
|
112
|
+
│ ├─ Voice Mode │
|
|
113
|
+
│ └─ Video/Screen Share UI │
|
|
114
|
+
└─────────────────────────────────────────────────┘
|
|
115
|
+
▼
|
|
116
|
+
┌─────────────────────────────────────────────────┐
|
|
117
|
+
│ ConvaiClient (Core Logic) │
|
|
118
|
+
│ ├─ Connection Management │
|
|
119
|
+
│ ├─ Message Handling │
|
|
120
|
+
│ ├─ State Management │
|
|
121
|
+
│ └─ Audio/Video Controls │
|
|
122
|
+
└─────────────────────────────────────────────────┘
|
|
123
|
+
▼
|
|
124
|
+
┌─────────────────────────────────────────────────┐
|
|
125
|
+
│ WebRTC Room (Communication Layer) │
|
|
126
|
+
│ ├─ Real-time Audio/Video Streaming │
|
|
127
|
+
│ ├─ Track Management │
|
|
128
|
+
│ └─ Network Communication │
|
|
129
|
+
└─────────────────────────────────────────────────┘
|
|
130
|
+
▼
|
|
131
|
+
┌─────────────────────────────────────────────────┐
|
|
132
|
+
│ AudioRenderer (Critical for Playback) │
|
|
133
|
+
│ ├─ Attaches audio tracks to DOM │
|
|
134
|
+
│ ├─ Manages audio elements │
|
|
135
|
+
│ └─ Enables bot voice playback │
|
|
136
|
+
└─────────────────────────────────────────────────┘
|
|
137
|
+
```
|
|
55
138
|
|
|
56
|
-
|
|
139
|
+
### Key Principles
|
|
57
140
|
|
|
58
|
-
|
|
59
|
-
-
|
|
60
|
-
|
|
141
|
+
1. **ConvaiClient** - The brain. Manages connection, state, and communication with Convai servers.
|
|
142
|
+
2. **AudioRenderer** - **CRITICAL**: Without this, you won't hear the bot. It renders audio to the user's speakers.
|
|
143
|
+
3. **ConvaiWidget** - The complete UI. Uses both ConvaiClient and AudioRenderer internally.
|
|
144
|
+
4. **Connection Type** - Determines capabilities:
|
|
145
|
+
- `"audio"` (default) - Audio only
|
|
146
|
+
- `"video"` - Audio + Video + Screen Share
|
|
61
147
|
|
|
62
|
-
|
|
148
|
+
---
|
|
63
149
|
|
|
64
|
-
|
|
150
|
+
## React SDK
|
|
65
151
|
|
|
66
|
-
|
|
152
|
+
### useConvaiClient Hook
|
|
67
153
|
|
|
68
|
-
|
|
69
|
-
- `ConvaiClientState` - Client state interface
|
|
70
|
-
- `ChatMessage` - Message interface
|
|
71
|
-
- `IConvaiClient` - Client interface
|
|
72
|
-
- `AudioControls` - Audio control interface
|
|
73
|
-
- `VideoControls` - Video control interface
|
|
74
|
-
- `ScreenShareControls` - Screen share control interface
|
|
154
|
+
**Purpose**: Returns a fully configured `ConvaiClient` instance with reactive state updates.
|
|
75
155
|
|
|
76
|
-
**
|
|
156
|
+
**When to Use**: Every React app using Convai needs this hook.
|
|
77
157
|
|
|
78
|
-
|
|
79
|
-
- `AudioContext` - Audio context provider
|
|
158
|
+
**What It Does**:
|
|
80
159
|
|
|
81
|
-
|
|
160
|
+
- Creates and manages a ConvaiClient instance
|
|
161
|
+
- Provides reactive state (connection, messages, activity)
|
|
162
|
+
- Handles connection lifecycle
|
|
163
|
+
- Exposes audio/video/screen share controls
|
|
164
|
+
|
|
165
|
+
```tsx
|
|
166
|
+
import { useConvaiClient } from "@convai/web-sdk/react";
|
|
82
167
|
|
|
83
|
-
|
|
168
|
+
function ChatbotWrapper() {
|
|
169
|
+
const convaiClient = useConvaiClient({
|
|
170
|
+
apiKey: "your-api-key",
|
|
171
|
+
characterId: "your-character-id",
|
|
172
|
+
enableVideo: false, // Default: audio only
|
|
173
|
+
startWithAudioOn: false, // Mic starts muted
|
|
174
|
+
});
|
|
84
175
|
|
|
85
|
-
|
|
86
|
-
|
|
176
|
+
// Access reactive state
|
|
177
|
+
const { state, chatMessages, userTranscription, isBotReady } = convaiClient;
|
|
87
178
|
|
|
88
|
-
|
|
179
|
+
// Use controls
|
|
180
|
+
const handleMute = () => convaiClient.audioControls.muteAudio();
|
|
181
|
+
const handleSend = () =>
|
|
182
|
+
convaiClient.sendUserTextMessage("Hello, character!");
|
|
89
183
|
|
|
90
|
-
|
|
91
|
-
|
|
184
|
+
return (
|
|
185
|
+
<div>
|
|
186
|
+
<p>Status: {state.agentState}</p>
|
|
187
|
+
<p>Messages: {chatMessages.length}</p>
|
|
188
|
+
<button onClick={handleMute}>Mute</button>
|
|
189
|
+
<button onClick={handleSend}>Send</button>
|
|
190
|
+
</div>
|
|
191
|
+
);
|
|
192
|
+
}
|
|
193
|
+
```
|
|
92
194
|
|
|
93
|
-
|
|
195
|
+
### AudioRenderer Component
|
|
94
196
|
|
|
95
|
-
|
|
96
|
-
- `VanillaWidgetOptions` - Widget options interface
|
|
97
|
-
- `IConvaiClient` - Client interface
|
|
98
|
-
- `ConvaiConfig` - Configuration interface
|
|
99
|
-
- `ConvaiClientState` - Client state interface
|
|
100
|
-
- `ChatMessage` - Message interface
|
|
197
|
+
**Purpose**: Renders remote audio tracks to the user's speakers.
|
|
101
198
|
|
|
102
|
-
|
|
199
|
+
**⚠️ CRITICAL**: Without `AudioRenderer`, you will NOT hear the bot's voice.
|
|
103
200
|
|
|
104
|
-
**
|
|
201
|
+
**When to Use**:
|
|
105
202
|
|
|
106
|
-
-
|
|
107
|
-
- `
|
|
108
|
-
- `VideoManager` - Video management
|
|
109
|
-
- `ScreenShareManager` - Screen share management
|
|
110
|
-
- `MessageHandler` - Message handling
|
|
111
|
-
- `EventEmitter` - Event emitter base class
|
|
203
|
+
- Always when building custom UIs
|
|
204
|
+
- Already included in `ConvaiWidget` (no need to add separately)
|
|
112
205
|
|
|
113
|
-
**
|
|
206
|
+
**How It Works**:
|
|
114
207
|
|
|
115
|
-
-
|
|
116
|
-
-
|
|
208
|
+
- Attaches to the WebRTC room
|
|
209
|
+
- Automatically creates `<audio>` elements for remote participants (the bot)
|
|
210
|
+
- Manages audio playback lifecycle
|
|
117
211
|
|
|
118
|
-
|
|
212
|
+
```tsx
|
|
213
|
+
import { useConvaiClient, AudioRenderer } from "@convai/web-sdk/react";
|
|
119
214
|
|
|
120
|
-
|
|
215
|
+
function CustomChatUI() {
|
|
216
|
+
const convaiClient = useConvaiClient({
|
|
217
|
+
apiKey: "your-api-key",
|
|
218
|
+
characterId: "your-character-id",
|
|
219
|
+
});
|
|
220
|
+
|
|
221
|
+
return (
|
|
222
|
+
<div>
|
|
223
|
+
{/* CRITICAL: This component renders bot audio to speakers */}
|
|
224
|
+
<AudioRenderer />
|
|
225
|
+
|
|
226
|
+
{/* Your custom UI */}
|
|
227
|
+
<div>
|
|
228
|
+
{convaiClient.chatMessages.map((msg) => (
|
|
229
|
+
<div key={msg.id}>{msg.content}</div>
|
|
230
|
+
))}
|
|
231
|
+
</div>
|
|
232
|
+
</div>
|
|
233
|
+
);
|
|
234
|
+
}
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
### AudioContext
|
|
238
|
+
|
|
239
|
+
**Purpose**: Provides the WebRTC Room to child components.
|
|
240
|
+
|
|
241
|
+
**When to Use**: When building deeply nested custom UIs that need direct access to the audio room.
|
|
242
|
+
|
|
243
|
+
**How It Works**: React Context that holds the active WebRTC room.
|
|
121
244
|
|
|
122
245
|
```tsx
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
246
|
+
import { useConvaiClient, AudioRenderer, AudioContext } from "@convai/web-sdk/react";
|
|
247
|
+
import { useContext } from "react";
|
|
248
|
+
|
|
249
|
+
function ChatbotWrapper() {
|
|
250
|
+
const convaiClient = useConvaiClient({
|
|
251
|
+
/* config */
|
|
252
|
+
});
|
|
253
|
+
|
|
254
|
+
return (
|
|
255
|
+
<AudioContext.Provider value={convaiClient.room}>
|
|
256
|
+
<AudioRenderer />
|
|
257
|
+
<ChildComponent />
|
|
258
|
+
</AudioContext.Provider>
|
|
259
|
+
);
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
function ChildComponent() {
|
|
263
|
+
const room = useContext(AudioContext);
|
|
264
|
+
// Access WebRTC room directly
|
|
265
|
+
console.log("Room state:", room?.state);
|
|
266
|
+
return <div>Child has access to Room</div>;
|
|
135
267
|
}
|
|
136
268
|
```
|
|
137
269
|
|
|
138
|
-
###
|
|
270
|
+
### React Exports Reference
|
|
271
|
+
|
|
272
|
+
```tsx
|
|
273
|
+
// Components
|
|
274
|
+
import { ConvaiWidget } from "@convai/web-sdk/react";
|
|
275
|
+
|
|
276
|
+
// Hooks
|
|
277
|
+
import { useConvaiClient, useCharacterInfo } from "@convai/web-sdk/react";
|
|
278
|
+
|
|
279
|
+
// Audio Rendering (Critical)
|
|
280
|
+
import { AudioRenderer, AudioContext } from "@convai/web-sdk/react";
|
|
281
|
+
|
|
282
|
+
// Core Client (for advanced usage)
|
|
283
|
+
import { ConvaiClient } from "@convai/web-sdk/react";
|
|
284
|
+
|
|
285
|
+
// Types
|
|
286
|
+
import type {
|
|
287
|
+
ConvaiConfig,
|
|
288
|
+
ConvaiClientState,
|
|
289
|
+
ChatMessage,
|
|
290
|
+
IConvaiClient,
|
|
291
|
+
AudioControls,
|
|
292
|
+
VideoControls,
|
|
293
|
+
ScreenShareControls,
|
|
294
|
+
} from "@convai/web-sdk/react";
|
|
295
|
+
```
|
|
296
|
+
|
|
297
|
+
---
|
|
298
|
+
|
|
299
|
+
## Vanilla SDK
|
|
300
|
+
|
|
301
|
+
### ConvaiClient Class
|
|
302
|
+
|
|
303
|
+
**Purpose**: Core client for managing Convai connections in vanilla JavaScript/TypeScript.
|
|
304
|
+
|
|
305
|
+
**When to Use**: Any non-React application or when you need full control.
|
|
306
|
+
|
|
307
|
+
**What It Provides**:
|
|
308
|
+
|
|
309
|
+
- Connection management
|
|
310
|
+
- Message handling
|
|
311
|
+
- State management (via events)
|
|
312
|
+
- Audio/video/screen share controls
|
|
139
313
|
|
|
140
314
|
```typescript
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
315
|
+
import { ConvaiClient } from "@convai/web-sdk/vanilla";
|
|
316
|
+
|
|
317
|
+
const client = new ConvaiClient({
|
|
318
|
+
apiKey: "your-api-key",
|
|
319
|
+
characterId: "your-character-id",
|
|
320
|
+
});
|
|
321
|
+
|
|
322
|
+
// Connect
|
|
323
|
+
await client.connect();
|
|
324
|
+
|
|
325
|
+
// Listen to events
|
|
326
|
+
client.on("stateChange", (state) => {
|
|
327
|
+
console.log("Agent state:", state.agentState);
|
|
328
|
+
});
|
|
329
|
+
|
|
330
|
+
client.on("message", (message) => {
|
|
331
|
+
console.log("New message:", message.content);
|
|
332
|
+
});
|
|
333
|
+
|
|
334
|
+
// Send messages
|
|
335
|
+
client.sendUserTextMessage("Hello!");
|
|
336
|
+
|
|
337
|
+
// Control audio
|
|
338
|
+
await client.audioControls.muteAudio();
|
|
339
|
+
await client.audioControls.unmuteAudio();
|
|
340
|
+
|
|
341
|
+
// Disconnect
|
|
342
|
+
await client.disconnect();
|
|
152
343
|
```
|
|
153
344
|
|
|
154
|
-
###
|
|
345
|
+
### AudioRenderer Class
|
|
346
|
+
|
|
347
|
+
**Purpose**: Manages audio playback for vanilla JavaScript/TypeScript applications.
|
|
348
|
+
|
|
349
|
+
**⚠️ CRITICAL**: Without this, you will NOT hear the bot's voice.
|
|
350
|
+
|
|
351
|
+
**When to Use**:
|
|
352
|
+
|
|
353
|
+
- Always when building custom vanilla UIs
|
|
354
|
+
- Already included in vanilla `ConvaiWidget` (no need to add separately)
|
|
355
|
+
|
|
356
|
+
**How It Works**:
|
|
357
|
+
|
|
358
|
+
- Attaches to the WebRTC room
|
|
359
|
+
- Automatically creates hidden `<audio>` elements
|
|
360
|
+
- Manages audio playback for remote participants (the bot)
|
|
155
361
|
|
|
156
362
|
```typescript
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
characterId:
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
*/
|
|
175
|
-
enableVideo?: boolean;
|
|
176
|
-
/**
|
|
177
|
-
* Start with video camera on when connecting (default: false).
|
|
178
|
-
* Only works if enableVideo is true.
|
|
179
|
-
*/
|
|
180
|
-
startWithVideoOn?: boolean;
|
|
181
|
-
/**
|
|
182
|
-
* Start with microphone on when connecting (default: false).
|
|
183
|
-
* If false, microphone stays off until user enables it.
|
|
184
|
-
*/
|
|
185
|
-
startWithAudioOn?: boolean;
|
|
186
|
-
/** Enable text-to-speech audio generation (default: true) */
|
|
187
|
-
ttsEnabled?: boolean;
|
|
188
|
-
}
|
|
363
|
+
import { ConvaiClient, AudioRenderer } from "@convai/web-sdk/vanilla";
|
|
364
|
+
|
|
365
|
+
const client = new ConvaiClient({
|
|
366
|
+
apiKey: "your-api-key",
|
|
367
|
+
characterId: "your-character-id",
|
|
368
|
+
});
|
|
369
|
+
|
|
370
|
+
await client.connect();
|
|
371
|
+
|
|
372
|
+
// CRITICAL: Create AudioRenderer to hear bot audio
|
|
373
|
+
const audioRenderer = new AudioRenderer(client.room);
|
|
374
|
+
|
|
375
|
+
// Your custom UI logic...
|
|
376
|
+
|
|
377
|
+
// Cleanup
|
|
378
|
+
audioRenderer.destroy();
|
|
379
|
+
await client.disconnect();
|
|
189
380
|
```
|
|
190
381
|
|
|
191
|
-
|
|
382
|
+
### Vanilla Exports Reference
|
|
383
|
+
|
|
384
|
+
```typescript
|
|
385
|
+
// Widget
|
|
386
|
+
import { createConvaiWidget, destroyConvaiWidget } from "@convai/web-sdk/vanilla";
|
|
192
387
|
|
|
193
|
-
|
|
388
|
+
// Core Client
|
|
389
|
+
import { ConvaiClient } from "@convai/web-sdk/vanilla";
|
|
194
390
|
|
|
195
|
-
|
|
391
|
+
// Audio Rendering (Critical)
|
|
392
|
+
import { AudioRenderer } from "@convai/web-sdk/vanilla";
|
|
196
393
|
|
|
197
|
-
|
|
394
|
+
// Types
|
|
395
|
+
import type {
|
|
396
|
+
VanillaWidget,
|
|
397
|
+
VanillaWidgetOptions,
|
|
398
|
+
ConvaiConfig,
|
|
399
|
+
ConvaiClientState,
|
|
400
|
+
ChatMessage,
|
|
401
|
+
IConvaiClient,
|
|
402
|
+
} from "@convai/web-sdk/vanilla";
|
|
403
|
+
```
|
|
404
|
+
|
|
405
|
+
---
|
|
406
|
+
|
|
407
|
+
## Video & Screen Share
|
|
408
|
+
|
|
409
|
+
### Critical Requirements
|
|
410
|
+
|
|
411
|
+
> ⚠️ **IMPORTANT**: Video and Screen Share features require **TWO** configuration changes:
|
|
412
|
+
|
|
413
|
+
#### 1. Set `enableVideo: true` in Client Configuration
|
|
414
|
+
|
|
415
|
+
This sets the connection type to `"video"` which enables video capabilities.
|
|
416
|
+
|
|
417
|
+
#### 2. Set `showVideo` and/or `showScreenShare` in Widget Props
|
|
418
|
+
|
|
419
|
+
This shows the UI controls for video/screen share.
|
|
420
|
+
|
|
421
|
+
**Without both, video features will NOT work.**
|
|
422
|
+
|
|
423
|
+
---
|
|
424
|
+
|
|
425
|
+
### Enabling Video
|
|
426
|
+
|
|
427
|
+
#### React
|
|
198
428
|
|
|
199
429
|
```tsx
|
|
200
|
-
import { useConvaiClient, ConvaiWidget } from "@convai/web-sdk";
|
|
430
|
+
import { useConvaiClient, ConvaiWidget } from "@convai/web-sdk/react";
|
|
201
431
|
|
|
202
432
|
function App() {
|
|
433
|
+
// ✅ STEP 1: Enable video in client config
|
|
203
434
|
const convaiClient = useConvaiClient({
|
|
204
435
|
apiKey: "your-api-key",
|
|
205
436
|
characterId: "your-character-id",
|
|
206
|
-
enableVideo: true,
|
|
437
|
+
enableVideo: true, // ← REQUIRED for video
|
|
207
438
|
startWithVideoOn: false, // Camera off by default
|
|
208
439
|
});
|
|
209
440
|
|
|
210
441
|
return (
|
|
211
442
|
<ConvaiWidget
|
|
212
443
|
convaiClient={convaiClient}
|
|
213
|
-
showVideo={true}
|
|
214
|
-
showScreenShare={
|
|
444
|
+
showVideo={true} // ← STEP 2: Show video controls
|
|
445
|
+
showScreenShare={false} // Optional: hide screen share
|
|
215
446
|
/>
|
|
216
447
|
);
|
|
217
448
|
}
|
|
218
449
|
```
|
|
219
450
|
|
|
220
|
-
|
|
451
|
+
#### Vanilla
|
|
221
452
|
|
|
222
453
|
```typescript
|
|
223
454
|
import { ConvaiClient, createConvaiWidget } from "@convai/web-sdk/vanilla";
|
|
224
455
|
|
|
456
|
+
// ✅ STEP 1: Enable video in client config
|
|
225
457
|
const client = new ConvaiClient({
|
|
226
458
|
apiKey: "your-api-key",
|
|
227
459
|
characterId: "your-character-id",
|
|
228
|
-
enableVideo: true,
|
|
460
|
+
enableVideo: true, // ← REQUIRED for video
|
|
229
461
|
startWithVideoOn: false,
|
|
230
462
|
});
|
|
231
463
|
|
|
232
464
|
const widget = createConvaiWidget(document.body, {
|
|
233
465
|
convaiClient: client,
|
|
234
|
-
showVideo: true,
|
|
235
|
-
showScreenShare:
|
|
466
|
+
showVideo: true, // ← STEP 2: Show video controls
|
|
467
|
+
showScreenShare: false,
|
|
236
468
|
});
|
|
237
469
|
```
|
|
238
470
|
|
|
239
|
-
|
|
471
|
+
#### Manual Video Control
|
|
240
472
|
|
|
241
473
|
```typescript
|
|
242
|
-
// Enable
|
|
474
|
+
// Enable camera
|
|
243
475
|
await convaiClient.videoControls.enableVideo();
|
|
244
476
|
|
|
245
|
-
// Disable
|
|
477
|
+
// Disable camera
|
|
246
478
|
await convaiClient.videoControls.disableVideo();
|
|
247
479
|
|
|
248
|
-
// Toggle
|
|
480
|
+
// Toggle camera
|
|
249
481
|
await convaiClient.videoControls.toggleVideo();
|
|
250
482
|
|
|
251
|
-
// Check
|
|
252
|
-
|
|
483
|
+
// Check state
|
|
484
|
+
console.log(convaiClient.isVideoEnabled);
|
|
253
485
|
|
|
254
486
|
// Set video quality
|
|
255
487
|
await convaiClient.videoControls.setVideoQuality("high"); // 'low' | 'medium' | 'high'
|
|
256
488
|
|
|
257
|
-
// Get available
|
|
489
|
+
// Get available cameras
|
|
258
490
|
const devices = await convaiClient.videoControls.getVideoDevices();
|
|
259
491
|
|
|
260
|
-
//
|
|
492
|
+
// Switch camera
|
|
261
493
|
await convaiClient.videoControls.setVideoDevice(deviceId);
|
|
262
494
|
```
|
|
263
495
|
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
```typescript
|
|
267
|
-
// Enable screen share
|
|
268
|
-
await convaiClient.screenShareControls.enableScreenShare();
|
|
269
|
-
|
|
270
|
-
// Enable screen share with audio
|
|
271
|
-
await convaiClient.screenShareControls.enableScreenShareWithAudio();
|
|
272
|
-
|
|
273
|
-
// Disable screen share
|
|
274
|
-
await convaiClient.screenShareControls.disableScreenShare();
|
|
275
|
-
|
|
276
|
-
// Toggle screen share
|
|
277
|
-
await convaiClient.screenShareControls.toggleScreenShare();
|
|
278
|
-
|
|
279
|
-
// Check screen share state
|
|
280
|
-
const isActive = convaiClient.screenShareControls.isScreenShareActive;
|
|
281
|
-
```
|
|
282
|
-
|
|
283
|
-
**Video State Monitoring:**
|
|
284
|
-
|
|
285
|
-
```typescript
|
|
286
|
-
// React
|
|
287
|
-
const { isVideoEnabled } = convaiClient;
|
|
288
|
-
|
|
289
|
-
// Core API (event-based)
|
|
290
|
-
convaiClient.videoControls.on("videoStateChange", (state) => {
|
|
291
|
-
console.log("Video enabled:", state.isVideoEnabled);
|
|
292
|
-
console.log("Video hidden:", state.isVideoHidden);
|
|
293
|
-
});
|
|
294
|
-
```
|
|
496
|
+
---
|
|
295
497
|
|
|
296
|
-
###
|
|
498
|
+
### Enabling Screen Share
|
|
297
499
|
|
|
298
|
-
|
|
500
|
+
Screen sharing **requires** `enableVideo: true` (connection type must be `"video"`).
|
|
299
501
|
|
|
300
|
-
|
|
502
|
+
#### React
|
|
301
503
|
|
|
302
504
|
```tsx
|
|
303
|
-
|
|
505
|
+
import { useConvaiClient, ConvaiWidget } from "@convai/web-sdk/react";
|
|
506
|
+
|
|
507
|
+
function App() {
|
|
508
|
+
// ✅ STEP 1: Enable video (required for screen share)
|
|
304
509
|
const convaiClient = useConvaiClient({
|
|
305
|
-
|
|
510
|
+
apiKey: "your-api-key",
|
|
511
|
+
characterId: "your-character-id",
|
|
512
|
+
enableVideo: true, // ← REQUIRED for screen share
|
|
306
513
|
});
|
|
307
514
|
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
515
|
+
return (
|
|
516
|
+
<ConvaiWidget
|
|
517
|
+
convaiClient={convaiClient}
|
|
518
|
+
showVideo={true} // Optional: show video controls
|
|
519
|
+
showScreenShare={true} // ← STEP 2: Show screen share controls
|
|
520
|
+
/>
|
|
521
|
+
);
|
|
314
522
|
}
|
|
315
523
|
```
|
|
316
524
|
|
|
317
|
-
|
|
525
|
+
#### Vanilla
|
|
318
526
|
|
|
319
527
|
```typescript
|
|
320
|
-
|
|
528
|
+
import { ConvaiClient, createConvaiWidget } from "@convai/web-sdk/vanilla";
|
|
529
|
+
|
|
530
|
+
// ✅ STEP 1: Enable video (required for screen share)
|
|
531
|
+
const client = new ConvaiClient({
|
|
532
|
+
apiKey: "your-api-key",
|
|
533
|
+
characterId: "your-character-id",
|
|
534
|
+
enableVideo: true, // ← REQUIRED for screen share
|
|
535
|
+
});
|
|
321
536
|
|
|
322
|
-
|
|
323
|
-
client
|
|
537
|
+
const widget = createConvaiWidget(document.body, {
|
|
538
|
+
convaiClient: client,
|
|
539
|
+
showVideo: true,
|
|
540
|
+
showScreenShare: true, // ← STEP 2: Show screen share controls
|
|
324
541
|
});
|
|
325
542
|
```
|
|
326
543
|
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
When implementing voice mode, interrupt the bot when the user starts speaking:
|
|
544
|
+
#### Manual Screen Share Control
|
|
330
545
|
|
|
331
546
|
```typescript
|
|
332
|
-
//
|
|
333
|
-
|
|
334
|
-
// Interrupt any ongoing bot response
|
|
335
|
-
convaiClient.sendInterruptMessage();
|
|
547
|
+
// Start screen share
|
|
548
|
+
await convaiClient.screenShareControls.enableScreenShare();
|
|
336
549
|
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
550
|
+
// Start screen share with audio
|
|
551
|
+
await convaiClient.screenShareControls.enableScreenShareWithAudio();
|
|
552
|
+
|
|
553
|
+
// Stop screen share
|
|
554
|
+
await convaiClient.screenShareControls.disableScreenShare();
|
|
340
555
|
|
|
341
|
-
//
|
|
342
|
-
|
|
343
|
-
// Interrupt any ongoing bot response
|
|
344
|
-
convaiClient.sendInterruptMessage();
|
|
556
|
+
// Toggle screen share
|
|
557
|
+
await convaiClient.screenShareControls.toggleScreenShare();
|
|
345
558
|
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
};
|
|
559
|
+
// Check state
|
|
560
|
+
console.log(convaiClient.isScreenShareActive);
|
|
349
561
|
```
|
|
350
562
|
|
|
351
|
-
|
|
563
|
+
---
|
|
352
564
|
|
|
353
|
-
|
|
565
|
+
## Building Custom UIs
|
|
354
566
|
|
|
355
|
-
|
|
567
|
+
### Custom Chat Interface
|
|
568
|
+
|
|
569
|
+
Use the `chatMessages` array from ConvaiClient to build your own chat UI.
|
|
570
|
+
|
|
571
|
+
#### React Example
|
|
356
572
|
|
|
357
573
|
```tsx
|
|
358
|
-
|
|
574
|
+
import { useConvaiClient, AudioRenderer } from "@convai/web-sdk/react";
|
|
575
|
+
import { useState } from "react";
|
|
576
|
+
|
|
577
|
+
function CustomChatUI() {
|
|
359
578
|
const convaiClient = useConvaiClient({
|
|
360
|
-
|
|
579
|
+
apiKey: "your-api-key",
|
|
580
|
+
characterId: "your-character-id",
|
|
361
581
|
});
|
|
362
582
|
|
|
363
|
-
const
|
|
364
|
-
|
|
365
|
-
};
|
|
366
|
-
|
|
367
|
-
const handleUnmute = async () => {
|
|
368
|
-
await convaiClient.audioControls.unmuteAudio();
|
|
369
|
-
};
|
|
583
|
+
const { chatMessages, state } = convaiClient;
|
|
584
|
+
const [inputValue, setInputValue] = useState("");
|
|
370
585
|
|
|
371
|
-
const
|
|
372
|
-
|
|
586
|
+
const handleSend = () => {
|
|
587
|
+
if (inputValue.trim() && state.isConnected) {
|
|
588
|
+
convaiClient.sendUserTextMessage(inputValue);
|
|
589
|
+
setInputValue("");
|
|
590
|
+
}
|
|
373
591
|
};
|
|
374
592
|
|
|
375
593
|
return (
|
|
376
594
|
<div>
|
|
377
|
-
|
|
378
|
-
<
|
|
379
|
-
|
|
380
|
-
|
|
595
|
+
{/* CRITICAL: AudioRenderer for bot voice */}
|
|
596
|
+
<AudioRenderer />
|
|
597
|
+
|
|
598
|
+
{/* Chat Messages */}
|
|
599
|
+
<div className="chat-container">
|
|
600
|
+
{chatMessages.map((msg) => {
|
|
601
|
+
const isUser = msg.type.includes("user");
|
|
602
|
+
const displayMessage =
|
|
603
|
+
msg.type === "user-llm-text" || msg.type === "bot-llm-text";
|
|
604
|
+
|
|
605
|
+
if (!displayMessage) return null;
|
|
606
|
+
|
|
607
|
+
return (
|
|
608
|
+
<div
|
|
609
|
+
key={msg.id}
|
|
610
|
+
className={isUser ? "user-message" : "bot-message"}
|
|
611
|
+
>
|
|
612
|
+
<span className="sender">
|
|
613
|
+
{isUser ? "You" : "Character"}
|
|
614
|
+
</span>
|
|
615
|
+
<p>{msg.content}</p>
|
|
616
|
+
<span className="timestamp">
|
|
617
|
+
{new Date(msg.timestamp).toLocaleTimeString()}
|
|
618
|
+
</span>
|
|
619
|
+
</div>
|
|
620
|
+
);
|
|
621
|
+
})}
|
|
622
|
+
</div>
|
|
623
|
+
|
|
624
|
+
{/* Input */}
|
|
625
|
+
<div className="input-container">
|
|
626
|
+
<input
|
|
627
|
+
type="text"
|
|
628
|
+
value={inputValue}
|
|
629
|
+
onChange={(e) => setInputValue(e.target.value)}
|
|
630
|
+
onKeyPress={(e) => e.key === "Enter" && handleSend()}
|
|
631
|
+
placeholder="Type a message..."
|
|
632
|
+
disabled={!state.isConnected}
|
|
633
|
+
/>
|
|
634
|
+
<button onClick={handleSend} disabled={!state.isConnected}>
|
|
635
|
+
Send
|
|
636
|
+
</button>
|
|
637
|
+
</div>
|
|
638
|
+
|
|
639
|
+
{/* Status Indicator */}
|
|
640
|
+
<div className="status">
|
|
641
|
+
{state.isConnecting && "Connecting..."}
|
|
642
|
+
{state.isConnected && state.agentState}
|
|
643
|
+
{!state.isConnected && "Disconnected"}
|
|
644
|
+
</div>
|
|
381
645
|
</div>
|
|
382
646
|
);
|
|
383
647
|
}
|
|
384
648
|
```
|
|
385
649
|
|
|
386
|
-
|
|
650
|
+
#### Vanilla Example
|
|
387
651
|
|
|
388
652
|
```typescript
|
|
389
|
-
|
|
390
|
-
await client.audioControls.muteAudio();
|
|
653
|
+
import { ConvaiClient, AudioRenderer } from "@convai/web-sdk/vanilla";
|
|
391
654
|
|
|
392
|
-
|
|
393
|
-
|
|
655
|
+
const client = new ConvaiClient({
|
|
656
|
+
apiKey: "your-api-key",
|
|
657
|
+
characterId: "your-character-id",
|
|
658
|
+
});
|
|
394
659
|
|
|
395
|
-
|
|
396
|
-
await client.audioControls.toggleAudio();
|
|
660
|
+
await client.connect();
|
|
397
661
|
|
|
398
|
-
//
|
|
399
|
-
const
|
|
662
|
+
// CRITICAL: Create AudioRenderer for bot voice
|
|
663
|
+
const audioRenderer = new AudioRenderer(client.room);
|
|
400
664
|
|
|
401
|
-
|
|
402
|
-
|
|
665
|
+
const chatContainer = document.getElementById("chat-container");
|
|
666
|
+
const inputElement = document.getElementById("message-input");
|
|
667
|
+
const sendButton = document.getElementById("send-button");
|
|
403
668
|
|
|
404
|
-
//
|
|
405
|
-
|
|
406
|
-
|
|
669
|
+
// Render messages
|
|
670
|
+
client.on("messagesChange", (messages) => {
|
|
671
|
+
chatContainer.innerHTML = "";
|
|
407
672
|
|
|
408
|
-
|
|
673
|
+
messages.forEach((msg) => {
|
|
674
|
+
const isUser = msg.type.includes("user");
|
|
675
|
+
const displayMessage =
|
|
676
|
+
msg.type === "user-llm-text" || msg.type === "bot-llm-text";
|
|
409
677
|
|
|
410
|
-
|
|
411
|
-
// Get available audio devices
|
|
412
|
-
const devices = await convaiClient.audioControls.getAudioDevices();
|
|
678
|
+
if (!displayMessage) return;
|
|
413
679
|
|
|
414
|
-
|
|
415
|
-
|
|
680
|
+
const messageDiv = document.createElement("div");
|
|
681
|
+
messageDiv.className = isUser ? "user-message" : "bot-message";
|
|
416
682
|
|
|
417
|
-
|
|
418
|
-
|
|
683
|
+
const sender = document.createElement("span");
|
|
684
|
+
sender.textContent = isUser ? "You" : "Character";
|
|
685
|
+
sender.className = "sender";
|
|
419
686
|
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
// level is a number between 0 and 1
|
|
423
|
-
});
|
|
687
|
+
const content = document.createElement("p");
|
|
688
|
+
content.textContent = msg.content;
|
|
424
689
|
|
|
425
|
-
|
|
426
|
-
|
|
690
|
+
const timestamp = document.createElement("span");
|
|
691
|
+
timestamp.textContent = new Date(msg.timestamp).toLocaleTimeString();
|
|
692
|
+
timestamp.className = "timestamp";
|
|
427
693
|
|
|
428
|
-
|
|
694
|
+
messageDiv.appendChild(sender);
|
|
695
|
+
messageDiv.appendChild(content);
|
|
696
|
+
messageDiv.appendChild(timestamp);
|
|
697
|
+
chatContainer.appendChild(messageDiv);
|
|
698
|
+
});
|
|
429
699
|
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
const { isAudioMuted } = convaiClient;
|
|
433
|
-
|
|
434
|
-
// Core API (event-based)
|
|
435
|
-
convaiClient.audioControls.on("audioStateChange", (state) => {
|
|
436
|
-
console.log("Audio enabled:", state.isAudioEnabled);
|
|
437
|
-
console.log("Audio muted:", state.isAudioMuted);
|
|
438
|
-
console.log("Audio level:", state.audioLevel);
|
|
700
|
+
// Auto-scroll
|
|
701
|
+
chatContainer.scrollTop = chatContainer.scrollHeight;
|
|
439
702
|
});
|
|
703
|
+
|
|
704
|
+
// Send message
|
|
705
|
+
sendButton.addEventListener("click", () => {
|
|
706
|
+
const text = inputElement.value.trim();
|
|
707
|
+
if (text && client.state.isConnected) {
|
|
708
|
+
client.sendUserTextMessage(text);
|
|
709
|
+
inputElement.value = "";
|
|
710
|
+
}
|
|
711
|
+
});
|
|
712
|
+
|
|
713
|
+
inputElement.addEventListener("keypress", (e) => {
|
|
714
|
+
if (e.key === "Enter") {
|
|
715
|
+
sendButton.click();
|
|
716
|
+
}
|
|
717
|
+
});
|
|
718
|
+
|
|
719
|
+
// Cleanup
|
|
720
|
+
// audioRenderer.destroy();
|
|
721
|
+
// await client.disconnect();
|
|
440
722
|
```
|
|
441
723
|
|
|
442
|
-
|
|
724
|
+
---
|
|
443
725
|
|
|
444
|
-
|
|
726
|
+
### Audio Visualizer
|
|
445
727
|
|
|
446
|
-
|
|
728
|
+
Create real-time audio visualizers using the WebRTC room's audio tracks.
|
|
729
|
+
|
|
730
|
+
#### React Example
|
|
447
731
|
|
|
448
732
|
```tsx
|
|
449
|
-
|
|
733
|
+
import { useConvaiClient } from "@convai/web-sdk/react";
|
|
734
|
+
import { useEffect, useRef, useState } from "react";
|
|
735
|
+
|
|
736
|
+
function AudioVisualizer() {
|
|
450
737
|
const convaiClient = useConvaiClient({
|
|
451
|
-
|
|
738
|
+
apiKey: "your-api-key",
|
|
739
|
+
characterId: "your-character-id",
|
|
452
740
|
});
|
|
453
741
|
|
|
454
|
-
const
|
|
455
|
-
|
|
456
|
-
};
|
|
742
|
+
const canvasRef = useRef<HTMLCanvasElement>(null);
|
|
743
|
+
const [audioLevel, setAudioLevel] = useState(0);
|
|
457
744
|
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
<button onClick={() => handleToggleTTS(true)}>Enable TTS</button>
|
|
461
|
-
<button onClick={() => handleToggleTTS(false)}>Disable TTS</button>
|
|
462
|
-
</div>
|
|
463
|
-
);
|
|
464
|
-
}
|
|
465
|
-
```
|
|
745
|
+
useEffect(() => {
|
|
746
|
+
if (!convaiClient.room) return;
|
|
466
747
|
|
|
467
|
-
|
|
748
|
+
let animationId: number;
|
|
749
|
+
let analyzer: AnalyserNode | null = null;
|
|
750
|
+
let dataArray: Uint8Array | null = null;
|
|
468
751
|
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
client.toggleTts(true);
|
|
752
|
+
const setupAnalyzer = async () => {
|
|
753
|
+
const audioContext = new AudioContext();
|
|
472
754
|
|
|
473
|
-
//
|
|
474
|
-
|
|
475
|
-
|
|
755
|
+
// Get remote participant (bot)
|
|
756
|
+
const remoteParticipants = Array.from(
|
|
757
|
+
convaiClient.room.remoteParticipants.values()
|
|
758
|
+
);
|
|
476
759
|
|
|
477
|
-
|
|
760
|
+
if (remoteParticipants.length === 0) return;
|
|
478
761
|
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
characterId: "your-character-id",
|
|
484
|
-
ttsEnabled: true, // Enable TTS by default
|
|
485
|
-
});
|
|
762
|
+
const participant = remoteParticipants[0];
|
|
763
|
+
const audioTracks = Array.from(
|
|
764
|
+
participant.audioTrackPublications.values()
|
|
765
|
+
);
|
|
486
766
|
|
|
487
|
-
|
|
488
|
-
const client = new ConvaiClient({
|
|
489
|
-
apiKey: "your-api-key",
|
|
490
|
-
characterId: "your-character-id",
|
|
491
|
-
ttsEnabled: false, // Disable TTS
|
|
492
|
-
});
|
|
493
|
-
```
|
|
767
|
+
if (audioTracks.length === 0) return;
|
|
494
768
|
|
|
495
|
-
|
|
769
|
+
const audioTrack = audioTracks[0].track;
|
|
770
|
+
if (!audioTrack) return;
|
|
496
771
|
|
|
497
|
-
|
|
772
|
+
// Get MediaStream from track
|
|
773
|
+
const mediaStream = new MediaStream([audioTrack.mediaStreamTrack]);
|
|
498
774
|
|
|
499
|
-
|
|
775
|
+
// Create analyzer
|
|
776
|
+
const source = audioContext.createMediaStreamSource(mediaStream);
|
|
777
|
+
analyzer = audioContext.createAnalyser();
|
|
778
|
+
analyzer.fftSize = 256;
|
|
500
779
|
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
import { useState, useEffect } from "react";
|
|
780
|
+
source.connect(analyzer);
|
|
781
|
+
dataArray = new Uint8Array(analyzer.frequencyBinCount);
|
|
504
782
|
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
});
|
|
509
|
-
const [isVoiceMode, setIsVoiceMode] = useState(false);
|
|
783
|
+
// Animate
|
|
784
|
+
const animate = () => {
|
|
785
|
+
if (!analyzer || !dataArray) return;
|
|
510
786
|
|
|
511
|
-
|
|
512
|
-
// Interrupt any ongoing bot response
|
|
513
|
-
convaiClient.sendInterruptMessage();
|
|
787
|
+
analyzer.getByteFrequencyData(dataArray);
|
|
514
788
|
|
|
515
|
-
|
|
516
|
-
|
|
789
|
+
// Calculate average volume
|
|
790
|
+
const sum = dataArray.reduce((a, b) => a + b, 0);
|
|
791
|
+
const average = sum / dataArray.length;
|
|
792
|
+
const normalizedLevel = average / 255;
|
|
517
793
|
|
|
518
|
-
|
|
519
|
-
};
|
|
794
|
+
setAudioLevel(normalizedLevel);
|
|
520
795
|
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
convaiClient.sendInterruptMessage();
|
|
796
|
+
// Draw visualization
|
|
797
|
+
drawVisualizer(dataArray);
|
|
524
798
|
|
|
525
|
-
|
|
526
|
-
|
|
799
|
+
animationId = requestAnimationFrame(animate);
|
|
800
|
+
};
|
|
527
801
|
|
|
528
|
-
|
|
529
|
-
|
|
802
|
+
animate();
|
|
803
|
+
};
|
|
530
804
|
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
805
|
+
const drawVisualizer = (dataArray: Uint8Array) => {
|
|
806
|
+
const canvas = canvasRef.current;
|
|
807
|
+
if (!canvas) return;
|
|
808
|
+
|
|
809
|
+
const ctx = canvas.getContext("2d");
|
|
810
|
+
if (!ctx) return;
|
|
811
|
+
|
|
812
|
+
const width = canvas.width;
|
|
813
|
+
const height = canvas.height;
|
|
814
|
+
|
|
815
|
+
ctx.clearRect(0, 0, width, height);
|
|
816
|
+
|
|
817
|
+
const barWidth = (width / dataArray.length) * 2.5;
|
|
818
|
+
let x = 0;
|
|
819
|
+
|
|
820
|
+
for (let i = 0; i < dataArray.length; i++) {
|
|
821
|
+
const barHeight = (dataArray[i] / 255) * height;
|
|
822
|
+
|
|
823
|
+
ctx.fillStyle = `rgb(${barHeight + 100}, 50, 150)`;
|
|
824
|
+
ctx.fillRect(x, height - barHeight, barWidth, barHeight);
|
|
825
|
+
|
|
826
|
+
x += barWidth + 1;
|
|
827
|
+
}
|
|
828
|
+
};
|
|
829
|
+
|
|
830
|
+
if (convaiClient.state.isConnected) {
|
|
831
|
+
setupAnalyzer();
|
|
537
832
|
}
|
|
538
|
-
|
|
833
|
+
|
|
834
|
+
return () => {
|
|
835
|
+
if (animationId) cancelAnimationFrame(animationId);
|
|
836
|
+
};
|
|
837
|
+
}, [convaiClient.room, convaiClient.state.isConnected]);
|
|
539
838
|
|
|
540
839
|
return (
|
|
541
840
|
<div>
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
841
|
+
<canvas
|
|
842
|
+
ref={canvasRef}
|
|
843
|
+
width={800}
|
|
844
|
+
height={200}
|
|
845
|
+
style={{ border: "1px solid #ccc" }}
|
|
846
|
+
/>
|
|
847
|
+
<div>Audio Level: {(audioLevel * 100).toFixed(0)}%</div>
|
|
848
|
+
<div>
|
|
849
|
+
Bot is {convaiClient.state.isSpeaking ? "speaking" : "silent"}
|
|
850
|
+
</div>
|
|
550
851
|
</div>
|
|
551
852
|
);
|
|
552
853
|
}
|
|
553
854
|
```
|
|
554
855
|
|
|
555
|
-
|
|
856
|
+
#### Vanilla Example
|
|
556
857
|
|
|
557
858
|
```typescript
|
|
558
|
-
|
|
859
|
+
import { ConvaiClient, AudioRenderer } from "@convai/web-sdk/vanilla";
|
|
559
860
|
|
|
560
|
-
const
|
|
561
|
-
|
|
562
|
-
|
|
861
|
+
const client = new ConvaiClient({
|
|
862
|
+
apiKey: "your-api-key",
|
|
863
|
+
characterId: "your-character-id",
|
|
864
|
+
});
|
|
563
865
|
|
|
564
|
-
|
|
565
|
-
await client.audioControls.unmuteAudio();
|
|
866
|
+
await client.connect();
|
|
566
867
|
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
};
|
|
868
|
+
// CRITICAL: AudioRenderer for playback
|
|
869
|
+
const audioRenderer = new AudioRenderer(client.room);
|
|
570
870
|
|
|
571
|
-
const
|
|
572
|
-
|
|
573
|
-
client.sendInterruptMessage();
|
|
871
|
+
const canvas = document.getElementById("visualizer") as HTMLCanvasElement;
|
|
872
|
+
const ctx = canvas.getContext("2d")!;
|
|
574
873
|
|
|
575
|
-
|
|
576
|
-
|
|
874
|
+
let analyzer: AnalyserNode | null = null;
|
|
875
|
+
let dataArray: Uint8Array | null = null;
|
|
876
|
+
let animationId: number;
|
|
577
877
|
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
};
|
|
878
|
+
// Setup analyzer
|
|
879
|
+
const audioContext = new AudioContext();
|
|
581
880
|
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
document.getElementById("transcription").textContent = transcription;
|
|
587
|
-
}
|
|
588
|
-
});
|
|
881
|
+
const remoteParticipants = Array.from(client.room.remoteParticipants.values());
|
|
882
|
+
const participant = remoteParticipants[0];
|
|
883
|
+
const audioTracks = Array.from(participant.audioTrackPublications.values());
|
|
884
|
+
const audioTrack = audioTracks[0].track;
|
|
589
885
|
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
const transcriptionDiv = document.getElementById("transcription");
|
|
886
|
+
const mediaStream = new MediaStream([audioTrack.mediaStreamTrack]);
|
|
887
|
+
const source = audioContext.createMediaStreamSource(mediaStream);
|
|
593
888
|
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
} else {
|
|
598
|
-
voiceButton.textContent = "Start Voice Mode";
|
|
599
|
-
transcriptionDiv.style.display = "none";
|
|
600
|
-
}
|
|
601
|
-
}
|
|
602
|
-
```
|
|
889
|
+
analyzer = audioContext.createAnalyser();
|
|
890
|
+
analyzer.fftSize = 256;
|
|
891
|
+
source.connect(analyzer);
|
|
603
892
|
|
|
604
|
-
|
|
893
|
+
dataArray = new Uint8Array(analyzer.frequencyBinCount);
|
|
605
894
|
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
if (isVoiceMode) {
|
|
610
|
-
switch (state.agentState) {
|
|
611
|
-
case "listening":
|
|
612
|
-
// User can speak
|
|
613
|
-
console.log("Bot is listening");
|
|
614
|
-
break;
|
|
615
|
-
case "thinking":
|
|
616
|
-
// Bot is processing
|
|
617
|
-
console.log("Bot is thinking");
|
|
618
|
-
break;
|
|
619
|
-
case "speaking":
|
|
620
|
-
// Bot is responding
|
|
621
|
-
console.log("Bot is speaking");
|
|
622
|
-
// Optionally interrupt if user wants to speak
|
|
623
|
-
break;
|
|
624
|
-
}
|
|
625
|
-
}
|
|
626
|
-
});
|
|
627
|
-
```
|
|
895
|
+
// Animate
|
|
896
|
+
function animate() {
|
|
897
|
+
if (!analyzer || !dataArray) return;
|
|
628
898
|
|
|
629
|
-
|
|
899
|
+
analyzer.getByteFrequencyData(dataArray);
|
|
630
900
|
|
|
631
|
-
|
|
901
|
+
// Clear canvas
|
|
902
|
+
ctx.clearRect(0, 0, canvas.width, canvas.height);
|
|
632
903
|
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
const convaiClient = useConvaiClient({
|
|
636
|
-
apiKey: "your-api-key",
|
|
637
|
-
characterId: "your-character-id",
|
|
638
|
-
});
|
|
904
|
+
const barWidth = (canvas.width / dataArray.length) * 2.5;
|
|
905
|
+
let x = 0;
|
|
639
906
|
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
apiKey: "your-api-key",
|
|
643
|
-
characterId: "your-character-id",
|
|
644
|
-
});
|
|
907
|
+
for (let i = 0; i < dataArray.length; i++) {
|
|
908
|
+
const barHeight = (dataArray[i] / 255) * canvas.height;
|
|
645
909
|
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
910
|
+
ctx.fillStyle = `rgb(${barHeight + 100}, 50, 150)`;
|
|
911
|
+
ctx.fillRect(x, canvas.height - barHeight, barWidth, barHeight);
|
|
912
|
+
|
|
913
|
+
x += barWidth + 1;
|
|
914
|
+
}
|
|
915
|
+
|
|
916
|
+
animationId = requestAnimationFrame(animate);
|
|
917
|
+
}
|
|
918
|
+
|
|
919
|
+
animate();
|
|
920
|
+
|
|
921
|
+
// Cleanup
|
|
922
|
+
// cancelAnimationFrame(animationId);
|
|
923
|
+
// audioRenderer.destroy();
|
|
924
|
+
// await client.disconnect();
|
|
652
925
|
```
|
|
653
926
|
|
|
654
|
-
|
|
927
|
+
---
|
|
928
|
+
|
|
929
|
+
### Message Types
|
|
930
|
+
|
|
931
|
+
All messages from `convaiClient.chatMessages` have a `type` field:
|
|
655
932
|
|
|
656
933
|
```typescript
|
|
657
|
-
|
|
934
|
+
type ChatMessageType =
|
|
935
|
+
| "user" // User's sent message (raw)
|
|
936
|
+
| "user-transcription" // Real-time speech-to-text from user
|
|
937
|
+
| "user-llm-text" // User text processed by LLM (final)
|
|
938
|
+
| "convai" // Character's response (raw)
|
|
939
|
+
| "bot-llm-text" // Character's LLM-generated text (final)
|
|
940
|
+
| "bot-emotion" // Character's emotional state
|
|
941
|
+
| "emotion" // Generic emotion
|
|
942
|
+
| "behavior-tree" // Behavior tree response
|
|
943
|
+
| "action" // Action execution
|
|
944
|
+
| "interrupt-bot"; // Interrupt message
|
|
658
945
|
```
|
|
659
946
|
|
|
660
|
-
**
|
|
947
|
+
**For Chat UIs, filter to:**
|
|
661
948
|
|
|
662
949
|
```typescript
|
|
663
|
-
|
|
950
|
+
const displayMessages = chatMessages.filter(
|
|
951
|
+
(msg) => msg.type === "user-llm-text" || msg.type === "bot-llm-text"
|
|
952
|
+
);
|
|
664
953
|
```
|
|
665
954
|
|
|
666
|
-
|
|
955
|
+
---
|
|
956
|
+
|
|
957
|
+
## API Reference
|
|
958
|
+
|
|
959
|
+
### Configuration
|
|
667
960
|
|
|
668
961
|
```typescript
|
|
669
|
-
|
|
670
|
-
|
|
962
|
+
interface ConvaiConfig {
|
|
963
|
+
/** Your Convai API key from convai.com dashboard (required) */
|
|
964
|
+
apiKey: string;
|
|
965
|
+
|
|
966
|
+
/** The Character ID to connect to (required) */
|
|
967
|
+
characterId: string;
|
|
968
|
+
|
|
969
|
+
/**
|
|
970
|
+
* End user identifier for speaker management (optional).
|
|
971
|
+
* When provided: enables long-term memory and analytics
|
|
972
|
+
* When not provided: anonymous mode, no persistent memory
|
|
973
|
+
*/
|
|
974
|
+
endUserId?: string;
|
|
975
|
+
|
|
976
|
+
/** Custom Convai API URL (optional) */
|
|
977
|
+
url?: string;
|
|
978
|
+
|
|
979
|
+
/**
|
|
980
|
+
* Enable video capability (default: false).
|
|
981
|
+
* If true, connection_type will be "video" (supports audio, video, screenshare).
|
|
982
|
+
* If false, connection_type will be "audio" (audio only).
|
|
983
|
+
* ⚠️ REQUIRED for video and screen share features.
|
|
984
|
+
*/
|
|
985
|
+
enableVideo?: boolean;
|
|
986
|
+
|
|
987
|
+
/**
|
|
988
|
+
* Start with video camera on when connecting (default: false).
|
|
989
|
+
* Only works if enableVideo is true.
|
|
990
|
+
*/
|
|
991
|
+
startWithVideoOn?: boolean;
|
|
992
|
+
|
|
993
|
+
/**
|
|
994
|
+
* Start with microphone on when connecting (default: false).
|
|
995
|
+
* If false, microphone stays off until user enables it.
|
|
996
|
+
*/
|
|
997
|
+
startWithAudioOn?: boolean;
|
|
998
|
+
|
|
999
|
+
/**
|
|
1000
|
+
* Enable text-to-speech audio generation (default: true).
|
|
1001
|
+
*/
|
|
1002
|
+
ttsEnabled?: boolean;
|
|
1003
|
+
}
|
|
671
1004
|
```
|
|
672
1005
|
|
|
673
|
-
|
|
1006
|
+
### Connection Management
|
|
674
1007
|
|
|
675
1008
|
```typescript
|
|
676
|
-
//
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
// Core API (event-based)
|
|
683
|
-
convaiClient.on("stateChange", (state) => {
|
|
684
|
-
console.log("State changed:", state);
|
|
1009
|
+
// Connect
|
|
1010
|
+
await convaiClient.connect({
|
|
1011
|
+
apiKey: "your-api-key",
|
|
1012
|
+
characterId: "your-character-id",
|
|
1013
|
+
enableVideo: true,
|
|
685
1014
|
});
|
|
686
1015
|
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
});
|
|
1016
|
+
// Disconnect
|
|
1017
|
+
await convaiClient.disconnect();
|
|
690
1018
|
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
1019
|
+
// Reconnect
|
|
1020
|
+
await convaiClient.reconnect();
|
|
1021
|
+
|
|
1022
|
+
// Reset session (clear conversation history)
|
|
1023
|
+
convaiClient.resetSession();
|
|
1024
|
+
|
|
1025
|
+
// Check connection state
|
|
1026
|
+
console.log(convaiClient.state.isConnected);
|
|
1027
|
+
console.log(convaiClient.state.isConnecting);
|
|
1028
|
+
console.log(convaiClient.state.agentState); // 'disconnected' | 'connected' | 'listening' | 'thinking' | 'speaking'
|
|
1029
|
+
console.log(convaiClient.isBotReady); // Bot ready to receive messages
|
|
694
1030
|
```
|
|
695
1031
|
|
|
696
1032
|
### Messaging
|
|
697
1033
|
|
|
698
|
-
**Send Text Message:**
|
|
699
|
-
|
|
700
1034
|
```typescript
|
|
1035
|
+
// Send text message
|
|
701
1036
|
convaiClient.sendUserTextMessage("Hello, how are you?");
|
|
702
|
-
```
|
|
703
1037
|
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
```typescript
|
|
707
|
-
// Trigger specific character action
|
|
1038
|
+
// Send trigger message (invoke character action)
|
|
708
1039
|
convaiClient.sendTriggerMessage("greet", "User entered the room");
|
|
709
1040
|
|
|
710
|
-
//
|
|
711
|
-
convaiClient.
|
|
712
|
-
```
|
|
1041
|
+
// Interrupt character's current response
|
|
1042
|
+
convaiClient.sendInterruptMessage();
|
|
713
1043
|
|
|
714
|
-
|
|
1044
|
+
// Update context
|
|
1045
|
+
convaiClient.updateTemplateKeys({ user_name: "John" });
|
|
1046
|
+
convaiClient.updateDynamicInfo({ text: "User is browsing products" });
|
|
715
1047
|
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
convaiClient.updateTemplateKeys({
|
|
719
|
-
user_name: "John",
|
|
720
|
-
location: "New York",
|
|
721
|
-
});
|
|
1048
|
+
// Access messages
|
|
1049
|
+
console.log(convaiClient.chatMessages);
|
|
722
1050
|
|
|
723
|
-
//
|
|
724
|
-
convaiClient.
|
|
725
|
-
text: "User is currently browsing the products page",
|
|
726
|
-
});
|
|
1051
|
+
// Access real-time user transcription
|
|
1052
|
+
console.log(convaiClient.userTranscription);
|
|
727
1053
|
```
|
|
728
1054
|
|
|
729
|
-
|
|
1055
|
+
### Audio Controls
|
|
730
1056
|
|
|
731
1057
|
```typescript
|
|
732
|
-
//
|
|
733
|
-
|
|
1058
|
+
// Mute/unmute microphone
|
|
1059
|
+
await convaiClient.audioControls.muteAudio();
|
|
1060
|
+
await convaiClient.audioControls.unmuteAudio();
|
|
1061
|
+
await convaiClient.audioControls.toggleAudio();
|
|
734
1062
|
|
|
735
|
-
//
|
|
736
|
-
convaiClient.
|
|
737
|
-
console.log("New message:", message.content);
|
|
738
|
-
console.log("Message type:", message.type);
|
|
739
|
-
});
|
|
1063
|
+
// Check mute state
|
|
1064
|
+
console.log(convaiClient.isAudioMuted);
|
|
740
1065
|
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
});
|
|
744
|
-
```
|
|
1066
|
+
// Get available microphones
|
|
1067
|
+
const devices = await convaiClient.audioControls.getAudioDevices();
|
|
745
1068
|
|
|
746
|
-
|
|
1069
|
+
// Set microphone
|
|
1070
|
+
await convaiClient.audioControls.setAudioDevice(deviceId);
|
|
747
1071
|
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
| "emotion" // Character's emotional state
|
|
755
|
-
| "behavior-tree" // Behavior tree response
|
|
756
|
-
| "action" // Action execution
|
|
757
|
-
| "bot-emotion" // Bot emotional response
|
|
758
|
-
| "user-llm-text" // User text processed by LLM
|
|
759
|
-
| "interrupt-bot"; // Interrupt message
|
|
1072
|
+
// Monitor audio level
|
|
1073
|
+
convaiClient.audioControls.startAudioLevelMonitoring();
|
|
1074
|
+
convaiClient.audioControls.on("audioLevelChange", (level) => {
|
|
1075
|
+
console.log("Audio level:", level); // 0 to 1
|
|
1076
|
+
});
|
|
1077
|
+
convaiClient.audioControls.stopAudioLevelMonitoring();
|
|
760
1078
|
```
|
|
761
1079
|
|
|
762
|
-
###
|
|
1080
|
+
### Video Controls
|
|
763
1081
|
|
|
764
|
-
|
|
1082
|
+
**⚠️ Requires `enableVideo: true` in config.**
|
|
765
1083
|
|
|
766
1084
|
```typescript
|
|
767
|
-
//
|
|
768
|
-
|
|
1085
|
+
// Enable/disable camera
|
|
1086
|
+
await convaiClient.videoControls.enableVideo();
|
|
1087
|
+
await convaiClient.videoControls.disableVideo();
|
|
1088
|
+
await convaiClient.videoControls.toggleVideo();
|
|
769
1089
|
|
|
770
|
-
// Check
|
|
771
|
-
|
|
772
|
-
console.log("Bot is listening");
|
|
773
|
-
}
|
|
1090
|
+
// Check video state
|
|
1091
|
+
console.log(convaiClient.isVideoEnabled);
|
|
774
1092
|
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
}
|
|
1093
|
+
// Set video quality
|
|
1094
|
+
await convaiClient.videoControls.setVideoQuality("high"); // 'low' | 'medium' | 'high'
|
|
778
1095
|
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
}
|
|
1096
|
+
// Get available cameras
|
|
1097
|
+
const devices = await convaiClient.videoControls.getVideoDevices();
|
|
782
1098
|
|
|
783
|
-
//
|
|
784
|
-
|
|
1099
|
+
// Switch camera
|
|
1100
|
+
await convaiClient.videoControls.setVideoDevice(deviceId);
|
|
785
1101
|
```
|
|
786
1102
|
|
|
787
|
-
|
|
1103
|
+
### Screen Share Controls
|
|
788
1104
|
|
|
789
|
-
|
|
790
|
-
// React
|
|
791
|
-
const { userTranscription } = convaiClient;
|
|
792
|
-
|
|
793
|
-
// Core API (event-based)
|
|
794
|
-
convaiClient.on("userTranscriptionChange", (transcription: string) => {
|
|
795
|
-
console.log("User is saying:", transcription);
|
|
796
|
-
});
|
|
797
|
-
```
|
|
798
|
-
|
|
799
|
-
**Bot Ready State:**
|
|
1105
|
+
**⚠️ Requires `enableVideo: true` in config.**
|
|
800
1106
|
|
|
801
1107
|
```typescript
|
|
802
|
-
//
|
|
803
|
-
|
|
1108
|
+
// Start/stop screen share
|
|
1109
|
+
await convaiClient.screenShareControls.enableScreenShare();
|
|
1110
|
+
await convaiClient.screenShareControls.enableScreenShareWithAudio();
|
|
1111
|
+
await convaiClient.screenShareControls.disableScreenShare();
|
|
1112
|
+
await convaiClient.screenShareControls.toggleScreenShare();
|
|
804
1113
|
|
|
805
|
-
//
|
|
806
|
-
convaiClient.
|
|
807
|
-
console.log("Bot is ready to receive messages");
|
|
808
|
-
});
|
|
1114
|
+
// Check screen share state
|
|
1115
|
+
console.log(convaiClient.isScreenShareActive);
|
|
809
1116
|
```
|
|
810
1117
|
|
|
811
|
-
|
|
1118
|
+
---
|
|
1119
|
+
|
|
1120
|
+
## Getting Credentials
|
|
812
1121
|
|
|
813
1122
|
1. Visit [convai.com](https://convai.com) and create an account
|
|
814
1123
|
2. Navigate to your dashboard
|
|
@@ -816,41 +1125,71 @@ convaiClient.on("botReady", () => {
|
|
|
816
1125
|
4. Copy your **API Key** from the dashboard
|
|
817
1126
|
5. Copy your **Character ID** from the character details
|
|
818
1127
|
|
|
819
|
-
|
|
1128
|
+
---
|
|
1129
|
+
|
|
1130
|
+
## TypeScript Support
|
|
1131
|
+
|
|
1132
|
+
All exports are fully typed:
|
|
1133
|
+
|
|
1134
|
+
**React:**
|
|
820
1135
|
|
|
821
1136
|
```typescript
|
|
822
|
-
|
|
823
|
-
|
|
1137
|
+
import type {
|
|
1138
|
+
// Configuration
|
|
1139
|
+
ConvaiConfig,
|
|
824
1140
|
|
|
825
|
-
//
|
|
826
|
-
|
|
1141
|
+
// State
|
|
1142
|
+
ConvaiClientState,
|
|
827
1143
|
|
|
828
|
-
//
|
|
829
|
-
|
|
1144
|
+
// Messages
|
|
1145
|
+
ChatMessage,
|
|
1146
|
+
ChatMessageType,
|
|
830
1147
|
|
|
831
|
-
//
|
|
832
|
-
|
|
833
|
-
|
|
1148
|
+
// Client
|
|
1149
|
+
IConvaiClient,
|
|
1150
|
+
ConvaiClient,
|
|
834
1151
|
|
|
835
|
-
|
|
1152
|
+
// Controls
|
|
1153
|
+
AudioControls,
|
|
1154
|
+
VideoControls,
|
|
1155
|
+
ScreenShareControls,
|
|
1156
|
+
} from "@convai/web-sdk/react";
|
|
1157
|
+
```
|
|
836
1158
|
|
|
837
|
-
|
|
1159
|
+
**Vanilla:**
|
|
838
1160
|
|
|
839
1161
|
```typescript
|
|
840
1162
|
import type {
|
|
841
|
-
|
|
1163
|
+
// Configuration
|
|
842
1164
|
ConvaiConfig,
|
|
1165
|
+
|
|
1166
|
+
// State
|
|
843
1167
|
ConvaiClientState,
|
|
1168
|
+
|
|
1169
|
+
// Messages
|
|
844
1170
|
ChatMessage,
|
|
1171
|
+
ChatMessageType,
|
|
1172
|
+
|
|
1173
|
+
// Client
|
|
1174
|
+
IConvaiClient,
|
|
1175
|
+
ConvaiClient,
|
|
1176
|
+
|
|
1177
|
+
// Controls
|
|
845
1178
|
AudioControls,
|
|
846
1179
|
VideoControls,
|
|
847
1180
|
ScreenShareControls,
|
|
848
|
-
|
|
849
|
-
|
|
1181
|
+
|
|
1182
|
+
// Widget
|
|
1183
|
+
VanillaWidget,
|
|
1184
|
+
VanillaWidgetOptions,
|
|
1185
|
+
} from "@convai/web-sdk/vanilla";
|
|
850
1186
|
```
|
|
851
1187
|
|
|
1188
|
+
---
|
|
1189
|
+
|
|
852
1190
|
## Support
|
|
853
1191
|
|
|
854
|
-
- [
|
|
855
|
-
- [
|
|
856
|
-
- [
|
|
1192
|
+
- **Documentation**: [API Reference](./API_REFERENCE.md)
|
|
1193
|
+
- **Forum**: [Convai Forum](https://forum.convai.com)
|
|
1194
|
+
- **Website**: [convai.com](https://convai.com)
|
|
1195
|
+
- **Issues**: [GitHub Issues](https://github.com/convai/web-sdk/issues)
|