@convai/web-sdk 0.2.3 → 0.3.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +529 -868
- package/dist/core/AudioManager.d.ts.map +1 -1
- package/dist/core/AudioManager.js +0 -5
- package/dist/core/AudioManager.js.map +1 -1
- package/dist/core/BlendshapeQueue.d.ts +128 -0
- package/dist/core/BlendshapeQueue.d.ts.map +1 -0
- package/dist/core/BlendshapeQueue.js +229 -0
- package/dist/core/BlendshapeQueue.js.map +1 -0
- package/dist/core/ConvaiClient.d.ts +19 -0
- package/dist/core/ConvaiClient.d.ts.map +1 -1
- package/dist/core/ConvaiClient.js +67 -29
- package/dist/core/ConvaiClient.js.map +1 -1
- package/dist/core/MessageHandler.d.ts +7 -0
- package/dist/core/MessageHandler.d.ts.map +1 -1
- package/dist/core/MessageHandler.js +35 -2
- package/dist/core/MessageHandler.js.map +1 -1
- package/dist/core/ScreenShareManager.d.ts.map +1 -1
- package/dist/core/ScreenShareManager.js +0 -3
- package/dist/core/ScreenShareManager.js.map +1 -1
- package/dist/core/VideoManager.d.ts.map +1 -1
- package/dist/core/VideoManager.js +0 -4
- package/dist/core/VideoManager.js.map +1 -1
- package/dist/core/index.d.ts +2 -0
- package/dist/core/index.d.ts.map +1 -1
- package/dist/core/index.js +2 -0
- package/dist/core/index.js.map +1 -1
- package/dist/core/types.d.ts +19 -0
- package/dist/core/types.d.ts.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/lipsync-helpers/arkitBlendshapeHelpers.d.ts +80 -0
- package/dist/lipsync-helpers/arkitBlendshapeHelpers.d.ts.map +1 -0
- package/dist/lipsync-helpers/arkitBlendshapeHelpers.js +201 -0
- package/dist/lipsync-helpers/arkitBlendshapeHelpers.js.map +1 -0
- package/dist/lipsync-helpers/arkitOrder61.d.ts +121 -0
- package/dist/lipsync-helpers/arkitOrder61.d.ts.map +1 -0
- package/dist/lipsync-helpers/arkitOrder61.js +287 -0
- package/dist/lipsync-helpers/arkitOrder61.js.map +1 -0
- package/dist/lipsync-helpers/arkitPhonemeReference.d.ts +155 -0
- package/dist/lipsync-helpers/arkitPhonemeReference.d.ts.map +1 -0
- package/dist/lipsync-helpers/arkitPhonemeReference.js +362 -0
- package/dist/lipsync-helpers/arkitPhonemeReference.js.map +1 -0
- package/dist/lipsync-helpers/index.d.ts +10 -0
- package/dist/lipsync-helpers/index.d.ts.map +1 -0
- package/dist/lipsync-helpers/index.js +21 -0
- package/dist/lipsync-helpers/index.js.map +1 -0
- package/dist/lipsync-helpers/metahumanOrder251.d.ts +115 -0
- package/dist/lipsync-helpers/metahumanOrder251.d.ts.map +1 -0
- package/dist/lipsync-helpers/metahumanOrder251.js +432 -0
- package/dist/lipsync-helpers/metahumanOrder251.js.map +1 -0
- package/dist/lipsync-helpers/neurosyncBlendshapeMapper.d.ts +30 -0
- package/dist/lipsync-helpers/neurosyncBlendshapeMapper.d.ts.map +1 -0
- package/dist/lipsync-helpers/neurosyncBlendshapeMapper.js +315 -0
- package/dist/lipsync-helpers/neurosyncBlendshapeMapper.js.map +1 -0
- package/dist/react/components/ConvaiWidget.d.ts.map +1 -1
- package/dist/react/components/ConvaiWidget.js +43 -3
- package/dist/react/components/ConvaiWidget.js.map +1 -1
- package/dist/react/hooks/useCharacterInfo.d.ts.map +1 -1
- package/dist/react/hooks/useCharacterInfo.js +1 -1
- package/dist/react/hooks/useCharacterInfo.js.map +1 -1
- package/dist/react/hooks/useConvaiClient.d.ts.map +1 -1
- package/dist/react/hooks/useConvaiClient.js +5 -0
- package/dist/react/hooks/useConvaiClient.js.map +1 -1
- package/dist/utils/speakerManagement.d.ts.map +1 -1
- package/dist/utils/speakerManagement.js +0 -5
- package/dist/utils/speakerManagement.js.map +1 -1
- package/dist/vanilla/AudioRenderer.d.ts +5 -0
- package/dist/vanilla/AudioRenderer.d.ts.map +1 -1
- package/dist/vanilla/AudioRenderer.js +27 -18
- package/dist/vanilla/AudioRenderer.js.map +1 -1
- package/package.json +4 -4
package/README.md
CHANGED
|
@@ -1,47 +1,6 @@
|
|
|
1
1
|
# @convai/web-sdk
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
JavaScript/TypeScript SDK for building AI voice assistants with real-time audio/video streaming. Drop-in widgets for **React** and **Vanilla JavaScript/TypeScript** with customizable UI components.
|
|
6
|
-
|
|
7
|
-
---
|
|
8
|
-
|
|
9
|
-
## 📑 Table of Contents
|
|
10
|
-
|
|
11
|
-
- [Installation](#installation)
|
|
12
|
-
- [Quick Start](#quick-start)
|
|
13
|
-
- [React - ConvaiWidget](#react---convaiwidget)
|
|
14
|
-
- [Vanilla JS/TS - ConvaiWidget](#vanilla-jsts---convaiwidget)
|
|
15
|
-
- [Core Concepts](#core-concepts)
|
|
16
|
-
- [React SDK](#react-sdk)
|
|
17
|
-
- [useConvaiClient Hook](#useconvaiclient-hook)
|
|
18
|
-
- [AudioRenderer Component](#audiorenderer-component)
|
|
19
|
-
- [AudioContext](#audiocontext)
|
|
20
|
-
- [React Exports Reference](#react-exports-reference)
|
|
21
|
-
- [Vanilla SDK](#vanilla-sdk)
|
|
22
|
-
- [ConvaiClient Class](#convaiclient-class)
|
|
23
|
-
- [AudioRenderer Class](#audiorenderer-class)
|
|
24
|
-
- [Vanilla Exports Reference](#vanilla-exports-reference)
|
|
25
|
-
- [Video & Screen Share](#video--screen-share)
|
|
26
|
-
- [Critical Requirements](#critical-requirements)
|
|
27
|
-
- [Enabling Video](#enabling-video)
|
|
28
|
-
- [Enabling Screen Share](#enabling-screen-share)
|
|
29
|
-
- [Building Custom UIs](#building-custom-uis)
|
|
30
|
-
- [Custom Chat Interface](#custom-chat-interface)
|
|
31
|
-
- [Audio Visualizer](#audio-visualizer)
|
|
32
|
-
- [Message Types](#message-types)
|
|
33
|
-
- [API Reference](#api-reference)
|
|
34
|
-
- [Configuration](#configuration)
|
|
35
|
-
- [Connection Management](#connection-management)
|
|
36
|
-
- [Messaging](#messaging)
|
|
37
|
-
- [Audio Controls](#audio-controls)
|
|
38
|
-
- [Video Controls](#video-controls)
|
|
39
|
-
- [Screen Share Controls](#screen-share-controls)
|
|
40
|
-
- [Getting Credentials](#getting-credentials)
|
|
41
|
-
- [TypeScript Support](#typescript-support)
|
|
42
|
-
- [Support](#support)
|
|
43
|
-
|
|
44
|
-
---
|
|
3
|
+
JavaScript/TypeScript SDK for Convai AI voice assistants. Build voice-powered AI interactions for web applications with real-time audio/video streaming. Supports both React and Vanilla JavaScript/TypeScript.
|
|
45
4
|
|
|
46
5
|
## Installation
|
|
47
6
|
|
|
@@ -49,25 +8,14 @@ JavaScript/TypeScript SDK for building AI voice assistants with real-time audio/
|
|
|
49
8
|
npm install @convai/web-sdk
|
|
50
9
|
```
|
|
51
10
|
|
|
52
|
-
|
|
11
|
+
## Basic Setup
|
|
53
12
|
|
|
54
|
-
|
|
55
|
-
npm install react@^18.0.0 react-dom@^18.0.0
|
|
56
|
-
```
|
|
57
|
-
|
|
58
|
-
---
|
|
59
|
-
|
|
60
|
-
## Quick Start
|
|
61
|
-
|
|
62
|
-
### React - ConvaiWidget
|
|
63
|
-
|
|
64
|
-
The `ConvaiWidget` is a complete, pre-built chat interface with voice/video capabilities.
|
|
13
|
+
### React
|
|
65
14
|
|
|
66
15
|
```tsx
|
|
67
|
-
import { useConvaiClient, ConvaiWidget } from "@convai/web-sdk
|
|
16
|
+
import { useConvaiClient, ConvaiWidget } from "@convai/web-sdk";
|
|
68
17
|
|
|
69
18
|
function App() {
|
|
70
|
-
// Initialize the Convai client
|
|
71
19
|
const convaiClient = useConvaiClient({
|
|
72
20
|
apiKey: "your-api-key",
|
|
73
21
|
characterId: "your-character-id",
|
|
@@ -77,9 +25,7 @@ function App() {
|
|
|
77
25
|
}
|
|
78
26
|
```
|
|
79
27
|
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
### Vanilla JS/TS - ConvaiWidget
|
|
28
|
+
### Vanilla TypeScript
|
|
83
29
|
|
|
84
30
|
```typescript
|
|
85
31
|
import { ConvaiClient, createConvaiWidget } from "@convai/web-sdk/vanilla";
|
|
@@ -90,7 +36,7 @@ const client = new ConvaiClient({
|
|
|
90
36
|
characterId: "your-character-id",
|
|
91
37
|
});
|
|
92
38
|
|
|
93
|
-
// Create
|
|
39
|
+
// Create widget - auto-connects on first user click
|
|
94
40
|
const widget = createConvaiWidget(document.body, {
|
|
95
41
|
convaiClient: client,
|
|
96
42
|
});
|
|
@@ -99,1025 +45,770 @@ const widget = createConvaiWidget(document.body, {
|
|
|
99
45
|
widget.destroy();
|
|
100
46
|
```
|
|
101
47
|
|
|
102
|
-
|
|
48
|
+
## Exports
|
|
103
49
|
|
|
104
|
-
|
|
50
|
+
### React Exports (`@convai/web-sdk` or `@convai/web-sdk/react`)
|
|
105
51
|
|
|
106
|
-
|
|
52
|
+
**Components:**
|
|
107
53
|
|
|
108
|
-
|
|
109
|
-
┌─────────────────────────────────────────────────┐
|
|
110
|
-
│ ConvaiWidget (UI Layer) │
|
|
111
|
-
│ ├─ Chat Interface │
|
|
112
|
-
│ ├─ Voice Mode │
|
|
113
|
-
│ └─ Video/Screen Share UI │
|
|
114
|
-
└─────────────────────────────────────────────────┘
|
|
115
|
-
▼
|
|
116
|
-
┌─────────────────────────────────────────────────┐
|
|
117
|
-
│ ConvaiClient (Core Logic) │
|
|
118
|
-
│ ├─ Connection Management │
|
|
119
|
-
│ ├─ Message Handling │
|
|
120
|
-
│ ├─ State Management │
|
|
121
|
-
│ └─ Audio/Video Controls │
|
|
122
|
-
└─────────────────────────────────────────────────┘
|
|
123
|
-
▼
|
|
124
|
-
┌─────────────────────────────────────────────────┐
|
|
125
|
-
│ WebRTC Room (Communication Layer) │
|
|
126
|
-
│ ├─ Real-time Audio/Video Streaming │
|
|
127
|
-
│ ├─ Track Management │
|
|
128
|
-
│ └─ Network Communication │
|
|
129
|
-
└─────────────────────────────────────────────────┘
|
|
130
|
-
▼
|
|
131
|
-
┌─────────────────────────────────────────────────┐
|
|
132
|
-
│ AudioRenderer (Critical for Playback) │
|
|
133
|
-
│ ├─ Attaches audio tracks to DOM │
|
|
134
|
-
│ ├─ Manages audio elements │
|
|
135
|
-
│ └─ Enables bot voice playback │
|
|
136
|
-
└─────────────────────────────────────────────────┘
|
|
137
|
-
```
|
|
54
|
+
- `ConvaiWidget` - Main chat widget component
|
|
138
55
|
|
|
139
|
-
|
|
56
|
+
**Hooks:**
|
|
140
57
|
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
4. **Connection Type** - Determines capabilities:
|
|
145
|
-
- `"audio"` (default) - Audio only
|
|
146
|
-
- `"video"` - Audio + Video + Screen Share
|
|
58
|
+
- `useConvaiClient(config?)` - Main client hook
|
|
59
|
+
- `useCharacterInfo(characterId, apiKey)` - Fetch character metadata
|
|
60
|
+
- `useLocalCameraTrack()` - Get local camera track
|
|
147
61
|
|
|
148
|
-
|
|
62
|
+
**Core Client:**
|
|
149
63
|
|
|
150
|
-
|
|
64
|
+
- `ConvaiClient` - Core client class
|
|
151
65
|
|
|
152
|
-
|
|
66
|
+
**Types:**
|
|
153
67
|
|
|
154
|
-
|
|
68
|
+
- `ConvaiConfig` - Configuration interface
|
|
69
|
+
- `ConvaiClientState` - Client state interface
|
|
70
|
+
- `ChatMessage` - Message interface
|
|
71
|
+
- `IConvaiClient` - Client interface
|
|
72
|
+
- `AudioControls` - Audio control interface
|
|
73
|
+
- `VideoControls` - Video control interface
|
|
74
|
+
- `ScreenShareControls` - Screen share control interface
|
|
155
75
|
|
|
156
|
-
**
|
|
76
|
+
**Components:**
|
|
157
77
|
|
|
158
|
-
|
|
78
|
+
- `AudioRenderer` - Audio playback component
|
|
79
|
+
- `AudioContext` - Audio context provider
|
|
159
80
|
|
|
160
|
-
|
|
161
|
-
- Provides reactive state (connection, messages, activity)
|
|
162
|
-
- Handles connection lifecycle
|
|
163
|
-
- Exposes audio/video/screen share controls
|
|
164
|
-
|
|
165
|
-
```tsx
|
|
166
|
-
import { useConvaiClient } from "@convai/web-sdk/react";
|
|
81
|
+
### Vanilla Exports (`@convai/web-sdk/vanilla`)
|
|
167
82
|
|
|
168
|
-
|
|
169
|
-
const convaiClient = useConvaiClient({
|
|
170
|
-
apiKey: "your-api-key",
|
|
171
|
-
characterId: "your-character-id",
|
|
172
|
-
enableVideo: false, // Default: audio only
|
|
173
|
-
startWithAudioOn: false, // Mic starts muted
|
|
174
|
-
});
|
|
83
|
+
**Functions:**
|
|
175
84
|
|
|
176
|
-
|
|
177
|
-
|
|
85
|
+
- `createConvaiWidget(container, options)` - Create widget instance
|
|
86
|
+
- `destroyConvaiWidget(widget)` - Destroy widget instance
|
|
178
87
|
|
|
179
|
-
|
|
180
|
-
const handleMute = () => convaiClient.audioControls.muteAudio();
|
|
181
|
-
const handleSend = () =>
|
|
182
|
-
convaiClient.sendUserTextMessage("Hello, character!");
|
|
88
|
+
**Classes:**
|
|
183
89
|
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
<p>Status: {state.agentState}</p>
|
|
187
|
-
<p>Messages: {chatMessages.length}</p>
|
|
188
|
-
<button onClick={handleMute}>Mute</button>
|
|
189
|
-
<button onClick={handleSend}>Send</button>
|
|
190
|
-
</div>
|
|
191
|
-
);
|
|
192
|
-
}
|
|
193
|
-
```
|
|
90
|
+
- `ConvaiClient` - Core client class
|
|
91
|
+
- `AudioRenderer` - Audio playback handler
|
|
194
92
|
|
|
195
|
-
|
|
93
|
+
**Types:**
|
|
196
94
|
|
|
197
|
-
|
|
95
|
+
- `VanillaWidget` - Widget instance interface
|
|
96
|
+
- `VanillaWidgetOptions` - Widget options interface
|
|
97
|
+
- `IConvaiClient` - Client interface
|
|
98
|
+
- `ConvaiConfig` - Configuration interface
|
|
99
|
+
- `ConvaiClientState` - Client state interface
|
|
100
|
+
- `ChatMessage` - Message interface
|
|
198
101
|
|
|
199
|
-
|
|
102
|
+
### Core Exports (`@convai/web-sdk/core`)
|
|
200
103
|
|
|
201
|
-
**
|
|
104
|
+
**Classes:**
|
|
202
105
|
|
|
203
|
-
-
|
|
204
|
-
-
|
|
106
|
+
- `ConvaiClient` - Main client class
|
|
107
|
+
- `AudioManager` - Audio management
|
|
108
|
+
- `VideoManager` - Video management
|
|
109
|
+
- `ScreenShareManager` - Screen share management
|
|
110
|
+
- `MessageHandler` - Message handling
|
|
111
|
+
- `EventEmitter` - Event emitter base class
|
|
205
112
|
|
|
206
|
-
**
|
|
113
|
+
**Types:**
|
|
207
114
|
|
|
208
|
-
-
|
|
209
|
-
-
|
|
210
|
-
- Manages audio playback lifecycle
|
|
115
|
+
- All types from React/Vanilla exports
|
|
116
|
+
- `ConvaiClientType` - Type alias for ConvaiClient
|
|
211
117
|
|
|
212
|
-
|
|
213
|
-
import { useConvaiClient, AudioRenderer } from "@convai/web-sdk/react";
|
|
118
|
+
## Props and Configuration
|
|
214
119
|
|
|
215
|
-
|
|
216
|
-
const convaiClient = useConvaiClient({
|
|
217
|
-
apiKey: "your-api-key",
|
|
218
|
-
characterId: "your-character-id",
|
|
219
|
-
});
|
|
220
|
-
|
|
221
|
-
return (
|
|
222
|
-
<div>
|
|
223
|
-
{/* CRITICAL: This component renders bot audio to speakers */}
|
|
224
|
-
<AudioRenderer />
|
|
225
|
-
|
|
226
|
-
{/* Your custom UI */}
|
|
227
|
-
<div>
|
|
228
|
-
{convaiClient.chatMessages.map((msg) => (
|
|
229
|
-
<div key={msg.id}>{msg.content}</div>
|
|
230
|
-
))}
|
|
231
|
-
</div>
|
|
232
|
-
</div>
|
|
233
|
-
);
|
|
234
|
-
}
|
|
235
|
-
```
|
|
236
|
-
|
|
237
|
-
### AudioContext
|
|
238
|
-
|
|
239
|
-
**Purpose**: Provides the WebRTC Room to child components.
|
|
240
|
-
|
|
241
|
-
**When to Use**: When building deeply nested custom UIs that need direct access to the audio room.
|
|
242
|
-
|
|
243
|
-
**How It Works**: React Context that holds the active WebRTC room.
|
|
120
|
+
### ConvaiWidget Props (React)
|
|
244
121
|
|
|
245
122
|
```tsx
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
</AudioContext.Provider>
|
|
259
|
-
);
|
|
260
|
-
}
|
|
261
|
-
|
|
262
|
-
function ChildComponent() {
|
|
263
|
-
const room = useContext(AudioContext);
|
|
264
|
-
// Access WebRTC room directly
|
|
265
|
-
console.log("Room state:", room?.state);
|
|
266
|
-
return <div>Child has access to Room</div>;
|
|
123
|
+
interface ConvaiWidgetProps {
|
|
124
|
+
/** Convai client instance (required) */
|
|
125
|
+
convaiClient: IConvaiClient & {
|
|
126
|
+
activity?: string;
|
|
127
|
+
isAudioMuted: boolean;
|
|
128
|
+
isVideoEnabled: boolean;
|
|
129
|
+
isScreenShareActive: boolean;
|
|
130
|
+
};
|
|
131
|
+
/** Show video toggle button in settings (default: true) */
|
|
132
|
+
showVideo?: boolean;
|
|
133
|
+
/** Show screen share toggle button in settings (default: true) */
|
|
134
|
+
showScreenShare?: boolean;
|
|
267
135
|
}
|
|
268
136
|
```
|
|
269
137
|
|
|
270
|
-
###
|
|
271
|
-
|
|
272
|
-
```tsx
|
|
273
|
-
// Components
|
|
274
|
-
import { ConvaiWidget } from "@convai/web-sdk/react";
|
|
275
|
-
|
|
276
|
-
// Hooks
|
|
277
|
-
import { useConvaiClient, useCharacterInfo } from "@convai/web-sdk/react";
|
|
278
|
-
|
|
279
|
-
// Audio Rendering (Critical)
|
|
280
|
-
import { AudioRenderer, AudioContext } from "@convai/web-sdk/react";
|
|
281
|
-
|
|
282
|
-
// Core Client (for advanced usage)
|
|
283
|
-
import { ConvaiClient } from "@convai/web-sdk/react";
|
|
284
|
-
|
|
285
|
-
// Types
|
|
286
|
-
import type {
|
|
287
|
-
ConvaiConfig,
|
|
288
|
-
ConvaiClientState,
|
|
289
|
-
ChatMessage,
|
|
290
|
-
IConvaiClient,
|
|
291
|
-
AudioControls,
|
|
292
|
-
VideoControls,
|
|
293
|
-
ScreenShareControls,
|
|
294
|
-
} from "@convai/web-sdk/react";
|
|
295
|
-
```
|
|
296
|
-
|
|
297
|
-
---
|
|
298
|
-
|
|
299
|
-
## Vanilla SDK
|
|
300
|
-
|
|
301
|
-
### ConvaiClient Class
|
|
302
|
-
|
|
303
|
-
**Purpose**: Core client for managing Convai connections in vanilla JavaScript/TypeScript.
|
|
304
|
-
|
|
305
|
-
**When to Use**: Any non-React application or when you need full control.
|
|
306
|
-
|
|
307
|
-
**What It Provides**:
|
|
308
|
-
|
|
309
|
-
- Connection management
|
|
310
|
-
- Message handling
|
|
311
|
-
- State management (via events)
|
|
312
|
-
- Audio/video/screen share controls
|
|
313
|
-
|
|
314
|
-
```typescript
|
|
315
|
-
import { ConvaiClient } from "@convai/web-sdk/vanilla";
|
|
316
|
-
|
|
317
|
-
const client = new ConvaiClient({
|
|
318
|
-
apiKey: "your-api-key",
|
|
319
|
-
characterId: "your-character-id",
|
|
320
|
-
});
|
|
321
|
-
|
|
322
|
-
// Connect
|
|
323
|
-
await client.connect();
|
|
324
|
-
|
|
325
|
-
// Listen to events
|
|
326
|
-
client.on("stateChange", (state) => {
|
|
327
|
-
console.log("Agent state:", state.agentState);
|
|
328
|
-
});
|
|
329
|
-
|
|
330
|
-
client.on("message", (message) => {
|
|
331
|
-
console.log("New message:", message.content);
|
|
332
|
-
});
|
|
333
|
-
|
|
334
|
-
// Send messages
|
|
335
|
-
client.sendUserTextMessage("Hello!");
|
|
336
|
-
|
|
337
|
-
// Control audio
|
|
338
|
-
await client.audioControls.muteAudio();
|
|
339
|
-
await client.audioControls.unmuteAudio();
|
|
340
|
-
|
|
341
|
-
// Disconnect
|
|
342
|
-
await client.disconnect();
|
|
343
|
-
```
|
|
344
|
-
|
|
345
|
-
### AudioRenderer Class
|
|
346
|
-
|
|
347
|
-
**Purpose**: Manages audio playback for vanilla JavaScript/TypeScript applications.
|
|
348
|
-
|
|
349
|
-
**⚠️ CRITICAL**: Without this, you will NOT hear the bot's voice.
|
|
350
|
-
|
|
351
|
-
**When to Use**:
|
|
352
|
-
|
|
353
|
-
- Always when building custom vanilla UIs
|
|
354
|
-
- Already included in vanilla `ConvaiWidget` (no need to add separately)
|
|
355
|
-
|
|
356
|
-
**How It Works**:
|
|
357
|
-
|
|
358
|
-
- Attaches to the WebRTC room
|
|
359
|
-
- Automatically creates hidden `<audio>` elements
|
|
360
|
-
- Manages audio playback for remote participants (the bot)
|
|
138
|
+
### createConvaiWidget Options (Vanilla)
|
|
361
139
|
|
|
362
140
|
```typescript
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
}
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
// Your custom UI logic...
|
|
376
|
-
|
|
377
|
-
// Cleanup
|
|
378
|
-
audioRenderer.destroy();
|
|
379
|
-
await client.disconnect();
|
|
141
|
+
interface VanillaWidgetOptions {
|
|
142
|
+
/** Convai client instance (required) */
|
|
143
|
+
convaiClient: IConvaiClient & {
|
|
144
|
+
activity?: string;
|
|
145
|
+
chatMessages: ChatMessage[];
|
|
146
|
+
};
|
|
147
|
+
/** Show video toggle button in settings (default: true) */
|
|
148
|
+
showVideo?: boolean;
|
|
149
|
+
/** Show screen share toggle button in settings (default: true) */
|
|
150
|
+
showScreenShare?: boolean;
|
|
151
|
+
}
|
|
380
152
|
```
|
|
381
153
|
|
|
382
|
-
###
|
|
154
|
+
### ConvaiConfig
|
|
383
155
|
|
|
384
156
|
```typescript
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
157
|
+
interface ConvaiConfig {
|
|
158
|
+
/** Your Convai API key from convai.com dashboard (required) */
|
|
159
|
+
apiKey: string;
|
|
160
|
+
/** The Character ID to connect to (required) */
|
|
161
|
+
characterId: string;
|
|
162
|
+
/**
|
|
163
|
+
* End user identifier for speaker management (optional).
|
|
164
|
+
* When provided: enables long-term memory and analytics
|
|
165
|
+
* When not provided: anonymous mode, no persistent memory
|
|
166
|
+
*/
|
|
167
|
+
endUserId?: string;
|
|
168
|
+
/** Custom Convai API URL (optional, defaults to production endpoint) */
|
|
169
|
+
url?: string;
|
|
170
|
+
/**
|
|
171
|
+
* Enable video capability (default: false).
|
|
172
|
+
* If true, connection_type will be "video" (supports audio, video, and screenshare).
|
|
173
|
+
* If false, connection_type will be "audio" (audio only).
|
|
174
|
+
*/
|
|
175
|
+
enableVideo?: boolean;
|
|
176
|
+
/**
|
|
177
|
+
* Start with video camera on when connecting (default: false).
|
|
178
|
+
* Only works if enableVideo is true.
|
|
179
|
+
*/
|
|
180
|
+
startWithVideoOn?: boolean;
|
|
181
|
+
/**
|
|
182
|
+
* Start with microphone on when connecting (default: false).
|
|
183
|
+
* If false, microphone stays off until user enables it.
|
|
184
|
+
*/
|
|
185
|
+
startWithAudioOn?: boolean;
|
|
186
|
+
/** Enable text-to-speech audio generation (default: true) */
|
|
187
|
+
ttsEnabled?: boolean;
|
|
188
|
+
}
|
|
403
189
|
```
|
|
404
190
|
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
## Video & Screen Share
|
|
191
|
+
## Features
|
|
408
192
|
|
|
409
|
-
###
|
|
193
|
+
### Video Enabled Chat
|
|
410
194
|
|
|
411
|
-
|
|
195
|
+
To enable video capabilities, set `enableVideo: true` in your configuration. This enables audio, video, and screen sharing.
|
|
412
196
|
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
This sets the connection type to `"video"` which enables video capabilities.
|
|
416
|
-
|
|
417
|
-
#### 2. Set `showVideo` and/or `showScreenShare` in Widget Props
|
|
418
|
-
|
|
419
|
-
This shows the UI controls for video/screen share.
|
|
420
|
-
|
|
421
|
-
**Without both, video features will NOT work.**
|
|
422
|
-
|
|
423
|
-
---
|
|
424
|
-
|
|
425
|
-
### Enabling Video
|
|
426
|
-
|
|
427
|
-
#### React
|
|
197
|
+
**React:**
|
|
428
198
|
|
|
429
199
|
```tsx
|
|
430
|
-
import { useConvaiClient, ConvaiWidget } from "@convai/web-sdk
|
|
200
|
+
import { useConvaiClient, ConvaiWidget } from "@convai/web-sdk";
|
|
431
201
|
|
|
432
202
|
function App() {
|
|
433
|
-
// ✅ STEP 1: Enable video in client config
|
|
434
203
|
const convaiClient = useConvaiClient({
|
|
435
204
|
apiKey: "your-api-key",
|
|
436
205
|
characterId: "your-character-id",
|
|
437
|
-
enableVideo: true,
|
|
206
|
+
enableVideo: true,
|
|
438
207
|
startWithVideoOn: false, // Camera off by default
|
|
439
208
|
});
|
|
440
209
|
|
|
441
210
|
return (
|
|
442
211
|
<ConvaiWidget
|
|
443
212
|
convaiClient={convaiClient}
|
|
444
|
-
showVideo={true}
|
|
445
|
-
showScreenShare={
|
|
213
|
+
showVideo={true}
|
|
214
|
+
showScreenShare={true}
|
|
446
215
|
/>
|
|
447
216
|
);
|
|
448
217
|
}
|
|
449
218
|
```
|
|
450
219
|
|
|
451
|
-
|
|
220
|
+
**Vanilla:**
|
|
452
221
|
|
|
453
222
|
```typescript
|
|
454
223
|
import { ConvaiClient, createConvaiWidget } from "@convai/web-sdk/vanilla";
|
|
455
224
|
|
|
456
|
-
// ✅ STEP 1: Enable video in client config
|
|
457
225
|
const client = new ConvaiClient({
|
|
458
226
|
apiKey: "your-api-key",
|
|
459
227
|
characterId: "your-character-id",
|
|
460
|
-
enableVideo: true,
|
|
228
|
+
enableVideo: true,
|
|
461
229
|
startWithVideoOn: false,
|
|
462
230
|
});
|
|
463
231
|
|
|
464
232
|
const widget = createConvaiWidget(document.body, {
|
|
465
233
|
convaiClient: client,
|
|
466
|
-
showVideo: true,
|
|
467
|
-
showScreenShare:
|
|
234
|
+
showVideo: true,
|
|
235
|
+
showScreenShare: true,
|
|
468
236
|
});
|
|
469
237
|
```
|
|
470
238
|
|
|
471
|
-
|
|
239
|
+
**Manual Video Controls:**
|
|
472
240
|
|
|
473
241
|
```typescript
|
|
474
|
-
// Enable camera
|
|
242
|
+
// Enable video camera
|
|
475
243
|
await convaiClient.videoControls.enableVideo();
|
|
476
244
|
|
|
477
|
-
// Disable camera
|
|
245
|
+
// Disable video camera
|
|
478
246
|
await convaiClient.videoControls.disableVideo();
|
|
479
247
|
|
|
480
|
-
// Toggle
|
|
248
|
+
// Toggle video
|
|
481
249
|
await convaiClient.videoControls.toggleVideo();
|
|
482
250
|
|
|
483
|
-
// Check state
|
|
484
|
-
|
|
251
|
+
// Check video state
|
|
252
|
+
const isVideoEnabled = convaiClient.videoControls.isVideoEnabled;
|
|
485
253
|
|
|
486
254
|
// Set video quality
|
|
487
255
|
await convaiClient.videoControls.setVideoQuality("high"); // 'low' | 'medium' | 'high'
|
|
488
256
|
|
|
489
|
-
// Get available
|
|
257
|
+
// Get available video devices
|
|
490
258
|
const devices = await convaiClient.videoControls.getVideoDevices();
|
|
491
259
|
|
|
492
|
-
//
|
|
260
|
+
// Set specific video device
|
|
493
261
|
await convaiClient.videoControls.setVideoDevice(deviceId);
|
|
494
262
|
```
|
|
495
263
|
|
|
496
|
-
|
|
264
|
+
**Screen Sharing:**
|
|
497
265
|
|
|
498
|
-
|
|
266
|
+
```typescript
|
|
267
|
+
// Enable screen share
|
|
268
|
+
await convaiClient.screenShareControls.enableScreenShare();
|
|
499
269
|
|
|
500
|
-
|
|
270
|
+
// Enable screen share with audio
|
|
271
|
+
await convaiClient.screenShareControls.enableScreenShareWithAudio();
|
|
501
272
|
|
|
502
|
-
|
|
273
|
+
// Disable screen share
|
|
274
|
+
await convaiClient.screenShareControls.disableScreenShare();
|
|
503
275
|
|
|
504
|
-
|
|
505
|
-
|
|
276
|
+
// Toggle screen share
|
|
277
|
+
await convaiClient.screenShareControls.toggleScreenShare();
|
|
506
278
|
|
|
507
|
-
|
|
508
|
-
|
|
279
|
+
// Check screen share state
|
|
280
|
+
const isActive = convaiClient.screenShareControls.isScreenShareActive;
|
|
281
|
+
```
|
|
282
|
+
|
|
283
|
+
**Video State Monitoring:**
|
|
284
|
+
|
|
285
|
+
```typescript
|
|
286
|
+
// React
|
|
287
|
+
const { isVideoEnabled } = convaiClient;
|
|
288
|
+
|
|
289
|
+
// Core API (event-based)
|
|
290
|
+
convaiClient.videoControls.on("videoStateChange", (state) => {
|
|
291
|
+
console.log("Video enabled:", state.isVideoEnabled);
|
|
292
|
+
console.log("Video hidden:", state.isVideoHidden);
|
|
293
|
+
});
|
|
294
|
+
```
|
|
295
|
+
|
|
296
|
+
### Interruption
|
|
297
|
+
|
|
298
|
+
Interrupt the character's current response to allow the user to speak immediately.
|
|
299
|
+
|
|
300
|
+
**React:**
|
|
301
|
+
|
|
302
|
+
```tsx
|
|
303
|
+
function ChatInterface() {
|
|
509
304
|
const convaiClient = useConvaiClient({
|
|
510
|
-
|
|
511
|
-
characterId: "your-character-id",
|
|
512
|
-
enableVideo: true, // ← REQUIRED for screen share
|
|
305
|
+
/* config */
|
|
513
306
|
});
|
|
514
307
|
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
);
|
|
308
|
+
const handleInterrupt = () => {
|
|
309
|
+
// Interrupt the bot's current response
|
|
310
|
+
convaiClient.sendInterruptMessage();
|
|
311
|
+
};
|
|
312
|
+
|
|
313
|
+
return <button onClick={handleInterrupt}>Interrupt</button>;
|
|
522
314
|
}
|
|
523
315
|
```
|
|
524
316
|
|
|
525
|
-
|
|
317
|
+
**Vanilla:**
|
|
526
318
|
|
|
527
319
|
```typescript
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
// ✅ STEP 1: Enable video (required for screen share)
|
|
531
|
-
const client = new ConvaiClient({
|
|
532
|
-
apiKey: "your-api-key",
|
|
533
|
-
characterId: "your-character-id",
|
|
534
|
-
enableVideo: true, // ← REQUIRED for screen share
|
|
535
|
-
});
|
|
320
|
+
const interruptButton = document.getElementById("interrupt-btn");
|
|
536
321
|
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
showVideo: true,
|
|
540
|
-
showScreenShare: true, // ← STEP 2: Show screen share controls
|
|
322
|
+
interruptButton.addEventListener("click", () => {
|
|
323
|
+
client.sendInterruptMessage();
|
|
541
324
|
});
|
|
542
325
|
```
|
|
543
326
|
|
|
544
|
-
|
|
327
|
+
**Voice Mode Interruption Pattern:**
|
|
545
328
|
|
|
546
|
-
|
|
547
|
-
// Start screen share
|
|
548
|
-
await convaiClient.screenShareControls.enableScreenShare();
|
|
329
|
+
When implementing voice mode, interrupt the bot when the user starts speaking:
|
|
549
330
|
|
|
550
|
-
|
|
551
|
-
|
|
331
|
+
```typescript
|
|
332
|
+
// When user enters voice mode
|
|
333
|
+
const enterVoiceMode = async () => {
|
|
334
|
+
// Interrupt any ongoing bot response
|
|
335
|
+
convaiClient.sendInterruptMessage();
|
|
552
336
|
|
|
553
|
-
//
|
|
554
|
-
await convaiClient.
|
|
337
|
+
// Unmute microphone
|
|
338
|
+
await convaiClient.audioControls.unmuteAudio();
|
|
339
|
+
};
|
|
555
340
|
|
|
556
|
-
//
|
|
557
|
-
|
|
341
|
+
// When user exits voice mode
|
|
342
|
+
const exitVoiceMode = async () => {
|
|
343
|
+
// Interrupt any ongoing bot response
|
|
344
|
+
convaiClient.sendInterruptMessage();
|
|
558
345
|
|
|
559
|
-
//
|
|
560
|
-
|
|
346
|
+
// Mute microphone
|
|
347
|
+
await convaiClient.audioControls.muteAudio();
|
|
348
|
+
};
|
|
561
349
|
```
|
|
562
350
|
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
## Building Custom UIs
|
|
351
|
+
### User Microphone Mute/Unmute
|
|
566
352
|
|
|
567
|
-
|
|
353
|
+
Control the user's microphone input.
|
|
568
354
|
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
#### React Example
|
|
355
|
+
**React:**
|
|
572
356
|
|
|
573
357
|
```tsx
|
|
574
|
-
|
|
575
|
-
import { useState } from "react";
|
|
576
|
-
|
|
577
|
-
function CustomChatUI() {
|
|
358
|
+
function AudioControls() {
|
|
578
359
|
const convaiClient = useConvaiClient({
|
|
579
|
-
|
|
580
|
-
characterId: "your-character-id",
|
|
360
|
+
/* config */
|
|
581
361
|
});
|
|
582
362
|
|
|
583
|
-
const
|
|
584
|
-
|
|
363
|
+
const handleMute = async () => {
|
|
364
|
+
await convaiClient.audioControls.muteAudio();
|
|
365
|
+
};
|
|
585
366
|
|
|
586
|
-
const
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
367
|
+
const handleUnmute = async () => {
|
|
368
|
+
await convaiClient.audioControls.unmuteAudio();
|
|
369
|
+
};
|
|
370
|
+
|
|
371
|
+
const handleToggle = async () => {
|
|
372
|
+
await convaiClient.audioControls.toggleAudio();
|
|
591
373
|
};
|
|
592
374
|
|
|
593
375
|
return (
|
|
594
376
|
<div>
|
|
595
|
-
{
|
|
596
|
-
<
|
|
597
|
-
|
|
598
|
-
{
|
|
599
|
-
<div className="chat-container">
|
|
600
|
-
{chatMessages.map((msg) => {
|
|
601
|
-
const isUser = msg.type.includes("user");
|
|
602
|
-
const displayMessage =
|
|
603
|
-
msg.type === "user-llm-text" || msg.type === "bot-llm-text";
|
|
604
|
-
|
|
605
|
-
if (!displayMessage) return null;
|
|
606
|
-
|
|
607
|
-
return (
|
|
608
|
-
<div
|
|
609
|
-
key={msg.id}
|
|
610
|
-
className={isUser ? "user-message" : "bot-message"}
|
|
611
|
-
>
|
|
612
|
-
<span className="sender">
|
|
613
|
-
{isUser ? "You" : "Character"}
|
|
614
|
-
</span>
|
|
615
|
-
<p>{msg.content}</p>
|
|
616
|
-
<span className="timestamp">
|
|
617
|
-
{new Date(msg.timestamp).toLocaleTimeString()}
|
|
618
|
-
</span>
|
|
619
|
-
</div>
|
|
620
|
-
);
|
|
621
|
-
})}
|
|
622
|
-
</div>
|
|
623
|
-
|
|
624
|
-
{/* Input */}
|
|
625
|
-
<div className="input-container">
|
|
626
|
-
<input
|
|
627
|
-
type="text"
|
|
628
|
-
value={inputValue}
|
|
629
|
-
onChange={(e) => setInputValue(e.target.value)}
|
|
630
|
-
onKeyPress={(e) => e.key === "Enter" && handleSend()}
|
|
631
|
-
placeholder="Type a message..."
|
|
632
|
-
disabled={!state.isConnected}
|
|
633
|
-
/>
|
|
634
|
-
<button onClick={handleSend} disabled={!state.isConnected}>
|
|
635
|
-
Send
|
|
636
|
-
</button>
|
|
637
|
-
</div>
|
|
638
|
-
|
|
639
|
-
{/* Status Indicator */}
|
|
640
|
-
<div className="status">
|
|
641
|
-
{state.isConnecting && "Connecting..."}
|
|
642
|
-
{state.isConnected && state.agentState}
|
|
643
|
-
{!state.isConnected && "Disconnected"}
|
|
644
|
-
</div>
|
|
377
|
+
<button onClick={handleMute}>Mute</button>
|
|
378
|
+
<button onClick={handleUnmute}>Unmute</button>
|
|
379
|
+
<button onClick={handleToggle}>Toggle</button>
|
|
380
|
+
<p>Muted: {convaiClient.audioControls.isAudioMuted ? "Yes" : "No"}</p>
|
|
645
381
|
</div>
|
|
646
382
|
);
|
|
647
383
|
}
|
|
648
384
|
```
|
|
649
385
|
|
|
650
|
-
|
|
386
|
+
**Vanilla:**
|
|
651
387
|
|
|
652
388
|
```typescript
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
const client = new ConvaiClient({
|
|
656
|
-
apiKey: "your-api-key",
|
|
657
|
-
characterId: "your-character-id",
|
|
658
|
-
});
|
|
659
|
-
|
|
660
|
-
await client.connect();
|
|
661
|
-
|
|
662
|
-
// CRITICAL: Create AudioRenderer for bot voice
|
|
663
|
-
const audioRenderer = new AudioRenderer(client.room);
|
|
389
|
+
// Mute microphone
|
|
390
|
+
await client.audioControls.muteAudio();
|
|
664
391
|
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
const sendButton = document.getElementById("send-button");
|
|
392
|
+
// Unmute microphone
|
|
393
|
+
await client.audioControls.unmuteAudio();
|
|
668
394
|
|
|
669
|
-
//
|
|
670
|
-
client.
|
|
671
|
-
chatContainer.innerHTML = "";
|
|
395
|
+
// Toggle mute state
|
|
396
|
+
await client.audioControls.toggleAudio();
|
|
672
397
|
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
const displayMessage =
|
|
676
|
-
msg.type === "user-llm-text" || msg.type === "bot-llm-text";
|
|
398
|
+
// Check mute state
|
|
399
|
+
const isMuted = client.audioControls.isAudioMuted;
|
|
677
400
|
|
|
678
|
-
|
|
401
|
+
// Enable audio (request permissions if needed)
|
|
402
|
+
await client.audioControls.enableAudio();
|
|
679
403
|
|
|
680
|
-
|
|
681
|
-
|
|
404
|
+
// Disable audio
|
|
405
|
+
await client.audioControls.disableAudio();
|
|
406
|
+
```
|
|
682
407
|
|
|
683
|
-
|
|
684
|
-
sender.textContent = isUser ? "You" : "Character";
|
|
685
|
-
sender.className = "sender";
|
|
408
|
+
**Audio Device Management:**
|
|
686
409
|
|
|
687
|
-
|
|
688
|
-
|
|
410
|
+
```typescript
|
|
411
|
+
// Get available audio devices
|
|
412
|
+
const devices = await convaiClient.audioControls.getAudioDevices();
|
|
689
413
|
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
timestamp.className = "timestamp";
|
|
414
|
+
// Set specific audio device
|
|
415
|
+
await convaiClient.audioControls.setAudioDevice(deviceId);
|
|
693
416
|
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
messageDiv.appendChild(timestamp);
|
|
697
|
-
chatContainer.appendChild(messageDiv);
|
|
698
|
-
});
|
|
417
|
+
// Monitor audio level
|
|
418
|
+
convaiClient.audioControls.startAudioLevelMonitoring();
|
|
699
419
|
|
|
700
|
-
|
|
701
|
-
|
|
420
|
+
convaiClient.audioControls.on("audioLevelChange", (level) => {
|
|
421
|
+
console.log("Audio level:", level);
|
|
422
|
+
// level is a number between 0 and 1
|
|
702
423
|
});
|
|
703
424
|
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
const text = inputElement.value.trim();
|
|
707
|
-
if (text && client.state.isConnected) {
|
|
708
|
-
client.sendUserTextMessage(text);
|
|
709
|
-
inputElement.value = "";
|
|
710
|
-
}
|
|
711
|
-
});
|
|
425
|
+
convaiClient.audioControls.stopAudioLevelMonitoring();
|
|
426
|
+
```
|
|
712
427
|
|
|
713
|
-
|
|
714
|
-
if (e.key === "Enter") {
|
|
715
|
-
sendButton.click();
|
|
716
|
-
}
|
|
717
|
-
});
|
|
428
|
+
**Audio State Monitoring:**
|
|
718
429
|
|
|
719
|
-
|
|
720
|
-
//
|
|
721
|
-
|
|
430
|
+
```typescript
|
|
431
|
+
// React
|
|
432
|
+
const { isAudioMuted } = convaiClient;
|
|
433
|
+
|
|
434
|
+
// Core API (event-based)
|
|
435
|
+
convaiClient.audioControls.on("audioStateChange", (state) => {
|
|
436
|
+
console.log("Audio enabled:", state.isAudioEnabled);
|
|
437
|
+
console.log("Audio muted:", state.isAudioMuted);
|
|
438
|
+
console.log("Audio level:", state.audioLevel);
|
|
439
|
+
});
|
|
722
440
|
```
|
|
723
441
|
|
|
724
|
-
|
|
442
|
+
### Character TTS Mute/Unmute
|
|
725
443
|
|
|
726
|
-
|
|
444
|
+
Control whether the character's responses are spoken aloud (text-to-speech).
|
|
727
445
|
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
#### React Example
|
|
446
|
+
**React:**
|
|
731
447
|
|
|
732
448
|
```tsx
|
|
733
|
-
|
|
734
|
-
import { useEffect, useRef, useState } from "react";
|
|
735
|
-
|
|
736
|
-
function AudioVisualizer() {
|
|
449
|
+
function TTSControls() {
|
|
737
450
|
const convaiClient = useConvaiClient({
|
|
738
|
-
|
|
739
|
-
characterId: "your-character-id",
|
|
451
|
+
/* config */
|
|
740
452
|
});
|
|
741
453
|
|
|
742
|
-
const
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
useEffect(() => {
|
|
746
|
-
if (!convaiClient.room) return;
|
|
747
|
-
|
|
748
|
-
let animationId: number;
|
|
749
|
-
let analyzer: AnalyserNode | null = null;
|
|
750
|
-
let dataArray: Uint8Array | null = null;
|
|
751
|
-
|
|
752
|
-
const setupAnalyzer = async () => {
|
|
753
|
-
const audioContext = new AudioContext();
|
|
754
|
-
|
|
755
|
-
// Get remote participant (bot)
|
|
756
|
-
const remoteParticipants = Array.from(
|
|
757
|
-
convaiClient.room.remoteParticipants.values()
|
|
758
|
-
);
|
|
759
|
-
|
|
760
|
-
if (remoteParticipants.length === 0) return;
|
|
761
|
-
|
|
762
|
-
const participant = remoteParticipants[0];
|
|
763
|
-
const audioTracks = Array.from(
|
|
764
|
-
participant.audioTrackPublications.values()
|
|
765
|
-
);
|
|
766
|
-
|
|
767
|
-
if (audioTracks.length === 0) return;
|
|
768
|
-
|
|
769
|
-
const audioTrack = audioTracks[0].track;
|
|
770
|
-
if (!audioTrack) return;
|
|
454
|
+
const handleToggleTTS = (enabled: boolean) => {
|
|
455
|
+
convaiClient.toggleTts(enabled);
|
|
456
|
+
};
|
|
771
457
|
|
|
772
|
-
|
|
773
|
-
|
|
458
|
+
return (
|
|
459
|
+
<div>
|
|
460
|
+
<button onClick={() => handleToggleTTS(true)}>Enable TTS</button>
|
|
461
|
+
<button onClick={() => handleToggleTTS(false)}>Disable TTS</button>
|
|
462
|
+
</div>
|
|
463
|
+
);
|
|
464
|
+
}
|
|
465
|
+
```
|
|
774
466
|
|
|
775
|
-
|
|
776
|
-
const source = audioContext.createMediaStreamSource(mediaStream);
|
|
777
|
-
analyzer = audioContext.createAnalyser();
|
|
778
|
-
analyzer.fftSize = 256;
|
|
467
|
+
**Vanilla:**
|
|
779
468
|
|
|
780
|
-
|
|
781
|
-
|
|
469
|
+
```typescript
|
|
470
|
+
// Enable text-to-speech (character will speak responses)
|
|
471
|
+
client.toggleTts(true);
|
|
782
472
|
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
473
|
+
// Disable text-to-speech (character will only send text, no audio)
|
|
474
|
+
client.toggleTts(false);
|
|
475
|
+
```
|
|
786
476
|
|
|
787
|
-
|
|
477
|
+
**Initial TTS Configuration:**
|
|
788
478
|
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
479
|
+
```typescript
|
|
480
|
+
// Set TTS state during connection
|
|
481
|
+
const client = new ConvaiClient({
|
|
482
|
+
apiKey: "your-api-key",
|
|
483
|
+
characterId: "your-character-id",
|
|
484
|
+
ttsEnabled: true, // Enable TTS by default
|
|
485
|
+
});
|
|
793
486
|
|
|
794
|
-
|
|
487
|
+
// Or disable initially
|
|
488
|
+
const client = new ConvaiClient({
|
|
489
|
+
apiKey: "your-api-key",
|
|
490
|
+
characterId: "your-character-id",
|
|
491
|
+
ttsEnabled: false, // Disable TTS
|
|
492
|
+
});
|
|
493
|
+
```
|
|
795
494
|
|
|
796
|
-
|
|
797
|
-
drawVisualizer(dataArray);
|
|
495
|
+
### Voice Mode Implementation
|
|
798
496
|
|
|
799
|
-
|
|
800
|
-
};
|
|
497
|
+
Voice mode allows users to speak instead of typing. The widget automatically handles voice mode, but you can implement it manually.
|
|
801
498
|
|
|
802
|
-
|
|
803
|
-
};
|
|
499
|
+
**React - Manual Voice Mode:**
|
|
804
500
|
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
501
|
+
```tsx
|
|
502
|
+
import { useConvaiClient } from "@convai/web-sdk";
|
|
503
|
+
import { useState, useEffect } from "react";
|
|
808
504
|
|
|
809
|
-
|
|
810
|
-
|
|
505
|
+
function CustomChatInterface() {
|
|
506
|
+
const convaiClient = useConvaiClient({
|
|
507
|
+
/* config */
|
|
508
|
+
});
|
|
509
|
+
const [isVoiceMode, setIsVoiceMode] = useState(false);
|
|
811
510
|
|
|
812
|
-
|
|
813
|
-
|
|
511
|
+
const enterVoiceMode = async () => {
|
|
512
|
+
// Interrupt any ongoing bot response
|
|
513
|
+
convaiClient.sendInterruptMessage();
|
|
814
514
|
|
|
815
|
-
|
|
515
|
+
// Unmute microphone
|
|
516
|
+
await convaiClient.audioControls.unmuteAudio();
|
|
816
517
|
|
|
817
|
-
|
|
818
|
-
|
|
518
|
+
setIsVoiceMode(true);
|
|
519
|
+
};
|
|
819
520
|
|
|
820
|
-
|
|
821
|
-
|
|
521
|
+
const exitVoiceMode = async () => {
|
|
522
|
+
// Interrupt any ongoing bot response
|
|
523
|
+
convaiClient.sendInterruptMessage();
|
|
822
524
|
|
|
823
|
-
|
|
824
|
-
|
|
525
|
+
// Mute microphone
|
|
526
|
+
await convaiClient.audioControls.muteAudio();
|
|
825
527
|
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
};
|
|
528
|
+
setIsVoiceMode(false);
|
|
529
|
+
};
|
|
829
530
|
|
|
830
|
-
|
|
831
|
-
|
|
531
|
+
// Monitor user transcription for voice input
|
|
532
|
+
useEffect(() => {
|
|
533
|
+
const transcription = convaiClient.userTranscription;
|
|
534
|
+
if (transcription && isVoiceMode) {
|
|
535
|
+
// Display real-time transcription
|
|
536
|
+
console.log("User is saying:", transcription);
|
|
832
537
|
}
|
|
833
|
-
|
|
834
|
-
return () => {
|
|
835
|
-
if (animationId) cancelAnimationFrame(animationId);
|
|
836
|
-
};
|
|
837
|
-
}, [convaiClient.room, convaiClient.state.isConnected]);
|
|
538
|
+
}, [convaiClient.userTranscription, isVoiceMode]);
|
|
838
539
|
|
|
839
540
|
return (
|
|
840
541
|
<div>
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
Bot is {convaiClient.state.isSpeaking ? "speaking" : "silent"}
|
|
850
|
-
</div>
|
|
542
|
+
{isVoiceMode ? (
|
|
543
|
+
<div>
|
|
544
|
+
<p>Listening: {convaiClient.userTranscription}</p>
|
|
545
|
+
<button onClick={exitVoiceMode}>Stop Voice Mode</button>
|
|
546
|
+
</div>
|
|
547
|
+
) : (
|
|
548
|
+
<button onClick={enterVoiceMode}>Start Voice Mode</button>
|
|
549
|
+
)}
|
|
851
550
|
</div>
|
|
852
551
|
);
|
|
853
552
|
}
|
|
854
553
|
```
|
|
855
554
|
|
|
856
|
-
|
|
555
|
+
**Vanilla - Manual Voice Mode:**
|
|
857
556
|
|
|
858
557
|
```typescript
|
|
859
|
-
|
|
558
|
+
let isVoiceMode = false;
|
|
860
559
|
|
|
861
|
-
const
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
});
|
|
865
|
-
|
|
866
|
-
await client.connect();
|
|
867
|
-
|
|
868
|
-
// CRITICAL: AudioRenderer for playback
|
|
869
|
-
const audioRenderer = new AudioRenderer(client.room);
|
|
870
|
-
|
|
871
|
-
const canvas = document.getElementById("visualizer") as HTMLCanvasElement;
|
|
872
|
-
const ctx = canvas.getContext("2d")!;
|
|
873
|
-
|
|
874
|
-
let analyzer: AnalyserNode | null = null;
|
|
875
|
-
let dataArray: Uint8Array | null = null;
|
|
876
|
-
let animationId: number;
|
|
877
|
-
|
|
878
|
-
// Setup analyzer
|
|
879
|
-
const audioContext = new AudioContext();
|
|
560
|
+
const enterVoiceMode = async () => {
|
|
561
|
+
// Interrupt any ongoing bot response
|
|
562
|
+
client.sendInterruptMessage();
|
|
880
563
|
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
const audioTracks = Array.from(participant.audioTrackPublications.values());
|
|
884
|
-
const audioTrack = audioTracks[0].track;
|
|
564
|
+
// Unmute microphone
|
|
565
|
+
await client.audioControls.unmuteAudio();
|
|
885
566
|
|
|
886
|
-
|
|
887
|
-
|
|
567
|
+
isVoiceMode = true;
|
|
568
|
+
updateUI();
|
|
569
|
+
};
|
|
888
570
|
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
571
|
+
const exitVoiceMode = async () => {
|
|
572
|
+
// Interrupt any ongoing bot response
|
|
573
|
+
client.sendInterruptMessage();
|
|
892
574
|
|
|
893
|
-
|
|
575
|
+
// Mute microphone
|
|
576
|
+
await client.audioControls.muteAudio();
|
|
894
577
|
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
578
|
+
isVoiceMode = false;
|
|
579
|
+
updateUI();
|
|
580
|
+
};
|
|
898
581
|
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
for (let i = 0; i < dataArray.length; i++) {
|
|
908
|
-
const barHeight = (dataArray[i] / 255) * canvas.height;
|
|
582
|
+
// Monitor user transcription
|
|
583
|
+
client.on("userTranscriptionChange", (transcription) => {
|
|
584
|
+
if (isVoiceMode && transcription) {
|
|
585
|
+
// Display real-time transcription
|
|
586
|
+
document.getElementById("transcription").textContent = transcription;
|
|
587
|
+
}
|
|
588
|
+
});
|
|
909
589
|
|
|
910
|
-
|
|
911
|
-
|
|
590
|
+
function updateUI() {
|
|
591
|
+
const voiceButton = document.getElementById("voice-btn");
|
|
592
|
+
const transcriptionDiv = document.getElementById("transcription");
|
|
912
593
|
|
|
913
|
-
|
|
594
|
+
if (isVoiceMode) {
|
|
595
|
+
voiceButton.textContent = "Stop Voice Mode";
|
|
596
|
+
transcriptionDiv.style.display = "block";
|
|
597
|
+
} else {
|
|
598
|
+
voiceButton.textContent = "Start Voice Mode";
|
|
599
|
+
transcriptionDiv.style.display = "none";
|
|
914
600
|
}
|
|
915
|
-
|
|
916
|
-
animationId = requestAnimationFrame(animate);
|
|
917
601
|
}
|
|
918
|
-
|
|
919
|
-
animate();
|
|
920
|
-
|
|
921
|
-
// Cleanup
|
|
922
|
-
// cancelAnimationFrame(animationId);
|
|
923
|
-
// audioRenderer.destroy();
|
|
924
|
-
// await client.disconnect();
|
|
925
602
|
```
|
|
926
603
|
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
### Message Types
|
|
930
|
-
|
|
931
|
-
All messages from `convaiClient.chatMessages` have a `type` field:
|
|
604
|
+
**Voice Mode with State Monitoring:**
|
|
932
605
|
|
|
933
606
|
```typescript
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
607
|
+
// Monitor agent state to handle voice mode transitions
|
|
608
|
+
convaiClient.on("stateChange", (state) => {
|
|
609
|
+
if (isVoiceMode) {
|
|
610
|
+
switch (state.agentState) {
|
|
611
|
+
case "listening":
|
|
612
|
+
// User can speak
|
|
613
|
+
console.log("Bot is listening");
|
|
614
|
+
break;
|
|
615
|
+
case "thinking":
|
|
616
|
+
// Bot is processing
|
|
617
|
+
console.log("Bot is thinking");
|
|
618
|
+
break;
|
|
619
|
+
case "speaking":
|
|
620
|
+
// Bot is responding
|
|
621
|
+
console.log("Bot is speaking");
|
|
622
|
+
// Optionally interrupt if user wants to speak
|
|
623
|
+
break;
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
});
|
|
945
627
|
```
|
|
946
628
|
|
|
947
|
-
|
|
629
|
+
### Connection Management
|
|
630
|
+
|
|
631
|
+
**Connect:**
|
|
948
632
|
|
|
949
633
|
```typescript
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
634
|
+
// React - config passed to hook
|
|
635
|
+
const convaiClient = useConvaiClient({
|
|
636
|
+
apiKey: "your-api-key",
|
|
637
|
+
characterId: "your-character-id",
|
|
638
|
+
});
|
|
954
639
|
|
|
955
|
-
|
|
640
|
+
// Or connect manually
|
|
641
|
+
await convaiClient.connect({
|
|
642
|
+
apiKey: "your-api-key",
|
|
643
|
+
characterId: "your-character-id",
|
|
644
|
+
});
|
|
956
645
|
|
|
957
|
-
|
|
646
|
+
// Vanilla
|
|
647
|
+
const client = new ConvaiClient();
|
|
648
|
+
await client.connect({
|
|
649
|
+
apiKey: "your-api-key",
|
|
650
|
+
characterId: "your-character-id",
|
|
651
|
+
});
|
|
652
|
+
```
|
|
958
653
|
|
|
959
|
-
|
|
654
|
+
**Disconnect:**
|
|
960
655
|
|
|
961
656
|
```typescript
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
apiKey: string;
|
|
965
|
-
|
|
966
|
-
/** The Character ID to connect to (required) */
|
|
967
|
-
characterId: string;
|
|
968
|
-
|
|
969
|
-
/**
|
|
970
|
-
* End user identifier for speaker management (optional).
|
|
971
|
-
* When provided: enables long-term memory and analytics
|
|
972
|
-
* When not provided: anonymous mode, no persistent memory
|
|
973
|
-
*/
|
|
974
|
-
endUserId?: string;
|
|
657
|
+
await convaiClient.disconnect();
|
|
658
|
+
```
|
|
975
659
|
|
|
976
|
-
|
|
977
|
-
url?: string;
|
|
660
|
+
**Reconnect:**
|
|
978
661
|
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
* If false, connection_type will be "audio" (audio only).
|
|
983
|
-
* ⚠️ REQUIRED for video and screen share features.
|
|
984
|
-
*/
|
|
985
|
-
enableVideo?: boolean;
|
|
662
|
+
```typescript
|
|
663
|
+
await convaiClient.reconnect();
|
|
664
|
+
```
|
|
986
665
|
|
|
987
|
-
|
|
988
|
-
* Start with video camera on when connecting (default: false).
|
|
989
|
-
* Only works if enableVideo is true.
|
|
990
|
-
*/
|
|
991
|
-
startWithVideoOn?: boolean;
|
|
666
|
+
**Reset Session:**
|
|
992
667
|
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
*/
|
|
997
|
-
startWithAudioOn?: boolean;
|
|
998
|
-
|
|
999
|
-
/**
|
|
1000
|
-
* Enable text-to-speech audio generation (default: true).
|
|
1001
|
-
*/
|
|
1002
|
-
ttsEnabled?: boolean;
|
|
1003
|
-
}
|
|
668
|
+
```typescript
|
|
669
|
+
// Clear conversation history and start new session
|
|
670
|
+
convaiClient.resetSession();
|
|
1004
671
|
```
|
|
1005
672
|
|
|
1006
|
-
|
|
673
|
+
**Connection State:**
|
|
1007
674
|
|
|
1008
675
|
```typescript
|
|
1009
|
-
//
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
676
|
+
// React
|
|
677
|
+
const { state } = convaiClient;
|
|
678
|
+
console.log("Connected:", state.isConnected);
|
|
679
|
+
console.log("Connecting:", state.isConnecting);
|
|
680
|
+
console.log("Agent state:", state.agentState); // 'disconnected' | 'connected' | 'listening' | 'thinking' | 'speaking'
|
|
681
|
+
|
|
682
|
+
// Core API (event-based)
|
|
683
|
+
convaiClient.on("stateChange", (state) => {
|
|
684
|
+
console.log("State changed:", state);
|
|
1014
685
|
});
|
|
1015
686
|
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
// Reconnect
|
|
1020
|
-
await convaiClient.reconnect();
|
|
1021
|
-
|
|
1022
|
-
// Reset session (clear conversation history)
|
|
1023
|
-
convaiClient.resetSession();
|
|
687
|
+
convaiClient.on("connect", () => {
|
|
688
|
+
console.log("Connected");
|
|
689
|
+
});
|
|
1024
690
|
|
|
1025
|
-
|
|
1026
|
-
console.log(
|
|
1027
|
-
|
|
1028
|
-
console.log(convaiClient.state.agentState); // 'disconnected' | 'connected' | 'listening' | 'thinking' | 'speaking'
|
|
1029
|
-
console.log(convaiClient.isBotReady); // Bot ready to receive messages
|
|
691
|
+
convaiClient.on("disconnect", () => {
|
|
692
|
+
console.log("Disconnected");
|
|
693
|
+
});
|
|
1030
694
|
```
|
|
1031
695
|
|
|
1032
696
|
### Messaging
|
|
1033
697
|
|
|
698
|
+
**Send Text Message:**
|
|
699
|
+
|
|
1034
700
|
```typescript
|
|
1035
|
-
// Send text message
|
|
1036
701
|
convaiClient.sendUserTextMessage("Hello, how are you?");
|
|
702
|
+
```
|
|
1037
703
|
|
|
1038
|
-
|
|
704
|
+
**Send Trigger Message:**
|
|
705
|
+
|
|
706
|
+
```typescript
|
|
707
|
+
// Trigger specific character action
|
|
1039
708
|
convaiClient.sendTriggerMessage("greet", "User entered the room");
|
|
1040
709
|
|
|
1041
|
-
//
|
|
1042
|
-
convaiClient.
|
|
710
|
+
// Trigger without message
|
|
711
|
+
convaiClient.sendTriggerMessage("wave");
|
|
712
|
+
```
|
|
1043
713
|
|
|
1044
|
-
|
|
1045
|
-
convaiClient.updateTemplateKeys({ user_name: "John" });
|
|
1046
|
-
convaiClient.updateDynamicInfo({ text: "User is browsing products" });
|
|
714
|
+
**Update Context:**
|
|
1047
715
|
|
|
1048
|
-
|
|
1049
|
-
|
|
716
|
+
```typescript
|
|
717
|
+
// Update template keys (e.g., user name, location)
|
|
718
|
+
convaiClient.updateTemplateKeys({
|
|
719
|
+
user_name: "John",
|
|
720
|
+
location: "New York",
|
|
721
|
+
});
|
|
1050
722
|
|
|
1051
|
-
//
|
|
1052
|
-
|
|
723
|
+
// Update dynamic information
|
|
724
|
+
convaiClient.updateDynamicInfo({
|
|
725
|
+
text: "User is currently browsing the products page",
|
|
726
|
+
});
|
|
1053
727
|
```
|
|
1054
728
|
|
|
1055
|
-
|
|
729
|
+
**Message History:**
|
|
1056
730
|
|
|
1057
731
|
```typescript
|
|
1058
|
-
//
|
|
1059
|
-
|
|
1060
|
-
await convaiClient.audioControls.unmuteAudio();
|
|
1061
|
-
await convaiClient.audioControls.toggleAudio();
|
|
732
|
+
// React
|
|
733
|
+
const { chatMessages } = convaiClient;
|
|
1062
734
|
|
|
1063
|
-
//
|
|
1064
|
-
|
|
735
|
+
// Core API (event-based)
|
|
736
|
+
convaiClient.on("message", (message: ChatMessage) => {
|
|
737
|
+
console.log("New message:", message.content);
|
|
738
|
+
console.log("Message type:", message.type);
|
|
739
|
+
});
|
|
1065
740
|
|
|
1066
|
-
|
|
1067
|
-
|
|
741
|
+
convaiClient.on("messagesChange", (messages: ChatMessage[]) => {
|
|
742
|
+
console.log("All messages:", messages);
|
|
743
|
+
});
|
|
744
|
+
```
|
|
1068
745
|
|
|
1069
|
-
|
|
1070
|
-
await convaiClient.audioControls.setAudioDevice(deviceId);
|
|
746
|
+
**Message Types:**
|
|
1071
747
|
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
748
|
+
```typescript
|
|
749
|
+
type ChatMessageType =
|
|
750
|
+
| "user" // User's sent message
|
|
751
|
+
| "convai" // Character's response
|
|
752
|
+
| "user-transcription" // Real-time speech-to-text from user
|
|
753
|
+
| "bot-llm-text" // Character's LLM-generated text
|
|
754
|
+
| "emotion" // Character's emotional state
|
|
755
|
+
| "behavior-tree" // Behavior tree response
|
|
756
|
+
| "action" // Action execution
|
|
757
|
+
| "bot-emotion" // Bot emotional response
|
|
758
|
+
| "user-llm-text" // User text processed by LLM
|
|
759
|
+
| "interrupt-bot"; // Interrupt message
|
|
1078
760
|
```
|
|
1079
761
|
|
|
1080
|
-
###
|
|
762
|
+
### State Monitoring
|
|
1081
763
|
|
|
1082
|
-
|
|
764
|
+
**Agent State:**
|
|
1083
765
|
|
|
1084
766
|
```typescript
|
|
1085
|
-
//
|
|
1086
|
-
|
|
1087
|
-
await convaiClient.videoControls.disableVideo();
|
|
1088
|
-
await convaiClient.videoControls.toggleVideo();
|
|
767
|
+
// React
|
|
768
|
+
const { state } = convaiClient;
|
|
1089
769
|
|
|
1090
|
-
// Check
|
|
1091
|
-
|
|
770
|
+
// Check specific states
|
|
771
|
+
if (state.isListening) {
|
|
772
|
+
console.log("Bot is listening");
|
|
773
|
+
}
|
|
1092
774
|
|
|
1093
|
-
|
|
1094
|
-
|
|
775
|
+
if (state.isThinking) {
|
|
776
|
+
console.log("Bot is thinking");
|
|
777
|
+
}
|
|
1095
778
|
|
|
1096
|
-
|
|
1097
|
-
|
|
779
|
+
if (state.isSpeaking) {
|
|
780
|
+
console.log("Bot is speaking");
|
|
781
|
+
}
|
|
1098
782
|
|
|
1099
|
-
//
|
|
1100
|
-
|
|
783
|
+
// Combined state
|
|
784
|
+
console.log(state.agentState); // 'disconnected' | 'connected' | 'listening' | 'thinking' | 'speaking'
|
|
1101
785
|
```
|
|
1102
786
|
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
**⚠️ Requires `enableVideo: true` in config.**
|
|
787
|
+
**User Transcription:**
|
|
1106
788
|
|
|
1107
789
|
```typescript
|
|
1108
|
-
//
|
|
1109
|
-
|
|
1110
|
-
await convaiClient.screenShareControls.enableScreenShareWithAudio();
|
|
1111
|
-
await convaiClient.screenShareControls.disableScreenShare();
|
|
1112
|
-
await convaiClient.screenShareControls.toggleScreenShare();
|
|
790
|
+
// React
|
|
791
|
+
const { userTranscription } = convaiClient;
|
|
1113
792
|
|
|
1114
|
-
//
|
|
1115
|
-
|
|
793
|
+
// Core API (event-based)
|
|
794
|
+
convaiClient.on("userTranscriptionChange", (transcription: string) => {
|
|
795
|
+
console.log("User is saying:", transcription);
|
|
796
|
+
});
|
|
1116
797
|
```
|
|
1117
798
|
|
|
1118
|
-
|
|
799
|
+
**Bot Ready State:**
|
|
800
|
+
|
|
801
|
+
```typescript
|
|
802
|
+
// React
|
|
803
|
+
const { isBotReady } = convaiClient;
|
|
804
|
+
|
|
805
|
+
// Core API (event-based)
|
|
806
|
+
convaiClient.on("botReady", () => {
|
|
807
|
+
console.log("Bot is ready to receive messages");
|
|
808
|
+
});
|
|
809
|
+
```
|
|
1119
810
|
|
|
1120
|
-
## Getting Credentials
|
|
811
|
+
## Getting Convai Credentials
|
|
1121
812
|
|
|
1122
813
|
1. Visit [convai.com](https://convai.com) and create an account
|
|
1123
814
|
2. Navigate to your dashboard
|
|
@@ -1125,71 +816,41 @@ console.log(convaiClient.isScreenShareActive);
|
|
|
1125
816
|
4. Copy your **API Key** from the dashboard
|
|
1126
817
|
5. Copy your **Character ID** from the character details
|
|
1127
818
|
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
## TypeScript Support
|
|
1131
|
-
|
|
1132
|
-
All exports are fully typed:
|
|
1133
|
-
|
|
1134
|
-
**React:**
|
|
819
|
+
## Import Paths
|
|
1135
820
|
|
|
1136
821
|
```typescript
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
ConvaiConfig,
|
|
1140
|
-
|
|
1141
|
-
// State
|
|
1142
|
-
ConvaiClientState,
|
|
822
|
+
// Default: React version (backward compatible)
|
|
823
|
+
import { useConvaiClient, ConvaiWidget } from "@convai/web-sdk";
|
|
1143
824
|
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
ChatMessageType,
|
|
825
|
+
// Explicit React import
|
|
826
|
+
import { useConvaiClient, ConvaiWidget } from "@convai/web-sdk/react";
|
|
1147
827
|
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
ConvaiClient,
|
|
828
|
+
// Vanilla JS/TS
|
|
829
|
+
import { ConvaiClient, createConvaiWidget } from "@convai/web-sdk/vanilla";
|
|
1151
830
|
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
VideoControls,
|
|
1155
|
-
ScreenShareControls,
|
|
1156
|
-
} from "@convai/web-sdk/react";
|
|
831
|
+
// Core only (no UI, framework agnostic)
|
|
832
|
+
import { ConvaiClient } from "@convai/web-sdk/core";
|
|
1157
833
|
```
|
|
1158
834
|
|
|
1159
|
-
|
|
835
|
+
## TypeScript Support
|
|
836
|
+
|
|
837
|
+
All exports are fully typed:
|
|
1160
838
|
|
|
1161
839
|
```typescript
|
|
1162
840
|
import type {
|
|
1163
|
-
|
|
841
|
+
ConvaiClient,
|
|
1164
842
|
ConvaiConfig,
|
|
1165
|
-
|
|
1166
|
-
// State
|
|
1167
843
|
ConvaiClientState,
|
|
1168
|
-
|
|
1169
|
-
// Messages
|
|
1170
844
|
ChatMessage,
|
|
1171
|
-
ChatMessageType,
|
|
1172
|
-
|
|
1173
|
-
// Client
|
|
1174
|
-
IConvaiClient,
|
|
1175
|
-
ConvaiClient,
|
|
1176
|
-
|
|
1177
|
-
// Controls
|
|
1178
845
|
AudioControls,
|
|
1179
846
|
VideoControls,
|
|
1180
847
|
ScreenShareControls,
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
VanillaWidget,
|
|
1184
|
-
VanillaWidgetOptions,
|
|
1185
|
-
} from "@convai/web-sdk/vanilla";
|
|
848
|
+
IConvaiClient,
|
|
849
|
+
} from "@convai/web-sdk";
|
|
1186
850
|
```
|
|
1187
851
|
|
|
1188
|
-
---
|
|
1189
|
-
|
|
1190
852
|
## Support
|
|
1191
853
|
|
|
1192
|
-
-
|
|
1193
|
-
-
|
|
1194
|
-
-
|
|
1195
|
-
- **Issues**: [GitHub Issues](https://github.com/convai/web-sdk/issues)
|
|
854
|
+
- [Convai Forum](https://forum.convai.com)
|
|
855
|
+
- [API Reference](./API_REFERENCE.md)
|
|
856
|
+
- [Convai Website](https://convai.com)
|