@convai/web-sdk 1.2.1-beta.0 → 1.2.2-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +297 -43
- package/dist/core/ConvaiClient.d.ts +36 -7
- package/dist/core/ConvaiClient.d.ts.map +1 -1
- package/dist/core/ConvaiClient.js +190 -40
- package/dist/core/ConvaiClient.js.map +1 -1
- package/dist/core/MemoryManager.d.ts +179 -0
- package/dist/core/MemoryManager.d.ts.map +1 -0
- package/dist/core/MemoryManager.js +281 -0
- package/dist/core/MemoryManager.js.map +1 -0
- package/dist/core/MessageHandler.d.ts +1 -7
- package/dist/core/MessageHandler.d.ts.map +1 -1
- package/dist/core/MessageHandler.js +8 -21
- package/dist/core/MessageHandler.js.map +1 -1
- package/dist/core/index.d.ts +2 -0
- package/dist/core/index.d.ts.map +1 -1
- package/dist/core/index.js +3 -0
- package/dist/core/index.js.map +1 -1
- package/dist/core/types.d.ts +422 -13
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js +58 -1
- package/dist/core/types.js.map +1 -1
- package/dist/index.d.ts +0 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/lipsync-helpers/mappingTypes.d.ts +1 -1
- package/dist/lipsync-helpers/mappingTypes.js +4 -4
- package/dist/react/components/ConvaiWidget.d.ts.map +1 -1
- package/dist/react/components/ConvaiWidget.js +1 -1
- package/dist/react/components/ConvaiWidget.js.map +1 -1
- package/dist/react/hooks/useCharacterInfo.d.ts +1 -1
- package/dist/react/hooks/useCharacterInfo.d.ts.map +1 -1
- package/dist/react/hooks/useCharacterInfo.js +11 -5
- package/dist/react/hooks/useCharacterInfo.js.map +1 -1
- package/dist/react/hooks/useConvaiClient.d.ts.map +1 -1
- package/dist/react/hooks/useConvaiClient.js +4 -2
- package/dist/react/hooks/useConvaiClient.js.map +1 -1
- package/dist/react/index.d.ts +1 -0
- package/dist/react/index.d.ts.map +1 -1
- package/dist/react/index.js +2 -0
- package/dist/react/index.js.map +1 -1
- package/dist/vanilla/ConvaiWidget.d.ts.map +1 -1
- package/dist/vanilla/ConvaiWidget.js +6 -4
- package/dist/vanilla/ConvaiWidget.js.map +1 -1
- package/dist/vanilla/index.d.ts +2 -1
- package/dist/vanilla/index.d.ts.map +1 -1
- package/dist/vanilla/index.js +2 -0
- package/dist/vanilla/index.js.map +1 -1
- package/dist/vanilla/types.d.ts +3 -2
- package/dist/vanilla/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/dist/dev.d.ts +0 -12
- package/dist/dev.d.ts.map +0 -1
- package/dist/dev.js +0 -12
- package/dist/dev.js.map +0 -1
- package/dist/types/index.d.ts +0 -310
- package/dist/types/index.d.ts.map +0 -1
- package/dist/types/index.js +0 -2
- package/dist/types/index.js.map +0 -1
- package/dist/utils/LatencyMonitor.d.ts +0 -88
- package/dist/utils/LatencyMonitor.d.ts.map +0 -1
- package/dist/utils/LatencyMonitor.js +0 -136
- package/dist/utils/LatencyMonitor.js.map +0 -1
package/README.md
CHANGED
|
@@ -19,16 +19,15 @@ This document is written as a complete implementation reference, from first setu
|
|
|
19
19
|
- [6. Core Concepts and Lifecycle](#6-core-concepts-and-lifecycle)
|
|
20
20
|
- [7. Configuration Reference (`ConvaiConfig`)](#7-configuration-reference-convaiconfig)
|
|
21
21
|
- [8. Core API Reference (`ConvaiClient`)](#8-core-api-reference-convaiclient)
|
|
22
|
-
- [9.
|
|
23
|
-
- [10.
|
|
24
|
-
- [11.
|
|
25
|
-
- [12.
|
|
26
|
-
- [13.
|
|
22
|
+
- [9. Memory Management API](#9-memory-management-api)
|
|
23
|
+
- [10. Message Semantics and Turn Completion](#10-message-semantics-and-turn-completion)
|
|
24
|
+
- [11. React API Reference](#11-react-api-reference)
|
|
25
|
+
- [12. Vanilla API Reference](#12-vanilla-api-reference)
|
|
26
|
+
- [13. Audio Integration Best Practices (Vanilla TypeScript)](#13-audio-integration-best-practices-vanilla-typescript)
|
|
27
27
|
- [14. Error Handling and Reliability Patterns](#14-error-handling-and-reliability-patterns)
|
|
28
28
|
- [15. Troubleshooting](#15-troubleshooting)
|
|
29
|
-
- [16.
|
|
29
|
+
- [16. Lipsync Helpers Reference](#16-lipsync-helpers-reference)
|
|
30
30
|
- [17. Examples](#17-examples)
|
|
31
|
-
- [18. License](#18-license)
|
|
32
31
|
|
|
33
32
|
## 1. Package Entry Points
|
|
34
33
|
|
|
@@ -55,10 +54,6 @@ Primary exports:
|
|
|
55
54
|
- `ScreenShareControls`
|
|
56
55
|
- `IConvaiClient`
|
|
57
56
|
- All exports from `@convai/web-sdk/lipsync-helpers`
|
|
58
|
-
- Type exports for latency models:
|
|
59
|
-
- `LatencyMonitor` (type)
|
|
60
|
-
- `LatencyMeasurement`
|
|
61
|
-
- `LatencyStats`
|
|
62
57
|
|
|
63
58
|
### `@convai/web-sdk/react`
|
|
64
59
|
|
|
@@ -89,6 +84,7 @@ Framework-agnostic low-level API:
|
|
|
89
84
|
- `VideoManager`
|
|
90
85
|
- `ScreenShareManager`
|
|
91
86
|
- `MessageHandler`
|
|
87
|
+
- `MemoryManager`
|
|
92
88
|
- `BlendshapeQueue`
|
|
93
89
|
- `EventEmitter`
|
|
94
90
|
- Type alias: `ConvaiClientType`
|
|
@@ -212,6 +208,11 @@ const convaiClient = useConvaiClient({
|
|
|
212
208
|
apiKey: import.meta.env.VITE_CONVAI_API_KEY,
|
|
213
209
|
characterId: import.meta.env.VITE_CONVAI_CHARACTER_ID,
|
|
214
210
|
endUserId: "<UNIQUE_END_USER_ID>",
|
|
211
|
+
endUserMetadata: {
|
|
212
|
+
name: "John Doe",
|
|
213
|
+
age: "30",
|
|
214
|
+
// Add any additional metadata you want to send
|
|
215
|
+
},
|
|
215
216
|
enableVideo: true,
|
|
216
217
|
startWithVideoOn: false,
|
|
217
218
|
startWithAudioOn: false,
|
|
@@ -268,6 +269,16 @@ useEffect(() => {
|
|
|
268
269
|
|
|
269
270
|
const unsubState = convaiClient.on("stateChange", (state) => {
|
|
270
271
|
console.log("State:", state.agentState);
|
|
272
|
+
|
|
273
|
+
// Access end user information from connection response
|
|
274
|
+
if (state.endUserId) {
|
|
275
|
+
console.log("End User ID:", state.endUserId);
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
if (state.endUserMetadata) {
|
|
279
|
+
console.log("End User Metadata:", state.endUserMetadata);
|
|
280
|
+
// Example: { name: 'John', age: '30' }
|
|
281
|
+
}
|
|
271
282
|
});
|
|
272
283
|
|
|
273
284
|
const unsubMessages = convaiClient.on("messagesChange", (messages) => {
|
|
@@ -389,6 +400,9 @@ await client.disconnect();
|
|
|
389
400
|
- `state.isThinking`: model is generating response
|
|
390
401
|
- `state.isSpeaking`: model audio is currently speaking
|
|
391
402
|
- `state.agentState`: combined high-level state (`disconnected | connected | listening | thinking | speaking`)
|
|
403
|
+
- `state.endUserId`: end user ID returned from the connection response (if provided in config)
|
|
404
|
+
- `state.endUserMetadata`: end user metadata returned from the connection response (if provided in config)
|
|
405
|
+
- `state.metrics`: array of metrics events received during the current session (clears on `resetSession()`)
|
|
392
406
|
|
|
393
407
|
### Widget lifecycle
|
|
394
408
|
|
|
@@ -405,6 +419,7 @@ Both React and vanilla widgets:
|
|
|
405
419
|
| `apiKey` | `string` | Yes | - | Convai API key. |
|
|
406
420
|
| `characterId` | `string` | Yes | - | Target character identifier. |
|
|
407
421
|
| `endUserId` | `string` | No | `undefined` | Stable end-user identity for memory/analytics continuity. |
|
|
422
|
+
| `endUserMetadata` | `Record<string, unknown>` | No | `undefined` | Additional end-user metadata (e.g., name, age) sent with the connection request. |
|
|
408
423
|
| `url` | `string` | No | SDK internal default | Convai base URL. Set explicitly if your deployment requires a specific environment. |
|
|
409
424
|
| `enableVideo` | `boolean` | No | `false` | Enables video-capable connection type. |
|
|
410
425
|
| `startWithVideoOn` | `boolean` | No | `false` | Auto-enable camera after connect. |
|
|
@@ -428,15 +443,16 @@ import { ConvaiClient } from "@convai/web-sdk/core";
|
|
|
428
443
|
For TypeScript, use the types exported from the main package or core:
|
|
429
444
|
|
|
430
445
|
- **`IConvaiClient`** — Interface implemented by `ConvaiClient`. Use this when you need to type variables, props, or function parameters that accept the client (e.g. `client: IConvaiClient`).
|
|
431
|
-
- **`ConvaiClientState`** — Shape of `client.state` (connection and activity flags: `isConnected`, `isBotReady`, `agentState`, `emotion`, etc.).
|
|
446
|
+
- **`ConvaiClientState`** — Shape of `client.state` (connection and activity flags: `isConnected`, `isBotReady`, `agentState`, `emotion`, `endUserId`, `endUserMetadata`, `metrics`, etc.).
|
|
432
447
|
- **`ConvaiConfig`** — Configuration object for `new ConvaiClient(config)` and `client.connect(config)`.
|
|
433
448
|
- **`ChatMessage`** — Single message in `client.chatMessages`.
|
|
449
|
+
- **`ConvaiMetrics`** — Metrics event structure containing data, timestamp, and unique identifier.
|
|
434
450
|
- **`AudioControls`**, **`VideoControls`**, **`ScreenShareControls`** — Types for the control managers exposed as `client.audioControls`, `client.videoControls`, `client.screenShareControls`.
|
|
435
451
|
|
|
436
452
|
Example:
|
|
437
453
|
|
|
438
454
|
```ts
|
|
439
|
-
import type { IConvaiClient, ConvaiClientState, ConvaiConfig } from "@convai/web-sdk";
|
|
455
|
+
import type { IConvaiClient, ConvaiClientState, ConvaiConfig, ConvaiMetrics } from "@convai/web-sdk";
|
|
440
456
|
// or from "@convai/web-sdk/core" when using core-only
|
|
441
457
|
```
|
|
442
458
|
|
|
@@ -463,9 +479,9 @@ new ConvaiClient(config?: ConvaiConfig)
|
|
|
463
479
|
| `audioControls` | `AudioControls` | Microphone controls. |
|
|
464
480
|
| `videoControls` | `VideoControls` | Camera controls. |
|
|
465
481
|
| `screenShareControls` | `ScreenShareControls` | Screen sharing controls. |
|
|
466
|
-
| `latencyMonitor` | `LatencyMonitor` | Measurement manager used by the client for turn latency. |
|
|
467
482
|
| `blendshapeQueue` | `BlendshapeQueue` | Buffer queue for lipsync frames. |
|
|
468
483
|
| `conversationSessionId` | `number` | Incremental turn session ID used by conversation events. |
|
|
484
|
+
| `memoryManager` | `MemoryManager \| null` | Long-term memory API manager. See [Memory API docs](./docs/MEMORY_API.md). |
|
|
469
485
|
|
|
470
486
|
### Methods
|
|
471
487
|
|
|
@@ -479,7 +495,7 @@ new ConvaiClient(config?: ConvaiConfig)
|
|
|
479
495
|
| `sendTriggerMessage` | `(triggerName?: string, triggerMessage?: string) => void` | Send trigger/action message. |
|
|
480
496
|
| `sendInterruptMessage` | `() => void` | Interrupt current bot response. |
|
|
481
497
|
| `updateTemplateKeys` | `(templateKeys: Record<string, string>) => void` | Update runtime template variables. |
|
|
482
|
-
| `updateDynamicInfo` | `(dynamicInfo:
|
|
498
|
+
| `updateDynamicInfo` | `(dynamicInfo: string) => void` | Update dynamic context with a text description. |
|
|
483
499
|
| `toggleTts` | `(enabled: boolean) => void` | Enable/disable TTS for subsequent responses. |
|
|
484
500
|
| `on` | `(event: string, callback: (...args: any[]) => void) => () => void` | Subscribe to an event and receive an unsubscribe function. |
|
|
485
501
|
| `off` | `(event: string, callback: (...args: any[]) => void) => void` | Remove a specific listener. |
|
|
@@ -501,7 +517,7 @@ new ConvaiClient(config?: ConvaiConfig)
|
|
|
501
517
|
| `turnEnd` | `{ sessionId, duration, timestamp }` | Server signaled end of turn (bot stopped speaking). Same semantics as `BlendshapeQueue.hasReceivedEndSignal()`. |
|
|
502
518
|
| `blendshapes` | `unknown` | Incoming blendshape chunk payload. |
|
|
503
519
|
| `blendshapeStatsReceived` | `unknown` | End-of-turn blendshape stats marker. |
|
|
504
|
-
| `
|
|
520
|
+
| `metrics` | `Record<string, unknown>` | Metrics data received from server. Multiple metrics events may occur per conversation. |
|
|
505
521
|
|
|
506
522
|
### Control manager APIs
|
|
507
523
|
|
|
@@ -564,30 +580,174 @@ Methods:
|
|
|
564
580
|
- `on("screenShareStateChange", callback)`
|
|
565
581
|
- `off("screenShareStateChange", callback)`
|
|
566
582
|
|
|
567
|
-
### `
|
|
583
|
+
### Using `updateDynamicInfo` for real-time context
|
|
568
584
|
|
|
569
|
-
`
|
|
585
|
+
The `updateDynamicInfo` method allows you to send real-time context to the character using a simple text description.
|
|
570
586
|
|
|
571
|
-
|
|
587
|
+
#### Basic usage
|
|
588
|
+
|
|
589
|
+
```tsx
|
|
590
|
+
// Simple text description
|
|
591
|
+
convaiClient.updateDynamicInfo("Player health is low");
|
|
592
|
+
```
|
|
572
593
|
|
|
573
|
-
|
|
574
|
-
- `disable()`
|
|
575
|
-
- `startMeasurement(type, userMessage?)`
|
|
576
|
-
- `endMeasurement()`
|
|
577
|
-
- `cancelMeasurement()`
|
|
578
|
-
- `getMeasurements()`
|
|
579
|
-
- `getLatestMeasurement()`
|
|
580
|
-
- `getStats()`
|
|
581
|
-
- `clear()`
|
|
582
|
-
- `getPendingMeasurement()`
|
|
583
|
-
- `on("measurement", callback)`
|
|
584
|
-
- `on("measurementsChange", callback)`
|
|
585
|
-
- `on("enabledChange", callback)`
|
|
594
|
+
#### Advanced usage with detailed context
|
|
586
595
|
|
|
587
|
-
|
|
596
|
+
```tsx
|
|
597
|
+
// Game state example
|
|
598
|
+
convaiClient.updateDynamicInfo(
|
|
599
|
+
"Player is in combat with 25% health and 40% stamina at battlefield location, 3 enemies nearby, equipped with sword"
|
|
600
|
+
);
|
|
601
|
+
```
|
|
602
|
+
|
|
603
|
+
#### Using in connect config
|
|
604
|
+
|
|
605
|
+
```tsx
|
|
606
|
+
const config: ConvaiConfig = {
|
|
607
|
+
apiKey: 'your-api-key',
|
|
608
|
+
characterId: 'your-character-id',
|
|
609
|
+
dynamicInfo: "Initial game state: Player at spawn point, level 1, tutorial not complete",
|
|
610
|
+
// Other config fields...
|
|
611
|
+
};
|
|
612
|
+
|
|
613
|
+
await convaiClient.connect(config);
|
|
614
|
+
```
|
|
615
|
+
|
|
616
|
+
#### Real-time updates example
|
|
617
|
+
|
|
618
|
+
```tsx
|
|
619
|
+
// Update context as game state changes
|
|
620
|
+
function updateCharacterContext(player: Player) {
|
|
621
|
+
const contextText = `Player health: ${player.health}%, mana: ${player.mana}%, location: ${player.location}, status: ${player.status}`;
|
|
622
|
+
|
|
623
|
+
convaiClient.updateDynamicInfo(contextText);
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
// Call whenever game state changes
|
|
627
|
+
player.on('stateChange', () => updateCharacterContext(player));
|
|
628
|
+
```
|
|
629
|
+
|
|
630
|
+
### Accessing metrics data
|
|
588
631
|
|
|
589
|
-
|
|
590
|
-
|
|
632
|
+
The SDK exposes metrics events received from the server through the client state. Multiple metrics events can be received during a single conversation.
|
|
633
|
+
|
|
634
|
+
#### Accessing metrics from client state
|
|
635
|
+
|
|
636
|
+
```tsx
|
|
637
|
+
// Access metrics array from state
|
|
638
|
+
const metrics = convaiClient.state.metrics;
|
|
639
|
+
|
|
640
|
+
// Log all metrics
|
|
641
|
+
console.log('Total metrics received:', metrics.length);
|
|
642
|
+
metrics.forEach(metric => {
|
|
643
|
+
console.log('Metric ID:', metric.id);
|
|
644
|
+
console.log('Timestamp:', metric.timestamp);
|
|
645
|
+
console.log('Data:', metric.data);
|
|
646
|
+
});
|
|
647
|
+
|
|
648
|
+
// Get latest metric
|
|
649
|
+
const latestMetric = metrics[metrics.length - 1];
|
|
650
|
+
if (latestMetric) {
|
|
651
|
+
console.log('Latest metric data:', latestMetric.data);
|
|
652
|
+
}
|
|
653
|
+
```
|
|
654
|
+
|
|
655
|
+
#### Listening to metrics events in real-time
|
|
656
|
+
|
|
657
|
+
```tsx
|
|
658
|
+
// Subscribe to metrics events
|
|
659
|
+
const unsubMetrics = convaiClient.on('metrics', (metricsData) => {
|
|
660
|
+
console.log('New metrics received:', metricsData);
|
|
661
|
+
|
|
662
|
+
// Process metrics data
|
|
663
|
+
if (metricsData.processingTime) {
|
|
664
|
+
console.log('Processing time:', metricsData.processingTime);
|
|
665
|
+
}
|
|
666
|
+
});
|
|
667
|
+
|
|
668
|
+
// Cleanup
|
|
669
|
+
unsubMetrics();
|
|
670
|
+
```
|
|
671
|
+
|
|
672
|
+
#### Accessing metrics in React
|
|
673
|
+
|
|
674
|
+
```tsx
|
|
675
|
+
import { useConvaiClient } from '@convai/web-sdk';
|
|
676
|
+
import { useEffect, useState } from 'react';
|
|
677
|
+
|
|
678
|
+
function MetricsDisplay() {
|
|
679
|
+
const convaiClient = useConvaiClient();
|
|
680
|
+
const [metrics, setMetrics] = useState<ConvaiMetrics[]>([]);
|
|
681
|
+
|
|
682
|
+
useEffect(() => {
|
|
683
|
+
// Subscribe to state changes
|
|
684
|
+
const unsubState = convaiClient.on('stateChange', (state) => {
|
|
685
|
+
setMetrics(state.metrics);
|
|
686
|
+
});
|
|
687
|
+
|
|
688
|
+
// Or subscribe to individual metrics events
|
|
689
|
+
const unsubMetrics = convaiClient.on('metrics', (metricsData) => {
|
|
690
|
+
console.log('New metrics:', metricsData);
|
|
691
|
+
});
|
|
692
|
+
|
|
693
|
+
return () => {
|
|
694
|
+
unsubState();
|
|
695
|
+
unsubMetrics();
|
|
696
|
+
};
|
|
697
|
+
}, [convaiClient]);
|
|
698
|
+
|
|
699
|
+
return (
|
|
700
|
+
<div>
|
|
701
|
+
<h3>Metrics ({metrics.length})</h3>
|
|
702
|
+
{metrics.map(metric => (
|
|
703
|
+
<div key={metric.id}>
|
|
704
|
+
<p>Time: {new Date(metric.timestamp).toLocaleTimeString()}</p>
|
|
705
|
+
<pre>{JSON.stringify(metric.data, null, 2)}</pre>
|
|
706
|
+
</div>
|
|
707
|
+
))}
|
|
708
|
+
</div>
|
|
709
|
+
);
|
|
710
|
+
}
|
|
711
|
+
```
|
|
712
|
+
|
|
713
|
+
#### Clearing metrics
|
|
714
|
+
|
|
715
|
+
Metrics are automatically cleared when you call `resetSession()` or `disconnect()`:
|
|
716
|
+
|
|
717
|
+
```tsx
|
|
718
|
+
// Clear conversation history and metrics
|
|
719
|
+
convaiClient.resetSession();
|
|
720
|
+
|
|
721
|
+
// Or disconnect (also clears metrics)
|
|
722
|
+
await convaiClient.disconnect();
|
|
723
|
+
```
|
|
724
|
+
|
|
725
|
+
#### Analyzing aggregated metrics
|
|
726
|
+
|
|
727
|
+
```tsx
|
|
728
|
+
function analyzeMetrics(client: IConvaiClient) {
|
|
729
|
+
const metrics = client.state.metrics;
|
|
730
|
+
|
|
731
|
+
// Calculate average processing time if present
|
|
732
|
+
const processingTimes = metrics
|
|
733
|
+
.map(m => m.data.processingTime as number)
|
|
734
|
+
.filter(t => typeof t === 'number');
|
|
735
|
+
|
|
736
|
+
if (processingTimes.length > 0) {
|
|
737
|
+
const avgTime = processingTimes.reduce((a, b) => a + b, 0) / processingTimes.length;
|
|
738
|
+
console.log('Average processing time:', avgTime.toFixed(2), 'ms');
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
// Count metrics by type if available
|
|
742
|
+
const metricsByType = metrics.reduce((acc, metric) => {
|
|
743
|
+
const type = (metric.data.type as string) || 'unknown';
|
|
744
|
+
acc[type] = (acc[type] || 0) + 1;
|
|
745
|
+
return acc;
|
|
746
|
+
}, {} as Record<string, number>);
|
|
747
|
+
|
|
748
|
+
console.log('Metrics by type:', metricsByType);
|
|
749
|
+
}
|
|
750
|
+
```
|
|
591
751
|
|
|
592
752
|
### Advanced core classes (`@convai/web-sdk/core`)
|
|
593
753
|
|
|
@@ -631,7 +791,6 @@ Properties:
|
|
|
631
791
|
Methods:
|
|
632
792
|
|
|
633
793
|
- `getBlendshapeQueue()`
|
|
634
|
-
- `setLatencyMonitor(monitor)`
|
|
635
794
|
- `getChatMessages()`
|
|
636
795
|
- `getUserTranscription()`
|
|
637
796
|
- `getIsBotResponding()`
|
|
@@ -652,7 +811,102 @@ Methods:
|
|
|
652
811
|
- `removeAllListeners()`
|
|
653
812
|
- `listenerCount(event)`
|
|
654
813
|
|
|
655
|
-
## 9.
|
|
814
|
+
## 9. Memory Management API
|
|
815
|
+
|
|
816
|
+
The Convai SDK includes a comprehensive Memory Management API that enables long-term memory storage for characters. Memories are scoped to a `(character_id, end_user_id)` pair, allowing each user to have personalized experiences that persist across conversation sessions.
|
|
817
|
+
|
|
818
|
+
### Quick Start
|
|
819
|
+
|
|
820
|
+
```typescript
|
|
821
|
+
import { ConvaiClient } from "@convai/web-sdk/core";
|
|
822
|
+
|
|
823
|
+
const client = new ConvaiClient({
|
|
824
|
+
apiKey: 'your-api-key',
|
|
825
|
+
characterId: 'your-character-id',
|
|
826
|
+
endUserId: 'user@example.com', // Required for memory operations
|
|
827
|
+
});
|
|
828
|
+
|
|
829
|
+
await client.connect();
|
|
830
|
+
|
|
831
|
+
// Access memory manager
|
|
832
|
+
const memoryManager = client.memoryManager;
|
|
833
|
+
|
|
834
|
+
if (memoryManager) {
|
|
835
|
+
// Add memories
|
|
836
|
+
await memoryManager.addMemories([
|
|
837
|
+
'User prefers outdoor activities',
|
|
838
|
+
'User is allergic to peanuts'
|
|
839
|
+
]);
|
|
840
|
+
|
|
841
|
+
// List memories
|
|
842
|
+
const memories = await memoryManager.listMemories();
|
|
843
|
+
console.log(`Total memories: ${memories.total_count}`);
|
|
844
|
+
|
|
845
|
+
// Get a specific memory
|
|
846
|
+
const memory = await memoryManager.getMemory(memoryId);
|
|
847
|
+
|
|
848
|
+
// Delete a memory
|
|
849
|
+
await memoryManager.deleteMemory(memoryId);
|
|
850
|
+
}
|
|
851
|
+
```
|
|
852
|
+
|
|
853
|
+
### Requirements
|
|
854
|
+
|
|
855
|
+
The `memoryManager` is available when:
|
|
856
|
+
- You provide either an `apiKey` or `authToken` in the config
|
|
857
|
+
- You provide an `endUserId` in the config
|
|
858
|
+
|
|
859
|
+
### Memory Manager Methods
|
|
860
|
+
|
|
861
|
+
| Method | Description |
|
|
862
|
+
|--------|-------------|
|
|
863
|
+
| `addMemories(memories: string[])` | Add one or more memory strings |
|
|
864
|
+
| `listMemories(options?)` | List memories with pagination |
|
|
865
|
+
| `getMemory(memoryId: string)` | Fetch a single memory by ID |
|
|
866
|
+
| `deleteMemory(memoryId: string)` | Delete a single memory |
|
|
867
|
+
| `deleteAllMemories()` | Delete all memories for current user/character |
|
|
868
|
+
| `setEndUserId(endUserId: string)` | Switch to a different user context |
|
|
869
|
+
| `setCharacterId(characterId: string)` | Switch to a different character context |
|
|
870
|
+
|
|
871
|
+
### Standalone Usage
|
|
872
|
+
|
|
873
|
+
You can also use the `MemoryManager` independently:
|
|
874
|
+
|
|
875
|
+
```typescript
|
|
876
|
+
import { MemoryManager } from "@convai/web-sdk/core";
|
|
877
|
+
|
|
878
|
+
const memoryManager = new MemoryManager(
|
|
879
|
+
'your-api-key',
|
|
880
|
+
'your-character-id',
|
|
881
|
+
'user@example.com'
|
|
882
|
+
);
|
|
883
|
+
|
|
884
|
+
const result = await memoryManager.addMemories(['User likes coffee']);
|
|
885
|
+
```
|
|
886
|
+
|
|
887
|
+
### Complete Documentation
|
|
888
|
+
|
|
889
|
+
For detailed documentation, examples, and API reference, see:
|
|
890
|
+
- **[Memory API Documentation](./docs/MEMORY_API.md)** - Complete guide with examples
|
|
891
|
+
- **[Memory API Usage Examples](./examples/memory-api-usage.ts)** - Runnable code examples
|
|
892
|
+
|
|
893
|
+
### Memory Types
|
|
894
|
+
|
|
895
|
+
All memory-related types are exported from the core package:
|
|
896
|
+
|
|
897
|
+
```typescript
|
|
898
|
+
import type {
|
|
899
|
+
Memory,
|
|
900
|
+
MemoryAddResponse,
|
|
901
|
+
MemoryListResponse,
|
|
902
|
+
MemoryGetResponse,
|
|
903
|
+
MemoryDeleteResponse,
|
|
904
|
+
MemoryDeleteAllResponse,
|
|
905
|
+
MemoryError,
|
|
906
|
+
} from "@convai/web-sdk/core";
|
|
907
|
+
```
|
|
908
|
+
|
|
909
|
+
## 10. Message Semantics and Turn Completion
|
|
656
910
|
|
|
657
911
|
### `ChatMessage` model
|
|
658
912
|
|
|
@@ -723,7 +977,7 @@ function subscribeTurnCompletion(client: any, options: TurnCompletionOptions) {
|
|
|
723
977
|
|
|
724
978
|
When to use both signals: You only need to wait for both `turnEnd` and `blendshapeStatsReceived` when you use lipsync. Set `expectBlendshapes: false` when you do not use facial animation; then `animationDone` is effectively always true and completion runs as soon as `turnEnd` fires. Set `expectBlendshapes: true` when you drive lipsync from the queue; speech and blendshape data are separate pipelines and can finish in either order, so waiting for both ensures "turn complete" means both speech and animation are done before you run `onComplete`.
|
|
725
979
|
|
|
726
|
-
##
|
|
980
|
+
## 11. React API Reference
|
|
727
981
|
|
|
728
982
|
### `useConvaiClient(config?)`
|
|
729
983
|
|
|
@@ -776,7 +1030,7 @@ Returns a LiveKit `TrackReferenceOrPlaceholder` for local camera rendering in cu
|
|
|
776
1030
|
- `AudioRenderer` from LiveKit React components
|
|
777
1031
|
- `AudioContext` from LiveKit React components
|
|
778
1032
|
|
|
779
|
-
##
|
|
1033
|
+
## 12. Vanilla API Reference
|
|
780
1034
|
|
|
781
1035
|
### `createConvaiWidget(container, options)`
|
|
782
1036
|
|
|
@@ -821,7 +1075,7 @@ Convenience wrapper that calls `widget.destroy()`.
|
|
|
821
1075
|
|
|
822
1076
|
`AudioRenderer` listens to LiveKit room track subscriptions and auto-attaches remote audio tracks to hidden `audio` elements for playback. Use one renderer instance per active room session and destroy it during cleanup.
|
|
823
1077
|
|
|
824
|
-
##
|
|
1078
|
+
## 13. Audio Integration Best Practices (Vanilla TypeScript)
|
|
825
1079
|
|
|
826
1080
|
This section provides the recommended integration for stable audio playback.
|
|
827
1081
|
|
|
@@ -901,7 +1155,7 @@ Recommended shutdown order:
|
|
|
901
1155
|
| Works once, then silent | Incomplete cleanup on previous session | Destroy renderer and disconnect client on teardown; avoid reusing invalid room state. |
|
|
902
1156
|
| Random muted behavior | App-side muting of remote tracks | Verify no custom code is muting remote publications or media elements. |
|
|
903
1157
|
|
|
904
|
-
##
|
|
1158
|
+
## 14. Error Handling and Reliability Patterns
|
|
905
1159
|
|
|
906
1160
|
### Pattern 1: Centralized SDK error handling
|
|
907
1161
|
|
|
@@ -971,7 +1225,7 @@ function cleanupListeners() {
|
|
|
971
1225
|
}
|
|
972
1226
|
```
|
|
973
1227
|
|
|
974
|
-
##
|
|
1228
|
+
## 15. Troubleshooting
|
|
975
1229
|
|
|
976
1230
|
### Connection issues
|
|
977
1231
|
|
|
@@ -1011,7 +1265,7 @@ function cleanupListeners() {
|
|
|
1011
1265
|
- Align lipsync start and stop with the queue: start playback when the bot starts speaking (`isBotSpeaking()` true) and treat the turn as finished when `blendshapeQueue.isConversationEnded()` is true before resetting.
|
|
1012
1266
|
- Drive blendshape application from a single loop (e.g. `requestAnimationFrame`) and advance frame index at 60fps so mouth movement stays in sync with audio.
|
|
1013
1267
|
|
|
1014
|
-
##
|
|
1268
|
+
## 17. Examples
|
|
1015
1269
|
|
|
1016
1270
|
Repository examples:
|
|
1017
1271
|
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { Room } from "livekit-client";
|
|
2
|
-
import { ConvaiConfig, ConvaiClientState, ChatMessage, IConvaiClient, AudioControls, VideoControls, ScreenShareControls } from "./types";
|
|
2
|
+
import { ConvaiConfig, ConvaiClientState, ChatMessage, IConvaiClient, AudioControls, VideoControls, ScreenShareControls, DynamicInfo, ContextUpdateOptions } from "./types";
|
|
3
3
|
import { EventEmitter } from "./EventEmitter";
|
|
4
|
-
import { LatencyMonitor } from "../utils/LatencyMonitor";
|
|
5
4
|
import { BlendshapeQueue } from "./BlendshapeQueue";
|
|
5
|
+
import { MemoryManager } from "./MemoryManager";
|
|
6
6
|
/**
|
|
7
7
|
* Main Convai client class for managing AI voice assistant connections
|
|
8
8
|
* Provides complete interface for connecting to Convai's voice assistants,
|
|
@@ -44,6 +44,7 @@ export declare class ConvaiClient extends EventEmitter implements IConvaiClient
|
|
|
44
44
|
private _state;
|
|
45
45
|
private _connectionType;
|
|
46
46
|
private _apiKey;
|
|
47
|
+
private _authToken;
|
|
47
48
|
private _characterId;
|
|
48
49
|
private _speakerId;
|
|
49
50
|
private _characterSessionId;
|
|
@@ -51,17 +52,20 @@ export declare class ConvaiClient extends EventEmitter implements IConvaiClient
|
|
|
51
52
|
private _participantSid;
|
|
52
53
|
private _audioSettings;
|
|
53
54
|
private _storedConfig;
|
|
55
|
+
private _endUserId;
|
|
56
|
+
private _endUserMetadata;
|
|
54
57
|
private _audioManager;
|
|
55
58
|
private _videoManager;
|
|
56
59
|
private _screenShareManager;
|
|
57
60
|
private _messageHandler;
|
|
58
|
-
private _latencyMonitor;
|
|
59
61
|
private _conversationSessionId;
|
|
60
62
|
private _conversationStartTime;
|
|
63
|
+
private _memoryManager;
|
|
61
64
|
constructor(config?: ConvaiConfig);
|
|
62
65
|
get state(): ConvaiClientState;
|
|
63
66
|
get connectionType(): "audio" | "video" | null;
|
|
64
67
|
get apiKey(): string | null;
|
|
68
|
+
get authToken(): string | null;
|
|
65
69
|
get characterId(): string | null;
|
|
66
70
|
get speakerId(): string | null;
|
|
67
71
|
get room(): Room;
|
|
@@ -72,9 +76,9 @@ export declare class ConvaiClient extends EventEmitter implements IConvaiClient
|
|
|
72
76
|
get audioControls(): AudioControls;
|
|
73
77
|
get videoControls(): VideoControls;
|
|
74
78
|
get screenShareControls(): ScreenShareControls;
|
|
75
|
-
get latencyMonitor(): LatencyMonitor;
|
|
76
79
|
get blendshapeQueue(): BlendshapeQueue;
|
|
77
80
|
get conversationSessionId(): number;
|
|
81
|
+
get memoryManager(): MemoryManager | null;
|
|
78
82
|
/**
|
|
79
83
|
* Setup event listeners for room and managers
|
|
80
84
|
*/
|
|
@@ -138,9 +142,34 @@ export declare class ConvaiClient extends EventEmitter implements IConvaiClient
|
|
|
138
142
|
/**
|
|
139
143
|
* Update dynamic information about the current context
|
|
140
144
|
*/
|
|
141
|
-
updateDynamicInfo(dynamicInfo:
|
|
142
|
-
|
|
143
|
-
|
|
145
|
+
updateDynamicInfo(dynamicInfo: DynamicInfo): void;
|
|
146
|
+
/**
|
|
147
|
+
* Update the bot's temporary runtime context in a unified way.
|
|
148
|
+
* Provides flexible control over ephemeral context management.
|
|
149
|
+
*
|
|
150
|
+
* @param options - Context update configuration
|
|
151
|
+
*
|
|
152
|
+
* @example
|
|
153
|
+
* ```typescript
|
|
154
|
+
* // Append to existing context
|
|
155
|
+
* client.updateContext({
|
|
156
|
+
* text: "User completed dragon quest",
|
|
157
|
+
* mode: "append",
|
|
158
|
+
* run_llm: "false"
|
|
159
|
+
* });
|
|
160
|
+
*
|
|
161
|
+
* // Replace entire context
|
|
162
|
+
* client.updateContext({
|
|
163
|
+
* text: "New game state: level 5",
|
|
164
|
+
* mode: "replace",
|
|
165
|
+
* run_llm: "auto"
|
|
166
|
+
* });
|
|
167
|
+
*
|
|
168
|
+
* // Clear context
|
|
169
|
+
* client.updateContext({ mode: "reset" });
|
|
170
|
+
* ```
|
|
171
|
+
*/
|
|
172
|
+
updateContext(options: ContextUpdateOptions): void;
|
|
144
173
|
/**
|
|
145
174
|
* Toggle text-to-speech on or off
|
|
146
175
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ConvaiClient.d.ts","sourceRoot":"","sources":["../../src/core/ConvaiClient.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,
|
|
1
|
+
{"version":3,"file":"ConvaiClient.d.ts","sourceRoot":"","sources":["../../src/core/ConvaiClient.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAwE,MAAM,gBAAgB,CAAC;AAC5G,OAAO,EACL,YAAY,EACZ,iBAAiB,EACjB,WAAW,EAEX,aAAa,EACb,aAAa,EACb,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,oBAAoB,EACrB,MAAM,SAAS,CAAC;AAKjB,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAWhD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AACH,qBAAa,YAAa,SAAQ,YAAa,YAAW,aAAa;IACrE,OAAO,CAAC,KAAK,CAAO;IACpB,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,eAAe,CAAkC;IACzD,OAAO,CAAC,OAAO,CAAuB;IACtC,OAAO,CAAC,UAAU,CAAuB;IACzC,OAAO,CAAC,YAAY,CAAuB;IAC3C,OAAO,CAAC,UAAU,CAAuB;IACzC,OAAO,CAAC,mBAAmB,CAAuB;IAClD,OAAO,CAAC,WAAW,CAAkB;IACrC,OAAO,CAAC,eAAe,CAAc;IACrC,OAAO,CAAC,cAAc,CAAgB;IACtC,OAAO,CAAC,aAAa,CAA6B;IAClD,OAAO,CAAC,UAAU,CAAuB;IACzC,OAAO,CAAC,gBAAgB,CAAwC;IAGhE,OAAO,CAAC,aAAa,CAAe;IACpC,OAAO,CAAC,aAAa,CAAe;IACpC,OAAO,CAAC,mBAAmB,CAAqB;IAChD,OAAO,CAAC,eAAe,CAAiB;IAGxC,OAAO,CAAC,sBAAsB,CAAa;IAC3C,OAAO,CAAC,sBAAsB,CAAa;IAG3C,OAAO,CAAC,cAAc,CAA8B;gBAExC,MAAM,CAAC,EAAE,YAAY;IAmDjC,IAAI,KAAK,IAAI,iBAAiB,CAE7B;IAED,IAAI,cAAc,IAAI,OAAO,GAAG,OAAO,GAAG,IAAI,CAE7C;IAED,IAAI,MAAM,IAAI,MAAM,GAAG,IAAI,CAE1B;IAED,IAAI,SAAS,IAAI,MAAM,GAAG,IAAI,CAE7B;IAED,IAAI,WAAW,IAAI,MAAM,GAAG,IAAI,CAE/B;IAED,IAAI,SAAS,IAAI,MAAM,GAAG,IAAI,CAE7B;IAED,IAAI,IAAI,IAAI,IAAI,CAEf;IAED,IAAI,YAAY,IAAI,WAAW,EAAE,CAEhC;IAED,IAAI,iBAAiB,IAAI,MAAM,CAE9B;IAED,IAAI,kBAAkB,IAAI,MAAM,GAAG,IAAI,CAEtC;IAED,IAAI,UAAU,IAAI,OAAO,CAExB;IAED,IAAI,aAAa,IAAI,aAAa,CAEjC;IAED,IAAI,aAAa,IAAI,aAAa,CAEjC;IAED,IAAI,mBAAmB,IAAI,mBAAmB,CAE7C;IAED,IAAI,eAAe,IAAI,eAAe,CAErC;IAED,IAAI,qBAAqB,IAAI,MAAM,CAElC;IAED,IAAI,aAAa,IAAI,aAAa,GAAG,IAAI,CAExC;IAED;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAwF3B;;;;OAIG;IACH,OAAO,CAAC,yBAAyB;IAYjC;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAwB1B;;OAEG;IACH,OAAO,CAAC,4BAA4B;IAuBpC;;OAEG;IACH,OAAO,CAAC,WAAW;IAuBnB;;OAEG;IACG,OAAO,CAAC,MAAM,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IA8QnD;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAYjC;;OAEG;IACH,OAAO,CAAC,oBAAoB;IA2B5B;;OAEG;IACG,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAOhC;;OAEG;IACH,YAAY,IAAI,IAAI;IAOpB;;OAEG;IACH,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IA6CvC;;OAEG;IACH,kBAAkB,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI;IAmBvE;;OAEG;IACH,oBAAoB,IAAI,IAAI;IAgC5B;;OAEG;IACH,kBAAkB,CAAC,YAAY,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAE,GAAG,IAAI;IAsBjE;;OAEG;IACH,iBAAiB,CAAC,WAAW,EAAE,WAAW,GAAG,IAAI;IAkBjD;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,aAAa,CAAC,OAAO,EAAE,oBAAoB,GAAG,IAAI;IA4BlD;;OAEG;IACH,SAAS,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IA0BjC;;OAEG;IACH,SAAS,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;CA4BlC"}
|