@spatialwalk/avatarkit 1.0.0-beta.17 → 1.0.0-beta.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -1
- package/README.md +135 -107
- package/dist/{StreamingAudioPlayer-a8MwHQ3Q.js → StreamingAudioPlayer-Cw8IygPS.js} +2 -2
- package/dist/{StreamingAudioPlayer-a8MwHQ3Q.js.map → StreamingAudioPlayer-Cw8IygPS.js.map} +1 -1
- package/dist/animation/AnimationWebSocketClient.d.ts +5 -4
- package/dist/animation/AnimationWebSocketClient.d.ts.map +1 -1
- package/dist/core/AvatarController.d.ts +18 -18
- package/dist/core/AvatarController.d.ts.map +1 -1
- package/dist/core/AvatarDownloader.d.ts +0 -4
- package/dist/core/AvatarDownloader.d.ts.map +1 -1
- package/dist/core/AvatarManager.d.ts +1 -3
- package/dist/core/AvatarManager.d.ts.map +1 -1
- package/dist/core/AvatarView.d.ts.map +1 -1
- package/dist/{index-suaZGA5u.js → index-CRv3XrVE.js} +821 -837
- package/dist/index-CRv3XrVE.js.map +1 -0
- package/dist/index.js +11 -13
- package/dist/types/character.d.ts +0 -11
- package/dist/types/character.d.ts.map +1 -1
- package/dist/utils/{reqId.d.ts → conversationId.d.ts} +6 -6
- package/dist/utils/conversationId.d.ts.map +1 -0
- package/package.json +1 -1
- package/dist/index-suaZGA5u.js.map +0 -1
- package/dist/utils/reqId.d.ts.map +0 -1
package/CHANGELOG.md
CHANGED
|
@@ -5,11 +5,33 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [1.0.0-beta.19] - 2025-01-25
|
|
9
|
+
|
|
10
|
+
### 📚 Documentation
|
|
11
|
+
- **Fixed misleading documentation** - Corrected Host mode API documentation to accurately reflect that `yieldAudioData()` can be called directly to start a new session without requiring `playback()` first
|
|
12
|
+
- Removed misleading "after playback() is called" comments
|
|
13
|
+
- Clarified two ways to start a session: using `playback()` for existing data, or `yieldAudioData()` directly for streaming
|
|
14
|
+
- Updated all examples and workflow descriptions to reflect correct API usage
|
|
15
|
+
|
|
16
|
+
## [1.0.0-beta.18] - 2025-01-25
|
|
17
|
+
|
|
18
|
+
### 🔧 API Changes
|
|
19
|
+
- **Renamed `reqId` to `conversationId`** - Updated terminology for better clarity
|
|
20
|
+
- All methods and parameters that used `reqId` now use `conversationId`
|
|
21
|
+
- `getCurrentReqId()` → `getCurrentConversationId()`
|
|
22
|
+
- `generateReqId()` → `generateConversationId()`
|
|
23
|
+
- Updated all event logs and documentation to use `conversationId`
|
|
24
|
+
- Note: Protobuf protocol still uses `reqId` field name internally, but SDK API uses `conversationId`
|
|
25
|
+
|
|
26
|
+
### 📚 Documentation
|
|
27
|
+
- Enhanced Host mode documentation to clearly emphasize the workflow: send audio data first to get conversationId, then use that conversationId to send animation data
|
|
28
|
+
- Updated Host Mode Example and Host Mode Flow sections with clearer step-by-step instructions
|
|
29
|
+
|
|
8
30
|
## [1.0.0-beta.17] - 2025-01-24
|
|
9
31
|
|
|
10
32
|
### ✨ New Features
|
|
11
33
|
- **Audio-Only Fallback Mechanism** - SDK now includes automatic fallback to audio-only playback when animation data is unavailable
|
|
12
|
-
-
|
|
34
|
+
- SDK mode: Automatically enters audio-only mode when server returns an error
|
|
13
35
|
- Host mode: Automatically enters audio-only mode when empty animation data is provided
|
|
14
36
|
- Once in audio-only mode, subsequent animation data for that session is ignored
|
|
15
37
|
- Fallback mode is interruptible, just like normal playback mode
|
|
@@ -20,6 +42,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
20
42
|
- `AvatarView` constructor now only requires `avatar` and `container` parameters
|
|
21
43
|
- Removed `AvatarViewOptions` interface
|
|
22
44
|
- `container` parameter is now required (no longer optional)
|
|
45
|
+
- **Method Renames** - Renamed methods in `AvatarController` for Host mode to better reflect their purpose
|
|
46
|
+
- `play()` → `playback()`: Renamed to better reflect that the method is used for playback of existing data (replay mode)
|
|
47
|
+
- Old API: `avatarController.play(initialAudioChunks, initialKeyframes)`
|
|
48
|
+
- New API: `avatarController.playback(initialAudioChunks, initialKeyframes)`
|
|
49
|
+
- `sendAudioChunk()` → `yieldAudioData()`: Renamed to better reflect that the method yields/streams audio data
|
|
50
|
+
- Old API: `avatarController.sendAudioChunk(data, isLast)`
|
|
51
|
+
- New API: `avatarController.yieldAudioData(data, isLast)`
|
|
52
|
+
- `sendKeyframes()` → `yieldFramesData()`: Renamed to better reflect that the method yields/streams animation keyframes
|
|
53
|
+
- Old API: `avatarController.sendKeyframes(keyframes, reqId)`
|
|
54
|
+
- New API: `avatarController.yieldFramesData(keyframes, conversationId)`
|
|
23
55
|
|
|
24
56
|
### 🔧 Improvements
|
|
25
57
|
- Extended transition animation duration from 200ms to 400ms for smoother end-of-playback transitions
|
package/README.md
CHANGED
|
@@ -37,8 +37,8 @@ import { DrivingServiceMode } from '@spatialwalk/avatarkit'
|
|
|
37
37
|
const configuration: Configuration = {
|
|
38
38
|
environment: Environment.test,
|
|
39
39
|
drivingServiceMode: DrivingServiceMode.sdk, // Optional, 'sdk' is default
|
|
40
|
-
// - DrivingServiceMode.sdk: SDK mode
|
|
41
|
-
// - DrivingServiceMode.host: Host mode
|
|
40
|
+
// - DrivingServiceMode.sdk: SDK mode - SDK handles WebSocket communication
|
|
41
|
+
// - DrivingServiceMode.host: Host mode - Host app provides audio and animation data
|
|
42
42
|
}
|
|
43
43
|
|
|
44
44
|
await AvatarKit.initialize('your-app-id', configuration)
|
|
@@ -47,59 +47,58 @@ await AvatarKit.initialize('your-app-id', configuration)
|
|
|
47
47
|
// AvatarKit.setSessionToken('your-session-token')
|
|
48
48
|
|
|
49
49
|
// 2. Load character
|
|
50
|
-
const avatarManager =
|
|
50
|
+
const avatarManager = AvatarManager.shared
|
|
51
51
|
const avatar = await avatarManager.load('character-id', (progress) => {
|
|
52
52
|
console.log(`Loading progress: ${progress.progress}%`)
|
|
53
53
|
})
|
|
54
54
|
|
|
55
55
|
// 3. Create view (automatically creates Canvas and AvatarController)
|
|
56
56
|
// The playback mode is determined by drivingServiceMode in AvatarKit configuration
|
|
57
|
-
// - DrivingServiceMode.sdk: SDK mode
|
|
58
|
-
// - DrivingServiceMode.host: Host mode
|
|
57
|
+
// - DrivingServiceMode.sdk: SDK mode - SDK handles WebSocket communication
|
|
58
|
+
// - DrivingServiceMode.host: Host mode - Host app provides audio and animation data
|
|
59
59
|
const container = document.getElementById('avatar-container')
|
|
60
60
|
const avatarView = new AvatarView(avatar, container)
|
|
61
61
|
|
|
62
|
-
// 4. Start real-time communication (
|
|
62
|
+
// 4. Start real-time communication (SDK mode only)
|
|
63
63
|
await avatarView.avatarController.start()
|
|
64
64
|
|
|
65
|
-
// 5. Send audio data (
|
|
65
|
+
// 5. Send audio data (SDK mode)
|
|
66
66
|
// ⚠️ Important: Audio must be 16kHz mono PCM16 format
|
|
67
67
|
// If audio is Uint8Array, you can use slice().buffer to convert to ArrayBuffer
|
|
68
68
|
const audioUint8 = new Uint8Array(1024) // Example: 16kHz PCM16 audio data (512 samples = 1024 bytes)
|
|
69
69
|
const audioData = audioUint8.slice().buffer // Simplified conversion, works for ArrayBuffer and SharedArrayBuffer
|
|
70
70
|
avatarView.avatarController.send(audioData, false) // Send audio data, will automatically start playing after accumulating enough data
|
|
71
|
-
avatarView.avatarController.send(audioData, true) // end=true
|
|
71
|
+
avatarView.avatarController.send(audioData, true) // end=true marks the end of current conversation round
|
|
72
72
|
```
|
|
73
73
|
|
|
74
|
-
###
|
|
74
|
+
### Host Mode Example
|
|
75
75
|
|
|
76
76
|
```typescript
|
|
77
77
|
import { AvatarPlaybackMode } from '@spatialwalk/avatarkit'
|
|
78
78
|
|
|
79
|
-
// 1-3. Same as
|
|
79
|
+
// 1-3. Same as SDK mode (initialize SDK, load character)
|
|
80
80
|
|
|
81
|
-
// 3. Create view with
|
|
81
|
+
// 3. Create view with Host mode
|
|
82
82
|
const container = document.getElementById('avatar-container')
|
|
83
83
|
const avatarView = new AvatarView(avatar, container)
|
|
84
84
|
|
|
85
|
-
// 4.
|
|
86
|
-
//
|
|
87
|
-
|
|
88
|
-
|
|
85
|
+
// 4. Host Mode Workflow:
|
|
86
|
+
// ⚠️ IMPORTANT: In Host mode, you MUST send audio data FIRST to get a conversationId,
|
|
87
|
+
// then use that conversationId to send animation data.
|
|
88
|
+
// Animation data with mismatched conversationId will be discarded.
|
|
89
89
|
|
|
90
|
-
//
|
|
91
|
-
// Note: Audio and animation data should be obtained from your backend service
|
|
90
|
+
// Option A: Playback existing audio and animation data (replay mode)
|
|
92
91
|
const initialAudioChunks = [{ data: audioData1, isLast: false }, { data: audioData2, isLast: false }]
|
|
93
92
|
const initialKeyframes = animationData1 // Animation keyframes from your service
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
//
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
93
|
+
// Step 1: Send audio first to get conversationId
|
|
94
|
+
const conversationId = await avatarView.avatarController.playback(initialAudioChunks, initialKeyframes)
|
|
95
|
+
|
|
96
|
+
// Option B: Stream new audio and animation data (start a new session directly)
|
|
97
|
+
// Step 1: Send audio data first to get conversationId (automatically generates conversationId if starting new session)
|
|
98
|
+
const currentConversationId = avatarView.avatarController.yieldAudioData(audioData3, false)
|
|
99
|
+
// Step 2: Use the conversationId to send animation data (mismatched conversationId will be discarded)
|
|
100
|
+
avatarView.avatarController.yieldFramesData(animationData2, currentConversationId || conversationId)
|
|
101
|
+
// Note: To start playback, you need to call playback() with the accumulated data, or ensure enough audio data is sent
|
|
103
102
|
```
|
|
104
103
|
|
|
105
104
|
### Complete Examples
|
|
@@ -109,8 +108,8 @@ Check the example code in the GitHub repository for complete usage flows for bot
|
|
|
109
108
|
**Example Project:** [AvatarKit-Web-Demo](https://github.com/spatialwalk/AvatarKit-Web-Demo)
|
|
110
109
|
|
|
111
110
|
This repository contains complete examples for Vanilla JS, Vue 3, and React, demonstrating:
|
|
112
|
-
-
|
|
113
|
-
-
|
|
111
|
+
- SDK mode: Real-time audio input with automatic animation data reception
|
|
112
|
+
- Host mode: Custom data sources with manual audio/animation data management
|
|
114
113
|
|
|
115
114
|
## 🏗️ Architecture Overview
|
|
116
115
|
|
|
@@ -120,7 +119,7 @@ The SDK uses a three-layer architecture for clear separation of concerns:
|
|
|
120
119
|
|
|
121
120
|
1. **Rendering Layer (AvatarView)** - Responsible for 3D rendering only
|
|
122
121
|
2. **Playback Layer (AvatarController)** - Manages audio/animation synchronization and playback
|
|
123
|
-
3. **Network Layer** - Handles WebSocket communication (only in
|
|
122
|
+
3. **Network Layer** - Handles WebSocket communication (only in SDK mode, internal implementation)
|
|
124
123
|
|
|
125
124
|
### Core Components
|
|
126
125
|
|
|
@@ -153,14 +152,14 @@ The SDK supports two playback modes, configured in `AvatarKit.initialize()`:
|
|
|
153
152
|
|
|
154
153
|
The SDK includes a fallback mechanism to ensure audio playback continues even when animation data is unavailable:
|
|
155
154
|
|
|
156
|
-
- **
|
|
155
|
+
- **SDK Mode**: If the server returns an error or fails to provide animation data, the SDK automatically enters audio-only mode and continues playing audio independently
|
|
157
156
|
- **Host Mode**: If empty animation data is provided (empty array or undefined), the SDK automatically enters audio-only mode
|
|
158
157
|
- Once in audio-only mode, any subsequent animation data for that session will be ignored, and only audio will continue playing
|
|
159
158
|
- The fallback mode is interruptible, just like normal playback mode
|
|
160
159
|
|
|
161
160
|
### Data Flow
|
|
162
161
|
|
|
163
|
-
####
|
|
162
|
+
#### SDK Mode Flow
|
|
164
163
|
|
|
165
164
|
```
|
|
166
165
|
User audio input (16kHz mono PCM16)
|
|
@@ -180,15 +179,20 @@ AvatarController (playback loop) → AvatarView.renderRealtimeFrame()
|
|
|
180
179
|
RenderSystem → WebGPU/WebGL → Canvas rendering
|
|
181
180
|
```
|
|
182
181
|
|
|
183
|
-
####
|
|
182
|
+
#### Host Mode Flow
|
|
184
183
|
|
|
185
184
|
```
|
|
186
185
|
External data source (audio + animation)
|
|
187
186
|
↓
|
|
188
|
-
|
|
187
|
+
Step 1: Send audio data FIRST to get conversationId
|
|
188
|
+
↓
|
|
189
|
+
AvatarController.playback(initialAudio, initialKeyframes) // Returns conversationId
|
|
190
|
+
OR
|
|
191
|
+
AvatarController.yieldAudioData(audioChunk) // Returns conversationId
|
|
192
|
+
↓
|
|
193
|
+
Step 2: Use conversationId to send animation data
|
|
189
194
|
↓
|
|
190
|
-
AvatarController.
|
|
191
|
-
AvatarController.yieldFramesData() // Stream additional animation
|
|
195
|
+
AvatarController.yieldFramesData(keyframes, conversationId) // Requires conversationId
|
|
192
196
|
↓
|
|
193
197
|
AvatarController → AnimationPlayer (synchronized playback)
|
|
194
198
|
↓
|
|
@@ -200,8 +204,8 @@ RenderSystem → WebGPU/WebGL → Canvas rendering
|
|
|
200
204
|
```
|
|
201
205
|
|
|
202
206
|
**Note:**
|
|
203
|
-
- In
|
|
204
|
-
- In
|
|
207
|
+
- In SDK mode, users provide audio data, SDK handles network communication and animation data reception
|
|
208
|
+
- In Host mode, users provide both audio and animation data, SDK handles synchronized playback only
|
|
205
209
|
|
|
206
210
|
### Audio Format Requirements
|
|
207
211
|
|
|
@@ -262,10 +266,11 @@ AvatarKit.cleanup()
|
|
|
262
266
|
|
|
263
267
|
### AvatarManager
|
|
264
268
|
|
|
265
|
-
Character resource manager, responsible for downloading, caching, and loading character data.
|
|
269
|
+
Character resource manager, responsible for downloading, caching, and loading character data. Use the singleton instance via `AvatarManager.shared`.
|
|
266
270
|
|
|
267
271
|
```typescript
|
|
268
|
-
|
|
272
|
+
// Get singleton instance
|
|
273
|
+
const manager = AvatarManager.shared
|
|
269
274
|
|
|
270
275
|
// Load character
|
|
271
276
|
const avatar = await manager.load(
|
|
@@ -296,6 +301,9 @@ const avatarView = new AvatarView(avatar, container)
|
|
|
296
301
|
// Get playback mode
|
|
297
302
|
const mode = avatarView.playbackMode // 'network' | 'external'
|
|
298
303
|
|
|
304
|
+
// Wait for first frame to render
|
|
305
|
+
await avatarView.ready // Promise that resolves when the first frame is rendered
|
|
306
|
+
|
|
299
307
|
// Cleanup resources (must be called before switching characters)
|
|
300
308
|
avatarView.dispose()
|
|
301
309
|
```
|
|
@@ -314,7 +322,7 @@ const newAvatar = await avatarManager.load('new-character-id')
|
|
|
314
322
|
// Create new AvatarView
|
|
315
323
|
currentAvatarView = new AvatarView(newAvatar, container)
|
|
316
324
|
|
|
317
|
-
//
|
|
325
|
+
// SDK mode: start connection
|
|
318
326
|
if (currentAvatarView.playbackMode === AvatarPlaybackMode.network) {
|
|
319
327
|
await currentAvatarView.controller.start()
|
|
320
328
|
}
|
|
@@ -322,77 +330,86 @@ if (currentAvatarView.playbackMode === AvatarPlaybackMode.network) {
|
|
|
322
330
|
|
|
323
331
|
### AvatarController
|
|
324
332
|
|
|
325
|
-
Audio/animation playback controller (playback layer), manages synchronized playback of audio and animation. Automatically handles WebSocket communication in
|
|
333
|
+
Audio/animation playback controller (playback layer), manages synchronized playback of audio and animation. Automatically handles WebSocket communication in SDK mode.
|
|
326
334
|
|
|
327
335
|
**Two Usage Patterns:**
|
|
328
336
|
|
|
329
|
-
####
|
|
337
|
+
#### SDK Mode Methods
|
|
330
338
|
|
|
331
339
|
```typescript
|
|
332
340
|
// Start WebSocket service
|
|
333
341
|
await avatarView.avatarController.start()
|
|
334
342
|
|
|
335
|
-
// Send audio data
|
|
336
|
-
avatarView.avatarController.send(audioData: ArrayBuffer, end: boolean)
|
|
343
|
+
// Send audio data
|
|
344
|
+
const conversationId = avatarView.avatarController.send(audioData: ArrayBuffer, end: boolean)
|
|
345
|
+
// Returns: conversationId - Conversation ID for this conversation session (used to distinguish each conversation round)
|
|
337
346
|
// audioData: Audio data (ArrayBuffer format, must be 16kHz mono PCM16)
|
|
338
347
|
// - Sample rate: 16kHz (16000 Hz) - backend requirement
|
|
339
348
|
// - Format: PCM16 (16-bit signed integer, little-endian)
|
|
340
349
|
// - Channels: Mono (single channel)
|
|
341
350
|
// - Example: 1 second = 16000 samples × 2 bytes = 32000 bytes
|
|
342
|
-
// end: false (default) -
|
|
343
|
-
// end: true -
|
|
351
|
+
// end: false (default) - Continue sending audio data for current conversation
|
|
352
|
+
// end: true - Mark the end of current conversation round. After end=true, sending new audio data will interrupt any ongoing playback from the previous conversation round
|
|
344
353
|
|
|
345
354
|
// Close WebSocket service
|
|
346
355
|
avatarView.avatarController.close()
|
|
347
356
|
```
|
|
348
357
|
|
|
349
|
-
####
|
|
358
|
+
#### Host Mode Methods
|
|
350
359
|
|
|
351
360
|
```typescript
|
|
352
361
|
// Playback existing audio and animation data (starts a new conversation)
|
|
353
|
-
const
|
|
362
|
+
const conversationId = await avatarView.avatarController.playback(
|
|
354
363
|
initialAudioChunks?: Array<{ data: Uint8Array, isLast: boolean }>, // Existing audio chunks (16kHz mono PCM16)
|
|
355
364
|
initialKeyframes?: any[] // Existing animation keyframes (obtained from your service)
|
|
356
365
|
)
|
|
357
|
-
// Returns:
|
|
366
|
+
// Returns: conversationId - New conversation ID for this conversation session
|
|
358
367
|
|
|
359
|
-
// Stream
|
|
360
|
-
const
|
|
368
|
+
// Stream audio chunks (can be called directly to start a new session, or after playback() to add more data)
|
|
369
|
+
const conversationId = avatarView.avatarController.yieldAudioData(
|
|
361
370
|
data: Uint8Array, // Audio chunk data
|
|
362
371
|
isLast: boolean = false // Whether this is the last chunk
|
|
363
372
|
)
|
|
364
|
-
// Returns:
|
|
373
|
+
// Returns: conversationId - Conversation ID for this audio session
|
|
374
|
+
// Note: If no conversationId exists, a new one will be automatically generated
|
|
365
375
|
|
|
366
|
-
// Stream
|
|
376
|
+
// Stream animation keyframes (requires conversationId from audio data)
|
|
367
377
|
avatarView.avatarController.yieldFramesData(
|
|
368
|
-
keyframes: any[], //
|
|
369
|
-
|
|
378
|
+
keyframes: any[], // Animation keyframes (obtained from your service)
|
|
379
|
+
conversationId: string // Conversation ID (required). Use getCurrentConversationId() or yieldAudioData() to get conversationId.
|
|
370
380
|
)
|
|
371
381
|
```
|
|
372
382
|
|
|
373
|
-
**⚠️ Important:
|
|
383
|
+
**⚠️ Important: Conversation ID (conversationId) Management**
|
|
384
|
+
|
|
385
|
+
**SDK Mode:**
|
|
386
|
+
- `send()` returns a conversationId to distinguish each conversation round
|
|
387
|
+
- `end=true` marks the end of a conversation round. After `end=true`, sending new audio data will interrupt any ongoing playback from the previous conversation round
|
|
374
388
|
|
|
389
|
+
**Host Mode:**
|
|
375
390
|
For each conversation session, you **must**:
|
|
376
|
-
1. **First send audio data** to get a
|
|
377
|
-
- `playback()` returns a
|
|
378
|
-
- `yieldAudioData()` returns a
|
|
379
|
-
2. **Then use that
|
|
380
|
-
- `yieldFramesData()` requires a valid
|
|
381
|
-
- Animation data with mismatched
|
|
382
|
-
- Use `
|
|
383
|
-
|
|
384
|
-
**Example Flow:**
|
|
391
|
+
1. **First send audio data** to get a conversationId (used to distinguish each conversation round):
|
|
392
|
+
- `playback()` returns a conversationId when playback existing audio and animation data (replay mode)
|
|
393
|
+
- `yieldAudioData()` returns a conversationId for streaming new audio data
|
|
394
|
+
2. **Then use that conversationId** to send animation data:
|
|
395
|
+
- `yieldFramesData()` requires a valid conversationId parameter
|
|
396
|
+
- Animation data with mismatched conversationId will be **discarded**
|
|
397
|
+
- Use `getCurrentConversationId()` to retrieve the current active conversationId
|
|
398
|
+
|
|
399
|
+
**Example Flow (Host Mode):**
|
|
385
400
|
```typescript
|
|
386
|
-
//
|
|
387
|
-
const
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
401
|
+
// Option A: Playback existing complete data (replay mode)
|
|
402
|
+
const conversationId = await avatarView.avatarController.playback(initialAudioChunks, initialKeyframes)
|
|
403
|
+
|
|
404
|
+
// Option B: Start streaming new data directly
|
|
405
|
+
// Step 1: Send audio data first to get conversationId (automatically generates if starting new session)
|
|
406
|
+
const conversationId = avatarView.avatarController.yieldAudioData(audioChunk, false)
|
|
407
|
+
// Step 2: Use the conversationId to send animation data
|
|
408
|
+
avatarView.avatarController.yieldFramesData(keyframes, conversationId)
|
|
409
|
+
// Note: To start playback with Option B, call playback() with accumulated data or ensure enough audio is sent
|
|
393
410
|
```
|
|
394
411
|
|
|
395
|
-
**Why
|
|
412
|
+
**Why conversationId is required:**
|
|
396
413
|
- Ensures audio and animation data belong to the same conversation session
|
|
397
414
|
- Prevents data from different sessions from being mixed
|
|
398
415
|
- Automatically discards mismatched animation data for data integrity
|
|
@@ -412,18 +429,22 @@ avatarView.avatarController.interrupt()
|
|
|
412
429
|
// Clear all data and resources
|
|
413
430
|
avatarView.avatarController.clear()
|
|
414
431
|
|
|
432
|
+
// Get current conversation ID (for Host mode)
|
|
433
|
+
const conversationId = avatarView.avatarController.getCurrentConversationId()
|
|
434
|
+
// Returns: Current conversationId for the active audio session, or null if no active session
|
|
435
|
+
|
|
415
436
|
// Set event callbacks
|
|
416
|
-
avatarView.avatarController.onConnectionState = (state: ConnectionState) => {} //
|
|
437
|
+
avatarView.avatarController.onConnectionState = (state: ConnectionState) => {} // SDK mode only
|
|
417
438
|
avatarView.avatarController.onAvatarState = (state: AvatarState) => {}
|
|
418
439
|
avatarView.avatarController.onError = (error: Error) => {}
|
|
419
440
|
```
|
|
420
441
|
|
|
421
442
|
**Important Notes:**
|
|
422
|
-
- `start()` and `close()` are only available in
|
|
423
|
-
- `playback()`, `yieldAudioData()`, and `yieldFramesData()` are only available in
|
|
424
|
-
- `pause()`, `resume()`, `interrupt()`, and `
|
|
443
|
+
- `start()` and `close()` are only available in SDK mode
|
|
444
|
+
- `playback()`, `yieldAudioData()`, and `yieldFramesData()` are only available in Host mode
|
|
445
|
+
- `pause()`, `resume()`, `interrupt()`, `clear()`, and `getCurrentConversationId()` are available in both modes
|
|
425
446
|
- The playback mode is determined when creating `AvatarView` and cannot be changed
|
|
426
|
-
- **
|
|
447
|
+
- **Conversation ID**: In Host mode, always send audio data first to obtain a conversationId, then use that conversationId when sending animation data. Animation data with mismatched conversationId will be discarded. Use `getCurrentConversationId()` to retrieve the current active conversationId.
|
|
427
448
|
|
|
428
449
|
## 🔧 Configuration
|
|
429
450
|
|
|
@@ -432,7 +453,7 @@ avatarView.avatarController.onError = (error: Error) => {}
|
|
|
432
453
|
```typescript
|
|
433
454
|
interface Configuration {
|
|
434
455
|
environment: Environment
|
|
435
|
-
drivingServiceMode?: DrivingServiceMode // Optional, default is 'sdk' (
|
|
456
|
+
drivingServiceMode?: DrivingServiceMode // Optional, default is 'sdk' (SDK mode)
|
|
436
457
|
}
|
|
437
458
|
```
|
|
438
459
|
|
|
@@ -469,8 +490,8 @@ constructor(avatar: Avatar, container: HTMLElement)
|
|
|
469
490
|
|
|
470
491
|
```typescript
|
|
471
492
|
enum AvatarPlaybackMode {
|
|
472
|
-
network = 'network', //
|
|
473
|
-
external = 'external' //
|
|
493
|
+
network = 'network', // SDK mode: SDK handles WebSocket communication
|
|
494
|
+
external = 'external' // Host mode: Host provides data, SDK handles playback
|
|
474
495
|
}
|
|
475
496
|
```
|
|
476
497
|
|
|
@@ -554,14 +575,12 @@ avatarView.avatarController.onError = (error: Error) => {
|
|
|
554
575
|
|
|
555
576
|
### Lifecycle Management
|
|
556
577
|
|
|
557
|
-
####
|
|
578
|
+
#### SDK Mode Lifecycle
|
|
558
579
|
|
|
559
580
|
```typescript
|
|
560
581
|
// Initialize
|
|
561
582
|
const container = document.getElementById('avatar-container')
|
|
562
583
|
const avatarView = new AvatarView(avatar, container)
|
|
563
|
-
playbackMode: AvatarPlaybackMode.network
|
|
564
|
-
})
|
|
565
584
|
await avatarView.avatarController.start()
|
|
566
585
|
|
|
567
586
|
// Use
|
|
@@ -572,7 +591,7 @@ avatarView.avatarController.close()
|
|
|
572
591
|
avatarView.dispose() // Automatically cleans up all resources
|
|
573
592
|
```
|
|
574
593
|
|
|
575
|
-
####
|
|
594
|
+
#### Host Mode Lifecycle
|
|
576
595
|
|
|
577
596
|
```typescript
|
|
578
597
|
// Initialize
|
|
@@ -581,12 +600,12 @@ const avatarView = new AvatarView(avatar, container)
|
|
|
581
600
|
|
|
582
601
|
// Use
|
|
583
602
|
const initialAudioChunks = [{ data: audioData1, isLast: false }]
|
|
584
|
-
// Step 1: Send audio first to get
|
|
585
|
-
const
|
|
586
|
-
// Step 2: Stream additional audio (returns
|
|
587
|
-
const
|
|
588
|
-
// Step 3: Use
|
|
589
|
-
avatarView.avatarController.yieldFramesData(keyframes,
|
|
603
|
+
// Step 1: Send audio first to get conversationId
|
|
604
|
+
const conversationId = await avatarView.avatarController.playback(initialAudioChunks, initialKeyframes)
|
|
605
|
+
// Step 2: Stream additional audio (returns conversationId)
|
|
606
|
+
const currentConversationId = avatarView.avatarController.yieldAudioData(audioChunk, false)
|
|
607
|
+
// Step 3: Use conversationId to send animation data (mismatched conversationId will be discarded)
|
|
608
|
+
avatarView.avatarController.yieldFramesData(keyframes, currentConversationId || conversationId)
|
|
590
609
|
|
|
591
610
|
// Cleanup
|
|
592
611
|
avatarView.avatarController.clear() // Clear all data and resources
|
|
@@ -596,8 +615,8 @@ avatarView.dispose() // Automatically cleans up all resources
|
|
|
596
615
|
**⚠️ Important Notes:**
|
|
597
616
|
- When disposing AvatarView instances, must call `dispose()` to properly clean up resources
|
|
598
617
|
- Not properly cleaning up may cause resource leaks and rendering errors
|
|
599
|
-
- In
|
|
600
|
-
- In
|
|
618
|
+
- In SDK mode, call `close()` before `dispose()` to properly close WebSocket connections
|
|
619
|
+
- In Host mode, call `clear()` before `dispose()` to clear all playback data
|
|
601
620
|
|
|
602
621
|
### Memory Optimization
|
|
603
622
|
|
|
@@ -607,7 +626,7 @@ avatarView.dispose() // Automatically cleans up all resources
|
|
|
607
626
|
|
|
608
627
|
### Audio Data Sending
|
|
609
628
|
|
|
610
|
-
####
|
|
629
|
+
#### SDK Mode
|
|
611
630
|
|
|
612
631
|
The `send()` method receives audio data in `ArrayBuffer` format:
|
|
613
632
|
|
|
@@ -619,16 +638,22 @@ The `send()` method receives audio data in `ArrayBuffer` format:
|
|
|
619
638
|
|
|
620
639
|
**Usage:**
|
|
621
640
|
- `audioData`: Audio data (ArrayBuffer format, must be 16kHz mono PCM16)
|
|
622
|
-
- `end=false` (default) -
|
|
623
|
-
- `end=true` -
|
|
641
|
+
- `end=false` (default) - Continue sending audio data for current conversation
|
|
642
|
+
- `end=true` - Mark the end of current conversation round. After `end=true`, sending new audio data will interrupt any ongoing playback from the previous conversation round
|
|
624
643
|
- **Important**: No need to wait for `end=true` to start playing, it will automatically start playing after accumulating enough audio data
|
|
625
644
|
|
|
626
|
-
####
|
|
645
|
+
#### Host Mode
|
|
646
|
+
|
|
647
|
+
The `playback()` method is used to playback existing audio and animation data (replay mode), generating a new conversationId and interrupting any existing conversation.
|
|
648
|
+
|
|
649
|
+
**Two ways to start a session in Host mode:**
|
|
650
|
+
1. **Use `playback()`** - For replaying existing complete audio and animation data
|
|
651
|
+
2. **Use `yieldAudioData()` directly** - For streaming new audio data (automatically generates conversationId if needed)
|
|
627
652
|
|
|
628
|
-
|
|
653
|
+
Then use `yieldAudioData()` to stream additional audio:
|
|
629
654
|
|
|
630
655
|
**Audio Format Requirements:**
|
|
631
|
-
- Same as
|
|
656
|
+
- Same as SDK mode: 16kHz mono PCM16 format
|
|
632
657
|
- Audio data should be provided as `Uint8Array` in chunks with `isLast` flag
|
|
633
658
|
|
|
634
659
|
**Usage:**
|
|
@@ -639,18 +664,21 @@ const initialAudioChunks = [
|
|
|
639
664
|
{ data: audioData1, isLast: false },
|
|
640
665
|
{ data: audioData2, isLast: false }
|
|
641
666
|
]
|
|
642
|
-
const
|
|
643
|
-
// Returns:
|
|
667
|
+
const conversationId = await avatarController.playback(initialAudioChunks, initialKeyframes)
|
|
668
|
+
// Returns: conversationId - New conversation ID for this conversation session
|
|
644
669
|
|
|
645
670
|
// Stream additional audio chunks
|
|
646
|
-
const
|
|
647
|
-
// Returns:
|
|
671
|
+
const conversationId = avatarController.yieldAudioData(audioChunk, isLast)
|
|
672
|
+
// Returns: conversationId - Conversation ID for this audio session
|
|
648
673
|
```
|
|
649
674
|
|
|
650
|
-
**⚠️
|
|
651
|
-
1. **
|
|
652
|
-
|
|
653
|
-
|
|
675
|
+
**⚠️ Conversation ID Workflow:**
|
|
676
|
+
1. **Start a session** → Choose one of two ways:
|
|
677
|
+
- **Option A**: Use `playback(initialAudioChunks, initialKeyframes)` to replay existing complete data
|
|
678
|
+
- **Option B**: Use `yieldAudioData(audioChunk)` directly to start streaming (automatically generates conversationId)
|
|
679
|
+
2. **Get conversationId** → Both methods return a conversationId
|
|
680
|
+
3. **Send animation with conversationId** → Use the conversationId from step 1 in `yieldFramesData()`
|
|
681
|
+
4. **Data matching** → Only animation data with matching conversationId will be accepted
|
|
654
682
|
|
|
655
683
|
**Resampling (Both Modes):**
|
|
656
684
|
- If your audio source is at a different sample rate (e.g., 24kHz, 48kHz), you **must** resample it to 16kHz before sending
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
var C = Object.defineProperty;
|
|
2
2
|
var g = (h, t, e) => t in h ? C(h, t, { enumerable: !0, configurable: !0, writable: !0, value: e }) : h[t] = e;
|
|
3
3
|
var i = (h, t, e) => g(h, typeof t != "symbol" ? t + "" : t, e);
|
|
4
|
-
import { A as m, e as f, a as c, l as u } from "./index-
|
|
4
|
+
import { A as m, e as f, a as c, l as u } from "./index-CRv3XrVE.js";
|
|
5
5
|
class y {
|
|
6
6
|
constructor(t) {
|
|
7
7
|
// AudioContext is managed internally
|
|
@@ -331,4 +331,4 @@ class y {
|
|
|
331
331
|
export {
|
|
332
332
|
y as StreamingAudioPlayer
|
|
333
333
|
};
|
|
334
|
-
//# sourceMappingURL=StreamingAudioPlayer-
|
|
334
|
+
//# sourceMappingURL=StreamingAudioPlayer-Cw8IygPS.js.map
|