@spatialwalk/avatarkit 1.0.0-beta.67 → 1.0.0-beta.69
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -11
- package/README.md +102 -18
- package/dist/{StreamingAudioPlayer-CD9jBs6B.js → StreamingAudioPlayer-DiIRp5nx.js} +109 -1
- package/dist/animation/AnimationWebSocketClient.d.ts +26 -0
- package/dist/animation/utils/eventEmitter.d.ts +3 -0
- package/dist/animation/utils/flameConverter.d.ts +10 -3
- package/dist/audio/AnimationPlayer.d.ts +46 -0
- package/dist/audio/StreamingAudioPlayer.d.ts +93 -0
- package/dist/config/app-config.d.ts +5 -1
- package/dist/config/constants.d.ts +7 -1
- package/dist/config/sdk-config-loader.d.ts +11 -3
- package/dist/core/Avatar.d.ts +10 -0
- package/dist/core/AvatarController.d.ts +164 -2
- package/dist/core/AvatarDownloader.d.ts +10 -0
- package/dist/core/AvatarManager.d.ts +27 -1
- package/dist/core/AvatarSDK.d.ts +27 -0
- package/dist/core/AvatarView.d.ts +148 -3
- package/dist/core/NetworkLayer.d.ts +6 -0
- package/dist/generated/common/v1/models.d.ts +8 -1
- package/dist/generated/driveningress/v1/driveningress.d.ts +11 -1
- package/dist/generated/driveningress/v2/driveningress.d.ts +5 -2
- package/dist/generated/google/protobuf/struct.d.ts +38 -5
- package/dist/generated/google/protobuf/timestamp.d.ts +102 -1
- package/dist/{index-GRm00rtd.js → index-BT9yxWW8.js} +1468 -30
- package/dist/index.d.ts +3 -0
- package/dist/index.js +1 -1
- package/dist/renderer/RenderSystem.d.ts +8 -0
- package/dist/renderer/covariance.d.ts +11 -0
- package/dist/renderer/sortSplats.d.ts +10 -0
- package/dist/renderer/webgl/reorderData.d.ts +12 -0
- package/dist/renderer/webgl/webglRenderer.d.ts +53 -0
- package/dist/renderer/webgpu/webgpuRenderer.d.ts +38 -0
- package/dist/types/character-settings.d.ts +4 -0
- package/dist/types/character.d.ts +9 -3
- package/dist/types/index.d.ts +56 -23
- package/dist/utils/animation-interpolation.d.ts +30 -5
- package/dist/utils/client-id.d.ts +5 -0
- package/dist/utils/conversationId.d.ts +18 -0
- package/dist/utils/error-utils.d.ts +24 -1
- package/dist/utils/id-manager.d.ts +26 -0
- package/dist/utils/logger.d.ts +4 -1
- package/dist/utils/posthog-tracker.d.ts +27 -5
- package/dist/utils/pwa-cache-manager.d.ts +36 -0
- package/dist/utils/usage-tracker.d.ts +17 -2
- package/dist/vite.d.ts +16 -1
- package/dist/wasm/avatarCoreAdapter.d.ts +145 -0
- package/dist/wasm/avatarCoreMemory.d.ts +52 -0
- package/package.json +3 -3
- package/vite.js +45 -29
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,28 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to this project will be documented in this file.
|
|
4
4
|
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
## [1.0.0-beta.69] - 2026-01-17
|
|
9
|
+
|
|
10
|
+
### 🔧 Improvements
|
|
11
|
+
- **API Documentation** - All public API comments are now in English with JSDoc support for better IDE IntelliSense
|
|
12
|
+
- **Type Safety** - Removed internal types from public API exports, using `KeyframeData` instead of `Flame`
|
|
13
|
+
- **Build Configuration** - JSDoc comments are now preserved in generated `.d.ts` files
|
|
14
|
+
|
|
15
|
+
### 📚 Documentation
|
|
16
|
+
- Enhanced audio data format documentation with WAV and MP3 processing examples
|
|
17
|
+
|
|
18
|
+
## [1.0.0-beta.68] - 2026-01-17
|
|
19
|
+
|
|
20
|
+
### 🐛 Bugfixes
|
|
21
|
+
- **Vite Plugin WASM File Detection** - Fixed Vite plugin not finding WASM files with hash in consuming projects
|
|
22
|
+
- Plugin now reads JS glue file to extract referenced WASM filename (including hash)
|
|
23
|
+
- Ensures correct WASM file is copied to match JS glue file references
|
|
24
|
+
- Prevents 404 errors when WASM files have content-based hashes
|
|
25
|
+
- Fixes issue where `avatar_core_wasm.wasm` was not found after adding hash support
|
|
26
|
+
|
|
5
27
|
## [1.0.0-beta.67] - 2026-01-17
|
|
6
28
|
|
|
7
29
|
### 🐛 Bugfixes
|
|
@@ -123,7 +145,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
123
145
|
|
|
124
146
|
### ✨ New Features
|
|
125
147
|
- **Eye Tracking Control** - Added `eyeTrackingEnabled` parameter to `PostProcessingConfig`, allowing external control of eye tracking enable/disable state in real-time
|
|
126
|
-
- **Point Count API** - Added `getPointCount()` method to `AvatarController` to get the point cloud count of the current
|
|
148
|
+
- **Point Count API** - Added `getPointCount()` method to `AvatarController` to get the point cloud count of the current avatar
|
|
127
149
|
|
|
128
150
|
## [1.0.0-beta.54] - 2025-01-05
|
|
129
151
|
|
|
@@ -153,7 +175,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
153
175
|
## [1.0.0-beta.50] - 2025-01-05
|
|
154
176
|
|
|
155
177
|
### 🔧 Performance Improvements
|
|
156
|
-
- **Removed CORS Preflight for Meta API** - Removed authentication headers (`X-App-Id` and `Authorization`) and unnecessary `Content-Type` header from
|
|
178
|
+
- **Removed CORS Preflight for Meta API** - Removed authentication headers (`X-App-Id` and `Authorization`) and unnecessary `Content-Type` header from avatar metadata API requests. This eliminates CORS preflight requests for simple GET requests, improving loading performance.
|
|
157
179
|
|
|
158
180
|
## [1.0.0-beta.49] - 2025-01-05
|
|
159
181
|
|
|
@@ -163,7 +185,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
163
185
|
## [1.0.0-beta.48] - 2025-01-05
|
|
164
186
|
|
|
165
187
|
### ✨ New Features
|
|
166
|
-
- **PWA Cache Management** - Added automatic PWA cache management for
|
|
188
|
+
- **PWA Cache Management** - Added automatic PWA cache management for avatar and template resources to improve loading performance
|
|
167
189
|
|
|
168
190
|
### 🔧 Performance Improvements
|
|
169
191
|
- **Cache Hit Rate Metrics** - Resource downloads now report cache status for analytics
|
|
@@ -171,7 +193,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
171
193
|
## [1.0.0-beta.47] - 2025-12-29
|
|
172
194
|
|
|
173
195
|
### 🐛 Bugfix
|
|
174
|
-
- **Avatar Meta Update** - Fixed issue where cached Avatar instances would return stale
|
|
196
|
+
- **Avatar Meta Update** - Fixed issue where cached Avatar instances would return stale avatar metadata even when the latest metadata was fetched. Now the avatar metadata is always updated to the latest version when loading an avatar, even if the version number hasn't changed.
|
|
175
197
|
|
|
176
198
|
## [1.0.0-beta.46] - 2025-12-29
|
|
177
199
|
|
|
@@ -273,12 +295,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
273
295
|
## [1.0.0-beta.31] - 2025-12-16
|
|
274
296
|
|
|
275
297
|
### 🐛 Bugfix
|
|
276
|
-
- **Environment CORS Issue** - Fixed CORS issues when requesting configuration and
|
|
298
|
+
- **Environment CORS Issue** - Fixed CORS issues when requesting configuration and avatar data APIs. SDK now provides default configuration fallback when config requests fail, ensuring smooth operation across different environments.
|
|
277
299
|
|
|
278
300
|
## [1.0.0-beta.30] - 2025-12-15
|
|
279
301
|
|
|
280
302
|
### 🐛 Bugfix
|
|
281
|
-
- **Template Resources Duplicate Download** - Fixed issue where template resources were being re-downloaded for each new
|
|
303
|
+
- **Template Resources Duplicate Download** - Fixed issue where template resources were being re-downloaded for each new avatar load. Template resources are now only loaded once during SDK initialization and reused for all avatars.
|
|
282
304
|
|
|
283
305
|
## [1.0.0-beta.29] - 2025-12-15
|
|
284
306
|
|
|
@@ -451,14 +473,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
451
473
|
|
|
452
474
|
### ✨ New Features
|
|
453
475
|
- **Multi-AvatarView Support** - SDK now supports multiple `AvatarView` instances simultaneously
|
|
454
|
-
- Each `AvatarView` instance can manage its own
|
|
455
|
-
- Multiple
|
|
476
|
+
- Each `AvatarView` instance can manage its own avatar independently
|
|
477
|
+
- Multiple avatars can be displayed and controlled at the same time
|
|
456
478
|
- Each instance maintains its own rendering context, playback state, and network connection
|
|
457
479
|
- Removed the previous limitation of "only one AvatarView instance at a time"
|
|
458
480
|
|
|
459
481
|
### 📚 Documentation
|
|
460
|
-
- Updated README.md to reflect multi-
|
|
461
|
-
- Added multi-
|
|
482
|
+
- Updated README.md to reflect multi-avatar support capabilities
|
|
483
|
+
- Added multi-avatar usage examples
|
|
462
484
|
- Removed outdated limitation notes about single AvatarView instance
|
|
463
485
|
|
|
464
486
|
---
|
|
@@ -469,7 +491,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
469
491
|
- Added authentication support for API requests
|
|
470
492
|
- `appId` is now automatically included in HTTP headers (`X-App-Id`) and WebSocket URL query parameters
|
|
471
493
|
- `sessionToken` is now automatically included in HTTP headers (`Authorization: Bearer {token}`) and WebSocket URL query parameters
|
|
472
|
-
- All network requests (
|
|
494
|
+
- All network requests (avatar loading, resource downloads, WebSocket connections) now include authentication credentials
|
|
473
495
|
|
|
474
496
|
### 🔧 Improvements
|
|
475
497
|
- Fixed canvas resize handling for different aspect ratios
|
package/README.md
CHANGED
|
@@ -6,7 +6,7 @@ Real-time virtual avatar rendering SDK based on 3D Gaussian Splatting, supportin
|
|
|
6
6
|
|
|
7
7
|
- **3D Gaussian Splatting Rendering** - Based on the latest point cloud rendering technology, providing high-quality 3D virtual avatars
|
|
8
8
|
- **Audio-Driven Real-Time Animation Rendering** - Users provide audio data, SDK handles receiving animation data and rendering
|
|
9
|
-
- **Multi-
|
|
9
|
+
- **Multi-Avatar Support** - Support multiple avatar instances simultaneously, each with independent state and rendering
|
|
10
10
|
- **WebGPU/WebGL Dual Rendering Backend** - Automatically selects the best rendering backend for compatibility
|
|
11
11
|
- **WASM High-Performance Computing** - Uses C++ compiled WebAssembly modules for geometric calculations
|
|
12
12
|
- **TypeScript Support** - Complete type definitions and IntelliSense
|
|
@@ -43,7 +43,11 @@ export default defineConfig({
|
|
|
43
43
|
|
|
44
44
|
- ✅ **开发服务器**:自动设置 WASM 文件的正确 MIME 类型 (`application/wasm`)
|
|
45
45
|
- ✅ **构建时**:自动复制 WASM 文件到 `dist/assets/` 目录
|
|
46
|
-
-
|
|
46
|
+
- 智能检测:从 JS glue 文件中提取引用的 WASM 文件名(包括 hash)
|
|
47
|
+
- 自动匹配:确保复制的 WASM 文件与 JS glue 文件中的引用匹配
|
|
48
|
+
- 支持 hash:正确处理带 hash 的 WASM 文件(如 `avatar_core_wasm-{hash}.wasm`)
|
|
49
|
+
- ✅ **WASM JS Glue**:自动复制 WASM JS glue 文件到 `dist/assets/` 目录
|
|
50
|
+
- ✅ **Cloudflare Pages**:自动生成 `_headers` 文件,确保 WASM 文件使用正确的 MIME 类型
|
|
47
51
|
- ✅ **Vite 配置**:自动配置 `optimizeDeps`、`assetsInclude`、`assetsInlineLimit` 等选项
|
|
48
52
|
|
|
49
53
|
### 手动配置(不使用插件)
|
|
@@ -125,7 +129,7 @@ await AvatarSDK.initialize('your-app-id', configuration)
|
|
|
125
129
|
// Set sessionToken (if needed, call separately)
|
|
126
130
|
// AvatarSDK.setSessionToken('your-session-token')
|
|
127
131
|
|
|
128
|
-
// 2. Load
|
|
132
|
+
// 2. Load avatar
|
|
129
133
|
const avatarManager = AvatarManager.shared
|
|
130
134
|
const avatar = await avatarManager.load('character-id', (progress) => {
|
|
131
135
|
console.log(`Loading progress: ${progress.progress}%`)
|
|
@@ -149,7 +153,11 @@ button.addEventListener('click', async () => {
|
|
|
149
153
|
await avatarView.controller.start()
|
|
150
154
|
|
|
151
155
|
// 6. Send audio data (SDK mode, must be mono PCM16 format matching configured sample rate)
|
|
152
|
-
|
|
156
|
+
// audioData: ArrayBuffer or Uint8Array containing PCM16 audio samples
|
|
157
|
+
// - PCM files: Can be directly read as ArrayBuffer
|
|
158
|
+
// - WAV files: Extract PCM data from WAV format (may require resampling)
|
|
159
|
+
// - MP3 files: Decode first (e.g., using AudioContext.decodeAudioData()), then convert to PCM16
|
|
160
|
+
const audioData = new ArrayBuffer(1024) // Placeholder: Replace with actual PCM16 audio data
|
|
153
161
|
avatarView.controller.send(audioData, false) // Send audio data
|
|
154
162
|
avatarView.controller.send(audioData, true) // end=true marks the end of current conversation round
|
|
155
163
|
})
|
|
@@ -159,7 +167,7 @@ button.addEventListener('click', async () => {
|
|
|
159
167
|
|
|
160
168
|
```typescript
|
|
161
169
|
|
|
162
|
-
// 1-3. Same as SDK mode (initialize SDK, load
|
|
170
|
+
// 1-3. Same as SDK mode (initialize SDK, load avatar)
|
|
163
171
|
|
|
164
172
|
// 3. Create view with Host mode
|
|
165
173
|
const container = document.getElementById('avatar-container')
|
|
@@ -197,7 +205,7 @@ The SDK uses a three-layer architecture for clear separation of concerns:
|
|
|
197
205
|
### Core Components
|
|
198
206
|
|
|
199
207
|
- **AvatarSDK** - SDK initialization and management
|
|
200
|
-
- **AvatarManager** -
|
|
208
|
+
- **AvatarManager** - Avatar resource loading and management
|
|
201
209
|
- **AvatarView** - 3D rendering view (rendering layer)
|
|
202
210
|
- **AvatarController** - Audio/animation playback controller (playback layer)
|
|
203
211
|
|
|
@@ -284,11 +292,87 @@ RenderSystem → WebGPU/WebGL → Canvas rendering
|
|
|
284
292
|
- **Byte Order**: Little-endian
|
|
285
293
|
|
|
286
294
|
**Audio Data Format:**
|
|
287
|
-
- Each sample is 2 bytes (16-bit)
|
|
295
|
+
- Each sample is 2 bytes (16-bit signed integer, little-endian)
|
|
288
296
|
- Audio data should be provided as `ArrayBuffer` or `Uint8Array`
|
|
289
297
|
- For example, with 16kHz sample rate: 1 second of audio = 16000 samples × 2 bytes = 32000 bytes
|
|
290
298
|
- For 48kHz sample rate: 1 second of audio = 48000 samples × 2 bytes = 96000 bytes
|
|
291
299
|
|
|
300
|
+
**Audio Data Source:**
|
|
301
|
+
The `audioData` parameter represents raw PCM16 audio samples in the configured sample rate and mono format. Common audio sources include:
|
|
302
|
+
- **PCM files**: Raw PCM16 files can be directly read as `ArrayBuffer` or `Uint8Array` and sent to the SDK (ensure sample rate matches configuration)
|
|
303
|
+
- **WAV files**: WAV files contain PCM16 audio data in their data chunk. After extracting the PCM data from the WAV file format, it can be sent to the SDK (may require resampling if sample rate differs)
|
|
304
|
+
- **MP3 files**: MP3 files need to be decoded first (e.g., using `AudioContext.decodeAudioData()` or a decoder library), then converted from the decoded format to PCM16 before sending to the SDK
|
|
305
|
+
- **Microphone input**: Real-time microphone audio needs to be captured and converted to PCM16 format at the configured sample rate before sending
|
|
306
|
+
- **Other audio sources**: Any audio source must be converted to mono PCM16 format at the configured sample rate before sending
|
|
307
|
+
|
|
308
|
+
**Example: Processing WAV and MP3 Files:**
|
|
309
|
+
```typescript
|
|
310
|
+
// WAV file processing
|
|
311
|
+
async function processWAVFile(wavFile: File): Promise<ArrayBuffer> {
|
|
312
|
+
const arrayBuffer = await wavFile.arrayBuffer()
|
|
313
|
+
const view = new DataView(arrayBuffer)
|
|
314
|
+
|
|
315
|
+
// WAV format: Skip header (usually 44 bytes for standard WAV)
|
|
316
|
+
// Check RIFF header
|
|
317
|
+
if (view.getUint32(0, true) !== 0x46464952) { // "RIFF"
|
|
318
|
+
throw new Error('Invalid WAV file')
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
// Find "data" chunk (offset may vary)
|
|
322
|
+
let dataOffset = 44 // Standard WAV header size
|
|
323
|
+
// For non-standard WAV files, you may need to search for "data" chunk
|
|
324
|
+
// This is a simplified example - production code should parse chunks properly
|
|
325
|
+
|
|
326
|
+
const pcmData = arrayBuffer.slice(dataOffset)
|
|
327
|
+
return pcmData
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
// MP3 file processing
|
|
331
|
+
async function processMP3File(mp3File: File, targetSampleRate: number): Promise<ArrayBuffer> {
|
|
332
|
+
const arrayBuffer = await mp3File.arrayBuffer()
|
|
333
|
+
const audioContext = new AudioContext({ sampleRate: targetSampleRate })
|
|
334
|
+
|
|
335
|
+
// Decode MP3 to AudioBuffer
|
|
336
|
+
const audioBuffer = await audioContext.decodeAudioData(arrayBuffer.slice(0))
|
|
337
|
+
|
|
338
|
+
// Convert AudioBuffer to PCM16 ArrayBuffer
|
|
339
|
+
const length = audioBuffer.length
|
|
340
|
+
const channels = audioBuffer.numberOfChannels
|
|
341
|
+
const pcm16Buffer = new ArrayBuffer(length * 2)
|
|
342
|
+
const pcm16View = new DataView(pcm16Buffer)
|
|
343
|
+
|
|
344
|
+
// Mix down to mono if stereo
|
|
345
|
+
const sourceData = channels === 1
|
|
346
|
+
? audioBuffer.getChannelData(0)
|
|
347
|
+
: new Float32Array(length)
|
|
348
|
+
|
|
349
|
+
if (channels > 1) {
|
|
350
|
+
const leftChannel = audioBuffer.getChannelData(0)
|
|
351
|
+
const rightChannel = audioBuffer.getChannelData(1)
|
|
352
|
+
for (let i = 0; i < length; i++) {
|
|
353
|
+
sourceData[i] = (leftChannel[i] + rightChannel[i]) / 2 // Mix to mono
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
// Convert float32 (-1.0 to 1.0) to int16 (-32768 to 32767)
|
|
358
|
+
for (let i = 0; i < length; i++) {
|
|
359
|
+
const sample = Math.max(-1, Math.min(1, sourceData[i])) // Clamp
|
|
360
|
+
const int16Sample = sample < 0 ? sample * 0x8000 : sample * 0x7FFF
|
|
361
|
+
pcm16View.setInt16(i * 2, int16Sample, true) // little-endian
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
audioContext.close()
|
|
365
|
+
return pcm16Buffer
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
// Usage example:
|
|
369
|
+
// const wavPcmData = await processWAVFile(wavFile)
|
|
370
|
+
// avatarView.controller.send(wavPcmData, false)
|
|
371
|
+
//
|
|
372
|
+
// const mp3PcmData = await processMP3File(mp3File, 16000) // 16kHz
|
|
373
|
+
// avatarView.controller.send(mp3PcmData, false)
|
|
374
|
+
```
|
|
375
|
+
|
|
292
376
|
**Resampling:**
|
|
293
377
|
- If your audio source is at a different sample rate, you must resample it to match the configured sample rate before sending to the SDK
|
|
294
378
|
- For high-quality resampling, we recommend using Web Audio API's `OfflineAudioContext` with anti-aliasing filtering
|
|
@@ -345,20 +429,20 @@ AvatarSDK.cleanup()
|
|
|
345
429
|
|
|
346
430
|
### AvatarManager
|
|
347
431
|
|
|
348
|
-
|
|
432
|
+
Avatar resource manager, responsible for downloading, caching, and loading avatar data. Use the singleton instance via `AvatarManager.shared`.
|
|
349
433
|
|
|
350
434
|
```typescript
|
|
351
435
|
// Get singleton instance
|
|
352
436
|
const manager = AvatarManager.shared
|
|
353
437
|
|
|
354
|
-
// Load
|
|
438
|
+
// Load avatar
|
|
355
439
|
const avatar = await manager.load(
|
|
356
440
|
characterId: string,
|
|
357
441
|
onProgress?: (progress: LoadProgressInfo) => void
|
|
358
442
|
)
|
|
359
443
|
|
|
360
444
|
// Clear cache
|
|
361
|
-
manager.
|
|
445
|
+
manager.clearAll()
|
|
362
446
|
```
|
|
363
447
|
|
|
364
448
|
### AvatarView
|
|
@@ -402,19 +486,19 @@ avatarView.transform = { x, y, scale }
|
|
|
402
486
|
// - y: Vertical offset in normalized coordinates (-1 to 1, where -1 = bottom edge, 0 = center, 1 = top edge)
|
|
403
487
|
// - scale: Scale factor (1.0 = original size, 2.0 = double size, 0.5 = half size)
|
|
404
488
|
|
|
405
|
-
// Cleanup resources (must be called before switching
|
|
489
|
+
// Cleanup resources (must be called before switching avatars)
|
|
406
490
|
avatarView.dispose()
|
|
407
491
|
```
|
|
408
492
|
|
|
409
|
-
**
|
|
493
|
+
**Avatar Switching Example:**
|
|
410
494
|
|
|
411
495
|
```typescript
|
|
412
|
-
// To switch
|
|
496
|
+
// To switch avatars, simply dispose the old view and create a new one
|
|
413
497
|
if (currentAvatarView) {
|
|
414
498
|
currentAvatarView.dispose()
|
|
415
499
|
}
|
|
416
500
|
|
|
417
|
-
// Load new
|
|
501
|
+
// Load new avatar
|
|
418
502
|
const newAvatar = await avatarManager.load('new-character-id')
|
|
419
503
|
|
|
420
504
|
// Create new AvatarView
|
|
@@ -444,7 +528,7 @@ button.addEventListener('click', async () => {
|
|
|
444
528
|
// Start WebSocket service
|
|
445
529
|
await avatarView.controller.start()
|
|
446
530
|
|
|
447
|
-
// Send audio data (must be
|
|
531
|
+
// Send audio data (must be mono PCM16 format matching configured sample rate)
|
|
448
532
|
const conversationId = avatarView.controller.send(audioData: ArrayBuffer, end: boolean)
|
|
449
533
|
// Returns: conversationId - Conversation ID for this conversation session
|
|
450
534
|
// end: false (default) - Continue sending audio data for current conversation
|
|
@@ -466,9 +550,9 @@ button.addEventListener('click', async () => {
|
|
|
466
550
|
// Initialize audio context - MUST be in user gesture context
|
|
467
551
|
await avatarView.controller.initializeAudioContext()
|
|
468
552
|
|
|
469
|
-
// Stream audio chunks (must be
|
|
553
|
+
// Stream audio chunks (must be mono PCM16 format matching configured sample rate)
|
|
470
554
|
const conversationId = avatarView.controller.yieldAudioData(
|
|
471
|
-
data: Uint8Array, // Audio chunk data
|
|
555
|
+
data: Uint8Array, // Audio chunk data (PCM16 format)
|
|
472
556
|
isLast: boolean = false // Whether this is the last chunk
|
|
473
557
|
)
|
|
474
558
|
// Returns: conversationId - Conversation ID for this audio session
|
|
@@ -728,7 +812,7 @@ avatarView.dispose() // Automatically cleans up all resources
|
|
|
728
812
|
### Memory Optimization
|
|
729
813
|
|
|
730
814
|
- SDK automatically manages WASM memory allocation
|
|
731
|
-
- Supports dynamic loading/unloading of
|
|
815
|
+
- Supports dynamic loading/unloading of avatar and animation resources
|
|
732
816
|
- Provides memory usage monitoring interface
|
|
733
817
|
|
|
734
818
|
## 🌐 Browser Compatibility
|
|
@@ -1,32 +1,52 @@
|
|
|
1
1
|
var __defProp = Object.defineProperty;
|
|
2
2
|
var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
|
|
3
3
|
var __publicField = (obj, key, value) => __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
|
|
4
|
-
import { A as APP_CONFIG, l as logger, e as errorToMessage, a as logEvent } from "./index-
|
|
4
|
+
import { A as APP_CONFIG, l as logger, e as errorToMessage, a as logEvent } from "./index-BT9yxWW8.js";
|
|
5
5
|
class StreamingAudioPlayer {
|
|
6
|
+
// 标记是否正在恢复 AudioContext,避免并发恢复请求
|
|
6
7
|
constructor(options) {
|
|
8
|
+
// AudioContext is managed internally
|
|
7
9
|
__publicField(this, "audioContext", null);
|
|
8
10
|
__publicField(this, "sampleRate");
|
|
9
11
|
__publicField(this, "channelCount");
|
|
10
12
|
__publicField(this, "debug");
|
|
13
|
+
// Session-level state
|
|
11
14
|
__publicField(this, "sessionId");
|
|
12
15
|
__publicField(this, "sessionStartTime", 0);
|
|
16
|
+
// AudioContext time when session started
|
|
13
17
|
__publicField(this, "pausedTimeOffset", 0);
|
|
18
|
+
// Accumulated paused time
|
|
14
19
|
__publicField(this, "pausedAt", 0);
|
|
20
|
+
// Time when paused
|
|
15
21
|
__publicField(this, "pausedAudioContextTime", 0);
|
|
22
|
+
// audioContext.currentTime when paused (for resume calculation)
|
|
16
23
|
__publicField(this, "scheduledTime", 0);
|
|
24
|
+
// Next chunk schedule time in AudioContext time
|
|
25
|
+
// Playback state
|
|
17
26
|
__publicField(this, "isPlaying", false);
|
|
18
27
|
__publicField(this, "isPaused", false);
|
|
19
28
|
__publicField(this, "autoStartEnabled", true);
|
|
29
|
+
// Control whether to auto-start when buffer is ready
|
|
20
30
|
__publicField(this, "autoContinue", false);
|
|
31
|
+
// 标记是否应该自动继续播放(当 end=false 且无数据时自动暂停后使用)
|
|
32
|
+
// Audio buffer queue
|
|
21
33
|
__publicField(this, "audioChunks", []);
|
|
22
34
|
__publicField(this, "scheduledChunks", 0);
|
|
35
|
+
// Number of chunks already scheduled
|
|
23
36
|
__publicField(this, "activeSources", /* @__PURE__ */ new Set());
|
|
24
37
|
__publicField(this, "lastScheduledChunkEndTime", 0);
|
|
38
|
+
// 最后一个已调度 chunk 的结束时间(相对时间)
|
|
25
39
|
__publicField(this, "lastGetCurrentTimeLog", 0);
|
|
40
|
+
// 上次记录 getCurrentTime 日志的时间戳(用于节流)
|
|
41
|
+
// 跟踪每个已调度的 chunk 的开始时间(绝对时间)和持续时间,用于准确计算当前播放时间
|
|
26
42
|
__publicField(this, "scheduledChunkInfo", []);
|
|
43
|
+
// Volume control
|
|
27
44
|
__publicField(this, "gainNode", null);
|
|
28
45
|
__publicField(this, "volume", 1);
|
|
46
|
+
// Default volume 1.0 (0.0 - 1.0)
|
|
47
|
+
// Event callbacks
|
|
29
48
|
__publicField(this, "onEndedCallback");
|
|
49
|
+
// AudioContext state management
|
|
30
50
|
__publicField(this, "stateChangeHandler");
|
|
31
51
|
__publicField(this, "isResuming", false);
|
|
32
52
|
this.sessionId = `session_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
|
|
@@ -34,6 +54,9 @@ class StreamingAudioPlayer {
|
|
|
34
54
|
this.channelCount = (options == null ? void 0 : options.channelCount) ?? 1;
|
|
35
55
|
this.debug = (options == null ? void 0 : options.debug) ?? false;
|
|
36
56
|
}
|
|
57
|
+
/**
|
|
58
|
+
* Initialize audio context (create and ensure it's ready)
|
|
59
|
+
*/
|
|
37
60
|
async initialize() {
|
|
38
61
|
if (this.audioContext) {
|
|
39
62
|
return;
|
|
@@ -72,6 +95,15 @@ class StreamingAudioPlayer {
|
|
|
72
95
|
throw error instanceof Error ? error : new Error(message);
|
|
73
96
|
}
|
|
74
97
|
}
|
|
98
|
+
/**
|
|
99
|
+
* 确保 AudioContext 正在运行(如果被暂停则自动恢复)
|
|
100
|
+
* 只在正在播放且未暂停时自动恢复,避免干扰正常的暂停/恢复逻辑
|
|
101
|
+
*
|
|
102
|
+
* 优化:
|
|
103
|
+
* - 快速路径:如果已经是 running 状态,直接返回
|
|
104
|
+
* - 避免并发恢复:使用 isResuming 标志防止重复恢复请求
|
|
105
|
+
* - 处理 closed 状态:如果 AudioContext 已关闭,无法恢复
|
|
106
|
+
*/
|
|
75
107
|
async ensureAudioContextRunning() {
|
|
76
108
|
if (!this.audioContext) {
|
|
77
109
|
return;
|
|
@@ -120,6 +152,9 @@ class StreamingAudioPlayer {
|
|
|
120
152
|
}
|
|
121
153
|
}
|
|
122
154
|
}
|
|
155
|
+
/**
|
|
156
|
+
* Add audio chunk (16-bit PCM)
|
|
157
|
+
*/
|
|
123
158
|
addChunk(pcmData, isLast = false) {
|
|
124
159
|
if (!this.audioContext) {
|
|
125
160
|
logger.error("AudioContext not initialized");
|
|
@@ -157,6 +192,9 @@ class StreamingAudioPlayer {
|
|
|
157
192
|
this.log("[StreamingAudioPlayer] Not playing and no chunks, waiting for more chunks");
|
|
158
193
|
}
|
|
159
194
|
}
|
|
195
|
+
/**
|
|
196
|
+
* Start new session (stop current and start fresh)
|
|
197
|
+
*/
|
|
160
198
|
async startNewSession(audioChunks) {
|
|
161
199
|
this.stop();
|
|
162
200
|
this.sessionId = `session_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
|
|
@@ -173,6 +211,9 @@ class StreamingAudioPlayer {
|
|
|
173
211
|
this.addChunk(chunk.data, chunk.isLast);
|
|
174
212
|
}
|
|
175
213
|
}
|
|
214
|
+
/**
|
|
215
|
+
* Start playback
|
|
216
|
+
*/
|
|
176
217
|
async startPlayback() {
|
|
177
218
|
if (!this.audioContext) {
|
|
178
219
|
this.log("[StreamingAudioPlayer] Cannot start playback: AudioContext not initialized");
|
|
@@ -198,11 +239,17 @@ class StreamingAudioPlayer {
|
|
|
198
239
|
});
|
|
199
240
|
this.scheduleAllChunks();
|
|
200
241
|
}
|
|
242
|
+
/**
|
|
243
|
+
* Schedule all pending chunks
|
|
244
|
+
*/
|
|
201
245
|
scheduleAllChunks() {
|
|
202
246
|
while (this.scheduledChunks < this.audioChunks.length) {
|
|
203
247
|
this.scheduleNextChunk();
|
|
204
248
|
}
|
|
205
249
|
}
|
|
250
|
+
/**
|
|
251
|
+
* Schedule next audio chunk
|
|
252
|
+
*/
|
|
206
253
|
scheduleNextChunk() {
|
|
207
254
|
if (!this.audioContext) {
|
|
208
255
|
this.log("[StreamingAudioPlayer] Cannot schedule chunk: AudioContext not initialized");
|
|
@@ -284,6 +331,10 @@ class StreamingAudioPlayer {
|
|
|
284
331
|
});
|
|
285
332
|
}
|
|
286
333
|
}
|
|
334
|
+
/**
|
|
335
|
+
* Convert PCM data to AudioBuffer
|
|
336
|
+
* Input: 16-bit PCM (int16), Output: AudioBuffer (float32 [-1, 1])
|
|
337
|
+
*/
|
|
287
338
|
pcmToAudioBuffer(pcmData) {
|
|
288
339
|
if (!this.audioContext) {
|
|
289
340
|
return null;
|
|
@@ -319,6 +370,10 @@ class StreamingAudioPlayer {
|
|
|
319
370
|
}
|
|
320
371
|
return audioBuffer;
|
|
321
372
|
}
|
|
373
|
+
/**
|
|
374
|
+
* Get current playback time (seconds)
|
|
375
|
+
* 返回实际播放的音频总时长
|
|
376
|
+
*/
|
|
322
377
|
getCurrentTime() {
|
|
323
378
|
if (!this.audioContext || !this.isPlaying) {
|
|
324
379
|
return 0;
|
|
@@ -346,6 +401,10 @@ class StreamingAudioPlayer {
|
|
|
346
401
|
}
|
|
347
402
|
return Math.max(0, totalPlayedDuration);
|
|
348
403
|
}
|
|
404
|
+
/**
|
|
405
|
+
* Get total duration of buffered audio (seconds)
|
|
406
|
+
* 计算所有已缓冲 chunk 的总时长
|
|
407
|
+
*/
|
|
349
408
|
getBufferedDuration() {
|
|
350
409
|
if (!this.audioContext) {
|
|
351
410
|
return 0;
|
|
@@ -357,10 +416,17 @@ class StreamingAudioPlayer {
|
|
|
357
416
|
}
|
|
358
417
|
return totalDuration;
|
|
359
418
|
}
|
|
419
|
+
/**
|
|
420
|
+
* Get current AudioContext time
|
|
421
|
+
* @returns Current AudioContext time in seconds, or 0 if AudioContext is not initialized
|
|
422
|
+
*/
|
|
360
423
|
getAudioContextTime() {
|
|
361
424
|
var _a;
|
|
362
425
|
return ((_a = this.audioContext) == null ? void 0 : _a.currentTime) ?? 0;
|
|
363
426
|
}
|
|
427
|
+
/**
|
|
428
|
+
* Pause playback
|
|
429
|
+
*/
|
|
364
430
|
pause() {
|
|
365
431
|
if (!this.isPlaying || this.isPaused || !this.audioContext) {
|
|
366
432
|
return;
|
|
@@ -380,6 +446,9 @@ class StreamingAudioPlayer {
|
|
|
380
446
|
audioContextState: this.audioContext.state
|
|
381
447
|
});
|
|
382
448
|
}
|
|
449
|
+
/**
|
|
450
|
+
* Resume playback
|
|
451
|
+
*/
|
|
383
452
|
async resume() {
|
|
384
453
|
if (!this.isPaused || !this.audioContext || !this.isPlaying) {
|
|
385
454
|
return;
|
|
@@ -407,6 +476,9 @@ class StreamingAudioPlayer {
|
|
|
407
476
|
audioContextState: this.audioContext.state
|
|
408
477
|
});
|
|
409
478
|
}
|
|
479
|
+
/**
|
|
480
|
+
* Stop playback
|
|
481
|
+
*/
|
|
410
482
|
stop() {
|
|
411
483
|
if (!this.audioContext) {
|
|
412
484
|
return;
|
|
@@ -438,10 +510,17 @@ class StreamingAudioPlayer {
|
|
|
438
510
|
this.autoContinue = false;
|
|
439
511
|
this.log("[StreamingAudioPlayer] Playback stopped, state reset");
|
|
440
512
|
}
|
|
513
|
+
/**
|
|
514
|
+
* Enable or disable auto-start (for delayed start scenarios)
|
|
515
|
+
*/
|
|
441
516
|
setAutoStart(enabled) {
|
|
442
517
|
this.autoStartEnabled = enabled;
|
|
443
518
|
this.log(`Auto-start ${enabled ? "enabled" : "disabled"}`);
|
|
444
519
|
}
|
|
520
|
+
/**
|
|
521
|
+
* Start playback manually (for delayed start scenarios)
|
|
522
|
+
* This allows starting playback after transition animation completes
|
|
523
|
+
*/
|
|
445
524
|
play() {
|
|
446
525
|
if (this.isPlaying) {
|
|
447
526
|
return;
|
|
@@ -451,18 +530,30 @@ class StreamingAudioPlayer {
|
|
|
451
530
|
logger.errorWithError("[StreamingAudioPlayer] Failed to start playback from play():", err);
|
|
452
531
|
});
|
|
453
532
|
}
|
|
533
|
+
/**
|
|
534
|
+
* Mark playback as ended
|
|
535
|
+
*/
|
|
454
536
|
markEnded() {
|
|
455
537
|
var _a;
|
|
456
538
|
this.log("Playback ended");
|
|
457
539
|
this.isPlaying = false;
|
|
458
540
|
(_a = this.onEndedCallback) == null ? void 0 : _a.call(this);
|
|
459
541
|
}
|
|
542
|
+
/**
|
|
543
|
+
* Set ended callback
|
|
544
|
+
*/
|
|
460
545
|
onEnded(callback) {
|
|
461
546
|
this.onEndedCallback = callback;
|
|
462
547
|
}
|
|
548
|
+
/**
|
|
549
|
+
* Check if playing
|
|
550
|
+
*/
|
|
463
551
|
isPlayingNow() {
|
|
464
552
|
return this.isPlaying && !this.isPaused;
|
|
465
553
|
}
|
|
554
|
+
/**
|
|
555
|
+
* Dispose and cleanup
|
|
556
|
+
*/
|
|
466
557
|
dispose() {
|
|
467
558
|
this.stop();
|
|
468
559
|
if (this.audioContext && this.stateChangeHandler) {
|
|
@@ -484,6 +575,11 @@ class StreamingAudioPlayer {
|
|
|
484
575
|
this.onEndedCallback = void 0;
|
|
485
576
|
this.log("StreamingAudioPlayer disposed");
|
|
486
577
|
}
|
|
578
|
+
/**
|
|
579
|
+
* Flush buffered audio
|
|
580
|
+
* - hard: stops all playing sources and clears all chunks
|
|
581
|
+
* - soft (default): clears UNSCHEDULED chunks only
|
|
582
|
+
*/
|
|
487
583
|
flush(options) {
|
|
488
584
|
const hard = (options == null ? void 0 : options.hard) === true;
|
|
489
585
|
if (hard) {
|
|
@@ -501,6 +597,11 @@ class StreamingAudioPlayer {
|
|
|
501
597
|
}
|
|
502
598
|
this.log("Flushed (soft)", { remainingScheduled: this.scheduledChunks });
|
|
503
599
|
}
|
|
600
|
+
/**
|
|
601
|
+
* 设置音量 (0.0 - 1.0)
|
|
602
|
+
* 注意:这仅控制数字人音频播放器的音量,不影响系统音量
|
|
603
|
+
* @param volume 音量值,范围 0.0 到 1.0(0.0 为静音,1.0 为最大音量)
|
|
604
|
+
*/
|
|
504
605
|
setVolume(volume) {
|
|
505
606
|
if (volume < 0 || volume > 1) {
|
|
506
607
|
logger.warn(`[StreamingAudioPlayer] Volume out of range: ${volume}, clamping to [0, 1]`);
|
|
@@ -511,9 +612,16 @@ class StreamingAudioPlayer {
|
|
|
511
612
|
this.gainNode.gain.value = volume;
|
|
512
613
|
}
|
|
513
614
|
}
|
|
615
|
+
/**
|
|
616
|
+
* 获取当前音量
|
|
617
|
+
* @returns 当前音量值 (0.0 - 1.0)
|
|
618
|
+
*/
|
|
514
619
|
getVolume() {
|
|
515
620
|
return this.volume;
|
|
516
621
|
}
|
|
622
|
+
/**
|
|
623
|
+
* Debug logging
|
|
624
|
+
*/
|
|
517
625
|
log(message, data) {
|
|
518
626
|
if (this.debug) {
|
|
519
627
|
logger.log(`[StreamingAudioPlayer] ${message}`, data || "");
|
|
@@ -20,15 +20,41 @@ export declare class AnimationWebSocketClient extends EventEmitter {
|
|
|
20
20
|
private reconnectTimer;
|
|
21
21
|
private sessionConfigured;
|
|
22
22
|
constructor(options: AnimationWebSocketClientOptions);
|
|
23
|
+
/**
|
|
24
|
+
* 连接WebSocket
|
|
25
|
+
*/
|
|
23
26
|
connect(characterId: string): Promise<void>;
|
|
27
|
+
/**
|
|
28
|
+
* 断开连接
|
|
29
|
+
*/
|
|
24
30
|
disconnect(): void;
|
|
31
|
+
/**
|
|
32
|
+
* 发送音频数据
|
|
33
|
+
* @param conversationId - 会话ID(在 protobuf 协议中映射为 reqId 字段)
|
|
34
|
+
*/
|
|
25
35
|
sendAudioData(conversationId: string, audioData: ArrayBuffer, end: boolean): boolean;
|
|
36
|
+
/**
|
|
37
|
+
* 生成会话ID
|
|
38
|
+
* 使用统一的会话ID生成规则:YYYYMMDDHHmmss_nanoid
|
|
39
|
+
*/
|
|
26
40
|
generateConversationId(): string;
|
|
41
|
+
/**
|
|
42
|
+
* 获取连接状态
|
|
43
|
+
*/
|
|
27
44
|
isConnected(): boolean;
|
|
45
|
+
/**
|
|
46
|
+
* 获取当前角色ID
|
|
47
|
+
*/
|
|
28
48
|
getCurrentCharacterId(): string;
|
|
29
49
|
private buildWebSocketUrl;
|
|
30
50
|
private connectWebSocket;
|
|
51
|
+
/**
|
|
52
|
+
* 清理 URL 用于日志记录(隐藏敏感信息)
|
|
53
|
+
*/
|
|
31
54
|
private sanitizeUrlForLog;
|
|
55
|
+
/**
|
|
56
|
+
* v2 协议:配置会话(发送采样率等参数)
|
|
57
|
+
*/
|
|
32
58
|
private configureSession;
|
|
33
59
|
private handleMessage;
|
|
34
60
|
private scheduleReconnect;
|