@spatialwalk/avatarkit 1.0.0-beta.27 → 1.0.0-beta.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -4
- package/README.md +30 -31
- package/dist/StreamingAudioPlayer-C-_1X8K-.js +398 -0
- package/dist/animation/AnimationWebSocketClient.d.ts +0 -20
- package/dist/animation/utils/eventEmitter.d.ts +0 -3
- package/dist/animation/utils/flameConverter.d.ts +3 -10
- package/dist/audio/AnimationPlayer.d.ts +0 -46
- package/dist/audio/StreamingAudioPlayer.d.ts +0 -81
- package/dist/avatar_core_wasm-i0Ocpx6q.js +2693 -0
- package/dist/config/app-config.d.ts +1 -5
- package/dist/config/constants.d.ts +2 -10
- package/dist/config/sdk-config-loader.d.ts +2 -8
- package/dist/core/Avatar.d.ts +0 -6
- package/dist/core/AvatarController.d.ts +0 -111
- package/dist/core/AvatarDownloader.d.ts +0 -75
- package/dist/core/AvatarManager.d.ts +6 -13
- package/dist/core/AvatarSDK.d.ts +21 -0
- package/dist/core/AvatarView.d.ts +4 -103
- package/dist/core/NetworkLayer.d.ts +0 -6
- package/dist/generated/driveningress/v1/driveningress.d.ts +1 -11
- package/dist/generated/driveningress/v2/driveningress.d.ts +0 -2
- package/dist/generated/google/protobuf/struct.d.ts +5 -38
- package/dist/generated/google/protobuf/timestamp.d.ts +1 -102
- package/dist/index-BpVIIm3g.js +7921 -0
- package/dist/index.d.ts +1 -4
- package/dist/index.js +17 -17
- package/dist/renderer/RenderSystem.d.ts +0 -8
- package/dist/renderer/covariance.d.ts +0 -11
- package/dist/renderer/sortSplats.d.ts +0 -10
- package/dist/renderer/webgl/reorderData.d.ts +0 -12
- package/dist/renderer/webgl/webglRenderer.d.ts +3 -39
- package/dist/renderer/webgpu/webgpuRenderer.d.ts +3 -27
- package/dist/types/character-settings.d.ts +0 -4
- package/dist/types/character.d.ts +3 -9
- package/dist/types/index.d.ts +14 -21
- package/dist/utils/animation-interpolation.d.ts +3 -12
- package/dist/utils/client-id.d.ts +0 -5
- package/dist/utils/cls-tracker.d.ts +5 -26
- package/dist/utils/conversationId.d.ts +0 -18
- package/dist/utils/error-utils.d.ts +1 -24
- package/dist/utils/heartbeat-manager.d.ts +0 -26
- package/dist/utils/id-manager.d.ts +0 -23
- package/dist/utils/logger.d.ts +1 -4
- package/dist/utils/usage-tracker.d.ts +2 -17
- package/dist/wasm/avatarCoreAdapter.d.ts +0 -134
- package/dist/wasm/avatarCoreMemory.d.ts +0 -52
- package/package.json +1 -1
- package/dist/StreamingAudioPlayer-C6v9Ed55.js +0 -352
- package/dist/avatar_core_wasm-BPIbbUx_.js +0 -1663
- package/dist/core/AvatarKit.d.ts +0 -48
- package/dist/index-s9KqPWVW.js +0 -6770
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,39 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [1.0.0-beta.29] - 2025-12-15
|
|
9
|
+
|
|
10
|
+
### 🔄 Breaking Changes
|
|
11
|
+
- **Environment Enum** - Removed `Environment.test`. Only `Environment.cn` and `Environment.intl` are now supported. Environment configuration must be explicitly provided.
|
|
12
|
+
- **Log Level Default** - Default log level changed from `LogLevel.all` to `LogLevel.off`. Set `logLevel: LogLevel.all` in configuration to enable all logs.
|
|
13
|
+
|
|
14
|
+
### ✨ New Features
|
|
15
|
+
- **Avatar Caching** - Added avatar caching methods in `AvatarManager`:
|
|
16
|
+
- `retrieve(id: string): Avatar | undefined` - Get cached avatar by ID
|
|
17
|
+
- `clear(id: string): void` - Clear cached avatar for specific ID
|
|
18
|
+
- `clearAll(): void` - Clear all cached avatars
|
|
19
|
+
- `clearCache()` is now deprecated, use `clearAll()` instead
|
|
20
|
+
- **Background Image Support** - Added background image control in `AvatarView`:
|
|
21
|
+
- `isOpaque: boolean` - Getter/setter to control canvas background transparency
|
|
22
|
+
- `setBackgroundImage(image: HTMLImageElement | string | null): void` - Set or remove background image
|
|
23
|
+
|
|
24
|
+
### 🔧 Improvements
|
|
25
|
+
- **Avatar Version Checking** - Avatar cache now automatically checks version and reloads if cached avatar version differs from latest metadata
|
|
26
|
+
- **Concurrent Loading** - Multiple concurrent `load()` calls for the same avatar ID now reuse the same loading promise
|
|
27
|
+
|
|
28
|
+
## [1.0.0-beta.28] - 2025-12-08
|
|
29
|
+
|
|
30
|
+
### 🔄 Breaking Changes
|
|
31
|
+
- **Class Renamed** - `AvatarKit` class has been renamed to `AvatarSDK` for better consistency
|
|
32
|
+
- Update all imports: `import { AvatarKit } from '@spatialwalk/avatarkit'` → `import { AvatarSDK } from '@spatialwalk/avatarkit'`
|
|
33
|
+
- Update all API calls: `AvatarKit.initialize()` → `AvatarSDK.initialize()`
|
|
34
|
+
- All static methods and properties remain the same, only the class name has changed
|
|
35
|
+
|
|
36
|
+
### 🔧 Improvements
|
|
37
|
+
- **Rendering Optimizations** - Added fragment shader discard optimization to improve performance and edge quality
|
|
38
|
+
- **Shader Alignment** - Removed view matrix transpose to align with Android SDK implementation
|
|
39
|
+
- **Resource Cleanup** - Removed unused `frameMono` and `audioMono` resources from SDK and tests
|
|
40
|
+
|
|
8
41
|
## [1.0.0-beta.27] - 2025-12-04
|
|
9
42
|
|
|
10
43
|
### 🐛 Bugfix
|
|
@@ -115,8 +148,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
115
148
|
- Fallback mode is interruptible, just like normal playback mode
|
|
116
149
|
|
|
117
150
|
### 🔧 API Changes
|
|
118
|
-
- **Playback Mode Configuration** - Moved playback mode configuration from `AvatarView` constructor to `
|
|
119
|
-
- Playback mode is now determined by `drivingServiceMode` in `
|
|
151
|
+
- **Playback Mode Configuration** - Moved playback mode configuration from `AvatarView` constructor to `AvatarSDK.initialize()`
|
|
152
|
+
- Playback mode is now determined by `drivingServiceMode` in `AvatarSDK.initialize()` configuration
|
|
120
153
|
- `AvatarView` constructor now only requires `avatar` and `container` parameters
|
|
121
154
|
- Removed `AvatarViewOptions` interface
|
|
122
155
|
- `container` parameter is now required (no longer optional)
|
|
@@ -222,7 +255,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
222
255
|
|
|
223
256
|
### 📚 Documentation
|
|
224
257
|
- Updated demo repository link in README.md
|
|
225
|
-
- Changed example project link from `Avatarkit-web-demo` to `
|
|
258
|
+
- Changed example project link from `Avatarkit-web-demo` to `AvatarSDK-Web-Demo` to match the new repository name
|
|
226
259
|
|
|
227
260
|
---
|
|
228
261
|
|
|
@@ -232,7 +265,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
232
265
|
- Cleaned up public API to hide internal implementation details
|
|
233
266
|
- Marked internal methods with `@internal` JSDoc tag to exclude them from TypeScript declarations
|
|
234
267
|
- Removed unused environment configuration options (`realtimeApiBaseUrl`, `realtimeWsUrl`)
|
|
235
|
-
- Added `appId` getter to `
|
|
268
|
+
- Added `appId` getter to `AvatarSDK` for accessing initialized app ID
|
|
236
269
|
- Internal methods like `getEnvironmentConfig`, `logEvent`, `getCanvas`, `getCameraConfig`, `updateCameraConfig` are now properly hidden from public API
|
|
237
270
|
|
|
238
271
|
### 🐛 Bug Fixes
|
package/README.md
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
#
|
|
1
|
+
# SPAvatarSDK SDK
|
|
2
2
|
|
|
3
3
|
Real-time virtual avatar rendering SDK based on 3D Gaussian Splatting, supporting audio-driven animation rendering and high-quality 3D rendering.
|
|
4
4
|
|
|
@@ -24,7 +24,7 @@ npm install @spatialwalk/avatarkit
|
|
|
24
24
|
|
|
25
25
|
```typescript
|
|
26
26
|
import {
|
|
27
|
-
|
|
27
|
+
AvatarSDK,
|
|
28
28
|
AvatarManager,
|
|
29
29
|
AvatarView,
|
|
30
30
|
Configuration,
|
|
@@ -36,21 +36,21 @@ import {
|
|
|
36
36
|
// 1. Initialize SDK
|
|
37
37
|
|
|
38
38
|
const configuration: Configuration = {
|
|
39
|
-
environment: Environment.
|
|
39
|
+
environment: Environment.cn,
|
|
40
40
|
drivingServiceMode: DrivingServiceMode.sdk, // Optional, 'sdk' is default
|
|
41
41
|
// - DrivingServiceMode.sdk: SDK mode - SDK handles WebSocket communication
|
|
42
42
|
// - DrivingServiceMode.host: Host mode - Host app provides audio and animation data
|
|
43
|
-
logLevel: LogLevel.
|
|
43
|
+
logLevel: LogLevel.off, // Optional, 'off' is default
|
|
44
44
|
// - LogLevel.off: Disable all logs
|
|
45
45
|
// - LogLevel.error: Only error logs
|
|
46
46
|
// - LogLevel.warning: Warning and error logs
|
|
47
47
|
// - LogLevel.all: All logs (info, warning, error)
|
|
48
48
|
}
|
|
49
49
|
|
|
50
|
-
await
|
|
50
|
+
await AvatarSDK.initialize('your-app-id', configuration)
|
|
51
51
|
|
|
52
52
|
// Set sessionToken (if needed, call separately)
|
|
53
|
-
//
|
|
53
|
+
// AvatarSDK.setSessionToken('your-session-token')
|
|
54
54
|
|
|
55
55
|
// 2. Load character
|
|
56
56
|
const avatarManager = AvatarManager.shared
|
|
@@ -59,7 +59,7 @@ const avatar = await avatarManager.load('character-id', (progress) => {
|
|
|
59
59
|
})
|
|
60
60
|
|
|
61
61
|
// 3. Create view (automatically creates Canvas and AvatarController)
|
|
62
|
-
// The playback mode is determined by drivingServiceMode in
|
|
62
|
+
// The playback mode is determined by drivingServiceMode in AvatarSDK configuration
|
|
63
63
|
// - DrivingServiceMode.sdk: SDK mode - SDK handles WebSocket communication
|
|
64
64
|
// - DrivingServiceMode.host: Host mode - Host app provides audio and animation data
|
|
65
65
|
const container = document.getElementById('avatar-container')
|
|
@@ -94,7 +94,7 @@ avatarView.avatarController.yieldFramesData(animationData, conversationId)
|
|
|
94
94
|
|
|
95
95
|
Check the example code in the GitHub repository for complete usage flows for both modes.
|
|
96
96
|
|
|
97
|
-
**Example Project:** [
|
|
97
|
+
**Example Project:** [AvatarSDK-Web-Demo](https://github.com/spatialwalk/AvatarSDK-Web-Demo)
|
|
98
98
|
|
|
99
99
|
This repository contains complete examples for Vanilla JS, Vue 3, and React, demonstrating:
|
|
100
100
|
- SDK mode: Real-time audio input with automatic animation data reception
|
|
@@ -112,30 +112,30 @@ The SDK uses a three-layer architecture for clear separation of concerns:
|
|
|
112
112
|
|
|
113
113
|
### Core Components
|
|
114
114
|
|
|
115
|
-
- **
|
|
115
|
+
- **AvatarSDK** - SDK initialization and management
|
|
116
116
|
- **AvatarManager** - Character resource loading and management
|
|
117
117
|
- **AvatarView** - 3D rendering view (rendering layer)
|
|
118
118
|
- **AvatarController** - Audio/animation playback controller (playback layer)
|
|
119
119
|
|
|
120
120
|
### Playback Modes
|
|
121
121
|
|
|
122
|
-
The SDK supports two playback modes, configured in `
|
|
122
|
+
The SDK supports two playback modes, configured in `AvatarSDK.initialize()`:
|
|
123
123
|
|
|
124
124
|
#### 1. SDK Mode (Default)
|
|
125
|
-
- Configured via `drivingServiceMode: DrivingServiceMode.sdk` in `
|
|
125
|
+
- Configured via `drivingServiceMode: DrivingServiceMode.sdk` in `AvatarSDK.initialize()`
|
|
126
126
|
- SDK handles WebSocket communication automatically
|
|
127
127
|
- Send audio data via `AvatarController.send()`
|
|
128
128
|
- SDK receives animation data from backend and synchronizes playback
|
|
129
129
|
- Best for: Real-time audio input scenarios
|
|
130
130
|
|
|
131
131
|
#### 2. Host Mode
|
|
132
|
-
- Configured via `drivingServiceMode: DrivingServiceMode.host` in `
|
|
132
|
+
- Configured via `drivingServiceMode: DrivingServiceMode.host` in `AvatarSDK.initialize()`
|
|
133
133
|
- Host application manages its own network/data fetching
|
|
134
134
|
- Host application provides both audio and animation data
|
|
135
135
|
- SDK only handles synchronized playback
|
|
136
136
|
- Best for: Custom data sources, pre-recorded content, or custom network implementations
|
|
137
137
|
|
|
138
|
-
**Note:** The playback mode is determined by `drivingServiceMode` in `
|
|
138
|
+
**Note:** The playback mode is determined by `drivingServiceMode` in `AvatarSDK.initialize()` configuration.
|
|
139
139
|
|
|
140
140
|
### Fallback Mechanism
|
|
141
141
|
|
|
@@ -209,40 +209,40 @@ RenderSystem → WebGPU/WebGL → Canvas rendering
|
|
|
209
209
|
|
|
210
210
|
## 📚 API Reference
|
|
211
211
|
|
|
212
|
-
###
|
|
212
|
+
### AvatarSDK
|
|
213
213
|
|
|
214
214
|
The core management class of the SDK, responsible for initialization and global configuration.
|
|
215
215
|
|
|
216
216
|
```typescript
|
|
217
217
|
// Initialize SDK
|
|
218
|
-
await
|
|
218
|
+
await AvatarSDK.initialize(appId: string, configuration: Configuration)
|
|
219
219
|
|
|
220
220
|
// Check initialization status
|
|
221
|
-
const isInitialized =
|
|
221
|
+
const isInitialized = AvatarSDK.isInitialized
|
|
222
222
|
|
|
223
223
|
// Get initialized app ID
|
|
224
|
-
const appId =
|
|
224
|
+
const appId = AvatarSDK.appId
|
|
225
225
|
|
|
226
226
|
// Get configuration
|
|
227
|
-
const config =
|
|
227
|
+
const config = AvatarSDK.configuration
|
|
228
228
|
|
|
229
229
|
// Set sessionToken (if needed, call separately)
|
|
230
|
-
|
|
230
|
+
AvatarSDK.setSessionToken('your-session-token')
|
|
231
231
|
|
|
232
232
|
// Set userId (optional, for telemetry)
|
|
233
|
-
|
|
233
|
+
AvatarSDK.setUserId('user-id')
|
|
234
234
|
|
|
235
235
|
// Get sessionToken
|
|
236
|
-
const sessionToken =
|
|
236
|
+
const sessionToken = AvatarSDK.sessionToken
|
|
237
237
|
|
|
238
238
|
// Get userId
|
|
239
|
-
const userId =
|
|
239
|
+
const userId = AvatarSDK.userId
|
|
240
240
|
|
|
241
241
|
// Get SDK version
|
|
242
|
-
const version =
|
|
242
|
+
const version = AvatarSDK.version
|
|
243
243
|
|
|
244
244
|
// Cleanup resources (must be called when no longer in use)
|
|
245
|
-
|
|
245
|
+
AvatarSDK.cleanup()
|
|
246
246
|
```
|
|
247
247
|
|
|
248
248
|
### AvatarManager
|
|
@@ -280,7 +280,7 @@ constructor(avatar: Avatar, container: HTMLElement)
|
|
|
280
280
|
- SDK automatically handles resize events via ResizeObserver
|
|
281
281
|
|
|
282
282
|
**Playback Mode:**
|
|
283
|
-
- The playback mode is determined by `drivingServiceMode` in `
|
|
283
|
+
- The playback mode is determined by `drivingServiceMode` in `AvatarSDK.initialize()` configuration
|
|
284
284
|
- The playback mode is fixed when creating `AvatarView` and persists throughout its lifecycle
|
|
285
285
|
- Cannot be changed after creation
|
|
286
286
|
|
|
@@ -403,7 +403,7 @@ avatarView.avatarController.onError = (error: Error) => {}
|
|
|
403
403
|
interface Configuration {
|
|
404
404
|
environment: Environment
|
|
405
405
|
drivingServiceMode?: DrivingServiceMode // Optional, default is 'sdk' (SDK mode)
|
|
406
|
-
logLevel?: LogLevel // Optional, default is '
|
|
406
|
+
logLevel?: LogLevel // Optional, default is 'off' (no logs)
|
|
407
407
|
}
|
|
408
408
|
```
|
|
409
409
|
|
|
@@ -423,22 +423,21 @@ enum LogLevel {
|
|
|
423
423
|
**Note:** `LogLevel.off` completely disables all logging, including error logs. Use with caution in production environments.
|
|
424
424
|
|
|
425
425
|
**Description:**
|
|
426
|
-
- `environment`: Specifies the environment (cn/intl
|
|
426
|
+
- `environment`: Specifies the environment (cn/intl), SDK will automatically use the corresponding API address and WebSocket address based on the environment
|
|
427
427
|
- `drivingServiceMode`: Specifies the driving service mode
|
|
428
428
|
- `DrivingServiceMode.sdk` (default): SDK mode - SDK handles WebSocket communication automatically
|
|
429
429
|
- `DrivingServiceMode.host`: Host mode - Host application provides audio and animation data
|
|
430
430
|
- `logLevel`: Controls the verbosity of SDK logs
|
|
431
|
-
- `LogLevel.off
|
|
431
|
+
- `LogLevel.off` (default): Disable all logs
|
|
432
432
|
- `LogLevel.error`: Only error logs
|
|
433
433
|
- `LogLevel.warning`: Warning and error logs
|
|
434
|
-
- `LogLevel.all
|
|
435
|
-
- `sessionToken`: Set separately via `
|
|
434
|
+
- `LogLevel.all`: All logs (info, warning, error)
|
|
435
|
+
- `sessionToken`: Set separately via `AvatarSDK.setSessionToken()`, not in Configuration
|
|
436
436
|
|
|
437
437
|
```typescript
|
|
438
438
|
enum Environment {
|
|
439
439
|
cn = 'cn', // China region
|
|
440
440
|
intl = 'intl', // International region
|
|
441
|
-
test = 'test' // Test environment
|
|
442
441
|
}
|
|
443
442
|
```
|
|
444
443
|
|
|
@@ -0,0 +1,398 @@
|
|
|
1
|
+
var __defProp = Object.defineProperty;
|
|
2
|
+
var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
|
|
3
|
+
var __publicField = (obj, key, value) => __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
|
|
4
|
+
import { A as APP_CONFIG, e as errorToMessage, l as logEvent, a as logger } from "./index-BpVIIm3g.js";
|
|
5
|
+
class StreamingAudioPlayer {
|
|
6
|
+
constructor(options) {
|
|
7
|
+
__publicField(this, "audioContext", null);
|
|
8
|
+
__publicField(this, "sampleRate");
|
|
9
|
+
__publicField(this, "channelCount");
|
|
10
|
+
__publicField(this, "debug");
|
|
11
|
+
__publicField(this, "sessionId");
|
|
12
|
+
__publicField(this, "sessionStartTime", 0);
|
|
13
|
+
__publicField(this, "pausedTimeOffset", 0);
|
|
14
|
+
__publicField(this, "pausedAt", 0);
|
|
15
|
+
__publicField(this, "pausedAudioContextTime", 0);
|
|
16
|
+
__publicField(this, "scheduledTime", 0);
|
|
17
|
+
__publicField(this, "isPlaying", false);
|
|
18
|
+
__publicField(this, "isPaused", false);
|
|
19
|
+
__publicField(this, "autoStartEnabled", true);
|
|
20
|
+
__publicField(this, "audioChunks", []);
|
|
21
|
+
__publicField(this, "scheduledChunks", 0);
|
|
22
|
+
__publicField(this, "activeSources", /* @__PURE__ */ new Set());
|
|
23
|
+
__publicField(this, "gainNode", null);
|
|
24
|
+
__publicField(this, "volume", 1);
|
|
25
|
+
__publicField(this, "onEndedCallback");
|
|
26
|
+
this.sessionId = `session_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
|
|
27
|
+
this.sampleRate = (options == null ? void 0 : options.sampleRate) ?? APP_CONFIG.audio.sampleRate;
|
|
28
|
+
this.channelCount = (options == null ? void 0 : options.channelCount) ?? 1;
|
|
29
|
+
this.debug = (options == null ? void 0 : options.debug) ?? false;
|
|
30
|
+
}
|
|
31
|
+
async initialize() {
|
|
32
|
+
if (this.audioContext) {
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
try {
|
|
36
|
+
this.audioContext = new AudioContext({
|
|
37
|
+
sampleRate: this.sampleRate
|
|
38
|
+
});
|
|
39
|
+
this.gainNode = this.audioContext.createGain();
|
|
40
|
+
this.gainNode.gain.value = this.volume;
|
|
41
|
+
this.gainNode.connect(this.audioContext.destination);
|
|
42
|
+
if (this.audioContext.state === "suspended") {
|
|
43
|
+
await this.audioContext.resume();
|
|
44
|
+
}
|
|
45
|
+
this.log("AudioContext initialized", {
|
|
46
|
+
sessionId: this.sessionId,
|
|
47
|
+
sampleRate: this.audioContext.sampleRate,
|
|
48
|
+
state: this.audioContext.state
|
|
49
|
+
});
|
|
50
|
+
} catch (error) {
|
|
51
|
+
const message = errorToMessage(error);
|
|
52
|
+
logEvent("activeAudioSessionFailed", "warning", {
|
|
53
|
+
sessionId: this.sessionId,
|
|
54
|
+
reason: message
|
|
55
|
+
});
|
|
56
|
+
logger.error("Failed to initialize AudioContext:", message);
|
|
57
|
+
throw error instanceof Error ? error : new Error(message);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
addChunk(pcmData, isLast = false) {
|
|
61
|
+
if (!this.audioContext) {
|
|
62
|
+
logger.error("AudioContext not initialized");
|
|
63
|
+
return;
|
|
64
|
+
}
|
|
65
|
+
this.audioChunks.push({ data: pcmData, isLast });
|
|
66
|
+
this.log(`Added chunk ${this.audioChunks.length}`, {
|
|
67
|
+
size: pcmData.length,
|
|
68
|
+
totalChunks: this.audioChunks.length,
|
|
69
|
+
isLast,
|
|
70
|
+
isPlaying: this.isPlaying,
|
|
71
|
+
scheduledChunks: this.scheduledChunks
|
|
72
|
+
});
|
|
73
|
+
if (!this.isPlaying && this.autoStartEnabled && this.audioChunks.length > 0) {
|
|
74
|
+
this.log("[StreamingAudioPlayer] Auto-starting playback from addChunk");
|
|
75
|
+
this.startPlayback();
|
|
76
|
+
} else if (this.isPlaying && !this.isPaused) {
|
|
77
|
+
this.log("[StreamingAudioPlayer] Already playing, scheduling next chunk");
|
|
78
|
+
this.scheduleNextChunk();
|
|
79
|
+
} else {
|
|
80
|
+
this.log("[StreamingAudioPlayer] Not playing and no chunks, waiting for more chunks");
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
async startNewSession(audioChunks) {
|
|
84
|
+
this.stop();
|
|
85
|
+
this.sessionId = `session_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
|
|
86
|
+
this.audioChunks = [];
|
|
87
|
+
this.scheduledChunks = 0;
|
|
88
|
+
this.pausedTimeOffset = 0;
|
|
89
|
+
this.pausedAt = 0;
|
|
90
|
+
this.pausedAudioContextTime = 0;
|
|
91
|
+
this.log("Starting new session", {
|
|
92
|
+
chunks: audioChunks.length
|
|
93
|
+
});
|
|
94
|
+
for (const chunk of audioChunks) {
|
|
95
|
+
this.addChunk(chunk.data, chunk.isLast);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
startPlayback() {
|
|
99
|
+
if (!this.audioContext) {
|
|
100
|
+
this.log("[StreamingAudioPlayer] Cannot start playback: AudioContext not initialized");
|
|
101
|
+
return;
|
|
102
|
+
}
|
|
103
|
+
if (this.isPlaying) {
|
|
104
|
+
this.log("[StreamingAudioPlayer] Cannot start playback: Already playing");
|
|
105
|
+
return;
|
|
106
|
+
}
|
|
107
|
+
this.isPlaying = true;
|
|
108
|
+
this.sessionStartTime = this.audioContext.currentTime;
|
|
109
|
+
this.scheduledTime = this.sessionStartTime;
|
|
110
|
+
this.log("[StreamingAudioPlayer] Starting playback", {
|
|
111
|
+
sessionStartTime: this.sessionStartTime,
|
|
112
|
+
bufferedChunks: this.audioChunks.length,
|
|
113
|
+
scheduledChunks: this.scheduledChunks,
|
|
114
|
+
activeSources: this.activeSources.size
|
|
115
|
+
});
|
|
116
|
+
this.scheduleAllChunks();
|
|
117
|
+
}
|
|
118
|
+
scheduleAllChunks() {
|
|
119
|
+
while (this.scheduledChunks < this.audioChunks.length) {
|
|
120
|
+
this.scheduleNextChunk();
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
scheduleNextChunk() {
|
|
124
|
+
if (!this.audioContext) {
|
|
125
|
+
this.log("[StreamingAudioPlayer] Cannot schedule chunk: AudioContext not initialized");
|
|
126
|
+
return;
|
|
127
|
+
}
|
|
128
|
+
if (!this.isPlaying || this.isPaused) {
|
|
129
|
+
this.log("[StreamingAudioPlayer] Cannot schedule chunk: Not playing or paused");
|
|
130
|
+
return;
|
|
131
|
+
}
|
|
132
|
+
const chunkIndex = this.scheduledChunks;
|
|
133
|
+
if (chunkIndex >= this.audioChunks.length) {
|
|
134
|
+
this.log(`[StreamingAudioPlayer] No more chunks to schedule (chunkIndex: ${chunkIndex}, totalChunks: ${this.audioChunks.length})`);
|
|
135
|
+
return;
|
|
136
|
+
}
|
|
137
|
+
const chunk = this.audioChunks[chunkIndex];
|
|
138
|
+
if (chunk.data.length === 0 && !chunk.isLast) {
|
|
139
|
+
this.scheduledChunks++;
|
|
140
|
+
return;
|
|
141
|
+
}
|
|
142
|
+
const pcmData = chunk.data;
|
|
143
|
+
const isLast = chunk.isLast;
|
|
144
|
+
const audioBuffer = this.pcmToAudioBuffer(pcmData);
|
|
145
|
+
if (!audioBuffer) {
|
|
146
|
+
const errorMessage = "Failed to create AudioBuffer from PCM data";
|
|
147
|
+
logger.error(errorMessage);
|
|
148
|
+
logEvent("character_player", "error", {
|
|
149
|
+
sessionId: this.sessionId,
|
|
150
|
+
event: "audio_buffer_creation_failed"
|
|
151
|
+
});
|
|
152
|
+
return;
|
|
153
|
+
}
|
|
154
|
+
try {
|
|
155
|
+
const source = this.audioContext.createBufferSource();
|
|
156
|
+
source.buffer = audioBuffer;
|
|
157
|
+
source.connect(this.gainNode);
|
|
158
|
+
source.start(this.scheduledTime);
|
|
159
|
+
this.activeSources.add(source);
|
|
160
|
+
source.onended = () => {
|
|
161
|
+
this.activeSources.delete(source);
|
|
162
|
+
if (isLast && this.activeSources.size === 0) {
|
|
163
|
+
this.log("Last audio chunk ended, marking playback as ended");
|
|
164
|
+
this.markEnded();
|
|
165
|
+
}
|
|
166
|
+
};
|
|
167
|
+
this.scheduledTime += audioBuffer.duration;
|
|
168
|
+
this.scheduledChunks++;
|
|
169
|
+
this.log(`[StreamingAudioPlayer] Scheduled chunk ${chunkIndex + 1}/${this.audioChunks.length}`, {
|
|
170
|
+
startTime: this.scheduledTime - audioBuffer.duration,
|
|
171
|
+
duration: audioBuffer.duration,
|
|
172
|
+
nextScheduleTime: this.scheduledTime,
|
|
173
|
+
isLast,
|
|
174
|
+
activeSources: this.activeSources.size
|
|
175
|
+
});
|
|
176
|
+
} catch (err) {
|
|
177
|
+
logger.errorWithError("Failed to schedule audio chunk:", err);
|
|
178
|
+
logEvent("character_player", "error", {
|
|
179
|
+
sessionId: this.sessionId,
|
|
180
|
+
event: "schedule_chunk_failed",
|
|
181
|
+
reason: err instanceof Error ? err.message : String(err)
|
|
182
|
+
});
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
pcmToAudioBuffer(pcmData) {
|
|
186
|
+
if (!this.audioContext) {
|
|
187
|
+
return null;
|
|
188
|
+
}
|
|
189
|
+
if (pcmData.length === 0) {
|
|
190
|
+
const silenceDuration = 0.01;
|
|
191
|
+
const numSamples2 = Math.floor(this.sampleRate * silenceDuration);
|
|
192
|
+
const audioBuffer2 = this.audioContext.createBuffer(
|
|
193
|
+
this.channelCount,
|
|
194
|
+
numSamples2,
|
|
195
|
+
this.sampleRate
|
|
196
|
+
);
|
|
197
|
+
for (let channel = 0; channel < this.channelCount; channel++) {
|
|
198
|
+
const channelData = audioBuffer2.getChannelData(channel);
|
|
199
|
+
channelData.fill(0);
|
|
200
|
+
}
|
|
201
|
+
return audioBuffer2;
|
|
202
|
+
}
|
|
203
|
+
const alignedData = new Uint8Array(pcmData);
|
|
204
|
+
const int16Array = new Int16Array(alignedData.buffer, 0, alignedData.length / 2);
|
|
205
|
+
const numSamples = int16Array.length / this.channelCount;
|
|
206
|
+
const audioBuffer = this.audioContext.createBuffer(
|
|
207
|
+
this.channelCount,
|
|
208
|
+
numSamples,
|
|
209
|
+
this.sampleRate
|
|
210
|
+
);
|
|
211
|
+
for (let channel = 0; channel < this.channelCount; channel++) {
|
|
212
|
+
const channelData = audioBuffer.getChannelData(channel);
|
|
213
|
+
for (let i = 0; i < numSamples; i++) {
|
|
214
|
+
const sampleIndex = i * this.channelCount + channel;
|
|
215
|
+
channelData[i] = int16Array[sampleIndex] / 32768;
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
return audioBuffer;
|
|
219
|
+
}
|
|
220
|
+
getCurrentTime() {
|
|
221
|
+
if (!this.audioContext || !this.isPlaying) {
|
|
222
|
+
return 0;
|
|
223
|
+
}
|
|
224
|
+
if (this.isPaused) {
|
|
225
|
+
return this.pausedAt;
|
|
226
|
+
}
|
|
227
|
+
const currentAudioTime = this.audioContext.currentTime;
|
|
228
|
+
const elapsed = currentAudioTime - this.sessionStartTime - this.pausedTimeOffset;
|
|
229
|
+
return Math.max(0, elapsed);
|
|
230
|
+
}
|
|
231
|
+
pause() {
|
|
232
|
+
if (!this.isPlaying || this.isPaused || !this.audioContext) {
|
|
233
|
+
return;
|
|
234
|
+
}
|
|
235
|
+
this.pausedAt = this.getCurrentTime();
|
|
236
|
+
this.pausedAudioContextTime = this.audioContext.currentTime;
|
|
237
|
+
this.isPaused = true;
|
|
238
|
+
if (this.audioContext.state === "running") {
|
|
239
|
+
this.audioContext.suspend().catch((err) => {
|
|
240
|
+
logger.errorWithError("Failed to suspend AudioContext:", err);
|
|
241
|
+
this.isPaused = false;
|
|
242
|
+
});
|
|
243
|
+
}
|
|
244
|
+
this.log("Playback paused", {
|
|
245
|
+
pausedAt: this.pausedAt,
|
|
246
|
+
pausedAudioContextTime: this.pausedAudioContextTime,
|
|
247
|
+
audioContextState: this.audioContext.state
|
|
248
|
+
});
|
|
249
|
+
}
|
|
250
|
+
async resume() {
|
|
251
|
+
if (!this.isPaused || !this.audioContext || !this.isPlaying) {
|
|
252
|
+
return;
|
|
253
|
+
}
|
|
254
|
+
if (this.audioContext.state === "suspended") {
|
|
255
|
+
try {
|
|
256
|
+
await this.audioContext.resume();
|
|
257
|
+
} catch (err) {
|
|
258
|
+
logger.errorWithError("Failed to resume AudioContext:", err);
|
|
259
|
+
throw err;
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
const currentAudioTime = this.audioContext.currentTime;
|
|
263
|
+
this.sessionStartTime = this.pausedAudioContextTime - this.pausedAt - this.pausedTimeOffset;
|
|
264
|
+
this.isPaused = false;
|
|
265
|
+
if (this.scheduledChunks < this.audioChunks.length) {
|
|
266
|
+
this.scheduleAllChunks();
|
|
267
|
+
}
|
|
268
|
+
this.log("Playback resumed", {
|
|
269
|
+
pausedAt: this.pausedAt,
|
|
270
|
+
pausedAudioContextTime: this.pausedAudioContextTime,
|
|
271
|
+
currentAudioContextTime: currentAudioTime,
|
|
272
|
+
adjustedSessionStartTime: this.sessionStartTime,
|
|
273
|
+
audioContextState: this.audioContext.state
|
|
274
|
+
});
|
|
275
|
+
}
|
|
276
|
+
stop() {
|
|
277
|
+
if (!this.audioContext) {
|
|
278
|
+
return;
|
|
279
|
+
}
|
|
280
|
+
if (this.isPaused && this.audioContext.state === "suspended") {
|
|
281
|
+
this.audioContext.resume().catch(() => {
|
|
282
|
+
});
|
|
283
|
+
this.isPaused = false;
|
|
284
|
+
}
|
|
285
|
+
this.isPlaying = false;
|
|
286
|
+
this.isPaused = false;
|
|
287
|
+
this.sessionStartTime = 0;
|
|
288
|
+
this.scheduledTime = 0;
|
|
289
|
+
for (const source of this.activeSources) {
|
|
290
|
+
source.onended = null;
|
|
291
|
+
try {
|
|
292
|
+
source.stop(0);
|
|
293
|
+
} catch {
|
|
294
|
+
}
|
|
295
|
+
try {
|
|
296
|
+
source.disconnect();
|
|
297
|
+
} catch {
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
this.activeSources.clear();
|
|
301
|
+
this.audioChunks = [];
|
|
302
|
+
this.scheduledChunks = 0;
|
|
303
|
+
this.log("[StreamingAudioPlayer] Playback stopped, state reset");
|
|
304
|
+
}
|
|
305
|
+
setAutoStart(enabled) {
|
|
306
|
+
this.autoStartEnabled = enabled;
|
|
307
|
+
this.log(`Auto-start ${enabled ? "enabled" : "disabled"}`);
|
|
308
|
+
}
|
|
309
|
+
play() {
|
|
310
|
+
if (this.isPlaying) {
|
|
311
|
+
return;
|
|
312
|
+
}
|
|
313
|
+
this.autoStartEnabled = true;
|
|
314
|
+
this.startPlayback();
|
|
315
|
+
}
|
|
316
|
+
markEnded() {
|
|
317
|
+
var _a;
|
|
318
|
+
this.log("Playback ended");
|
|
319
|
+
this.isPlaying = false;
|
|
320
|
+
(_a = this.onEndedCallback) == null ? void 0 : _a.call(this);
|
|
321
|
+
}
|
|
322
|
+
onEnded(callback) {
|
|
323
|
+
this.onEndedCallback = callback;
|
|
324
|
+
}
|
|
325
|
+
isPlayingNow() {
|
|
326
|
+
return this.isPlaying && !this.isPaused;
|
|
327
|
+
}
|
|
328
|
+
getBufferedDuration() {
|
|
329
|
+
if (!this.audioContext) {
|
|
330
|
+
return 0;
|
|
331
|
+
}
|
|
332
|
+
let totalSamples = 0;
|
|
333
|
+
for (const chunk of this.audioChunks) {
|
|
334
|
+
totalSamples += chunk.data.length / 2 / this.channelCount;
|
|
335
|
+
}
|
|
336
|
+
return totalSamples / this.sampleRate;
|
|
337
|
+
}
|
|
338
|
+
getRemainingDuration() {
|
|
339
|
+
const total = this.getBufferedDuration();
|
|
340
|
+
const played = this.getCurrentTime();
|
|
341
|
+
return Math.max(0, total - played);
|
|
342
|
+
}
|
|
343
|
+
dispose() {
|
|
344
|
+
this.stop();
|
|
345
|
+
if (this.audioContext) {
|
|
346
|
+
this.audioContext.close();
|
|
347
|
+
this.audioContext = null;
|
|
348
|
+
this.gainNode = null;
|
|
349
|
+
}
|
|
350
|
+
this.audioChunks = [];
|
|
351
|
+
this.scheduledChunks = 0;
|
|
352
|
+
this.sessionStartTime = 0;
|
|
353
|
+
this.pausedTimeOffset = 0;
|
|
354
|
+
this.pausedAt = 0;
|
|
355
|
+
this.pausedAudioContextTime = 0;
|
|
356
|
+
this.scheduledTime = 0;
|
|
357
|
+
this.onEndedCallback = void 0;
|
|
358
|
+
this.log("StreamingAudioPlayer disposed");
|
|
359
|
+
}
|
|
360
|
+
flush(options) {
|
|
361
|
+
const hard = (options == null ? void 0 : options.hard) === true;
|
|
362
|
+
if (hard) {
|
|
363
|
+
this.stop();
|
|
364
|
+
this.audioChunks = [];
|
|
365
|
+
this.scheduledChunks = 0;
|
|
366
|
+
this.sessionStartTime = 0;
|
|
367
|
+
this.pausedAt = 0;
|
|
368
|
+
this.scheduledTime = 0;
|
|
369
|
+
this.log("Flushed (hard)");
|
|
370
|
+
return;
|
|
371
|
+
}
|
|
372
|
+
if (this.scheduledChunks < this.audioChunks.length) {
|
|
373
|
+
this.audioChunks.splice(this.scheduledChunks);
|
|
374
|
+
}
|
|
375
|
+
this.log("Flushed (soft)", { remainingScheduled: this.scheduledChunks });
|
|
376
|
+
}
|
|
377
|
+
setVolume(volume) {
|
|
378
|
+
if (volume < 0 || volume > 1) {
|
|
379
|
+
logger.warn(`[StreamingAudioPlayer] Volume out of range: ${volume}, clamping to [0, 1]`);
|
|
380
|
+
volume = Math.max(0, Math.min(1, volume));
|
|
381
|
+
}
|
|
382
|
+
this.volume = volume;
|
|
383
|
+
if (this.gainNode) {
|
|
384
|
+
this.gainNode.gain.value = volume;
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
getVolume() {
|
|
388
|
+
return this.volume;
|
|
389
|
+
}
|
|
390
|
+
log(message, data) {
|
|
391
|
+
if (this.debug) {
|
|
392
|
+
logger.log(`[StreamingAudioPlayer] ${message}`, data || "");
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
export {
|
|
397
|
+
StreamingAudioPlayer
|
|
398
|
+
};
|
|
@@ -19,31 +19,11 @@ export declare class AnimationWebSocketClient extends EventEmitter {
|
|
|
19
19
|
private isManuallyDisconnected;
|
|
20
20
|
private reconnectTimer;
|
|
21
21
|
constructor(options: AnimationWebSocketClientOptions);
|
|
22
|
-
/**
|
|
23
|
-
* 连接WebSocket
|
|
24
|
-
*/
|
|
25
22
|
connect(characterId: string): Promise<void>;
|
|
26
|
-
/**
|
|
27
|
-
* 断开连接
|
|
28
|
-
*/
|
|
29
23
|
disconnect(): void;
|
|
30
|
-
/**
|
|
31
|
-
* 发送音频数据
|
|
32
|
-
* @param conversationId - 会话ID(在 protobuf 协议中映射为 reqId 字段)
|
|
33
|
-
*/
|
|
34
24
|
sendAudioData(conversationId: string, audioData: ArrayBuffer, end: boolean): boolean;
|
|
35
|
-
/**
|
|
36
|
-
* 生成会话ID
|
|
37
|
-
* 使用统一的会话ID生成规则:YYYYMMDDHHmmss_nanoid
|
|
38
|
-
*/
|
|
39
25
|
generateConversationId(): string;
|
|
40
|
-
/**
|
|
41
|
-
* 获取连接状态
|
|
42
|
-
*/
|
|
43
26
|
isConnected(): boolean;
|
|
44
|
-
/**
|
|
45
|
-
* 获取当前角色ID
|
|
46
|
-
*/
|
|
47
27
|
getCurrentCharacterId(): string;
|
|
48
28
|
private buildWebSocketUrl;
|
|
49
29
|
private connectWebSocket;
|