@spatius/avatarkit 0.5.0 → 1.1.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -20
- package/README.md +30 -41
- package/dist/{StreamingAudioPlayer-D-7zAYjb.js → StreamingAudioPlayer-DrHXn-YA.js} +6 -6
- package/dist/core/Avatar.d.ts +1 -0
- package/dist/core/AvatarController.d.ts +12 -18
- package/dist/core/AvatarSDK.d.ts +16 -2
- package/dist/core/AvatarView.d.ts +89 -13
- package/dist/core/avatarViewRegistry.d.ts +9 -0
- package/dist/{index-DrRoews_.js → index-CVQn5uEB.js} +6224 -2982
- package/dist/index.js +12 -8
- package/dist/types/character-settings.d.ts +0 -1
- package/dist/types/character.d.ts +2 -0
- package/dist/types/index.d.ts +25 -12
- package/package.json +7 -1
- package/dist/performance/FrameRateMonitor.d.ts +0 -85
package/CHANGELOG.md
CHANGED
|
@@ -1,22 +1,36 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
-
|
|
19
|
-
-
|
|
20
|
-
|
|
21
|
-
###
|
|
22
|
-
-
|
|
3
|
+
All notable changes to this project will be documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
## [1.1.0-beta.1] - 2026-06-08
|
|
9
|
+
|
|
10
|
+
First 1.1 pre-release. Includes breaking changes — see below.
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
- Render quality tiers (`standard` / `high` / `ultra`), selectable at init via the `renderQuality` configuration option or at runtime via `setRenderQuality()`.
|
|
14
|
+
- Render output resolution cap, to bound rendering cost on high-DPI displays.
|
|
15
|
+
- `Configuration.region` is now part of the public API, so consumers can target non-default regions (e.g. `cn`).
|
|
16
|
+
|
|
17
|
+
### Changed
|
|
18
|
+
- Unified audio playback path with automatic recovery after a fallback.
|
|
19
|
+
- Smoother RTC-mode speak→idle transition with bezier end interpolation.
|
|
20
|
+
|
|
21
|
+
### Fixed
|
|
22
|
+
- Fixed unfriendly avatar-load error messages that could expose internal request details.
|
|
23
|
+
- Fixed `Configuration.region` being stripped from the published type declarations, which prevented TypeScript consumers from setting the region.
|
|
24
|
+
|
|
25
|
+
### Breaking Changes
|
|
26
|
+
- `DrivingServiceMode` values renamed from `sdk` / `host` to `direct` / `backend`.
|
|
27
|
+
- Removed the `eyefocus` option (no longer supported in `CharacterSettings` or post-processing configuration).
|
|
28
|
+
|
|
29
|
+
## [1.0.0] - 2026-05-17
|
|
30
|
+
|
|
31
|
+
First stable release.
|
|
32
|
+
|
|
33
|
+
- Real-time avatar rendering with WebGL and WebGPU backends.
|
|
34
|
+
- Audio-driven SDK mode and host-driven mode for custom animation pipelines.
|
|
35
|
+
- Public API: `AvatarSDK`, `AvatarView`, `AvatarController`, `AvatarManager`.
|
|
36
|
+
- See [README.md](README.md) for installation and usage.
|
package/README.md
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @spatius/avatarkit
|
|
2
2
|
|
|
3
3
|
Real-time virtual avatar rendering SDK for Web, supporting audio-driven animation and high-quality 3D rendering.
|
|
4
4
|
|
|
@@ -55,7 +55,7 @@ We provide complete example code and best practices to help you quickly integrat
|
|
|
55
55
|
|
|
56
56
|
**The demo repository includes:**
|
|
57
57
|
- ✅ Complete integration examples
|
|
58
|
-
- ✅ Usage examples for both
|
|
58
|
+
- ✅ Usage examples for both Direct mode and Backend mode
|
|
59
59
|
- ✅ Audio processing examples (PCM16, WAV, MP3, etc.)
|
|
60
60
|
- ✅ Vite configuration examples
|
|
61
61
|
- ✅ Next.js configuration examples
|
|
@@ -165,7 +165,7 @@ All environments require an **App ID** and **Session Token** for authentication.
|
|
|
165
165
|
The App ID is used to identify your application. You can obtain your App ID by:
|
|
166
166
|
|
|
167
167
|
1. **For Testing**: Use the default test App ID provided in demo repositories (paired with test Session Token, only works with publicly available test avatars like Rohan, Dr.Kellan, Priya, Josh, etc.)
|
|
168
|
-
2. **For Production**: Visit the [Developer Platform](https://
|
|
168
|
+
2. **For Production**: Visit the [Developer Platform](https://app.spatius.ai) to create your own App and avatars. You will receive your own App ID after creating an App.
|
|
169
169
|
|
|
170
170
|
### Session Token
|
|
171
171
|
|
|
@@ -175,7 +175,7 @@ The Session Token is required for authentication and must be obtained from your
|
|
|
175
175
|
- The Session Token must be valid and not expired
|
|
176
176
|
- In production applications, you **must** manually inject a valid Session Token obtained from your SDK provider
|
|
177
177
|
- The default Session Token provided in demo repositories is **only for demonstration purposes** and can only be used with test avatars
|
|
178
|
-
- If you want to create your own avatars and test them, please visit the [Developer Platform](https://
|
|
178
|
+
- If you want to create your own avatars and test them, please visit the [Developer Platform](https://app.spatius.ai) to create your own App and generate Session Tokens
|
|
179
179
|
|
|
180
180
|
**How to Set Session Token:**
|
|
181
181
|
|
|
@@ -224,9 +224,9 @@ import {
|
|
|
224
224
|
// 1. Initialize SDK
|
|
225
225
|
|
|
226
226
|
const configuration: Configuration = {
|
|
227
|
-
drivingServiceMode: DrivingServiceMode.
|
|
228
|
-
// - DrivingServiceMode.
|
|
229
|
-
// - DrivingServiceMode.
|
|
227
|
+
drivingServiceMode: DrivingServiceMode.direct, // Optional, `direct` is default
|
|
228
|
+
// - DrivingServiceMode.direct: Direct mode - SDK handles network communication
|
|
229
|
+
// - DrivingServiceMode.backend: Backend mode - Host app provides audio and animation data
|
|
230
230
|
logLevel: LogLevel.off, // Optional, 'off' is default
|
|
231
231
|
// - LogLevel.off: Disable all logs
|
|
232
232
|
// - LogLevel.error: Only error logs
|
|
@@ -237,7 +237,6 @@ const configuration: Configuration = {
|
|
|
237
237
|
sampleRate: 16000 // Supported: 8000, 16000, 22050, 24000, 32000, 44100, 48000 Hz
|
|
238
238
|
// ⚠️ Must match your actual audio sample rate. Mismatched sample rate will cause playback issues.
|
|
239
239
|
}
|
|
240
|
-
// characterApiBaseUrl: 'https://custom-api.example.com' // Optional, internal debug config, can be ignored
|
|
241
240
|
}
|
|
242
241
|
|
|
243
242
|
await AvatarSDK.initialize('your-app-id', configuration)
|
|
@@ -255,8 +254,8 @@ const avatar = await avatarManager.load('character-id', (progress) => {
|
|
|
255
254
|
|
|
256
255
|
// 3. Create view (automatically creates Canvas and AvatarController)
|
|
257
256
|
// The playback mode is determined by drivingServiceMode in AvatarSDK configuration
|
|
258
|
-
// - DrivingServiceMode.
|
|
259
|
-
// - DrivingServiceMode.
|
|
257
|
+
// - DrivingServiceMode.direct: Direct mode - SDK handles network communication
|
|
258
|
+
// - DrivingServiceMode.backend: Backend mode - Host app provides audio and animation data
|
|
260
259
|
const container = document.getElementById('avatar-container')
|
|
261
260
|
const avatarView = new AvatarView(avatar, container)
|
|
262
261
|
|
|
@@ -267,7 +266,7 @@ button.addEventListener('click', async () => {
|
|
|
267
266
|
// Initialize audio context - MUST be in user gesture context
|
|
268
267
|
await avatarView.controller.initializeAudioContext()
|
|
269
268
|
|
|
270
|
-
// 5. Start real-time communication (
|
|
269
|
+
// 5. Start real-time communication (Direct mode only)
|
|
271
270
|
// Note: start() initiates the WebSocket connection asynchronously.
|
|
272
271
|
// Wait for onConnectionState === 'connected' before calling send().
|
|
273
272
|
await avatarView.controller.start()
|
|
@@ -279,7 +278,7 @@ button.addEventListener('click', async () => {
|
|
|
279
278
|
}
|
|
280
279
|
})
|
|
281
280
|
|
|
282
|
-
// 7. Send audio data (
|
|
281
|
+
// 7. Send audio data (Direct mode, must be mono PCM16 format matching configured sample rate)
|
|
283
282
|
// audioData: ArrayBuffer or Uint8Array containing PCM16 (S16LE) audio samples
|
|
284
283
|
// ⚠️ Byte length MUST be even (2 bytes per sample). Odd-length data will cause server-side
|
|
285
284
|
// validation error and WebSocket disconnect.
|
|
@@ -296,9 +295,9 @@ button.addEventListener('click', async () => {
|
|
|
296
295
|
|
|
297
296
|
```typescript
|
|
298
297
|
|
|
299
|
-
// 1-3. Same as
|
|
298
|
+
// 1-3. Same as Direct mode (initialize SDK, load avatar)
|
|
300
299
|
|
|
301
|
-
// 3. Create view with
|
|
300
|
+
// 3. Create view with Backend mode
|
|
302
301
|
const container = document.getElementById('avatar-container')
|
|
303
302
|
const avatarView = new AvatarView(avatar, container)
|
|
304
303
|
|
|
@@ -318,8 +317,8 @@ button.addEventListener('click', async () => {
|
|
|
318
317
|
### Complete Examples
|
|
319
318
|
|
|
320
319
|
This SDK supports two usage modes:
|
|
321
|
-
-
|
|
322
|
-
-
|
|
320
|
+
- Direct mode: Real-time audio input with automatic animation data reception
|
|
321
|
+
- Backend mode: Custom data sources with manual audio/animation data management
|
|
323
322
|
|
|
324
323
|
## 🏗️ Architecture Overview
|
|
325
324
|
|
|
@@ -335,14 +334,14 @@ This SDK supports two usage modes:
|
|
|
335
334
|
The SDK supports two playback modes, configured in `AvatarSDK.initialize()`:
|
|
336
335
|
|
|
337
336
|
#### 1. SDK Mode (Default)
|
|
338
|
-
- Configured via `drivingServiceMode: DrivingServiceMode.
|
|
337
|
+
- Configured via `drivingServiceMode: DrivingServiceMode.direct` in `AvatarSDK.initialize()`
|
|
339
338
|
- SDK handles network communication automatically
|
|
340
339
|
- Send audio data via `AvatarController.send()`
|
|
341
340
|
- SDK receives animation data from backend and synchronizes playback
|
|
342
341
|
- Best for: Real-time audio input scenarios
|
|
343
342
|
|
|
344
343
|
#### 2. Host Mode
|
|
345
|
-
- Configured via `drivingServiceMode: DrivingServiceMode.
|
|
344
|
+
- Configured via `drivingServiceMode: DrivingServiceMode.backend` in `AvatarSDK.initialize()`
|
|
346
345
|
- Host application manages its own network/data fetching
|
|
347
346
|
- Host application provides both audio and animation data
|
|
348
347
|
- SDK only handles synchronized playback
|
|
@@ -619,13 +618,13 @@ const newAvatar = await AvatarManager.shared.load('new-character-id')
|
|
|
619
618
|
// 3. Create new AvatarView
|
|
620
619
|
currentAvatarView = new AvatarView(newAvatar, container)
|
|
621
620
|
|
|
622
|
-
// 4. Start connection if
|
|
621
|
+
// 4. Start connection if Direct mode
|
|
623
622
|
await currentAvatarView.controller.start()
|
|
624
623
|
```
|
|
625
624
|
|
|
626
625
|
### AvatarController
|
|
627
626
|
|
|
628
|
-
Audio/animation playback controller, manages synchronized playback of audio and animation. Automatically handles network communication in
|
|
627
|
+
Audio/animation playback controller, manages synchronized playback of audio and animation. Automatically handles network communication in Direct mode.
|
|
629
628
|
|
|
630
629
|
**Two Usage Patterns:**
|
|
631
630
|
|
|
@@ -690,7 +689,6 @@ button.addEventListener('click', async () => {
|
|
|
690
689
|
- `yieldAudioData()` returns a conversationId (automatically generates if starting new session)
|
|
691
690
|
- `yieldFramesData()` requires a valid conversationId parameter
|
|
692
691
|
- Animation data with mismatched conversationId will be **discarded**
|
|
693
|
-
- Use `getCurrentConversationId()` to retrieve the current active conversationId
|
|
694
692
|
|
|
695
693
|
#### Common Methods (Both Modes)
|
|
696
694
|
|
|
@@ -705,19 +703,12 @@ await avatarView.controller.resume()
|
|
|
705
703
|
// Interrupt current playback (stops and clears data)
|
|
706
704
|
avatarView.controller.interrupt()
|
|
707
705
|
|
|
708
|
-
// Clear all data and resources
|
|
709
|
-
avatarView.controller.clear()
|
|
710
|
-
|
|
711
|
-
// Get current conversation ID (for Host mode)
|
|
712
|
-
const conversationId = avatarView.controller.getCurrentConversationId()
|
|
713
|
-
// Returns: Current conversationId for the active audio session, or null if no active session
|
|
714
|
-
|
|
715
706
|
// Volume control (affects only avatar audio player, not system volume)
|
|
716
707
|
avatarView.controller.setVolume(0.5) // Set volume to 50% (0.0 to 1.0)
|
|
717
708
|
const currentVolume = avatarView.controller.getVolume() // Get current volume (0.0 to 1.0)
|
|
718
709
|
|
|
719
710
|
// Set event callbacks
|
|
720
|
-
avatarView.controller.onConnectionState = (state: ConnectionState) => {} //
|
|
711
|
+
avatarView.controller.onConnectionState = (state: ConnectionState) => {} // Direct mode only
|
|
721
712
|
avatarView.controller.onConversationState = (state: ConversationState) => {}
|
|
722
713
|
avatarView.controller.onError = (error: AvatarError) => {} // Includes error.code for specific error type
|
|
723
714
|
```
|
|
@@ -740,9 +731,9 @@ avatarView.avatarTransform = { x: 0.5, y: 0, scale: 2.0 } // Right half, double
|
|
|
740
731
|
```
|
|
741
732
|
|
|
742
733
|
**Important Notes:**
|
|
743
|
-
- `start()` and `close()` are only available in
|
|
744
|
-
- `yieldAudioData()` and `yieldFramesData()` are only available in
|
|
745
|
-
- `pause()`, `resume()`, `interrupt()`, `
|
|
734
|
+
- `start()` and `close()` are only available in Direct mode
|
|
735
|
+
- `yieldAudioData()` and `yieldFramesData()` are only available in Backend mode
|
|
736
|
+
- `pause()`, `resume()`, `interrupt()`, `setVolume()`, and `getVolume()` are available in both modes
|
|
746
737
|
- The playback mode is determined when creating `AvatarView` and cannot be changed
|
|
747
738
|
|
|
748
739
|
## 🔧 Configuration
|
|
@@ -751,10 +742,9 @@ avatarView.avatarTransform = { x: 0.5, y: 0, scale: 2.0 } // Right half, double
|
|
|
751
742
|
|
|
752
743
|
```typescript
|
|
753
744
|
interface Configuration {
|
|
754
|
-
drivingServiceMode?: DrivingServiceMode // Optional, default is
|
|
745
|
+
drivingServiceMode?: DrivingServiceMode // Optional, default is `direct` (Direct mode)
|
|
755
746
|
logLevel?: LogLevel // Optional, default is 'off' (no logs)
|
|
756
747
|
audioFormat?: AudioFormat // Optional, default is { channelCount: 1, sampleRate: 16000 }
|
|
757
|
-
characterApiBaseUrl?: string // Optional, internal debug config, can be ignored
|
|
758
748
|
}
|
|
759
749
|
|
|
760
750
|
interface AudioFormat {
|
|
@@ -780,8 +770,8 @@ enum LogLevel {
|
|
|
780
770
|
|
|
781
771
|
**Description:**
|
|
782
772
|
- `drivingServiceMode`: Specifies the driving service mode
|
|
783
|
-
- `DrivingServiceMode.
|
|
784
|
-
- `DrivingServiceMode.
|
|
773
|
+
- `DrivingServiceMode.direct` (default): Direct mode - SDK handles network communication automatically
|
|
774
|
+
- `DrivingServiceMode.backend`: Backend mode - Host application provides audio and animation data
|
|
785
775
|
- `logLevel`: Controls the verbosity of SDK logs
|
|
786
776
|
- `LogLevel.off` (default): Disable all logs
|
|
787
777
|
- `LogLevel.error`: Only error logs
|
|
@@ -792,7 +782,6 @@ enum LogLevel {
|
|
|
792
782
|
- `sampleRate`: Audio sample rate in Hz (default: 16000)
|
|
793
783
|
- Supported values: 8000, 16000, 22050, 24000, 32000, 44100, 48000
|
|
794
784
|
- The configured sample rate will be used for both audio recording and playback
|
|
795
|
-
- `characterApiBaseUrl`: Internal debug config, can be ignored
|
|
796
785
|
- `sessionToken`: **Required for authentication**. Set separately via `AvatarSDK.setSessionToken()`, not in Configuration. See [Authentication](#-authentication) section for details
|
|
797
786
|
|
|
798
787
|
### CameraConfig
|
|
@@ -896,7 +885,7 @@ avatarView.controller.onError = (error: AvatarError) => {
|
|
|
896
885
|
| `sessionTimeout` | Session timeout | WebSocket close code 4002 |
|
|
897
886
|
| `connectionInProgress` | Connection already in progress | Duplicate `start()` call |
|
|
898
887
|
| **Playback** | | |
|
|
899
|
-
| `networkLayerNotAvailable` | Network layer not available | `send()` in
|
|
888
|
+
| `networkLayerNotAvailable` | Network layer not available | `send()` in backend mode |
|
|
900
889
|
| `playbackStartFailed` | Failed to start playback | Internal error |
|
|
901
890
|
| `playbackInitFailed` | Playback initialization failed | Internal error |
|
|
902
891
|
| `audioOnlyInitFailed` | Audio-only playback init failed | Fallback mode error |
|
|
@@ -942,12 +931,12 @@ avatarView.dispose()
|
|
|
942
931
|
|
|
943
932
|
**⚠️ Important Notes:**
|
|
944
933
|
- `dispose()` automatically cleans up all resources, including:
|
|
945
|
-
- Network connections (
|
|
934
|
+
- Network connections (Direct mode)
|
|
946
935
|
- Playback data and animation resources (both modes)
|
|
947
936
|
- Render system and canvas elements
|
|
948
937
|
- All event listeners and callbacks
|
|
949
938
|
- Not properly calling `dispose()` may cause resource leaks and rendering errors
|
|
950
|
-
- If you need to manually close connections
|
|
939
|
+
- If you need to manually close connections before disposing, you can call `avatarView.controller.close()` (Direct mode) first, but it's not required as `dispose()` handles this automatically
|
|
951
940
|
|
|
952
941
|
### Memory Optimization
|
|
953
942
|
|
|
@@ -973,4 +962,4 @@ Issues and Pull Requests are welcome!
|
|
|
973
962
|
|
|
974
963
|
For questions, please contact:
|
|
975
964
|
- Email: code@spatius.net
|
|
976
|
-
- Documentation: https://docs.
|
|
965
|
+
- Documentation: https://docs.spatius.ai
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
var __defProp = Object.defineProperty;
|
|
2
2
|
var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
|
|
3
3
|
var __publicField = (obj, key, value) => __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
|
|
4
|
-
import { A as APP_CONFIG, l as logger, e as errorToMessage, a as logEvent } from "./index-
|
|
4
|
+
import { A as APP_CONFIG, l as logger, e as errorToMessage, a as logEvent } from "./index-CVQn5uEB.js";
|
|
5
5
|
class StreamingAudioPlayer {
|
|
6
6
|
// Mark if AudioContext is being resumed, avoid concurrent resume requests
|
|
7
7
|
constructor(options) {
|
|
@@ -87,8 +87,8 @@ class StreamingAudioPlayer {
|
|
|
87
87
|
});
|
|
88
88
|
} catch (error) {
|
|
89
89
|
const message = errorToMessage(error);
|
|
90
|
-
logEvent("
|
|
91
|
-
|
|
90
|
+
logEvent("audio_session_init_failed", "warning", {
|
|
91
|
+
session_id: this.sessionId,
|
|
92
92
|
reason: message
|
|
93
93
|
});
|
|
94
94
|
logger.error("Failed to initialize AudioContext:", message);
|
|
@@ -144,7 +144,7 @@ class StreamingAudioPlayer {
|
|
|
144
144
|
} catch (err) {
|
|
145
145
|
logger.errorWithError("[StreamingAudioPlayer] Failed to resume AudioContext:", err);
|
|
146
146
|
logEvent("audio_context_resume_failed", "error", {
|
|
147
|
-
|
|
147
|
+
session_id: this.sessionId,
|
|
148
148
|
reason: err instanceof Error ? err.message : String(err)
|
|
149
149
|
});
|
|
150
150
|
} finally {
|
|
@@ -281,7 +281,7 @@ class StreamingAudioPlayer {
|
|
|
281
281
|
const errorMessage = "Failed to create AudioBuffer from PCM data";
|
|
282
282
|
logger.error(errorMessage);
|
|
283
283
|
logEvent("audio_buffer_creation_failed", "error", {
|
|
284
|
-
|
|
284
|
+
session_id: this.sessionId
|
|
285
285
|
});
|
|
286
286
|
return;
|
|
287
287
|
}
|
|
@@ -328,7 +328,7 @@ class StreamingAudioPlayer {
|
|
|
328
328
|
} catch (err) {
|
|
329
329
|
logger.errorWithError("Failed to schedule audio chunk:", err);
|
|
330
330
|
logEvent("schedule_chunk_failed", "error", {
|
|
331
|
-
|
|
331
|
+
session_id: this.sessionId,
|
|
332
332
|
reason: err instanceof Error ? err.message : String(err)
|
|
333
333
|
});
|
|
334
334
|
}
|
package/dist/core/Avatar.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Avatar } from './Avatar';
|
|
2
|
-
import { ConnectionState, AvatarError, DrivingServiceMode, ConversationState, PostProcessingConfig } from '../types';
|
|
2
|
+
import { ConnectionState, AvatarError, DrivingServiceMode, ConversationState, AnimationType, PostProcessingConfig } from '../types';
|
|
3
3
|
import { FrameRateInfo } from '../performance/FrameRateMonitor';
|
|
4
4
|
export declare class AvatarController {
|
|
5
5
|
private networkLayer?;
|
|
@@ -10,6 +10,8 @@ export declare class AvatarController {
|
|
|
10
10
|
onConnectionState: ((state: ConnectionState) => void) | null;
|
|
11
11
|
onConversationState: ((state: ConversationState) => void) | null;
|
|
12
12
|
onError: ((error: AvatarError) => void) | null;
|
|
13
|
+
/** Callback for animation type changes (e.g., idle → mono in fallback mode). Aligned with iOS/Android AvatarController.onAnimationState. */
|
|
14
|
+
onAnimationState: ((type: AnimationType) => void) | null;
|
|
13
15
|
private eventListeners;
|
|
14
16
|
private readonly frameRateMonitor;
|
|
15
17
|
/** Frame rate monitoring callback. Fires with aggregated metrics from a 2-second sliding window. */
|
|
@@ -20,10 +22,7 @@ export declare class AvatarController {
|
|
|
20
22
|
set frameRateMonitorEnabled(value: boolean);
|
|
21
23
|
private renderCallback?;
|
|
22
24
|
private characterHandle;
|
|
23
|
-
private characterId;
|
|
24
25
|
private postProcessingConfig;
|
|
25
|
-
private playbackLoopId;
|
|
26
|
-
private playbackLoopGeneration;
|
|
27
26
|
private lastRenderedFrameIndex;
|
|
28
27
|
private keyframesOffset;
|
|
29
28
|
private readonly MAX_KEYFRAMES;
|
|
@@ -43,14 +42,13 @@ export declare class AvatarController {
|
|
|
43
42
|
playbackMode?: DrivingServiceMode;
|
|
44
43
|
});
|
|
45
44
|
private handleVisibilityChange;
|
|
46
|
-
private shouldReportPlaybackStats;
|
|
47
|
-
private _getDeviceScoreProps;
|
|
48
45
|
/**
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
* @returns Current conversation ID, or null if no active session
|
|
46
|
+
* Playback time of the current audio session, in seconds.
|
|
47
|
+
* Resets to 0 on each new playback round. Returns 0 when not playing.
|
|
52
48
|
*/
|
|
53
|
-
|
|
49
|
+
getAudioTime(): number;
|
|
50
|
+
private shouldReportPlaybackStats;
|
|
51
|
+
private _getDeviceScoreProps;
|
|
54
52
|
/**
|
|
55
53
|
* Initialize audio context (must be called in user gesture context)
|
|
56
54
|
*
|
|
@@ -92,7 +90,7 @@ export declare class AvatarController {
|
|
|
92
90
|
* Public API: accepts binary data array (protobuf encoded Message array)
|
|
93
91
|
* @param keyframesDataArray - Animation keyframes binary data array (each element is a protobuf encoded Message) or empty array to trigger audio-only mode
|
|
94
92
|
* @param conversationId - Conversation ID (required). If conversationId doesn't match current conversationId, keyframes will be discarded.
|
|
95
|
-
*
|
|
93
|
+
* The conversationId is returned by yieldAudioData().
|
|
96
94
|
* @returns `true` if the server has sent all animation data for this conversation (end signal received), `false` otherwise.
|
|
97
95
|
*/
|
|
98
96
|
yieldFramesData(keyframesDataArray: (Uint8Array | ArrayBuffer)[], conversationId: string): boolean;
|
|
@@ -112,14 +110,9 @@ export declare class AvatarController {
|
|
|
112
110
|
*/
|
|
113
111
|
interrupt(): void;
|
|
114
112
|
/**
|
|
115
|
-
*
|
|
116
|
-
*/
|
|
117
|
-
clear(): void;
|
|
118
|
-
/**
|
|
119
|
-
* Get point cloud count of the current avatar
|
|
120
|
-
* @returns Point cloud count, or null if avatar is not loaded
|
|
113
|
+
* The point count of current avatar, or null if avatar is not loaded.
|
|
121
114
|
*/
|
|
122
|
-
|
|
115
|
+
get pointCount(): number | null;
|
|
123
116
|
/**
|
|
124
117
|
* Set post-processing configuration
|
|
125
118
|
* These parameters will be applied in real-time to animation parameters returned by the server
|
|
@@ -137,4 +130,5 @@ export declare class AvatarController {
|
|
|
137
130
|
* @returns Current volume value (0.0 - 1.0)
|
|
138
131
|
*/
|
|
139
132
|
getVolume(): number;
|
|
133
|
+
protected stopPlayback(): void;
|
|
140
134
|
}
|
package/dist/core/AvatarSDK.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Configuration } from '../types';
|
|
1
|
+
import { Configuration, RenderQuality } from '../types';
|
|
2
2
|
export declare class AvatarSDK {
|
|
3
3
|
private static _initializationState;
|
|
4
4
|
private static _initializingPromise;
|
|
@@ -7,6 +7,13 @@ export declare class AvatarSDK {
|
|
|
7
7
|
private static _avatarCore;
|
|
8
8
|
private static _cachedDeviceScore;
|
|
9
9
|
private static _rendererBackend;
|
|
10
|
+
/** Render output resolution cap. Disabled by default; when enabled and the
|
|
11
|
+
* canvas backing height (css × DPR) exceeds maxHeight, the canvas is sized
|
|
12
|
+
* to maxHeight (aspect-preserving) and the browser scales it back up to
|
|
13
|
+
* the CSS box. Training material is 1080p so heights above ~1080-1440
|
|
14
|
+
* add no real visual information. */
|
|
15
|
+
static renderResolutionCapEnabled: boolean;
|
|
16
|
+
static renderResolutionMaxHeight: number;
|
|
10
17
|
/**
|
|
11
18
|
* Initialize SDK
|
|
12
19
|
* @param appId Application ID to be included in both HTTP Headers and WebSocket Headers
|
|
@@ -25,9 +32,16 @@ export declare class AvatarSDK {
|
|
|
25
32
|
* Optional interface for developers, SDK includes this in telemetry logs
|
|
26
33
|
*/
|
|
27
34
|
static setUserId(userId: string): void;
|
|
28
|
-
static get isInitialized(): boolean;
|
|
29
35
|
static get appId(): string | null;
|
|
30
36
|
static get configuration(): Configuration | null;
|
|
37
|
+
/**
|
|
38
|
+
* Update the global render quality tier. Takes effect on the next rendered
|
|
39
|
+
* frame across all active AvatarView instances. Throws if SDK isn't initialized.
|
|
40
|
+
*/
|
|
41
|
+
static setRenderQuality(quality: RenderQuality): void;
|
|
42
|
+
/** Toggle the render output resolution cap. Takes effect on the next layout
|
|
43
|
+
* pass across all active AvatarView instances. */
|
|
44
|
+
static setRenderResolutionCap(enabled: boolean, maxHeight?: number): void;
|
|
31
45
|
static get sessionToken(): string | null;
|
|
32
46
|
static get userId(): string | null;
|
|
33
47
|
static get version(): string;
|
|
@@ -9,26 +9,23 @@ export declare class AvatarView {
|
|
|
9
9
|
private renderSystem;
|
|
10
10
|
private isInitialized;
|
|
11
11
|
private cameraConfig;
|
|
12
|
-
private
|
|
13
|
-
private currentKeyframes;
|
|
14
|
-
private lastRenderedFrameIndex;
|
|
15
|
-
private lastRealtimeProtoFrame;
|
|
16
|
-
private idleAnimationLoopId;
|
|
17
|
-
private realtimeAnimationLoopId;
|
|
12
|
+
private renderLoopId;
|
|
18
13
|
private resizeObserver;
|
|
19
14
|
private onWindowResize;
|
|
15
|
+
private onVisibilityChange;
|
|
20
16
|
private frameCount;
|
|
21
17
|
private lastFpsUpdate;
|
|
22
18
|
private currentFPS;
|
|
23
|
-
private
|
|
24
|
-
private transitionStartTime;
|
|
25
|
-
private readonly startTransitionDurationMs;
|
|
26
|
-
private readonly endTransitionDurationMs;
|
|
19
|
+
private currentFrame;
|
|
27
20
|
private cachedIdleFirstFrame;
|
|
28
|
-
private idleCurrentFrameIndex;
|
|
29
|
-
private currentPlayingFrame;
|
|
30
21
|
private characterHandle;
|
|
31
22
|
private characterId;
|
|
23
|
+
private transitionFrames;
|
|
24
|
+
private onTransitionFramesConsumed;
|
|
25
|
+
private isConversationActive;
|
|
26
|
+
private lastRenderedFrameIndex;
|
|
27
|
+
private animationHandleMap;
|
|
28
|
+
private activeAnimationState;
|
|
32
29
|
private isPureRenderingMode;
|
|
33
30
|
private _renderingEnabled;
|
|
34
31
|
private avatarActiveTimer;
|
|
@@ -44,6 +41,42 @@ export declare class AvatarView {
|
|
|
44
41
|
* Get controller (public interface)
|
|
45
42
|
*/
|
|
46
43
|
get controller(): AvatarController;
|
|
44
|
+
/**
|
|
45
|
+
* Current native render surface size in pixels (canvas backing buffer
|
|
46
|
+
* size, post-DPR).
|
|
47
|
+
*
|
|
48
|
+
* Differs from the canvas CSS size (`offsetWidth / offsetHeight`): this
|
|
49
|
+
* reflects the actual pixel buffer the WebGPU renderer is targeting after
|
|
50
|
+
* the most recent resize has been applied.
|
|
51
|
+
*
|
|
52
|
+
* Returns `{ width: 0, height: 0 }` before the canvas has been initialized.
|
|
53
|
+
*/
|
|
54
|
+
get renderSize(): {
|
|
55
|
+
width: number;
|
|
56
|
+
height: number;
|
|
57
|
+
};
|
|
58
|
+
private _exportBitmapResolve;
|
|
59
|
+
/**
|
|
60
|
+
* Exports the current rendering as a Blob (PNG).
|
|
61
|
+
* Returns null if the canvas is not initialized or not rendering.
|
|
62
|
+
* Aligned with iOS exportBitmap() / Android exportBitmap().
|
|
63
|
+
*
|
|
64
|
+
* The capture happens synchronously inside the render loop (after renderFrame)
|
|
65
|
+
* to work correctly with WebGL preserveDrawingBuffer:false.
|
|
66
|
+
*/
|
|
67
|
+
exportBitmap(): Promise<Blob | null>;
|
|
68
|
+
/** @deprecated Use startRenderLoop() */
|
|
69
|
+
private startIdleAnimationLoop;
|
|
70
|
+
/**
|
|
71
|
+
* Render a specific idle frame by index for benchmark capture.
|
|
72
|
+
* Bypasses animation loop and _renderingEnabled check.
|
|
73
|
+
*/
|
|
74
|
+
renderIdleFrameForBenchmark(frameIndex: number): Promise<void>;
|
|
75
|
+
/**
|
|
76
|
+
* Render a single frame from FlameParams for benchmark capture (transition / speaking).
|
|
77
|
+
* Bypasses animation loop and _renderingEnabled check.
|
|
78
|
+
*/
|
|
79
|
+
renderFrameForBenchmark(flameParams: import('../wasm/avatarCoreAdapter').FlameParams): Promise<void>;
|
|
47
80
|
/**
|
|
48
81
|
* Cleanup view resources
|
|
49
82
|
* Closes avatarController and cleans up all related resources
|
|
@@ -107,7 +140,17 @@ export declare class AvatarView {
|
|
|
107
140
|
* @param options.useLinear - Use linear interpolation (default: true)
|
|
108
141
|
* @returns Array of opaque keyframe data for sequential playback
|
|
109
142
|
*/
|
|
110
|
-
|
|
143
|
+
generateTransitionToFrame(data: ArrayBuffer | Uint8Array, frameCount: number, options?: {
|
|
144
|
+
useLinear?: boolean;
|
|
145
|
+
}): Promise<unknown[]>;
|
|
146
|
+
/**
|
|
147
|
+
* RTC mode: generate `frameCount` interpolated frames from the most recently
|
|
148
|
+
* rendered frame back to the idle loop start. Caller plays them at 25fps
|
|
149
|
+
* via `renderFrame`, then hands control back with
|
|
150
|
+
* `renderFrame(undefined, true)`. Used for speaking → idle and disconnect
|
|
151
|
+
* soft transitions.
|
|
152
|
+
*/
|
|
153
|
+
generateTransitionToIdle(frameCount: number, options?: {
|
|
111
154
|
useLinear?: boolean;
|
|
112
155
|
}): Promise<unknown[]>;
|
|
113
156
|
/**
|
|
@@ -115,6 +158,12 @@ export declare class AvatarView {
|
|
|
115
158
|
* Called by renderFromProtobuf when streaming frames arrive during transition.
|
|
116
159
|
*/
|
|
117
160
|
cancelFrameSequence(): void;
|
|
161
|
+
/** 计算 canvas backing-store 像素尺寸. 默认 css × dpr;
|
|
162
|
+
* AvatarSDK.setRenderResolutionCap 启用且高度超过阈值时,
|
|
163
|
+
* 按比例缩到阈值, css 尺寸不变 (浏览器自动拉伸). */
|
|
164
|
+
private computeCappedBacking;
|
|
165
|
+
/** Registry hook: AvatarSDK.setRenderResolutionCap 调用时重算 backing size. */
|
|
166
|
+
applyResolutionCapFromSdk(): void;
|
|
118
167
|
/**
|
|
119
168
|
* Pause rendering loop
|
|
120
169
|
*
|
|
@@ -167,4 +216,31 @@ export declare class AvatarView {
|
|
|
167
216
|
y: number;
|
|
168
217
|
scale: number;
|
|
169
218
|
});
|
|
219
|
+
/**
|
|
220
|
+
* Get the approximate bounding rectangle of the avatar in canvas pixel coordinates.
|
|
221
|
+
* Projects idle first frame positions through current view/projection matrices and transform.
|
|
222
|
+
* No caching is performed — each call recomputes from scratch.
|
|
223
|
+
*
|
|
224
|
+
* **Important:** The result depends on the current canvas size and `avatarTransform`.
|
|
225
|
+
* You must call this method again after any of the following changes to get an up-to-date result:
|
|
226
|
+
* - Canvas / container size changes (e.g. window resize)
|
|
227
|
+
* - `avatarTransform` changes (offset or scale)
|
|
228
|
+
*
|
|
229
|
+
* **Performance note:** Each call iterates ~70k splat points. Avoid calling every frame;
|
|
230
|
+
* call on-demand (e.g. after init, on resize, after transform change).
|
|
231
|
+
*
|
|
232
|
+
* @returns Bounding rectangle { x, y, width, height } in CSS pixels (top-left origin), or null if not ready
|
|
233
|
+
*
|
|
234
|
+
* @example
|
|
235
|
+
* const rect = avatarView.getBoundingRect()
|
|
236
|
+
* if (rect) {
|
|
237
|
+
* console.log(`Avatar at (${rect.x}, ${rect.y}), size ${rect.width}x${rect.height}`)
|
|
238
|
+
* }
|
|
239
|
+
*/
|
|
240
|
+
getBoundingRect(): {
|
|
241
|
+
x: number;
|
|
242
|
+
y: number;
|
|
243
|
+
width: number;
|
|
244
|
+
height: number;
|
|
245
|
+
} | null;
|
|
170
246
|
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { AvatarView } from './AvatarView';
|
|
2
|
+
declare class AvatarViewRegistry {
|
|
3
|
+
private entries;
|
|
4
|
+
register(view: AvatarView): void;
|
|
5
|
+
unregister(view: AvatarView): void;
|
|
6
|
+
applyResolutionCapToAll(): void;
|
|
7
|
+
}
|
|
8
|
+
export declare const avatarViewRegistry: AvatarViewRegistry;
|
|
9
|
+
export {};
|