@spatialwalk/avatarkit 1.0.0-beta.86 → 1.0.0-beta.88
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/README.md +29 -0
- package/dist/{StreamingAudioPlayer-pfCQ3Bn7.js → StreamingAudioPlayer-D0H3vuGE.js} +1 -1
- package/dist/core/AvatarController.d.ts +1 -0
- package/dist/core/AvatarSDK.d.ts +3 -1
- package/dist/{index-Z9AXSJw3.js → index-Ow7xDTS3.js} +228 -101
- package/dist/index.js +1 -1
- package/package.json +8 -3
- package/dist/marble-test/src/billboard.d.ts +0 -22
- package/dist/marble-test/src/main.d.ts +0 -6
- package/dist/marble-test/src/spzLoader.d.ts +0 -18
- package/dist/marble-test/vite.config.d.ts +0 -2
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,35 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [1.0.0-beta.88] - 2026-03-10
|
|
9
|
+
|
|
10
|
+
### 🐛 Bugfixes
|
|
11
|
+
|
|
12
|
+
- **Build Packaging Fix** - WASM binary was missing from dist in beta.87 npm package; added `benchmark-demo` and `marble-test` to dts exclude to prevent type declaration leakage
|
|
13
|
+
- **Release Workflow** - Added build artifact verification step (WASM presence + no leaked directories)
|
|
14
|
+
|
|
15
|
+
## [1.0.0-beta.87] - 2026-03-09
|
|
16
|
+
|
|
17
|
+
### 🔧 Performance
|
|
18
|
+
|
|
19
|
+
- **Playback Cadence Stabilization** - Replaced RAF + `Math.floor(audioTime*25)` frame dispatch with fixed 25fps tick scheduler and bounded async queue, eliminating bursty frame production
|
|
20
|
+
- **Display Decoupling** - Speaking state uses RAF-driven `presentFrameOnly()` to present latest computed frame (WebGPU only), reducing display-side pacing jitter
|
|
21
|
+
- **WebGL Speaking-State Loop Suspension** - WebGL backend pauses realtime animation loop during Speaking state to avoid redundant `reorderPackedData` CPU work and RAF callback competition
|
|
22
|
+
|
|
23
|
+
### 🐛 Bugfixes
|
|
24
|
+
|
|
25
|
+
- **Frame Index Floating-Point Precision** - Added epsilon tolerance (`+1e-6`) to `Math.floor(audioTime * fps)` to prevent IEEE 754 precision errors from producing duplicate frame indices at frame boundaries
|
|
26
|
+
- **WebGL Regression Fix** - Fixed severe WebGL performance regression (jank 9.47%→0.16%) caused by WebGPU-only optimization that triggered 60fps `reorderPackedData` calls on WebGL path
|
|
27
|
+
- **SDK Duplicate Initialize** - Deduplicated SDK initialization to prevent repeated setup when `initialize()` is called multiple times
|
|
28
|
+
|
|
29
|
+
### ✅ Tests
|
|
30
|
+
|
|
31
|
+
- **Android Init Coverage** - Added Android duplicate-initialize regression coverage
|
|
32
|
+
|
|
33
|
+
### 📊 Benchmark
|
|
34
|
+
|
|
35
|
+
- **beta.86 vs beta.87 Baseline** - Full cross-backend performance comparison (WebGPU + WebGL), 1 warmup + 10 measure rounds, 10s/round. Jank rate reduced 94.6% (WebGPU) and 93.9% (WebGL); frame interval stability improved 64-67%
|
|
36
|
+
|
|
8
37
|
## [1.0.0-beta.86] - 2026-03-03
|
|
9
38
|
|
|
10
39
|
### 🐛 Bugfixes
|
package/README.md
CHANGED
|
@@ -16,6 +16,35 @@ Real-time virtual avatar rendering SDK for Web, supporting audio-driven animatio
|
|
|
16
16
|
npm install @spatialwalk/avatarkit
|
|
17
17
|
```
|
|
18
18
|
|
|
19
|
+
## 🚧 Release Gate (Hard Rule)
|
|
20
|
+
|
|
21
|
+
Release must pass gates before publish. Do not publish by manual ad-hoc commands.
|
|
22
|
+
|
|
23
|
+
Required gate checks:
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
pnpm typecheck
|
|
27
|
+
pnpm test
|
|
28
|
+
pnpm build
|
|
29
|
+
./tools/check_perf_baseline_release_gate.sh
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
If iteration includes bugfixes, `docs/bugfix-history.md` must have completed rows (test mapping + red/green evidence).
|
|
33
|
+
|
|
34
|
+
Hotfix bypass is allowed only for emergency and must be recorded:
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
HOTFIX_BYPASS=1 ./tools/check_perf_baseline_release_gate.sh
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## 🧪 Benchmark Demo (Web SDK)
|
|
41
|
+
|
|
42
|
+
Use the dedicated benchmark demo (independent from `vanilla/`) for perf/render baseline runs:
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
pnpm demo:benchmark
|
|
46
|
+
```
|
|
47
|
+
|
|
19
48
|
## 🚀 Demo Repository
|
|
20
49
|
|
|
21
50
|
<div align="center">
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
var __defProp = Object.defineProperty;
|
|
2
2
|
var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
|
|
3
3
|
var __publicField = (obj, key, value) => __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
|
|
4
|
-
import { A as APP_CONFIG, l as logger, e as errorToMessage, a as logEvent } from "./index-
|
|
4
|
+
import { A as APP_CONFIG, l as logger, e as errorToMessage, a as logEvent } from "./index-Ow7xDTS3.js";
|
|
5
5
|
class StreamingAudioPlayer {
|
|
6
6
|
// Mark if AudioContext is being resumed, avoid concurrent resume requests
|
|
7
7
|
constructor(options) {
|
package/dist/core/AvatarSDK.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { Configuration } from '../types';
|
|
2
2
|
export declare class AvatarSDK {
|
|
3
|
-
private static
|
|
3
|
+
private static _initializationState;
|
|
4
|
+
private static _initializingPromise;
|
|
4
5
|
private static _configuration;
|
|
5
6
|
private static readonly _version;
|
|
6
7
|
private static _avatarCore;
|
|
@@ -11,6 +12,7 @@ export declare class AvatarSDK {
|
|
|
11
12
|
* @param configuration Configuration parameters
|
|
12
13
|
*/
|
|
13
14
|
static initialize(appId: string, configuration: Configuration): Promise<void>;
|
|
15
|
+
private static _initializeInternal;
|
|
14
16
|
/**
|
|
15
17
|
* Set sessionToken
|
|
16
18
|
* Developer Client -> Developer Server -> AvatarKit Server -> return sessionToken (max 1 hour validity)
|
|
@@ -9491,7 +9491,7 @@ const _AnimationPlayer = class _AnimationPlayer {
|
|
|
9491
9491
|
if (this.streamingPlayer) {
|
|
9492
9492
|
return;
|
|
9493
9493
|
}
|
|
9494
|
-
const { StreamingAudioPlayer } = await import("./StreamingAudioPlayer-
|
|
9494
|
+
const { StreamingAudioPlayer } = await import("./StreamingAudioPlayer-D0H3vuGE.js");
|
|
9495
9495
|
const { AvatarSDK: AvatarSDK2 } = await Promise.resolve().then(() => AvatarSDK$1);
|
|
9496
9496
|
const audioFormat = AvatarSDK2.getAudioFormat();
|
|
9497
9497
|
this.streamingPlayer = new StreamingAudioPlayer({
|
|
@@ -10959,17 +10959,36 @@ class AvatarSDK {
|
|
|
10959
10959
|
* @param configuration Configuration parameters
|
|
10960
10960
|
*/
|
|
10961
10961
|
static async initialize(appId, configuration) {
|
|
10962
|
+
logEvent("sdk_init_start", "info", {
|
|
10963
|
+
env: configuration.environment,
|
|
10964
|
+
dsm: configuration.drivingServiceMode || DrivingServiceMode.sdk,
|
|
10965
|
+
init_state: this._initializationState,
|
|
10966
|
+
already_initialized: this._initializationState === "initialized"
|
|
10967
|
+
/* Initialized */
|
|
10968
|
+
});
|
|
10969
|
+
if (!appId) {
|
|
10970
|
+
throw new Error("Failed to initialize AvatarSDK: appId is required. Get your appId at https://docs.spatialreal.ai/");
|
|
10971
|
+
}
|
|
10972
|
+
if (this._initializationState === "initialized") {
|
|
10973
|
+
logger.log(`[AvatarSDK] Re-initializing with new environment: ${configuration.environment}`);
|
|
10974
|
+
idManager.setAppId(appId);
|
|
10975
|
+
this._configuration = configuration;
|
|
10976
|
+
return;
|
|
10977
|
+
}
|
|
10978
|
+
if (this._initializationState === "initializing") {
|
|
10979
|
+
logger.log("[AvatarSDK] Initialization already in progress, waiting for existing initialization...");
|
|
10980
|
+
if (this._initializingPromise) {
|
|
10981
|
+
return this._initializingPromise;
|
|
10982
|
+
}
|
|
10983
|
+
return;
|
|
10984
|
+
}
|
|
10985
|
+
this._initializationState = "initializing";
|
|
10986
|
+
this._initializingPromise = this._initializeInternal(appId, configuration);
|
|
10987
|
+
return this._initializingPromise;
|
|
10988
|
+
}
|
|
10989
|
+
static async _initializeInternal(appId, configuration) {
|
|
10962
10990
|
var _a;
|
|
10963
10991
|
try {
|
|
10964
|
-
if (!appId) {
|
|
10965
|
-
throw new Error("Failed to initialize AvatarSDK: appId is required. Get your appId at https://docs.spatialreal.ai/");
|
|
10966
|
-
}
|
|
10967
|
-
if (this._isInitialized) {
|
|
10968
|
-
logger.log(`[AvatarSDK] Re-initializing with new environment: ${configuration.environment}`);
|
|
10969
|
-
idManager.setAppId(appId);
|
|
10970
|
-
this._configuration = configuration;
|
|
10971
|
-
return;
|
|
10972
|
-
}
|
|
10973
10992
|
logger.log(`[AvatarSDK] Initializing with appId: ${appId}, environment: ${configuration.environment}`);
|
|
10974
10993
|
this._configuration = configuration;
|
|
10975
10994
|
setLogLevel(configuration.logLevel ?? LogLevel.off);
|
|
@@ -10986,7 +11005,7 @@ class AvatarSDK {
|
|
|
10986
11005
|
});
|
|
10987
11006
|
await this.initializeWASMModule();
|
|
10988
11007
|
await this.initializeTemplateResources();
|
|
10989
|
-
this.
|
|
11008
|
+
this._initializationState = "initialized";
|
|
10990
11009
|
logEvent("sdk_initialized", "info", {
|
|
10991
11010
|
env: configuration.environment,
|
|
10992
11011
|
dsm: configuration.drivingServiceMode || DrivingServiceMode.sdk
|
|
@@ -11008,7 +11027,10 @@ class AvatarSDK {
|
|
|
11008
11027
|
logEvent("sdk_startup", "error", {
|
|
11009
11028
|
reason: errorMessage
|
|
11010
11029
|
});
|
|
11030
|
+
this._initializationState = "uninitialized";
|
|
11011
11031
|
throw error;
|
|
11032
|
+
} finally {
|
|
11033
|
+
this._initializingPromise = null;
|
|
11012
11034
|
}
|
|
11013
11035
|
}
|
|
11014
11036
|
/**
|
|
@@ -11059,6 +11081,9 @@ class AvatarSDK {
|
|
|
11059
11081
|
throw new Error("AvatarCore not available");
|
|
11060
11082
|
}
|
|
11061
11083
|
logger.log("[AvatarSDK] Loading template resources...");
|
|
11084
|
+
logEvent("template_init_start", "info", {
|
|
11085
|
+
stage: "sdk_init"
|
|
11086
|
+
});
|
|
11062
11087
|
try {
|
|
11063
11088
|
const { AvatarDownloader: AvatarDownloader2 } = await Promise.resolve().then(() => AvatarDownloader$1);
|
|
11064
11089
|
const downloader = new AvatarDownloader2();
|
|
@@ -11097,7 +11122,7 @@ class AvatarSDK {
|
|
|
11097
11122
|
}
|
|
11098
11123
|
// 只读属性
|
|
11099
11124
|
static get isInitialized() {
|
|
11100
|
-
return this.
|
|
11125
|
+
return this._initializationState === "initialized";
|
|
11101
11126
|
}
|
|
11102
11127
|
static get appId() {
|
|
11103
11128
|
return idManager.getAppId();
|
|
@@ -11141,7 +11166,7 @@ class AvatarSDK {
|
|
|
11141
11166
|
* Cleanup resources
|
|
11142
11167
|
*/
|
|
11143
11168
|
static cleanup() {
|
|
11144
|
-
if (
|
|
11169
|
+
if (this._initializationState === "uninitialized") {
|
|
11145
11170
|
return;
|
|
11146
11171
|
}
|
|
11147
11172
|
try {
|
|
@@ -11151,7 +11176,8 @@ class AvatarSDK {
|
|
|
11151
11176
|
}
|
|
11152
11177
|
this._configuration = null;
|
|
11153
11178
|
this._dynamicSdkConfig = null;
|
|
11154
|
-
this.
|
|
11179
|
+
this._initializationState = "uninitialized";
|
|
11180
|
+
this._initializingPromise = null;
|
|
11155
11181
|
idManager.clear();
|
|
11156
11182
|
clearSdkConfigCache();
|
|
11157
11183
|
cleanupPostHog();
|
|
@@ -11218,9 +11244,10 @@ class AvatarSDK {
|
|
|
11218
11244
|
}
|
|
11219
11245
|
}
|
|
11220
11246
|
}
|
|
11221
|
-
__publicField(AvatarSDK, "
|
|
11247
|
+
__publicField(AvatarSDK, "_initializationState", "uninitialized");
|
|
11248
|
+
__publicField(AvatarSDK, "_initializingPromise", null);
|
|
11222
11249
|
__publicField(AvatarSDK, "_configuration", null);
|
|
11223
|
-
__publicField(AvatarSDK, "_version", "1.0.0-beta.
|
|
11250
|
+
__publicField(AvatarSDK, "_version", "1.0.0-beta.88");
|
|
11224
11251
|
__publicField(AvatarSDK, "_avatarCore", null);
|
|
11225
11252
|
__publicField(AvatarSDK, "_dynamicSdkConfig", null);
|
|
11226
11253
|
const AvatarSDK$1 = /* @__PURE__ */ Object.freeze(/* @__PURE__ */ Object.defineProperty({
|
|
@@ -11796,17 +11823,19 @@ class NetworkLayer {
|
|
|
11796
11823
|
setupWebSocketListeners() {
|
|
11797
11824
|
this.wsClient.removeAllListeners();
|
|
11798
11825
|
this.wsClient.on("sessionConfirmed", (connectionId) => {
|
|
11826
|
+
var _a, _b;
|
|
11799
11827
|
if (connectionId) {
|
|
11800
11828
|
logger.log(`[NetworkLayer] Session confirmed, connection_id: ${connectionId}, ready to send audio`);
|
|
11801
11829
|
} else {
|
|
11802
11830
|
logger.log("[NetworkLayer] Session confirmed, ready to send audio");
|
|
11803
11831
|
}
|
|
11832
|
+
if (!this.dataController.connected) {
|
|
11833
|
+
this.isFallbackMode = false;
|
|
11834
|
+
this.dataController.setConnected(true);
|
|
11835
|
+
(_b = (_a = this.dataController).onConnectionState) == null ? void 0 : _b.call(_a, ConnectionState.connected);
|
|
11836
|
+
}
|
|
11804
11837
|
});
|
|
11805
11838
|
this.wsClient.on("connected", () => {
|
|
11806
|
-
var _a, _b;
|
|
11807
|
-
this.isFallbackMode = false;
|
|
11808
|
-
this.dataController.setConnected(true);
|
|
11809
|
-
(_b = (_a = this.dataController).onConnectionState) == null ? void 0 : _b.call(_a, ConnectionState.connected);
|
|
11810
11839
|
});
|
|
11811
11840
|
this.wsClient.on("disconnected", () => {
|
|
11812
11841
|
var _a, _b;
|
|
@@ -12114,6 +12143,8 @@ class AvatarController {
|
|
|
12114
12143
|
__publicField(this, "postProcessingConfig", null);
|
|
12115
12144
|
// ========== Playback Loop ==========
|
|
12116
12145
|
__publicField(this, "playbackLoopId", null);
|
|
12146
|
+
__publicField(this, "playbackLoopGeneration", 0);
|
|
12147
|
+
// invalidates stale async frame computes across stop/resume
|
|
12117
12148
|
__publicField(this, "lastRenderedFrameIndex", -1);
|
|
12118
12149
|
__publicField(this, "keyframesOffset", 0);
|
|
12119
12150
|
// Offset to track how many frames were removed from the beginning
|
|
@@ -12136,10 +12167,10 @@ class AvatarController {
|
|
|
12136
12167
|
audioTimeStuckCount: 0,
|
|
12137
12168
|
reported: false
|
|
12138
12169
|
});
|
|
12139
|
-
__publicField(this, "MAX_AUDIO_TIME_ZERO_COUNT",
|
|
12140
|
-
// 约 1 秒(
|
|
12141
|
-
__publicField(this, "MAX_AUDIO_TIME_STUCK_COUNT",
|
|
12142
|
-
// 约 1 秒(
|
|
12170
|
+
__publicField(this, "MAX_AUDIO_TIME_ZERO_COUNT", FLAME_FRAME_RATE);
|
|
12171
|
+
// 约 1 秒(25fps)
|
|
12172
|
+
__publicField(this, "MAX_AUDIO_TIME_STUCK_COUNT", FLAME_FRAME_RATE);
|
|
12173
|
+
// 约 1 秒(25fps)
|
|
12143
12174
|
__publicField(this, "AUDIO_TIME_STUCK_THRESHOLD", 1e-3);
|
|
12144
12175
|
// 1ms,小于此值视为卡住
|
|
12145
12176
|
// ========== Host Mode Latency Metrics ==========
|
|
@@ -13050,6 +13081,74 @@ class AvatarController {
|
|
|
13050
13081
|
}
|
|
13051
13082
|
return false;
|
|
13052
13083
|
}
|
|
13084
|
+
/**
|
|
13085
|
+
* Reset playback stuck detection counters
|
|
13086
|
+
* @internal
|
|
13087
|
+
*/
|
|
13088
|
+
resetPlaybackStuckCheckState() {
|
|
13089
|
+
this.playbackStuckCheckState.audioTimeZeroCount = 0;
|
|
13090
|
+
this.playbackStuckCheckState.audioTimeStuckCount = 0;
|
|
13091
|
+
this.playbackStuckCheckState.lastAudioTime = 0;
|
|
13092
|
+
this.playbackStuckCheckState.reported = false;
|
|
13093
|
+
}
|
|
13094
|
+
/**
|
|
13095
|
+
* Compute and render one animation frame by absolute frame index.
|
|
13096
|
+
* Uses loop generation guard to drop stale async outputs after stop/pause/resume.
|
|
13097
|
+
* @internal
|
|
13098
|
+
*/
|
|
13099
|
+
async computeAndRenderFrame(frameIndex, loopGeneration) {
|
|
13100
|
+
let arrayIndex = frameIndex - this.keyframesOffset;
|
|
13101
|
+
if (arrayIndex < 0) {
|
|
13102
|
+
arrayIndex = 0;
|
|
13103
|
+
}
|
|
13104
|
+
const isOutOfBounds = arrayIndex >= this.currentKeyframes.length;
|
|
13105
|
+
if (isOutOfBounds) {
|
|
13106
|
+
arrayIndex = this.currentKeyframes.length - 1;
|
|
13107
|
+
const now = Date.now();
|
|
13108
|
+
const stateChanged = isOutOfBounds !== this.lastOutOfBoundsState;
|
|
13109
|
+
const timeSinceLastLog = now - this.lastSyncLogTime;
|
|
13110
|
+
if (stateChanged || timeSinceLastLog >= 1e3) {
|
|
13111
|
+
logger.warn(`[PlaybackLoop] Frame index out of bounds! frameIndex: ${frameIndex}, maxAvailable: ${this.currentKeyframes.length - 1 + this.keyframesOffset}, using last frame`);
|
|
13112
|
+
this.lastSyncLogTime = now;
|
|
13113
|
+
this.lastOutOfBoundsState = isOutOfBounds;
|
|
13114
|
+
}
|
|
13115
|
+
} else if (isOutOfBounds !== this.lastOutOfBoundsState) {
|
|
13116
|
+
this.lastOutOfBoundsState = isOutOfBounds;
|
|
13117
|
+
}
|
|
13118
|
+
if (this.currentKeyframes.length > this.MAX_KEYFRAMES) {
|
|
13119
|
+
const framesToKeep = this.KEYFRAMES_CLEANUP_THRESHOLD;
|
|
13120
|
+
const framesToRemove = frameIndex - framesToKeep;
|
|
13121
|
+
if (framesToRemove > 0 && framesToRemove < frameIndex) {
|
|
13122
|
+
this.currentKeyframes.splice(0, framesToRemove);
|
|
13123
|
+
this.keyframesOffset += framesToRemove;
|
|
13124
|
+
arrayIndex = frameIndex - this.keyframesOffset;
|
|
13125
|
+
if (arrayIndex < 0) arrayIndex = 0;
|
|
13126
|
+
if (arrayIndex >= this.currentKeyframes.length) {
|
|
13127
|
+
arrayIndex = this.currentKeyframes.length - 1;
|
|
13128
|
+
}
|
|
13129
|
+
logger.log(`[AvatarController] Cleaned up ${framesToRemove} old frames (kept ${this.currentKeyframes.length} frames, offset: ${this.keyframesOffset})`);
|
|
13130
|
+
}
|
|
13131
|
+
}
|
|
13132
|
+
if (arrayIndex < 0 || arrayIndex >= this.currentKeyframes.length) {
|
|
13133
|
+
return;
|
|
13134
|
+
}
|
|
13135
|
+
const currentFrame = this.currentKeyframes[arrayIndex];
|
|
13136
|
+
let wasmParams = convertProtoFlameToWasmParams(currentFrame);
|
|
13137
|
+
if (this.postProcessingConfig) {
|
|
13138
|
+
wasmParams = this.applyPostProcessingToParams(wasmParams);
|
|
13139
|
+
}
|
|
13140
|
+
const avatarCore = AvatarSDK.getAvatarCore();
|
|
13141
|
+
if (!avatarCore) {
|
|
13142
|
+
return;
|
|
13143
|
+
}
|
|
13144
|
+
const splatData = await avatarCore.computeFrameFlatFromParams(wasmParams, this.characterHandle ?? void 0);
|
|
13145
|
+
if (loopGeneration !== this.playbackLoopGeneration || !this.isPlaying || this.currentState === AvatarState.paused) {
|
|
13146
|
+
return;
|
|
13147
|
+
}
|
|
13148
|
+
if (splatData && this.renderCallback) {
|
|
13149
|
+
this.renderCallback(splatData, frameIndex);
|
|
13150
|
+
}
|
|
13151
|
+
}
|
|
13053
13152
|
/**
|
|
13054
13153
|
* Start playback loop
|
|
13055
13154
|
* @internal
|
|
@@ -13059,92 +13158,72 @@ class AvatarController {
|
|
|
13059
13158
|
return;
|
|
13060
13159
|
}
|
|
13061
13160
|
const fps = FLAME_FRAME_RATE;
|
|
13062
|
-
const
|
|
13063
|
-
|
|
13161
|
+
const frameIntervalMs = 1e3 / fps;
|
|
13162
|
+
const loopGeneration = ++this.playbackLoopGeneration;
|
|
13163
|
+
let computeInFlight = false;
|
|
13164
|
+
let queuedFrameIndex = null;
|
|
13165
|
+
let nextTickTime = performance.now();
|
|
13166
|
+
const scheduleNextTick = () => {
|
|
13167
|
+
if (loopGeneration !== this.playbackLoopGeneration || !this.isPlaying || this.currentState === AvatarState.paused || !this.animationPlayer) {
|
|
13168
|
+
this.playbackLoopId = null;
|
|
13169
|
+
return;
|
|
13170
|
+
}
|
|
13171
|
+
nextTickTime += frameIntervalMs;
|
|
13172
|
+
const delay = Math.max(0, nextTickTime - performance.now());
|
|
13173
|
+
this.playbackLoopId = window.setTimeout(tick, delay);
|
|
13174
|
+
};
|
|
13175
|
+
const tryDispatchFrame = (frameIndex) => {
|
|
13176
|
+
if (frameIndex <= this.lastRenderedFrameIndex) {
|
|
13177
|
+
return;
|
|
13178
|
+
}
|
|
13179
|
+
if (computeInFlight) {
|
|
13180
|
+
if (queuedFrameIndex === null || frameIndex > queuedFrameIndex) {
|
|
13181
|
+
queuedFrameIndex = frameIndex;
|
|
13182
|
+
}
|
|
13183
|
+
return;
|
|
13184
|
+
}
|
|
13185
|
+
this.lastRenderedFrameIndex = frameIndex;
|
|
13186
|
+
computeInFlight = true;
|
|
13187
|
+
void this.computeAndRenderFrame(frameIndex, loopGeneration).catch((error) => {
|
|
13188
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
13189
|
+
logger.error("[AvatarController] Playback frame compute error:", errorMessage);
|
|
13190
|
+
}).finally(() => {
|
|
13191
|
+
computeInFlight = false;
|
|
13192
|
+
if (loopGeneration !== this.playbackLoopGeneration || !this.isPlaying || this.currentState === AvatarState.paused) {
|
|
13193
|
+
queuedFrameIndex = null;
|
|
13194
|
+
return;
|
|
13195
|
+
}
|
|
13196
|
+
if (queuedFrameIndex !== null) {
|
|
13197
|
+
const pending = queuedFrameIndex;
|
|
13198
|
+
queuedFrameIndex = null;
|
|
13199
|
+
tryDispatchFrame(pending);
|
|
13200
|
+
}
|
|
13201
|
+
});
|
|
13202
|
+
};
|
|
13203
|
+
const tick = () => {
|
|
13204
|
+
if (!this.isPlaying || this.currentState === AvatarState.paused || !this.animationPlayer || loopGeneration !== this.playbackLoopGeneration) {
|
|
13064
13205
|
this.playbackLoopId = null;
|
|
13065
|
-
this.
|
|
13066
|
-
this.playbackStuckCheckState.audioTimeStuckCount = 0;
|
|
13067
|
-
this.playbackStuckCheckState.lastAudioTime = 0;
|
|
13068
|
-
this.playbackStuckCheckState.reported = false;
|
|
13206
|
+
this.resetPlaybackStuckCheckState();
|
|
13069
13207
|
return;
|
|
13070
13208
|
}
|
|
13071
13209
|
try {
|
|
13072
13210
|
const audioTime = this.animationPlayer.getCurrentTime();
|
|
13073
13211
|
this.checkPlaybackStuck(audioTime);
|
|
13074
|
-
if (audioTime === 0) {
|
|
13075
|
-
this.playbackLoopId = requestAnimationFrame(playLoop);
|
|
13076
|
-
return;
|
|
13077
|
-
}
|
|
13078
13212
|
if (audioTime > 0 && Math.abs(audioTime - this.playbackStuckCheckState.lastAudioTime) >= this.AUDIO_TIME_STUCK_THRESHOLD) {
|
|
13079
13213
|
this.playbackStuckCheckState.audioTimeZeroCount = 0;
|
|
13080
13214
|
this.playbackStuckCheckState.audioTimeStuckCount = 0;
|
|
13081
13215
|
}
|
|
13082
|
-
|
|
13083
|
-
|
|
13084
|
-
|
|
13085
|
-
if (arrayIndex < 0) {
|
|
13086
|
-
arrayIndex = 0;
|
|
13087
|
-
}
|
|
13088
|
-
const isOutOfBounds = arrayIndex >= this.currentKeyframes.length;
|
|
13089
|
-
if (isOutOfBounds) {
|
|
13090
|
-
arrayIndex = this.currentKeyframes.length - 1;
|
|
13091
|
-
const now = Date.now();
|
|
13092
|
-
const stateChanged = isOutOfBounds !== this.lastOutOfBoundsState;
|
|
13093
|
-
const timeSinceLastLog = now - this.lastSyncLogTime;
|
|
13094
|
-
if (stateChanged || timeSinceLastLog >= 1e3) {
|
|
13095
|
-
logger.warn(`[PlaybackLoop] Frame index out of bounds! audioTime: ${audioTime.toFixed(3)}s, frameIndex: ${frameIndex}, maxAvailable: ${this.currentKeyframes.length - 1 + this.keyframesOffset}, using last frame`);
|
|
13096
|
-
this.lastSyncLogTime = now;
|
|
13097
|
-
this.lastOutOfBoundsState = isOutOfBounds;
|
|
13098
|
-
}
|
|
13099
|
-
} else {
|
|
13100
|
-
if (isOutOfBounds !== this.lastOutOfBoundsState) {
|
|
13101
|
-
this.lastOutOfBoundsState = isOutOfBounds;
|
|
13102
|
-
}
|
|
13103
|
-
}
|
|
13104
|
-
if (frameIndex === this.lastRenderedFrameIndex) {
|
|
13105
|
-
this.playbackLoopId = requestAnimationFrame(playLoop);
|
|
13106
|
-
return;
|
|
13107
|
-
}
|
|
13108
|
-
this.lastRenderedFrameIndex = frameIndex;
|
|
13109
|
-
if (this.currentKeyframes.length > this.MAX_KEYFRAMES) {
|
|
13110
|
-
const framesToKeep = this.KEYFRAMES_CLEANUP_THRESHOLD;
|
|
13111
|
-
const framesToRemove = frameIndex - framesToKeep;
|
|
13112
|
-
if (framesToRemove > 0 && framesToRemove < frameIndex) {
|
|
13113
|
-
this.currentKeyframes.splice(0, framesToRemove);
|
|
13114
|
-
this.keyframesOffset += framesToRemove;
|
|
13115
|
-
arrayIndex = frameIndex - this.keyframesOffset;
|
|
13116
|
-
if (arrayIndex < 0) arrayIndex = 0;
|
|
13117
|
-
if (arrayIndex >= this.currentKeyframes.length) {
|
|
13118
|
-
arrayIndex = this.currentKeyframes.length - 1;
|
|
13119
|
-
}
|
|
13120
|
-
logger.log(`[AvatarController] Cleaned up ${framesToRemove} old frames (kept ${this.currentKeyframes.length} frames, offset: ${this.keyframesOffset})`);
|
|
13121
|
-
}
|
|
13122
|
-
}
|
|
13123
|
-
if (arrayIndex >= 0 && arrayIndex < this.currentKeyframes.length) {
|
|
13124
|
-
const currentFrame = this.currentKeyframes[arrayIndex];
|
|
13125
|
-
let wasmParams = convertProtoFlameToWasmParams(currentFrame);
|
|
13126
|
-
if (this.postProcessingConfig) {
|
|
13127
|
-
wasmParams = this.applyPostProcessingToParams(wasmParams);
|
|
13128
|
-
}
|
|
13129
|
-
const avatarCore = AvatarSDK.getAvatarCore();
|
|
13130
|
-
if (avatarCore) {
|
|
13131
|
-
const splatData = await avatarCore.computeFrameFlatFromParams(wasmParams, this.characterHandle ?? void 0);
|
|
13132
|
-
if (splatData && this.renderCallback) {
|
|
13133
|
-
this.renderCallback(splatData, frameIndex);
|
|
13134
|
-
}
|
|
13135
|
-
}
|
|
13216
|
+
if (audioTime > 0) {
|
|
13217
|
+
const frameIndex = Math.max(0, Math.floor(audioTime * fps + 1e-6));
|
|
13218
|
+
tryDispatchFrame(frameIndex);
|
|
13136
13219
|
}
|
|
13137
13220
|
} catch (error) {
|
|
13138
13221
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
13139
13222
|
logger.error("[AvatarController] Playback loop error:", errorMessage);
|
|
13140
13223
|
}
|
|
13141
|
-
|
|
13142
|
-
this.playbackLoopId = requestAnimationFrame(playLoop);
|
|
13143
|
-
} else {
|
|
13144
|
-
this.playbackLoopId = null;
|
|
13145
|
-
}
|
|
13224
|
+
scheduleNextTick();
|
|
13146
13225
|
};
|
|
13147
|
-
this.playbackLoopId =
|
|
13226
|
+
this.playbackLoopId = window.setTimeout(tick, 0);
|
|
13148
13227
|
}
|
|
13149
13228
|
/**
|
|
13150
13229
|
* Stop playback loop
|
|
@@ -13152,9 +13231,11 @@ class AvatarController {
|
|
|
13152
13231
|
*/
|
|
13153
13232
|
stopPlaybackLoop() {
|
|
13154
13233
|
if (this.playbackLoopId) {
|
|
13234
|
+
clearTimeout(this.playbackLoopId);
|
|
13155
13235
|
cancelAnimationFrame(this.playbackLoopId);
|
|
13156
13236
|
this.playbackLoopId = null;
|
|
13157
13237
|
}
|
|
13238
|
+
this.playbackLoopGeneration++;
|
|
13158
13239
|
}
|
|
13159
13240
|
// ========== Fallback Mode ==========
|
|
13160
13241
|
/**
|
|
@@ -15784,6 +15865,18 @@ class RenderSystem {
|
|
|
15784
15865
|
// 总渲染耗时
|
|
15785
15866
|
__publicField(this, "sortTime", 0);
|
|
15786
15867
|
// 排序耗时
|
|
15868
|
+
__publicField(this, "reorderTime", 0);
|
|
15869
|
+
// 重排耗时(WebGL)
|
|
15870
|
+
__publicField(this, "renderSubmitTime", 0);
|
|
15871
|
+
// CPU 提交耗时(load + render 调用)
|
|
15872
|
+
__publicField(this, "frameSerial", 0);
|
|
15873
|
+
// 实际渲染帧序号(每次 renderFrame 成功执行后 +1)
|
|
15874
|
+
__publicField(this, "frameEndTimeMs", 0);
|
|
15875
|
+
// 实际渲染结束时间戳(performance.now)
|
|
15876
|
+
__publicField(this, "presentFrameSerial", 0);
|
|
15877
|
+
// 展示帧序号(每次实际提交 draw 后 +1)
|
|
15878
|
+
__publicField(this, "presentFrameEndTimeMs", 0);
|
|
15879
|
+
// 展示帧完成时间戳(performance.now)
|
|
15787
15880
|
// Transform for render texture blit
|
|
15788
15881
|
__publicField(this, "offsetX", 0);
|
|
15789
15882
|
__publicField(this, "offsetY", 0);
|
|
@@ -15858,10 +15951,14 @@ class RenderSystem {
|
|
|
15858
15951
|
);
|
|
15859
15952
|
const sortTime = performance.now() - startSort;
|
|
15860
15953
|
this.sortTime = sortTime;
|
|
15954
|
+
const startSubmit = performance.now();
|
|
15861
15955
|
if (this.backend === "webgpu") {
|
|
15956
|
+
this.reorderTime = 0;
|
|
15862
15957
|
this.renderer.loadSplatsFromPackedData(this.originalPackedData, pointCount, sortOrder);
|
|
15863
15958
|
} else {
|
|
15959
|
+
const startReorder = performance.now();
|
|
15864
15960
|
const reorderedData = reorderPackedData(this.originalPackedData, sortOrder);
|
|
15961
|
+
this.reorderTime = performance.now() - startReorder;
|
|
15865
15962
|
this.renderer.loadSplatsFromPackedData(reorderedData, pointCount);
|
|
15866
15963
|
}
|
|
15867
15964
|
const startRender = performance.now();
|
|
@@ -15874,6 +15971,28 @@ class RenderSystem {
|
|
|
15874
15971
|
);
|
|
15875
15972
|
const renderTime = performance.now() - startRender;
|
|
15876
15973
|
this.renderTime = renderTime;
|
|
15974
|
+
this.renderSubmitTime = performance.now() - startSubmit;
|
|
15975
|
+
this.frameEndTimeMs = performance.now();
|
|
15976
|
+
this.frameSerial += 1;
|
|
15977
|
+
this.presentFrameEndTimeMs = this.frameEndTimeMs;
|
|
15978
|
+
this.presentFrameSerial += 1;
|
|
15979
|
+
}
|
|
15980
|
+
/**
|
|
15981
|
+
* 仅提交展示(复用最新上传的 GPU 数据,不做排序/重排)
|
|
15982
|
+
*/
|
|
15983
|
+
presentFrameOnly() {
|
|
15984
|
+
if (!this.renderer || !this.originalPackedData) {
|
|
15985
|
+
return;
|
|
15986
|
+
}
|
|
15987
|
+
this.updateCameraMatrices();
|
|
15988
|
+
this.renderer.render(
|
|
15989
|
+
this.viewMatrix,
|
|
15990
|
+
this.projectionMatrix,
|
|
15991
|
+
[this.canvas.width, this.canvas.height],
|
|
15992
|
+
this.offsetX !== 0 || this.offsetY !== 0 || this.scale !== 1 ? { x: this.offsetX, y: this.offsetY, scale: this.scale } : void 0
|
|
15993
|
+
);
|
|
15994
|
+
this.presentFrameEndTimeMs = performance.now();
|
|
15995
|
+
this.presentFrameSerial += 1;
|
|
15877
15996
|
}
|
|
15878
15997
|
/**
|
|
15879
15998
|
* Set transform for render texture blit
|
|
@@ -16636,7 +16755,7 @@ class AvatarView {
|
|
|
16636
16755
|
}
|
|
16637
16756
|
}
|
|
16638
16757
|
/**
|
|
16639
|
-
* Update FPS statistics (called
|
|
16758
|
+
* Update FPS statistics (called only when a frame is actually rendered)
|
|
16640
16759
|
* @internal
|
|
16641
16760
|
*/
|
|
16642
16761
|
updateFPS() {
|
|
@@ -16679,7 +16798,6 @@ class AvatarView {
|
|
|
16679
16798
|
if (!this.renderSystem) {
|
|
16680
16799
|
return;
|
|
16681
16800
|
}
|
|
16682
|
-
this.updateFPS();
|
|
16683
16801
|
if (this.renderingState !== "idle") {
|
|
16684
16802
|
this.idleAnimationLoopId = requestAnimationFrame(renderFrame);
|
|
16685
16803
|
return;
|
|
@@ -16738,11 +16856,17 @@ class AvatarView {
|
|
|
16738
16856
|
const renderFrame = async (currentTime) => {
|
|
16739
16857
|
try {
|
|
16740
16858
|
const state = this.renderingState;
|
|
16741
|
-
this.updateFPS();
|
|
16742
16859
|
if (!this.renderSystem || state === "idle") {
|
|
16743
16860
|
this.realtimeAnimationLoopId = null;
|
|
16744
16861
|
return;
|
|
16745
16862
|
}
|
|
16863
|
+
if (state === "speaking") {
|
|
16864
|
+
if (this._renderingEnabled && !this.isPureRenderingMode) {
|
|
16865
|
+
this.renderSystem.presentFrameOnly();
|
|
16866
|
+
}
|
|
16867
|
+
this.realtimeAnimationLoopId = requestAnimationFrame(renderFrame);
|
|
16868
|
+
return;
|
|
16869
|
+
}
|
|
16746
16870
|
const elapsed = currentTime - lastTime;
|
|
16747
16871
|
if (elapsed < frameInterval) {
|
|
16748
16872
|
this.realtimeAnimationLoopId = requestAnimationFrame(renderFrame);
|
|
@@ -16814,10 +16938,6 @@ class AvatarView {
|
|
|
16814
16938
|
this.realtimeAnimationLoopId = requestAnimationFrame(renderFrame);
|
|
16815
16939
|
return;
|
|
16816
16940
|
}
|
|
16817
|
-
if (state === "speaking") {
|
|
16818
|
-
this.realtimeAnimationLoopId = requestAnimationFrame(renderFrame);
|
|
16819
|
-
return;
|
|
16820
|
-
}
|
|
16821
16941
|
} catch (error) {
|
|
16822
16942
|
logger.error("[AvatarView] Realtime animation loop error:", error instanceof Error ? error.message : String(error));
|
|
16823
16943
|
this.stopRealtimeAnimationLoop();
|
|
@@ -16868,6 +16988,7 @@ class AvatarView {
|
|
|
16868
16988
|
if (!this.renderSystem || !this._renderingEnabled) {
|
|
16869
16989
|
return;
|
|
16870
16990
|
}
|
|
16991
|
+
this.updateFPS();
|
|
16871
16992
|
this.renderSystem.loadSplatsFromPackedData(splatData);
|
|
16872
16993
|
this.renderSystem.renderFrame();
|
|
16873
16994
|
}
|
|
@@ -17406,6 +17527,12 @@ class AvatarView {
|
|
|
17406
17527
|
return {
|
|
17407
17528
|
renderTime: this.renderSystem.renderTime,
|
|
17408
17529
|
sortTime: this.renderSystem.sortTime,
|
|
17530
|
+
reorderTime: this.renderSystem.reorderTime,
|
|
17531
|
+
renderSubmitTime: this.renderSystem.renderSubmitTime,
|
|
17532
|
+
frameSerial: this.renderSystem.frameSerial,
|
|
17533
|
+
frameEndTimeMs: this.renderSystem.frameEndTimeMs,
|
|
17534
|
+
presentFrameSerial: this.renderSystem.presentFrameSerial,
|
|
17535
|
+
presentFrameEndTimeMs: this.renderSystem.presentFrameEndTimeMs,
|
|
17409
17536
|
backend: this.renderSystem.getBackend(),
|
|
17410
17537
|
fps: this.currentFPS
|
|
17411
17538
|
// pointCount 可后续通过 AvatarCoreAdapter 添加公开方法获取
|
package/dist/index.js
CHANGED
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@spatialwalk/avatarkit",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "1.0.0-beta.
|
|
4
|
+
"version": "1.0.0-beta.88",
|
|
5
5
|
"packageManager": "pnpm@10.18.2",
|
|
6
6
|
"description": "AvatarKit SDK - 3D Gaussian Splatting Avatar Rendering SDK",
|
|
7
7
|
"author": "AvatarKit Team",
|
|
@@ -49,6 +49,7 @@
|
|
|
49
49
|
"build:vite-plugin": "tsc vite.ts --outDir . --module esnext --target es2020 --moduleResolution bundler --esModuleInterop --skipLibCheck --declaration --declarationMap",
|
|
50
50
|
"build:next-plugin": "tsc next.ts --outDir . --module esnext --target es2020 --moduleResolution bundler --esModuleInterop --skipLibCheck --declaration --declarationMap",
|
|
51
51
|
"dev": "vite build --mode library --watch",
|
|
52
|
+
"demo:benchmark": "vite --config benchmark-demo/vite.config.mjs",
|
|
52
53
|
"clean": "rm -rf dist",
|
|
53
54
|
"typecheck": "tsc --noEmit",
|
|
54
55
|
"test": "cd tests && pnpm test",
|
|
@@ -62,8 +63,12 @@
|
|
|
62
63
|
"next": ">=13.0.0"
|
|
63
64
|
},
|
|
64
65
|
"peerDependenciesMeta": {
|
|
65
|
-
"vite": {
|
|
66
|
-
|
|
66
|
+
"vite": {
|
|
67
|
+
"optional": true
|
|
68
|
+
},
|
|
69
|
+
"next": {
|
|
70
|
+
"optional": true
|
|
71
|
+
}
|
|
67
72
|
},
|
|
68
73
|
"dependencies": {
|
|
69
74
|
"@bufbuild/protobuf": "^2.10.0",
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Billboard transform for avatar splats.
|
|
3
|
-
* Uses CYLINDRICAL billboard (Y-axis only rotation) so the avatar
|
|
4
|
-
* stays upright regardless of camera pitch, only rotating left/right.
|
|
5
|
-
*
|
|
6
|
-
* Packed format per splat: 13 floats = [pos3, color4, cov6]
|
|
7
|
-
* Covariance layout: [xx, xy, xz, yy, yz, zz]
|
|
8
|
-
*/
|
|
9
|
-
/**
|
|
10
|
-
* Apply billboard transform to avatar splat data.
|
|
11
|
-
*
|
|
12
|
-
* - Rotates each splat position by R, then translates to avatarWorldPos
|
|
13
|
-
* - Rotates covariance: C' = R * C * R^T
|
|
14
|
-
* - Color (4 floats) is copied unchanged
|
|
15
|
-
*
|
|
16
|
-
* @param avatarSplats Source packed Float32Array from SDK (13 floats/splat)
|
|
17
|
-
* @param avatarWorldPos Where to place the avatar in the scene
|
|
18
|
-
* @param cameraPos Current camera position (avatar faces this)
|
|
19
|
-
* @param scale Optional uniform scale for the avatar (default 1.0)
|
|
20
|
-
* @returns New Float32Array with transformed splats
|
|
21
|
-
*/
|
|
22
|
-
export declare function applyBillboardTransform(avatarSplats: Float32Array, avatarWorldPos: [number, number, number], cameraPos: [number, number, number], scale?: number): Float32Array;
|
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* SPZ (Splat Zip) decoder → packed format [pos3, color4, cov6] × N
|
|
3
|
-
*
|
|
4
|
-
* SPZ format (Niantic Labs):
|
|
5
|
-
* - gzip-compressed stream
|
|
6
|
-
* - 16-byte header + column-based attribute data
|
|
7
|
-
*
|
|
8
|
-
* Reference: https://github.com/nianticlabs/spz
|
|
9
|
-
*/
|
|
10
|
-
/**
|
|
11
|
-
* Load and decode an SPZ file into packed format [pos3, color4, cov6] × N
|
|
12
|
-
*/
|
|
13
|
-
export declare function loadSpz(buffer: ArrayBuffer, onProgress?: (msg: string) => void): Promise<{
|
|
14
|
-
packedData: Float32Array;
|
|
15
|
-
pointCount: number;
|
|
16
|
-
center: [number, number, number];
|
|
17
|
-
extent: number;
|
|
18
|
-
}>;
|