@spatialwalk/avatarkit 1.0.0-beta.84 → 1.0.0-beta.86
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md
CHANGED
|
@@ -5,22 +5,14 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
-
## [1.0.0-beta.
|
|
8
|
+
## [1.0.0-beta.86] - 2026-03-03
|
|
9
9
|
|
|
10
10
|
### 🐛 Bugfixes
|
|
11
|
-
- **
|
|
11
|
+
- **Transition Continuity** - Fixed start transition source frame selection to use the currently displayed idle frame, reducing first-frame jump.
|
|
12
|
+
- **Transition Robustness** - Added frame index clamping for realtime callback and speaking->idle transition source lookup, preventing end-transition fallback jumps caused by out-of-range indices.
|
|
12
13
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
### 🐛 Bugfixes
|
|
16
|
-
- **Playback Startup Sync** - Improved startup sequencing for SDK mode playback to reduce transition/audio desync at conversation start
|
|
17
|
-
- **Initialization Guard** - Added explicit `appId` validation in `AvatarSDK.initialize()` to fail fast when `appId` is empty
|
|
18
|
-
|
|
19
|
-
### 🔧 Improvements
|
|
20
|
-
- **Server Error Propagation** - `onError` now consistently receives `AvatarError` with server message and mapped error code for `MESSAGE_SERVER_ERROR` in SDK mode
|
|
21
|
-
|
|
22
|
-
### 📚 Documentation
|
|
23
|
-
- **README Error Callback** - Added detailed `onError` callback behavior and server error code mapping documentation
|
|
14
|
+
### ✅ Tests
|
|
15
|
+
- **State Machine Coverage** - Updated `AvatarView` rendering state tests to align with current dual-transition flow, and added boundary tests for start/end transition frame selection.
|
|
24
16
|
|
|
25
17
|
## [1.0.0-beta.82] - 2026-02-12
|
|
26
18
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
var __defProp = Object.defineProperty;
|
|
2
2
|
var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
|
|
3
3
|
var __publicField = (obj, key, value) => __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
|
|
4
|
-
import { A as APP_CONFIG, l as logger, e as errorToMessage, a as logEvent } from "./index-
|
|
4
|
+
import { A as APP_CONFIG, l as logger, e as errorToMessage, a as logEvent } from "./index-Z9AXSJw3.js";
|
|
5
5
|
class StreamingAudioPlayer {
|
|
6
6
|
// Mark if AudioContext is being resumed, avoid concurrent resume requests
|
|
7
7
|
constructor(options) {
|
|
@@ -13,14 +13,16 @@ export declare class AvatarView {
|
|
|
13
13
|
private currentKeyframes;
|
|
14
14
|
private lastRenderedFrameIndex;
|
|
15
15
|
private lastRealtimeProtoFrame;
|
|
16
|
-
private
|
|
17
|
-
private
|
|
18
|
-
private isConversationActive;
|
|
16
|
+
private idleAnimationLoopId;
|
|
17
|
+
private realtimeAnimationLoopId;
|
|
19
18
|
private resizeObserver;
|
|
20
19
|
private onWindowResize;
|
|
21
20
|
private frameCount;
|
|
22
21
|
private lastFpsUpdate;
|
|
23
22
|
private currentFPS;
|
|
23
|
+
private transitionKeyframes;
|
|
24
|
+
private transitionStartTime;
|
|
25
|
+
private readonly startTransitionDurationMs;
|
|
24
26
|
private readonly endTransitionDurationMs;
|
|
25
27
|
private cachedIdleFirstFrame;
|
|
26
28
|
private idleCurrentFrameIndex;
|
|
@@ -2648,6 +2648,32 @@ function isObject(value) {
|
|
|
2648
2648
|
function isSet(value) {
|
|
2649
2649
|
return value !== null && value !== void 0;
|
|
2650
2650
|
}
|
|
2651
|
+
function convertProtoFlameToWasmParams(protoFlame) {
|
|
2652
|
+
var _a;
|
|
2653
|
+
return {
|
|
2654
|
+
translation: protoFlame.translation || [0, 0, 0],
|
|
2655
|
+
rotation: protoFlame.rotation || [0, 0, 0],
|
|
2656
|
+
neck_pose: protoFlame.neckPose || [0, 0, 0],
|
|
2657
|
+
jaw_pose: protoFlame.jawPose || [0, 0, 0],
|
|
2658
|
+
eyes_pose: protoFlame.eyePose || [0, 0, 0, 0, 0, 0],
|
|
2659
|
+
eyelid: protoFlame.eyeLid || [0, 0],
|
|
2660
|
+
expr_params: protoFlame.expression || [],
|
|
2661
|
+
shape_params: [],
|
|
2662
|
+
// Realtime doesn't provide shape params, use default
|
|
2663
|
+
has_eyelid: (((_a = protoFlame.eyeLid) == null ? void 0 : _a.length) || 0) > 0
|
|
2664
|
+
};
|
|
2665
|
+
}
|
|
2666
|
+
function convertWasmParamsToProtoFlame(wasmParams) {
|
|
2667
|
+
return {
|
|
2668
|
+
translation: wasmParams.translation || [0, 0, 0],
|
|
2669
|
+
rotation: wasmParams.rotation || [0, 0, 0],
|
|
2670
|
+
neckPose: wasmParams.neck_pose || [0, 0, 0],
|
|
2671
|
+
jawPose: wasmParams.jaw_pose || [0, 0, 0],
|
|
2672
|
+
eyePose: wasmParams.eyes_pose || [0, 0, 0, 0, 0, 0],
|
|
2673
|
+
eyeLid: wasmParams.eyelid || [0, 0],
|
|
2674
|
+
expression: wasmParams.expr_params || []
|
|
2675
|
+
};
|
|
2676
|
+
}
|
|
2651
2677
|
const POSTHOG_HOST_INTL = "https://i.spatialwalk.ai";
|
|
2652
2678
|
const POSTHOG_API_KEY_INTL = "phc_IFTLa6Z6VhTaNvsxB7klvG2JeNwcSpnnwz8YvZRC96Q";
|
|
2653
2679
|
function getPostHogConfig(_environment) {
|
|
@@ -2700,32 +2726,6 @@ const APP_CONFIG = {
|
|
|
2700
2726
|
}
|
|
2701
2727
|
}
|
|
2702
2728
|
};
|
|
2703
|
-
function convertProtoFlameToWasmParams(protoFlame) {
|
|
2704
|
-
var _a;
|
|
2705
|
-
return {
|
|
2706
|
-
translation: protoFlame.translation || [0, 0, 0],
|
|
2707
|
-
rotation: protoFlame.rotation || [0, 0, 0],
|
|
2708
|
-
neck_pose: protoFlame.neckPose || [0, 0, 0],
|
|
2709
|
-
jaw_pose: protoFlame.jawPose || [0, 0, 0],
|
|
2710
|
-
eyes_pose: protoFlame.eyePose || [0, 0, 0, 0, 0, 0],
|
|
2711
|
-
eyelid: protoFlame.eyeLid || [0, 0],
|
|
2712
|
-
expr_params: protoFlame.expression || [],
|
|
2713
|
-
shape_params: [],
|
|
2714
|
-
// Realtime doesn't provide shape params, use default
|
|
2715
|
-
has_eyelid: (((_a = protoFlame.eyeLid) == null ? void 0 : _a.length) || 0) > 0
|
|
2716
|
-
};
|
|
2717
|
-
}
|
|
2718
|
-
function convertWasmParamsToProtoFlame(wasmParams) {
|
|
2719
|
-
return {
|
|
2720
|
-
translation: wasmParams.translation || [0, 0, 0],
|
|
2721
|
-
rotation: wasmParams.rotation || [0, 0, 0],
|
|
2722
|
-
neckPose: wasmParams.neck_pose || [0, 0, 0],
|
|
2723
|
-
jawPose: wasmParams.jaw_pose || [0, 0, 0],
|
|
2724
|
-
eyePose: wasmParams.eyes_pose || [0, 0, 0, 0, 0, 0],
|
|
2725
|
-
eyeLid: wasmParams.eyelid || [0, 0],
|
|
2726
|
-
expression: wasmParams.expr_params || []
|
|
2727
|
-
};
|
|
2728
|
-
}
|
|
2729
2729
|
var t = "undefined" != typeof window ? window : void 0, i = "undefined" != typeof globalThis ? globalThis : t;
|
|
2730
2730
|
"undefined" == typeof self && (i.self = i), "undefined" == typeof File && (i.File = function() {
|
|
2731
2731
|
});
|
|
@@ -9491,7 +9491,7 @@ const _AnimationPlayer = class _AnimationPlayer {
|
|
|
9491
9491
|
if (this.streamingPlayer) {
|
|
9492
9492
|
return;
|
|
9493
9493
|
}
|
|
9494
|
-
const { StreamingAudioPlayer } = await import("./StreamingAudioPlayer-
|
|
9494
|
+
const { StreamingAudioPlayer } = await import("./StreamingAudioPlayer-pfCQ3Bn7.js");
|
|
9495
9495
|
const { AvatarSDK: AvatarSDK2 } = await Promise.resolve().then(() => AvatarSDK$1);
|
|
9496
9496
|
const audioFormat = AvatarSDK2.getAudioFormat();
|
|
9497
9497
|
this.streamingPlayer = new StreamingAudioPlayer({
|
|
@@ -11220,7 +11220,7 @@ class AvatarSDK {
|
|
|
11220
11220
|
}
|
|
11221
11221
|
__publicField(AvatarSDK, "_isInitialized", false);
|
|
11222
11222
|
__publicField(AvatarSDK, "_configuration", null);
|
|
11223
|
-
__publicField(AvatarSDK, "_version", "1.0.0-beta.
|
|
11223
|
+
__publicField(AvatarSDK, "_version", "1.0.0-beta.86");
|
|
11224
11224
|
__publicField(AvatarSDK, "_avatarCore", null);
|
|
11225
11225
|
__publicField(AvatarSDK, "_dynamicSdkConfig", null);
|
|
11226
11226
|
const AvatarSDK$1 = /* @__PURE__ */ Object.freeze(/* @__PURE__ */ Object.defineProperty({
|
|
@@ -12072,117 +12072,6 @@ class NetworkLayer {
|
|
|
12072
12072
|
this.visibilityChangeHandler = null;
|
|
12073
12073
|
}
|
|
12074
12074
|
}
|
|
12075
|
-
function lerpArrays(from, to2, progress) {
|
|
12076
|
-
const length = Math.min(from.length, to2.length);
|
|
12077
|
-
const result2 = Array.from({ length });
|
|
12078
|
-
for (let i2 = 0; i2 < length; i2++) {
|
|
12079
|
-
result2[i2] = from[i2] + (to2[i2] - from[i2]) * progress;
|
|
12080
|
-
}
|
|
12081
|
-
return result2;
|
|
12082
|
-
}
|
|
12083
|
-
const clamp01 = (x2) => Math.max(0, Math.min(1, x2));
|
|
12084
|
-
function linearLerp(from, to2, progress) {
|
|
12085
|
-
return {
|
|
12086
|
-
translation: lerpArrays(from.translation || [0, 0, 0], to2.translation || [0, 0, 0], progress),
|
|
12087
|
-
rotation: lerpArrays(from.rotation || [0, 0, 0], to2.rotation || [0, 0, 0], progress),
|
|
12088
|
-
neckPose: lerpArrays(from.neckPose || [0, 0, 0], to2.neckPose || [0, 0, 0], progress),
|
|
12089
|
-
jawPose: lerpArrays(from.jawPose || [0, 0, 0], to2.jawPose || [0, 0, 0], progress),
|
|
12090
|
-
eyePose: lerpArrays(from.eyePose || [0, 0, 0, 0, 0, 0], to2.eyePose || [0, 0, 0, 0, 0, 0], progress),
|
|
12091
|
-
eyeLid: (() => {
|
|
12092
|
-
const fromEyelid = from.eyeLid;
|
|
12093
|
-
const toEyelid = to2.eyeLid;
|
|
12094
|
-
if ((fromEyelid == null ? void 0 : fromEyelid.length) && (toEyelid == null ? void 0 : toEyelid.length))
|
|
12095
|
-
return lerpArrays(fromEyelid, toEyelid, progress);
|
|
12096
|
-
return fromEyelid || toEyelid || [];
|
|
12097
|
-
})(),
|
|
12098
|
-
expression: lerpArrays(from.expression || [], to2.expression || [], progress)
|
|
12099
|
-
};
|
|
12100
|
-
}
|
|
12101
|
-
function generateTransitionFramesLinear(from, to2, durationMs, fps = FLAME_FRAME_RATE) {
|
|
12102
|
-
const steps = Math.max(1, Math.floor(durationMs / 1e3 * fps));
|
|
12103
|
-
const frames = Array.from({ length: steps });
|
|
12104
|
-
if (steps === 1) {
|
|
12105
|
-
frames[0] = to2;
|
|
12106
|
-
return frames;
|
|
12107
|
-
}
|
|
12108
|
-
for (let i2 = 0; i2 < steps; i2++) {
|
|
12109
|
-
const progress = i2 / (steps - 1);
|
|
12110
|
-
frames[i2] = linearLerp(from, to2, progress);
|
|
12111
|
-
}
|
|
12112
|
-
frames[0] = from;
|
|
12113
|
-
frames[frames.length - 1] = to2;
|
|
12114
|
-
return frames;
|
|
12115
|
-
}
|
|
12116
|
-
function createBezierEasing(x1, y1, x2, y2) {
|
|
12117
|
-
const cx = 3 * x1;
|
|
12118
|
-
const bx = 3 * (x2 - x1) - cx;
|
|
12119
|
-
const ax = 1 - cx - bx;
|
|
12120
|
-
const cy = 3 * y1;
|
|
12121
|
-
const by = 3 * (y2 - y1) - cy;
|
|
12122
|
-
const ay = 1 - cy - by;
|
|
12123
|
-
const sampleCurveX = (t2) => ((ax * t2 + bx) * t2 + cx) * t2;
|
|
12124
|
-
const sampleCurveY = (t2) => ((ay * t2 + by) * t2 + cy) * t2;
|
|
12125
|
-
const sampleCurveDerivativeX = (t2) => (3 * ax * t2 + 2 * bx) * t2 + cx;
|
|
12126
|
-
const solveCurveX = (x3) => {
|
|
12127
|
-
let t2 = x3;
|
|
12128
|
-
for (let i2 = 0; i2 < 8; i2++) {
|
|
12129
|
-
const error = sampleCurveX(t2) - x3;
|
|
12130
|
-
if (Math.abs(error) < 1e-6) break;
|
|
12131
|
-
const d2 = sampleCurveDerivativeX(t2);
|
|
12132
|
-
if (Math.abs(d2) < 1e-6) break;
|
|
12133
|
-
t2 -= error / d2;
|
|
12134
|
-
}
|
|
12135
|
-
return t2;
|
|
12136
|
-
};
|
|
12137
|
-
return (x3) => {
|
|
12138
|
-
if (x3 <= 0) return 0;
|
|
12139
|
-
if (x3 >= 1) return 1;
|
|
12140
|
-
return sampleCurveY(solveCurveX(x3));
|
|
12141
|
-
};
|
|
12142
|
-
}
|
|
12143
|
-
const BEZIER_CURVES = {
|
|
12144
|
-
jaw: createBezierEasing(...BEZIER_CURVES$1.jaw),
|
|
12145
|
-
expression: createBezierEasing(...BEZIER_CURVES$1.expression),
|
|
12146
|
-
eye: createBezierEasing(...BEZIER_CURVES$1.eye),
|
|
12147
|
-
neck: createBezierEasing(...BEZIER_CURVES$1.neck),
|
|
12148
|
-
global: createBezierEasing(...BEZIER_CURVES$1.global)
|
|
12149
|
-
};
|
|
12150
|
-
function bezierLerp(from, to2, progress) {
|
|
12151
|
-
const getT = (key) => {
|
|
12152
|
-
const scaledProgress = clamp01(progress * TIME_SCALE[key]);
|
|
12153
|
-
return BEZIER_CURVES[key](scaledProgress);
|
|
12154
|
-
};
|
|
12155
|
-
return {
|
|
12156
|
-
translation: lerpArrays(from.translation || [0, 0, 0], to2.translation || [0, 0, 0], getT("global")),
|
|
12157
|
-
rotation: lerpArrays(from.rotation || [0, 0, 0], to2.rotation || [0, 0, 0], getT("global")),
|
|
12158
|
-
neckPose: lerpArrays(from.neckPose || [0, 0, 0], to2.neckPose || [0, 0, 0], getT("neck")),
|
|
12159
|
-
jawPose: lerpArrays(from.jawPose || [0, 0, 0], to2.jawPose || [0, 0, 0], getT("jaw")),
|
|
12160
|
-
eyePose: lerpArrays(from.eyePose || [0, 0, 0, 0, 0, 0], to2.eyePose || [0, 0, 0, 0, 0, 0], getT("eye")),
|
|
12161
|
-
eyeLid: (() => {
|
|
12162
|
-
const fromEyelid = from.eyeLid;
|
|
12163
|
-
const toEyelid = to2.eyeLid;
|
|
12164
|
-
if ((fromEyelid == null ? void 0 : fromEyelid.length) && (toEyelid == null ? void 0 : toEyelid.length))
|
|
12165
|
-
return lerpArrays(fromEyelid, toEyelid, getT("eye"));
|
|
12166
|
-
return fromEyelid || toEyelid || [];
|
|
12167
|
-
})(),
|
|
12168
|
-
expression: lerpArrays(from.expression || [], to2.expression || [], getT("expression"))
|
|
12169
|
-
};
|
|
12170
|
-
}
|
|
12171
|
-
function generateTransitionFrames(from, to2, durationMs, fps = FLAME_FRAME_RATE) {
|
|
12172
|
-
const steps = Math.max(1, Math.floor(durationMs / 1e3 * fps));
|
|
12173
|
-
const frames = Array.from({ length: steps });
|
|
12174
|
-
if (steps === 1) {
|
|
12175
|
-
frames[0] = to2;
|
|
12176
|
-
return frames;
|
|
12177
|
-
}
|
|
12178
|
-
for (let i2 = 0; i2 < steps; i2++) {
|
|
12179
|
-
const progress = i2 / (steps - 1);
|
|
12180
|
-
frames[i2] = bezierLerp(from, to2, progress);
|
|
12181
|
-
}
|
|
12182
|
-
frames[0] = from;
|
|
12183
|
-
frames[frames.length - 1] = to2;
|
|
12184
|
-
return frames;
|
|
12185
|
-
}
|
|
12186
12075
|
class AvatarController {
|
|
12187
12076
|
// 16kHz * 2 bytes per sample
|
|
12188
12077
|
constructor(avatar, options) {
|
|
@@ -13022,7 +12911,7 @@ class AvatarController {
|
|
|
13022
12911
|
* @internal
|
|
13023
12912
|
*/
|
|
13024
12913
|
async startStreamingPlaybackInternal() {
|
|
13025
|
-
var _a, _b, _c
|
|
12914
|
+
var _a, _b, _c;
|
|
13026
12915
|
this.checkAudioContextInitialized();
|
|
13027
12916
|
if (this.isPlaying) {
|
|
13028
12917
|
this.isStartingPlayback = false;
|
|
@@ -13062,29 +12951,18 @@ class AvatarController {
|
|
|
13062
12951
|
});
|
|
13063
12952
|
});
|
|
13064
12953
|
this.emit("startRendering");
|
|
13065
|
-
const transitionFrames = await this.generateStartTransitionFrames();
|
|
13066
|
-
if (transitionFrames.length > 0) {
|
|
13067
|
-
this.currentKeyframes.unshift(...transitionFrames);
|
|
13068
|
-
}
|
|
13069
|
-
const silenceDurationS = START_TRANSITION_DURATION_MS / 1e3;
|
|
13070
|
-
const silenceChunk = this.createSilencePCMChunk(silenceDurationS);
|
|
13071
|
-
const audioChunksWithSilence = [
|
|
13072
|
-
{ data: silenceChunk, isLast: false },
|
|
13073
|
-
...this.pendingAudioChunks
|
|
13074
|
-
];
|
|
13075
12954
|
const streamingPlayer = this.animationPlayer.getStreamingPlayer();
|
|
13076
12955
|
if (streamingPlayer) {
|
|
13077
12956
|
streamingPlayer.setAutoStart(false);
|
|
13078
|
-
|
|
12957
|
+
}
|
|
12958
|
+
if (streamingPlayer) {
|
|
12959
|
+
await streamingPlayer.startNewSession(this.pendingAudioChunks);
|
|
13079
12960
|
}
|
|
13080
12961
|
this.pendingAudioChunks = [];
|
|
13081
12962
|
this.isPlaying = true;
|
|
13082
12963
|
this.currentState = AvatarState.playing;
|
|
13083
12964
|
(_a = this.onConversationState) == null ? void 0 : _a.call(this, this.mapToConversationState(AvatarState.playing));
|
|
13084
12965
|
this.startPlaybackLoop();
|
|
13085
|
-
if (streamingPlayer) {
|
|
13086
|
-
streamingPlayer.play();
|
|
13087
|
-
}
|
|
13088
12966
|
this.isStartingPlayback = false;
|
|
13089
12967
|
logEvent("character_player", "info", {
|
|
13090
12968
|
avatar_id: this.avatar.id,
|
|
@@ -13094,49 +12972,11 @@ class AvatarController {
|
|
|
13094
12972
|
} catch (error) {
|
|
13095
12973
|
const message = error instanceof Error ? error.message : String(error);
|
|
13096
12974
|
logger.error("[AvatarController] Failed to start streaming playback:", message);
|
|
13097
|
-
this.stopPlaybackLoop();
|
|
13098
12975
|
(_c = this.onError) == null ? void 0 : _c.call(this, new AvatarError("Failed to start streaming playback", "INIT_FAILED"));
|
|
13099
|
-
this.emit("stopRendering");
|
|
13100
12976
|
this.isPlaying = false;
|
|
13101
|
-
this.currentState = AvatarState.idle;
|
|
13102
|
-
(_d = this.onConversationState) == null ? void 0 : _d.call(this, this.mapToConversationState(AvatarState.idle));
|
|
13103
12977
|
this.isStartingPlayback = false;
|
|
13104
12978
|
}
|
|
13105
12979
|
}
|
|
13106
|
-
/**
|
|
13107
|
-
* Generate start-transition frames (idle → first conversation keyframe)
|
|
13108
|
-
* @returns transition frames array, or empty if generation fails
|
|
13109
|
-
* @internal
|
|
13110
|
-
*/
|
|
13111
|
-
async generateStartTransitionFrames() {
|
|
13112
|
-
try {
|
|
13113
|
-
const toFrame = this.currentKeyframes[0];
|
|
13114
|
-
if (!toFrame) return [];
|
|
13115
|
-
const toFrameWithPP = this.applyPostProcessingToFlame(toFrame);
|
|
13116
|
-
let fromFrame = null;
|
|
13117
|
-
const avatarCore = AvatarSDK.getAvatarCore();
|
|
13118
|
-
if (avatarCore && this.characterId) {
|
|
13119
|
-
const idleParams = await avatarCore.getCurrentFrameParams(0, this.characterId);
|
|
13120
|
-
fromFrame = convertWasmParamsToProtoFlame(idleParams);
|
|
13121
|
-
}
|
|
13122
|
-
if (!fromFrame) return [];
|
|
13123
|
-
return generateTransitionFramesLinear(fromFrame, toFrameWithPP, START_TRANSITION_DURATION_MS);
|
|
13124
|
-
} catch (e2) {
|
|
13125
|
-
logger.warn("[AvatarController] Failed to generate start transition:", e2 instanceof Error ? e2.message : String(e2));
|
|
13126
|
-
return [];
|
|
13127
|
-
}
|
|
13128
|
-
}
|
|
13129
|
-
/**
|
|
13130
|
-
* Create a silence PCM chunk (16-bit mono, 16kHz)
|
|
13131
|
-
* @internal
|
|
13132
|
-
*/
|
|
13133
|
-
createSilencePCMChunk(durationS) {
|
|
13134
|
-
const sampleRate = APP_CONFIG.audio.sampleRate;
|
|
13135
|
-
const channelCount = 1;
|
|
13136
|
-
const bytesPerSample = 2;
|
|
13137
|
-
const totalBytes = Math.round(sampleRate * channelCount * bytesPerSample * durationS);
|
|
13138
|
-
return new Uint8Array(totalBytes);
|
|
13139
|
-
}
|
|
13140
12980
|
/**
|
|
13141
12981
|
* Playback loop: Calculate animation frame based on audio time, notify render layer to render
|
|
13142
12982
|
*/
|
|
@@ -16243,6 +16083,117 @@ class RenderSystem {
|
|
|
16243
16083
|
m2[15] = 1;
|
|
16244
16084
|
}
|
|
16245
16085
|
}
|
|
16086
|
+
function lerpArrays(from, to2, progress) {
|
|
16087
|
+
const length = Math.min(from.length, to2.length);
|
|
16088
|
+
const result2 = Array.from({ length });
|
|
16089
|
+
for (let i2 = 0; i2 < length; i2++) {
|
|
16090
|
+
result2[i2] = from[i2] + (to2[i2] - from[i2]) * progress;
|
|
16091
|
+
}
|
|
16092
|
+
return result2;
|
|
16093
|
+
}
|
|
16094
|
+
const clamp01 = (x2) => Math.max(0, Math.min(1, x2));
|
|
16095
|
+
function linearLerp(from, to2, progress) {
|
|
16096
|
+
return {
|
|
16097
|
+
translation: lerpArrays(from.translation || [0, 0, 0], to2.translation || [0, 0, 0], progress),
|
|
16098
|
+
rotation: lerpArrays(from.rotation || [0, 0, 0], to2.rotation || [0, 0, 0], progress),
|
|
16099
|
+
neckPose: lerpArrays(from.neckPose || [0, 0, 0], to2.neckPose || [0, 0, 0], progress),
|
|
16100
|
+
jawPose: lerpArrays(from.jawPose || [0, 0, 0], to2.jawPose || [0, 0, 0], progress),
|
|
16101
|
+
eyePose: lerpArrays(from.eyePose || [0, 0, 0, 0, 0, 0], to2.eyePose || [0, 0, 0, 0, 0, 0], progress),
|
|
16102
|
+
eyeLid: (() => {
|
|
16103
|
+
const fromEyelid = from.eyeLid;
|
|
16104
|
+
const toEyelid = to2.eyeLid;
|
|
16105
|
+
if ((fromEyelid == null ? void 0 : fromEyelid.length) && (toEyelid == null ? void 0 : toEyelid.length))
|
|
16106
|
+
return lerpArrays(fromEyelid, toEyelid, progress);
|
|
16107
|
+
return fromEyelid || toEyelid || [];
|
|
16108
|
+
})(),
|
|
16109
|
+
expression: lerpArrays(from.expression || [], to2.expression || [], progress)
|
|
16110
|
+
};
|
|
16111
|
+
}
|
|
16112
|
+
function generateTransitionFramesLinear(from, to2, durationMs, fps = FLAME_FRAME_RATE) {
|
|
16113
|
+
const steps = Math.max(1, Math.floor(durationMs / 1e3 * fps));
|
|
16114
|
+
const frames = Array.from({ length: steps });
|
|
16115
|
+
if (steps === 1) {
|
|
16116
|
+
frames[0] = to2;
|
|
16117
|
+
return frames;
|
|
16118
|
+
}
|
|
16119
|
+
for (let i2 = 0; i2 < steps; i2++) {
|
|
16120
|
+
const progress = i2 / (steps - 1);
|
|
16121
|
+
frames[i2] = linearLerp(from, to2, progress);
|
|
16122
|
+
}
|
|
16123
|
+
frames[0] = from;
|
|
16124
|
+
frames[frames.length - 1] = to2;
|
|
16125
|
+
return frames;
|
|
16126
|
+
}
|
|
16127
|
+
function createBezierEasing(x1, y1, x2, y2) {
|
|
16128
|
+
const cx = 3 * x1;
|
|
16129
|
+
const bx = 3 * (x2 - x1) - cx;
|
|
16130
|
+
const ax = 1 - cx - bx;
|
|
16131
|
+
const cy = 3 * y1;
|
|
16132
|
+
const by = 3 * (y2 - y1) - cy;
|
|
16133
|
+
const ay = 1 - cy - by;
|
|
16134
|
+
const sampleCurveX = (t2) => ((ax * t2 + bx) * t2 + cx) * t2;
|
|
16135
|
+
const sampleCurveY = (t2) => ((ay * t2 + by) * t2 + cy) * t2;
|
|
16136
|
+
const sampleCurveDerivativeX = (t2) => (3 * ax * t2 + 2 * bx) * t2 + cx;
|
|
16137
|
+
const solveCurveX = (x3) => {
|
|
16138
|
+
let t2 = x3;
|
|
16139
|
+
for (let i2 = 0; i2 < 8; i2++) {
|
|
16140
|
+
const error = sampleCurveX(t2) - x3;
|
|
16141
|
+
if (Math.abs(error) < 1e-6) break;
|
|
16142
|
+
const d2 = sampleCurveDerivativeX(t2);
|
|
16143
|
+
if (Math.abs(d2) < 1e-6) break;
|
|
16144
|
+
t2 -= error / d2;
|
|
16145
|
+
}
|
|
16146
|
+
return t2;
|
|
16147
|
+
};
|
|
16148
|
+
return (x3) => {
|
|
16149
|
+
if (x3 <= 0) return 0;
|
|
16150
|
+
if (x3 >= 1) return 1;
|
|
16151
|
+
return sampleCurveY(solveCurveX(x3));
|
|
16152
|
+
};
|
|
16153
|
+
}
|
|
16154
|
+
const BEZIER_CURVES = {
|
|
16155
|
+
jaw: createBezierEasing(...BEZIER_CURVES$1.jaw),
|
|
16156
|
+
expression: createBezierEasing(...BEZIER_CURVES$1.expression),
|
|
16157
|
+
eye: createBezierEasing(...BEZIER_CURVES$1.eye),
|
|
16158
|
+
neck: createBezierEasing(...BEZIER_CURVES$1.neck),
|
|
16159
|
+
global: createBezierEasing(...BEZIER_CURVES$1.global)
|
|
16160
|
+
};
|
|
16161
|
+
function bezierLerp(from, to2, progress) {
|
|
16162
|
+
const getT = (key) => {
|
|
16163
|
+
const scaledProgress = clamp01(progress * TIME_SCALE[key]);
|
|
16164
|
+
return BEZIER_CURVES[key](scaledProgress);
|
|
16165
|
+
};
|
|
16166
|
+
return {
|
|
16167
|
+
translation: lerpArrays(from.translation || [0, 0, 0], to2.translation || [0, 0, 0], getT("global")),
|
|
16168
|
+
rotation: lerpArrays(from.rotation || [0, 0, 0], to2.rotation || [0, 0, 0], getT("global")),
|
|
16169
|
+
neckPose: lerpArrays(from.neckPose || [0, 0, 0], to2.neckPose || [0, 0, 0], getT("neck")),
|
|
16170
|
+
jawPose: lerpArrays(from.jawPose || [0, 0, 0], to2.jawPose || [0, 0, 0], getT("jaw")),
|
|
16171
|
+
eyePose: lerpArrays(from.eyePose || [0, 0, 0, 0, 0, 0], to2.eyePose || [0, 0, 0, 0, 0, 0], getT("eye")),
|
|
16172
|
+
eyeLid: (() => {
|
|
16173
|
+
const fromEyelid = from.eyeLid;
|
|
16174
|
+
const toEyelid = to2.eyeLid;
|
|
16175
|
+
if ((fromEyelid == null ? void 0 : fromEyelid.length) && (toEyelid == null ? void 0 : toEyelid.length))
|
|
16176
|
+
return lerpArrays(fromEyelid, toEyelid, getT("eye"));
|
|
16177
|
+
return fromEyelid || toEyelid || [];
|
|
16178
|
+
})(),
|
|
16179
|
+
expression: lerpArrays(from.expression || [], to2.expression || [], getT("expression"))
|
|
16180
|
+
};
|
|
16181
|
+
}
|
|
16182
|
+
function generateTransitionFrames(from, to2, durationMs, fps = FLAME_FRAME_RATE) {
|
|
16183
|
+
const steps = Math.max(1, Math.floor(durationMs / 1e3 * fps));
|
|
16184
|
+
const frames = Array.from({ length: steps });
|
|
16185
|
+
if (steps === 1) {
|
|
16186
|
+
frames[0] = to2;
|
|
16187
|
+
return frames;
|
|
16188
|
+
}
|
|
16189
|
+
for (let i2 = 0; i2 < steps; i2++) {
|
|
16190
|
+
const progress = i2 / (steps - 1);
|
|
16191
|
+
frames[i2] = bezierLerp(from, to2, progress);
|
|
16192
|
+
}
|
|
16193
|
+
frames[0] = from;
|
|
16194
|
+
frames[frames.length - 1] = to2;
|
|
16195
|
+
return frames;
|
|
16196
|
+
}
|
|
16246
16197
|
class AvatarView {
|
|
16247
16198
|
/**
|
|
16248
16199
|
* Constructor
|
|
@@ -16266,16 +16217,20 @@ class AvatarView {
|
|
|
16266
16217
|
__publicField(this, "currentKeyframes", []);
|
|
16267
16218
|
__publicField(this, "lastRenderedFrameIndex", -1);
|
|
16268
16219
|
__publicField(this, "lastRealtimeProtoFrame", null);
|
|
16269
|
-
// Animation loop
|
|
16270
|
-
__publicField(this, "
|
|
16271
|
-
__publicField(this, "
|
|
16272
|
-
__publicField(this, "isConversationActive", false);
|
|
16220
|
+
// Animation loop types
|
|
16221
|
+
__publicField(this, "idleAnimationLoopId", null);
|
|
16222
|
+
__publicField(this, "realtimeAnimationLoopId", null);
|
|
16273
16223
|
__publicField(this, "resizeObserver", null);
|
|
16274
16224
|
__publicField(this, "onWindowResize", () => this.handleResize());
|
|
16275
16225
|
// FPS 计算
|
|
16276
16226
|
__publicField(this, "frameCount", 0);
|
|
16277
16227
|
__publicField(this, "lastFpsUpdate", 0);
|
|
16278
16228
|
__publicField(this, "currentFPS", 0);
|
|
16229
|
+
// Transition animation data
|
|
16230
|
+
__publicField(this, "transitionKeyframes", []);
|
|
16231
|
+
__publicField(this, "transitionStartTime", 0);
|
|
16232
|
+
__publicField(this, "startTransitionDurationMs", START_TRANSITION_DURATION_MS);
|
|
16233
|
+
// Idle -> Speaking 过渡时长
|
|
16279
16234
|
__publicField(this, "endTransitionDurationMs", END_TRANSITION_DURATION_MS);
|
|
16280
16235
|
// Speaking -> Idle 过渡时长
|
|
16281
16236
|
__publicField(this, "cachedIdleFirstFrame", null);
|
|
@@ -16375,6 +16330,27 @@ class AvatarView {
|
|
|
16375
16330
|
keyframes[keyframes.length - 1] = aligned.to;
|
|
16376
16331
|
return keyframes;
|
|
16377
16332
|
}
|
|
16333
|
+
/**
|
|
16334
|
+
* Idle loop increments index after rendering, so the currently displayed idle frame is (idleCurrentFrameIndex - 1).
|
|
16335
|
+
* @internal
|
|
16336
|
+
*/
|
|
16337
|
+
getDisplayedIdleFrameIndex() {
|
|
16338
|
+
return Math.max(0, this.idleCurrentFrameIndex - 1);
|
|
16339
|
+
}
|
|
16340
|
+
/**
|
|
16341
|
+
* Clamp realtime frame index to current keyframes range to avoid out-of-bounds access.
|
|
16342
|
+
* @internal
|
|
16343
|
+
*/
|
|
16344
|
+
getClampedRealtimeFrameIndex(frameIndex) {
|
|
16345
|
+
const lastIndex = this.currentKeyframes.length - 1;
|
|
16346
|
+
if (lastIndex < 0) {
|
|
16347
|
+
return -1;
|
|
16348
|
+
}
|
|
16349
|
+
if (frameIndex < 0) {
|
|
16350
|
+
return 0;
|
|
16351
|
+
}
|
|
16352
|
+
return Math.min(frameIndex, lastIndex);
|
|
16353
|
+
}
|
|
16378
16354
|
/**
|
|
16379
16355
|
* Get cached Idle first frame, fetch and cache if not cached
|
|
16380
16356
|
* @internal
|
|
@@ -16508,7 +16484,7 @@ class AvatarView {
|
|
|
16508
16484
|
if (APP_CONFIG.debug)
|
|
16509
16485
|
logger.log("[AvatarView] Starting rendering...");
|
|
16510
16486
|
await this.renderFirstFrame();
|
|
16511
|
-
this.
|
|
16487
|
+
this.startIdleAnimationLoop();
|
|
16512
16488
|
this.isInitialized = true;
|
|
16513
16489
|
if (APP_CONFIG.debug)
|
|
16514
16490
|
logger.log("[AvatarView] Avatar view initialized successfully");
|
|
@@ -16681,12 +16657,16 @@ class AvatarView {
|
|
|
16681
16657
|
this.lastFpsUpdate = performance.now();
|
|
16682
16658
|
}
|
|
16683
16659
|
/**
|
|
16684
|
-
* Start
|
|
16685
|
-
* Priority: endTransitionFrames > conversation (Controller playbackLoop) > idle
|
|
16660
|
+
* Start idle animation loop
|
|
16686
16661
|
* @internal
|
|
16687
16662
|
*/
|
|
16688
|
-
|
|
16689
|
-
if (this.
|
|
16663
|
+
startIdleAnimationLoop() {
|
|
16664
|
+
if (this.idleAnimationLoopId) {
|
|
16665
|
+
this.stopIdleAnimationLoop();
|
|
16666
|
+
}
|
|
16667
|
+
if (this.renderingState !== "idle") {
|
|
16668
|
+
if (APP_CONFIG.debug)
|
|
16669
|
+
logger.log("[AvatarView] Skip starting idle loop because not in idle state");
|
|
16690
16670
|
return;
|
|
16691
16671
|
}
|
|
16692
16672
|
this.idleCurrentFrameIndex = 0;
|
|
@@ -16700,72 +16680,185 @@ class AvatarView {
|
|
|
16700
16680
|
return;
|
|
16701
16681
|
}
|
|
16702
16682
|
this.updateFPS();
|
|
16683
|
+
if (this.renderingState !== "idle") {
|
|
16684
|
+
this.idleAnimationLoopId = requestAnimationFrame(renderFrame);
|
|
16685
|
+
return;
|
|
16686
|
+
}
|
|
16703
16687
|
const elapsed = currentTime - lastTime;
|
|
16704
16688
|
if (elapsed < frameInterval) {
|
|
16705
|
-
this.
|
|
16689
|
+
this.idleAnimationLoopId = requestAnimationFrame(renderFrame);
|
|
16706
16690
|
return;
|
|
16707
16691
|
}
|
|
16708
16692
|
lastTime = currentTime - elapsed % frameInterval;
|
|
16709
|
-
if (this.isPureRenderingMode
|
|
16710
|
-
this.
|
|
16693
|
+
if (this.isPureRenderingMode) {
|
|
16694
|
+
this.idleAnimationLoopId = requestAnimationFrame(renderFrame);
|
|
16695
|
+
return;
|
|
16696
|
+
}
|
|
16697
|
+
if (!this._renderingEnabled) {
|
|
16698
|
+
this.idleAnimationLoopId = requestAnimationFrame(renderFrame);
|
|
16699
|
+
return;
|
|
16700
|
+
}
|
|
16701
|
+
const avatarCore = AvatarSDK.getAvatarCore();
|
|
16702
|
+
if (!avatarCore) {
|
|
16711
16703
|
return;
|
|
16712
16704
|
}
|
|
16713
|
-
|
|
16714
|
-
|
|
16715
|
-
|
|
16716
|
-
|
|
16717
|
-
|
|
16718
|
-
|
|
16719
|
-
|
|
16705
|
+
const splatData = await avatarCore.computeCompleteFrameFlat({ frameIndex: this.idleCurrentFrameIndex }, this.characterHandle ?? void 0);
|
|
16706
|
+
this.idleCurrentFrameIndex++;
|
|
16707
|
+
if (splatData) {
|
|
16708
|
+
if (this.renderingState !== "idle") {
|
|
16709
|
+
return;
|
|
16710
|
+
}
|
|
16711
|
+
if (this.isPureRenderingMode) {
|
|
16712
|
+
return;
|
|
16713
|
+
}
|
|
16714
|
+
this.doRender(splatData);
|
|
16715
|
+
}
|
|
16716
|
+
this.idleAnimationLoopId = requestAnimationFrame(renderFrame);
|
|
16717
|
+
} catch (error) {
|
|
16718
|
+
logger.error("[AvatarView] Idle animation loop error:", error instanceof Error ? error.message : String(error));
|
|
16719
|
+
this.stopIdleAnimationLoop();
|
|
16720
|
+
}
|
|
16721
|
+
};
|
|
16722
|
+
this.idleAnimationLoopId = requestAnimationFrame(renderFrame);
|
|
16723
|
+
if (APP_CONFIG.debug)
|
|
16724
|
+
logger.log("[AvatarView] Idle animation loop started");
|
|
16725
|
+
}
|
|
16726
|
+
/**
|
|
16727
|
+
* Start realtime conversation animation loop
|
|
16728
|
+
* @internal
|
|
16729
|
+
*/
|
|
16730
|
+
startRealtimeAnimationLoop() {
|
|
16731
|
+
if (this.realtimeAnimationLoopId) {
|
|
16732
|
+
this.stopRealtimeAnimationLoop();
|
|
16733
|
+
}
|
|
16734
|
+
let lastTime = 0;
|
|
16735
|
+
const targetFPS = FLAME_FRAME_RATE;
|
|
16736
|
+
const frameInterval = 1e3 / targetFPS;
|
|
16737
|
+
this.initFPS();
|
|
16738
|
+
const renderFrame = async (currentTime) => {
|
|
16739
|
+
try {
|
|
16740
|
+
const state = this.renderingState;
|
|
16741
|
+
this.updateFPS();
|
|
16742
|
+
if (!this.renderSystem || state === "idle") {
|
|
16743
|
+
this.realtimeAnimationLoopId = null;
|
|
16744
|
+
return;
|
|
16745
|
+
}
|
|
16746
|
+
const elapsed = currentTime - lastTime;
|
|
16747
|
+
if (elapsed < frameInterval) {
|
|
16748
|
+
this.realtimeAnimationLoopId = requestAnimationFrame(renderFrame);
|
|
16749
|
+
return;
|
|
16750
|
+
}
|
|
16751
|
+
lastTime = currentTime - elapsed % frameInterval;
|
|
16752
|
+
if (state === "transitioningToSpeaking" || state === "transitioningToIdle") {
|
|
16753
|
+
if (this.transitionKeyframes.length === 0) {
|
|
16754
|
+
if (state === "transitioningToSpeaking") {
|
|
16755
|
+
this.setState(
|
|
16756
|
+
"speaking"
|
|
16757
|
+
/* Speaking */
|
|
16758
|
+
);
|
|
16759
|
+
this.avatarController.onTransitionComplete();
|
|
16760
|
+
} else if (state === "transitioningToIdle") {
|
|
16761
|
+
this.setState(
|
|
16762
|
+
"idle"
|
|
16763
|
+
/* Idle */
|
|
16764
|
+
);
|
|
16765
|
+
this.stopRealtimeAnimationLoop();
|
|
16766
|
+
this.startIdleAnimationLoop();
|
|
16767
|
+
return;
|
|
16768
|
+
}
|
|
16769
|
+
this.realtimeAnimationLoopId = requestAnimationFrame(renderFrame);
|
|
16770
|
+
return;
|
|
16771
|
+
}
|
|
16772
|
+
const elapsed2 = performance.now() - this.transitionStartTime;
|
|
16773
|
+
const currentTransitionDurationMs = state === "transitioningToSpeaking" ? this.startTransitionDurationMs : this.endTransitionDurationMs;
|
|
16774
|
+
const progress = Math.min(1, Math.max(0, elapsed2 / currentTransitionDurationMs));
|
|
16775
|
+
const steps = this.transitionKeyframes.length;
|
|
16776
|
+
const idx = Math.min(steps - 1, Math.floor(progress * (steps - 1)));
|
|
16777
|
+
const currentFrame = this.transitionKeyframes[idx];
|
|
16778
|
+
this.currentPlayingFrame = currentFrame;
|
|
16779
|
+
const wasmParams = convertProtoFlameToWasmParams(currentFrame);
|
|
16780
|
+
const avatarCore = AvatarSDK.getAvatarCore();
|
|
16781
|
+
if (avatarCore) {
|
|
16782
|
+
const sd = await avatarCore.computeFrameFlatFromParams(wasmParams, this.characterHandle ?? void 0);
|
|
16720
16783
|
if (sd) {
|
|
16721
16784
|
this.doRender(sd);
|
|
16722
16785
|
}
|
|
16723
16786
|
}
|
|
16724
|
-
if (
|
|
16787
|
+
if (progress >= 1) {
|
|
16788
|
+
if (state === "transitioningToSpeaking") {
|
|
16789
|
+
this.setState(
|
|
16790
|
+
"speaking"
|
|
16791
|
+
/* Speaking */
|
|
16792
|
+
);
|
|
16793
|
+
this.transitionKeyframes = [];
|
|
16794
|
+
this.avatarController.onTransitionComplete();
|
|
16795
|
+
} else if (state === "transitioningToIdle") {
|
|
16796
|
+
this.setState(
|
|
16797
|
+
"idle"
|
|
16798
|
+
/* Idle */
|
|
16799
|
+
);
|
|
16800
|
+
this.transitionKeyframes = [];
|
|
16801
|
+
this.stopRealtimeAnimationLoop();
|
|
16802
|
+
this.startIdleAnimationLoop();
|
|
16803
|
+
return;
|
|
16804
|
+
}
|
|
16805
|
+
}
|
|
16806
|
+
if (state === "transitioningToSpeaking" && this.transitionStartTime > 0 && this.transitionKeyframes.length > 0 && elapsed2 >= this.startTransitionDurationMs + 100) {
|
|
16725
16807
|
this.setState(
|
|
16726
|
-
"
|
|
16727
|
-
/*
|
|
16808
|
+
"speaking"
|
|
16809
|
+
/* Speaking */
|
|
16728
16810
|
);
|
|
16811
|
+
this.transitionKeyframes = [];
|
|
16812
|
+
this.avatarController.onTransitionComplete();
|
|
16729
16813
|
}
|
|
16730
|
-
this.
|
|
16731
|
-
return;
|
|
16732
|
-
}
|
|
16733
|
-
if (this.isConversationActive) {
|
|
16734
|
-
this.renderLoopId = requestAnimationFrame(renderFrame);
|
|
16814
|
+
this.realtimeAnimationLoopId = requestAnimationFrame(renderFrame);
|
|
16735
16815
|
return;
|
|
16736
16816
|
}
|
|
16737
|
-
|
|
16738
|
-
|
|
16739
|
-
this.renderLoopId = requestAnimationFrame(renderFrame);
|
|
16817
|
+
if (state === "speaking") {
|
|
16818
|
+
this.realtimeAnimationLoopId = requestAnimationFrame(renderFrame);
|
|
16740
16819
|
return;
|
|
16741
16820
|
}
|
|
16742
|
-
const splatData = await avatarCore.computeCompleteFrameFlat({ frameIndex: this.idleCurrentFrameIndex }, this.characterHandle ?? void 0);
|
|
16743
|
-
this.idleCurrentFrameIndex++;
|
|
16744
|
-
if (splatData && this.renderingState === "idle" && !this.isPureRenderingMode && !this.isConversationActive) {
|
|
16745
|
-
this.doRender(splatData);
|
|
16746
|
-
}
|
|
16747
|
-
this.renderLoopId = requestAnimationFrame(renderFrame);
|
|
16748
16821
|
} catch (error) {
|
|
16749
|
-
logger.error("[AvatarView]
|
|
16750
|
-
this.
|
|
16822
|
+
logger.error("[AvatarView] Realtime animation loop error:", error instanceof Error ? error.message : String(error));
|
|
16823
|
+
this.stopRealtimeAnimationLoop();
|
|
16751
16824
|
}
|
|
16752
16825
|
};
|
|
16753
|
-
this.
|
|
16826
|
+
this.realtimeAnimationLoopId = requestAnimationFrame(renderFrame);
|
|
16754
16827
|
if (APP_CONFIG.debug)
|
|
16755
|
-
logger.log("[AvatarView]
|
|
16828
|
+
logger.log("[AvatarView] Realtime animation loop started");
|
|
16829
|
+
}
|
|
16830
|
+
/**
|
|
16831
|
+
* Stop idle animation loop
|
|
16832
|
+
* @internal
|
|
16833
|
+
*/
|
|
16834
|
+
stopIdleAnimationLoop() {
|
|
16835
|
+
if (this.idleAnimationLoopId) {
|
|
16836
|
+
cancelAnimationFrame(this.idleAnimationLoopId);
|
|
16837
|
+
this.idleAnimationLoopId = null;
|
|
16838
|
+
if (APP_CONFIG.debug)
|
|
16839
|
+
logger.log("[AvatarView] Idle animation loop stopped");
|
|
16840
|
+
}
|
|
16756
16841
|
}
|
|
16757
16842
|
/**
|
|
16758
|
-
* Stop
|
|
16843
|
+
* Stop realtime conversation animation loop
|
|
16759
16844
|
* @internal
|
|
16760
16845
|
*/
|
|
16761
|
-
|
|
16762
|
-
if (this.
|
|
16763
|
-
cancelAnimationFrame(this.
|
|
16764
|
-
this.
|
|
16846
|
+
stopRealtimeAnimationLoop() {
|
|
16847
|
+
if (this.realtimeAnimationLoopId) {
|
|
16848
|
+
cancelAnimationFrame(this.realtimeAnimationLoopId);
|
|
16849
|
+
this.realtimeAnimationLoopId = null;
|
|
16765
16850
|
if (APP_CONFIG.debug)
|
|
16766
|
-
logger.log("[AvatarView]
|
|
16851
|
+
logger.log("[AvatarView] Realtime animation loop stopped");
|
|
16767
16852
|
}
|
|
16768
16853
|
}
|
|
16854
|
+
/**
|
|
16855
|
+
* Stop all animation loops
|
|
16856
|
+
* @internal
|
|
16857
|
+
*/
|
|
16858
|
+
stopAllAnimationLoops() {
|
|
16859
|
+
this.stopIdleAnimationLoop();
|
|
16860
|
+
this.stopRealtimeAnimationLoop();
|
|
16861
|
+
}
|
|
16769
16862
|
/**
|
|
16770
16863
|
* Unified render method - all rendering goes through here
|
|
16771
16864
|
* This is the single point of control for renderingEnabled flag
|
|
@@ -16787,42 +16880,80 @@ class AvatarView {
|
|
|
16787
16880
|
return;
|
|
16788
16881
|
}
|
|
16789
16882
|
this.lastRenderedFrameIndex = frameIndex;
|
|
16790
|
-
|
|
16791
|
-
|
|
16883
|
+
const clampedFrameIndex = this.getClampedRealtimeFrameIndex(frameIndex);
|
|
16884
|
+
if (clampedFrameIndex >= 0) {
|
|
16885
|
+
this.lastRealtimeProtoFrame = this.currentKeyframes[clampedFrameIndex];
|
|
16792
16886
|
this.currentPlayingFrame = this.lastRealtimeProtoFrame;
|
|
16793
16887
|
}
|
|
16794
16888
|
this.doRender(splatData);
|
|
16795
16889
|
}
|
|
16796
16890
|
/**
|
|
16797
16891
|
* State transition method
|
|
16892
|
+
* Unified state transition management to ensure state consistency
|
|
16798
16893
|
* @internal
|
|
16799
16894
|
*/
|
|
16800
16895
|
setState(newState) {
|
|
16896
|
+
const oldState = this.renderingState;
|
|
16801
16897
|
this.renderingState = newState;
|
|
16898
|
+
if (oldState === "transitioningToIdle" && newState !== "transitioningToIdle") {
|
|
16899
|
+
this.transitionKeyframes = [];
|
|
16900
|
+
}
|
|
16901
|
+
if (oldState === "transitioningToSpeaking" && newState !== "transitioningToSpeaking") {
|
|
16902
|
+
this.transitionKeyframes = [];
|
|
16903
|
+
}
|
|
16802
16904
|
if (newState === "idle") {
|
|
16803
16905
|
this.currentKeyframes = [];
|
|
16804
16906
|
this.lastRenderedFrameIndex = -1;
|
|
16805
16907
|
this.lastRealtimeProtoFrame = null;
|
|
16806
|
-
this.
|
|
16807
|
-
this.
|
|
16908
|
+
this.transitionKeyframes = [];
|
|
16909
|
+
this.transitionStartTime = 0;
|
|
16808
16910
|
this.currentPlayingFrame = null;
|
|
16809
|
-
this.idleCurrentFrameIndex = 0;
|
|
16810
16911
|
}
|
|
16811
16912
|
}
|
|
16812
16913
|
/**
|
|
16813
|
-
*
|
|
16914
|
+
* Check if in realtime playing state (Speaking or transitioning to Speaking)
|
|
16915
|
+
* @internal
|
|
16916
|
+
*/
|
|
16917
|
+
get isRealtimePlaying() {
|
|
16918
|
+
return this.renderingState === "speaking" || this.renderingState === "transitioningToSpeaking";
|
|
16919
|
+
}
|
|
16920
|
+
/**
|
|
16921
|
+
* Check if in transition state
|
|
16922
|
+
* @internal
|
|
16923
|
+
*/
|
|
16924
|
+
get isTransitioning() {
|
|
16925
|
+
return this.renderingState === "transitioningToSpeaking" || this.renderingState === "transitioningToIdle";
|
|
16926
|
+
}
|
|
16927
|
+
/**
|
|
16928
|
+
* Check if will return to Idle after transition ends
|
|
16929
|
+
* @internal
|
|
16930
|
+
*/
|
|
16931
|
+
get endToIdleAfterTransition() {
|
|
16932
|
+
return this.renderingState === "transitioningToIdle";
|
|
16933
|
+
}
|
|
16934
|
+
/**
|
|
16935
|
+
* Handle interrupt
|
|
16936
|
+
* When interrupted, should generate transition animation instead of directly jumping back to Idle
|
|
16814
16937
|
* @internal
|
|
16815
16938
|
*/
|
|
16816
16939
|
handleInterrupt() {
|
|
16817
|
-
|
|
16940
|
+
const state = this.renderingState;
|
|
16941
|
+
if (state === "idle") {
|
|
16818
16942
|
return;
|
|
16819
16943
|
}
|
|
16820
|
-
|
|
16821
|
-
|
|
16822
|
-
|
|
16823
|
-
|
|
16824
|
-
|
|
16825
|
-
|
|
16944
|
+
if (state === "transitioningToIdle") {
|
|
16945
|
+
return;
|
|
16946
|
+
}
|
|
16947
|
+
if (state === "speaking" || state === "transitioningToSpeaking") {
|
|
16948
|
+
this.stopRealtimeRendering();
|
|
16949
|
+
} else {
|
|
16950
|
+
this.setState(
|
|
16951
|
+
"idle"
|
|
16952
|
+
/* Idle */
|
|
16953
|
+
);
|
|
16954
|
+
this.stopRealtimeAnimationLoop();
|
|
16955
|
+
this.startIdleAnimationLoop();
|
|
16956
|
+
}
|
|
16826
16957
|
}
|
|
16827
16958
|
/**
|
|
16828
16959
|
* Setup AvatarController event listeners
|
|
@@ -16831,24 +16962,13 @@ class AvatarView {
|
|
|
16831
16962
|
setupControllerEventListeners() {
|
|
16832
16963
|
this.avatarController.setupInternalEventListeners({
|
|
16833
16964
|
onKeyframesUpdate: (keyframes) => {
|
|
16834
|
-
this.
|
|
16965
|
+
this.prepareRealtimeRendering(keyframes);
|
|
16835
16966
|
},
|
|
16836
16967
|
onStartRendering: () => {
|
|
16837
|
-
this.
|
|
16838
|
-
this.isConversationActive = true;
|
|
16839
|
-
this.setState(
|
|
16840
|
-
"speaking"
|
|
16841
|
-
/* Speaking */
|
|
16842
|
-
);
|
|
16843
|
-
logEvent("character_view", "info", {
|
|
16844
|
-
avatar_id: this.avatar.id,
|
|
16845
|
-
event: "rendering_started",
|
|
16846
|
-
keyframesCount: this.currentKeyframes.length
|
|
16847
|
-
});
|
|
16968
|
+
this.startRealtimeRendering();
|
|
16848
16969
|
},
|
|
16849
16970
|
onStopRendering: () => {
|
|
16850
|
-
this.
|
|
16851
|
-
this.generateEndTransition();
|
|
16971
|
+
this.stopRealtimeRendering();
|
|
16852
16972
|
},
|
|
16853
16973
|
onInterrupt: () => {
|
|
16854
16974
|
this.handleInterrupt();
|
|
@@ -16856,47 +16976,183 @@ class AvatarView {
|
|
|
16856
16976
|
});
|
|
16857
16977
|
}
|
|
16858
16978
|
/**
|
|
16859
|
-
*
|
|
16979
|
+
* Prepare realtime rendering (generate transition to Speaking)
|
|
16980
|
+
* Unified logic: from current playing frame -> Speaking first frame
|
|
16860
16981
|
* @internal
|
|
16861
16982
|
*/
|
|
16862
|
-
async
|
|
16983
|
+
async prepareRealtimeRendering(keyframes) {
|
|
16984
|
+
const state = this.renderingState;
|
|
16985
|
+
if ((state === "speaking" || state === "transitioningToSpeaking") && this.currentKeyframes.length > 0) {
|
|
16986
|
+
this.currentKeyframes = keyframes;
|
|
16987
|
+
return;
|
|
16988
|
+
}
|
|
16989
|
+
this.stopIdleAnimationLoop();
|
|
16990
|
+
this.currentKeyframes = keyframes;
|
|
16863
16991
|
this.setState(
|
|
16864
|
-
"
|
|
16865
|
-
/*
|
|
16992
|
+
"transitioningToSpeaking"
|
|
16993
|
+
/* TransitioningToSpeaking */
|
|
16866
16994
|
);
|
|
16867
16995
|
try {
|
|
16868
|
-
|
|
16869
|
-
if (
|
|
16870
|
-
|
|
16871
|
-
}
|
|
16872
|
-
if (!fromFrame) {
|
|
16873
|
-
this.setState(
|
|
16874
|
-
"idle"
|
|
16875
|
-
/* Idle */
|
|
16876
|
-
);
|
|
16877
|
-
return;
|
|
16878
|
-
}
|
|
16879
|
-
const fromFrameWithPP = this.avatarController.applyPostProcessingToFlame(fromFrame);
|
|
16880
|
-
const idleFirstProto = await this.getCachedIdleFirstFrame();
|
|
16881
|
-
if (idleFirstProto) {
|
|
16882
|
-
const frames = this.generateAndAlignTransitionFrames(fromFrameWithPP, idleFirstProto, this.endTransitionDurationMs);
|
|
16883
|
-
if (frames.length > 0) {
|
|
16884
|
-
this.endTransitionFrames = frames;
|
|
16885
|
-
this.currentPlayingFrame = null;
|
|
16886
|
-
if (APP_CONFIG.debug)
|
|
16887
|
-
logger.log("[AvatarView] End transition started:", frames.length, "frames");
|
|
16996
|
+
const avatarCore = AvatarSDK.getAvatarCore();
|
|
16997
|
+
if (avatarCore && keyframes.length > 0) {
|
|
16998
|
+
if (this.renderingState !== "transitioningToSpeaking") {
|
|
16888
16999
|
return;
|
|
16889
17000
|
}
|
|
17001
|
+
let fromFrame = this.currentPlayingFrame;
|
|
17002
|
+
if (!fromFrame) {
|
|
17003
|
+
if (state === "idle") {
|
|
17004
|
+
const displayedIdleFrameIndex = this.getDisplayedIdleFrameIndex();
|
|
17005
|
+
const idleParams = await avatarCore.getCurrentFrameParams(displayedIdleFrameIndex, this.characterId);
|
|
17006
|
+
fromFrame = convertWasmParamsToProtoFlame(idleParams);
|
|
17007
|
+
} else if (state === "transitioningToIdle" || state === "transitioningToSpeaking") {
|
|
17008
|
+
if (this.transitionKeyframes.length > 0) {
|
|
17009
|
+
const elapsed = performance.now() - this.transitionStartTime;
|
|
17010
|
+
const currentTransitionDurationMs = state === "transitioningToSpeaking" ? this.startTransitionDurationMs : this.endTransitionDurationMs;
|
|
17011
|
+
const progress = Math.min(1, Math.max(0, elapsed / currentTransitionDurationMs));
|
|
17012
|
+
const steps = this.transitionKeyframes.length;
|
|
17013
|
+
const idx = Math.min(steps - 1, Math.floor(progress * (steps - 1)));
|
|
17014
|
+
fromFrame = this.transitionKeyframes[idx];
|
|
17015
|
+
}
|
|
17016
|
+
}
|
|
17017
|
+
}
|
|
17018
|
+
if (!fromFrame) {
|
|
17019
|
+
logger.warn("[AvatarView] Cannot get current playing frame, fallback to idle frame");
|
|
17020
|
+
const displayedIdleFrameIndex = this.getDisplayedIdleFrameIndex();
|
|
17021
|
+
const idleParams = await avatarCore.getCurrentFrameParams(displayedIdleFrameIndex, this.characterId);
|
|
17022
|
+
fromFrame = convertWasmParamsToProtoFlame(idleParams);
|
|
17023
|
+
}
|
|
17024
|
+
await this.getCachedIdleFirstFrame();
|
|
17025
|
+
const firstSpeaking = keyframes[0];
|
|
17026
|
+
const firstSpeakingWithPostProcessing = this.avatarController.applyPostProcessingToFlame(firstSpeaking);
|
|
17027
|
+
this.transitionKeyframes = this.generateAndAlignTransitionFrames(fromFrame, firstSpeakingWithPostProcessing, this.startTransitionDurationMs, true);
|
|
17028
|
+
this.transitionStartTime = performance.now();
|
|
17029
|
+
this.currentPlayingFrame = null;
|
|
17030
|
+
if (this.transitionKeyframes.length === 0) {
|
|
17031
|
+
this.setState(
|
|
17032
|
+
"speaking"
|
|
17033
|
+
/* Speaking */
|
|
17034
|
+
);
|
|
17035
|
+
this.avatarController.onTransitionComplete();
|
|
17036
|
+
} else {
|
|
17037
|
+
if (APP_CONFIG.debug)
|
|
17038
|
+
logger.log("[AvatarView] Transition started:", this.transitionKeyframes.length, "frames");
|
|
17039
|
+
}
|
|
16890
17040
|
}
|
|
16891
17041
|
} catch (e2) {
|
|
16892
|
-
logger.warn("[AvatarView]
|
|
17042
|
+
logger.warn("[AvatarView] Transition generation failed:", e2 instanceof Error ? e2.message : String(e2));
|
|
17043
|
+
if (this.renderingState === "transitioningToSpeaking") {
|
|
17044
|
+
this.setState(
|
|
17045
|
+
"speaking"
|
|
17046
|
+
/* Speaking */
|
|
17047
|
+
);
|
|
17048
|
+
this.avatarController.onTransitionComplete();
|
|
17049
|
+
}
|
|
17050
|
+
}
|
|
17051
|
+
this.startRealtimeAnimationLoop();
|
|
17052
|
+
}
|
|
17053
|
+
/**
|
|
17054
|
+
* Start realtime rendering loop
|
|
17055
|
+
* @internal
|
|
17056
|
+
*/
|
|
17057
|
+
startRealtimeRendering() {
|
|
17058
|
+
if (APP_CONFIG.debug)
|
|
17059
|
+
logger.log("[AvatarView] Starting realtime rendering with", this.currentKeyframes.length, "frames");
|
|
17060
|
+
logEvent("character_view", "info", {
|
|
17061
|
+
avatar_id: this.avatar.id,
|
|
17062
|
+
event: "rendering_started",
|
|
17063
|
+
keyframesCount: this.currentKeyframes.length
|
|
17064
|
+
});
|
|
17065
|
+
}
|
|
17066
|
+
/**
|
|
17067
|
+
* Stop realtime conversation rendering
|
|
17068
|
+
* @internal
|
|
17069
|
+
*/
|
|
17070
|
+
stopRealtimeRendering() {
|
|
17071
|
+
var _a, _b;
|
|
17072
|
+
const state = this.renderingState;
|
|
17073
|
+
if (state === "idle" || state === "transitioningToIdle") {
|
|
17074
|
+
return;
|
|
16893
17075
|
}
|
|
16894
|
-
if (
|
|
17076
|
+
if (state !== "speaking" && state !== "transitioningToSpeaking") {
|
|
17077
|
+
if (state === "transitioningToIdle") {
|
|
17078
|
+
return;
|
|
17079
|
+
}
|
|
16895
17080
|
this.setState(
|
|
16896
17081
|
"idle"
|
|
16897
17082
|
/* Idle */
|
|
16898
17083
|
);
|
|
17084
|
+
this.stopRealtimeAnimationLoop();
|
|
17085
|
+
this.startIdleAnimationLoop();
|
|
17086
|
+
return;
|
|
16899
17087
|
}
|
|
17088
|
+
this.setState(
|
|
17089
|
+
"transitioningToIdle"
|
|
17090
|
+
/* TransitioningToIdle */
|
|
17091
|
+
);
|
|
17092
|
+
(_b = (_a = this.avatarController).onConversationState) == null ? void 0 : _b.call(_a, ConversationState.idle);
|
|
17093
|
+
(async () => {
|
|
17094
|
+
try {
|
|
17095
|
+
if (this.renderingState !== "transitioningToIdle") {
|
|
17096
|
+
return;
|
|
17097
|
+
}
|
|
17098
|
+
const avatarCore = AvatarSDK.getAvatarCore();
|
|
17099
|
+
if (avatarCore) {
|
|
17100
|
+
let fromFrame = this.currentPlayingFrame;
|
|
17101
|
+
if (!fromFrame) {
|
|
17102
|
+
if (this.lastRealtimeProtoFrame) {
|
|
17103
|
+
fromFrame = this.lastRealtimeProtoFrame;
|
|
17104
|
+
} else if (this.currentKeyframes.length > 0) {
|
|
17105
|
+
const clampedFrameIndex = this.getClampedRealtimeFrameIndex(this.lastRenderedFrameIndex);
|
|
17106
|
+
if (clampedFrameIndex >= 0) {
|
|
17107
|
+
fromFrame = this.currentKeyframes[clampedFrameIndex];
|
|
17108
|
+
}
|
|
17109
|
+
}
|
|
17110
|
+
}
|
|
17111
|
+
if (!fromFrame && this.transitionKeyframes.length > 0) {
|
|
17112
|
+
const elapsed = performance.now() - this.transitionStartTime;
|
|
17113
|
+
const progress = Math.min(1, Math.max(0, elapsed / this.endTransitionDurationMs));
|
|
17114
|
+
const steps = this.transitionKeyframes.length;
|
|
17115
|
+
const idx = Math.min(steps - 1, Math.floor(progress * (steps - 1)));
|
|
17116
|
+
fromFrame = this.transitionKeyframes[idx];
|
|
17117
|
+
}
|
|
17118
|
+
if (!fromFrame) {
|
|
17119
|
+
logger.warn("[AvatarView] Cannot get current playing frame for transition to idle, fallback to idle frame");
|
|
17120
|
+
this.setState(
|
|
17121
|
+
"idle"
|
|
17122
|
+
/* Idle */
|
|
17123
|
+
);
|
|
17124
|
+
this.stopRealtimeAnimationLoop();
|
|
17125
|
+
this.startIdleAnimationLoop();
|
|
17126
|
+
return;
|
|
17127
|
+
}
|
|
17128
|
+
const fromFrameWithPostProcessing = this.avatarController.applyPostProcessingToFlame(fromFrame);
|
|
17129
|
+
const idleFirstProto = await this.getCachedIdleFirstFrame();
|
|
17130
|
+
if (idleFirstProto) {
|
|
17131
|
+
this.transitionKeyframes = this.generateAndAlignTransitionFrames(fromFrameWithPostProcessing, idleFirstProto, this.endTransitionDurationMs);
|
|
17132
|
+
this.transitionStartTime = performance.now();
|
|
17133
|
+
this.currentPlayingFrame = null;
|
|
17134
|
+
if (this.transitionKeyframes.length > 0 && this.renderingState === "transitioningToIdle") {
|
|
17135
|
+
if (APP_CONFIG.debug)
|
|
17136
|
+
logger.log("[AvatarView] Return transition started:", this.transitionKeyframes.length, "frames");
|
|
17137
|
+
if (!this.realtimeAnimationLoopId) {
|
|
17138
|
+
this.startRealtimeAnimationLoop();
|
|
17139
|
+
}
|
|
17140
|
+
return;
|
|
17141
|
+
}
|
|
17142
|
+
}
|
|
17143
|
+
}
|
|
17144
|
+
} catch (e2) {
|
|
17145
|
+
logger.warn("[AvatarView] Return transition generation failed:", e2 instanceof Error ? e2.message : String(e2));
|
|
17146
|
+
}
|
|
17147
|
+
if (this.renderingState === "transitioningToIdle") {
|
|
17148
|
+
this.setState(
|
|
17149
|
+
"idle"
|
|
17150
|
+
/* Idle */
|
|
17151
|
+
);
|
|
17152
|
+
this.stopRealtimeAnimationLoop();
|
|
17153
|
+
this.startIdleAnimationLoop();
|
|
17154
|
+
}
|
|
17155
|
+
})();
|
|
16900
17156
|
}
|
|
16901
17157
|
/**
|
|
16902
17158
|
* Cleanup view resources
|
|
@@ -16913,7 +17169,7 @@ class AvatarView {
|
|
|
16913
17169
|
this.avatarController.clear();
|
|
16914
17170
|
this.avatarController.dispose();
|
|
16915
17171
|
}
|
|
16916
|
-
this.
|
|
17172
|
+
this.stopAllAnimationLoops();
|
|
16917
17173
|
this.stopAvatarActiveHeartbeat();
|
|
16918
17174
|
this.setState(
|
|
16919
17175
|
"idle"
|
|
@@ -17070,14 +17326,14 @@ class AvatarView {
|
|
|
17070
17326
|
}
|
|
17071
17327
|
/**
|
|
17072
17328
|
* Pause rendering loop
|
|
17073
|
-
*
|
|
17329
|
+
*
|
|
17074
17330
|
* When called:
|
|
17075
17331
|
* - Rendering loop stops (no GPU/canvas updates)
|
|
17076
17332
|
* - Audio playback continues normally
|
|
17077
17333
|
* - Animation state machine continues running
|
|
17078
|
-
*
|
|
17334
|
+
*
|
|
17079
17335
|
* Use `resumeRendering()` to resume rendering.
|
|
17080
|
-
*
|
|
17336
|
+
*
|
|
17081
17337
|
* @example
|
|
17082
17338
|
* // Stop rendering to save GPU resources (audio continues)
|
|
17083
17339
|
* avatarView.pauseRendering()
|
|
@@ -17091,11 +17347,11 @@ class AvatarView {
|
|
|
17091
17347
|
}
|
|
17092
17348
|
/**
|
|
17093
17349
|
* Resume rendering loop
|
|
17094
|
-
*
|
|
17350
|
+
*
|
|
17095
17351
|
* When called:
|
|
17096
17352
|
* - Rendering loop resumes from current state
|
|
17097
17353
|
* - If in Idle state, immediately renders current frame to restore display
|
|
17098
|
-
*
|
|
17354
|
+
*
|
|
17099
17355
|
* @example
|
|
17100
17356
|
* // Resume rendering
|
|
17101
17357
|
* avatarView.resumeRendering()
|
|
@@ -17157,11 +17413,11 @@ class AvatarView {
|
|
|
17157
17413
|
}
|
|
17158
17414
|
/**
|
|
17159
17415
|
* Get or set avatar transform in canvas
|
|
17160
|
-
*
|
|
17416
|
+
*
|
|
17161
17417
|
* @example
|
|
17162
17418
|
* // Get current transform
|
|
17163
17419
|
* const current = avatarView.transform
|
|
17164
|
-
*
|
|
17420
|
+
*
|
|
17165
17421
|
* // Set transform
|
|
17166
17422
|
* avatarView.transform = { x: 0.5, y: 0, scale: 2.0 }
|
|
17167
17423
|
*/
|
package/dist/index.js
CHANGED
package/package.json
CHANGED