@spatius/avatarkit 1.3.1-beta.1 → 1.3.1-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [1.3.1-beta.2] - 2026-07-17
|
|
9
|
+
|
|
10
|
+
### Fixed
|
|
11
|
+
- Fixed playback getting stuck when audio was sent while the connection was still being established and the session later reconnected: the audio clock could freeze, leaving the avatar frozen on a single frame instead of animating in sync with the audio.
|
|
12
|
+
- Fixed the avatar not returning to idle after a playback round finished in that same send-then-connect scenario; it now correctly transitions back to idle when playback ends.
|
|
13
|
+
|
|
8
14
|
## [1.3.1-beta.1] - 2026-07-16
|
|
9
15
|
|
|
10
16
|
### Changed
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
var __defProp = Object.defineProperty;
|
|
2
2
|
var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
|
|
3
3
|
var __publicField = (obj, key, value) => __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
|
|
4
|
-
import { A as APP_CONFIG, l as logger, e as errorToMessage, a as logEvent } from "./index-
|
|
4
|
+
import { A as APP_CONFIG, l as logger, e as errorToMessage, a as logEvent } from "./index-CFPt9NwR.js";
|
|
5
5
|
class StreamingAudioPlayer {
|
|
6
6
|
// Mark if AudioContext is being resumed, avoid concurrent resume requests
|
|
7
7
|
constructor(options) {
|
|
@@ -46,6 +46,8 @@ class StreamingAudioPlayer {
|
|
|
46
46
|
// Default volume 1.0 (0.0 - 1.0)
|
|
47
47
|
// Event callbacks
|
|
48
48
|
__publicField(this, "onEndedCallback");
|
|
49
|
+
// 音频缓冲卡顿(缓冲耗尽自动暂停 / 有数据自动恢复)通知:true=进入卡顿,false=恢复。
|
|
50
|
+
__publicField(this, "onAudioStallCallback");
|
|
49
51
|
// AudioContext state management
|
|
50
52
|
__publicField(this, "stateChangeHandler");
|
|
51
53
|
__publicField(this, "isResuming", false);
|
|
@@ -156,6 +158,7 @@ class StreamingAudioPlayer {
|
|
|
156
158
|
* Add audio chunk (16-bit PCM)
|
|
157
159
|
*/
|
|
158
160
|
addChunk(pcmData, isLast = false) {
|
|
161
|
+
var _a;
|
|
159
162
|
if (!this.audioContext) {
|
|
160
163
|
logger.error("AudioContext not initialized");
|
|
161
164
|
return;
|
|
@@ -176,6 +179,7 @@ class StreamingAudioPlayer {
|
|
|
176
179
|
if (this.autoContinue && this.isPaused) {
|
|
177
180
|
this.log("[StreamingAudioPlayer] autoContinue=true, auto-resuming playback");
|
|
178
181
|
this.autoContinue = false;
|
|
182
|
+
(_a = this.onAudioStallCallback) == null ? void 0 : _a.call(this, false);
|
|
179
183
|
this.resume().catch((err) => {
|
|
180
184
|
logger.error("Failed to auto-resume playback:", err);
|
|
181
185
|
});
|
|
@@ -229,13 +233,13 @@ class StreamingAudioPlayer {
|
|
|
229
233
|
this.log("[StreamingAudioPlayer] Cannot start playback: Already playing");
|
|
230
234
|
return;
|
|
231
235
|
}
|
|
232
|
-
await this.ensureAudioContextRunning();
|
|
233
236
|
this.isPlaying = true;
|
|
234
237
|
this.sessionStartTime = this.audioContext.currentTime;
|
|
235
238
|
this.scheduledTime = this.sessionStartTime;
|
|
236
239
|
this.lastScheduledChunkEndTime = 0;
|
|
237
240
|
this.scheduledChunkInfo = [];
|
|
238
241
|
this.autoContinue = false;
|
|
242
|
+
await this.ensureAudioContextRunning();
|
|
239
243
|
this.log("[StreamingAudioPlayer] Starting playback", {
|
|
240
244
|
sessionStartTime: this.sessionStartTime,
|
|
241
245
|
bufferedChunks: this.audioChunks.length,
|
|
@@ -304,18 +308,17 @@ class StreamingAudioPlayer {
|
|
|
304
308
|
});
|
|
305
309
|
this.activeSources.add(source);
|
|
306
310
|
source.onended = () => {
|
|
311
|
+
var _a;
|
|
307
312
|
this.activeSources.delete(source);
|
|
308
313
|
if (this.activeSources.size === 0) {
|
|
309
314
|
const lastChunk = this.audioChunks[this.scheduledChunks - 1];
|
|
310
|
-
|
|
315
|
+
const sessionEnded = !!(lastChunk == null ? void 0 : lastChunk.isLast);
|
|
316
|
+
if (!sessionEnded) {
|
|
311
317
|
this.log("All audio chunks ended but end=false, pausing and setting autoContinue");
|
|
312
|
-
|
|
313
|
-
scheduled_chunks: this.scheduledChunks,
|
|
314
|
-
audio_time: this.getCurrentTime()
|
|
315
|
-
});
|
|
318
|
+
(_a = this.onAudioStallCallback) == null ? void 0 : _a.call(this, true);
|
|
316
319
|
this.autoContinue = true;
|
|
317
320
|
this.pause();
|
|
318
|
-
} else
|
|
321
|
+
} else {
|
|
319
322
|
this.log("Last audio chunk ended, marking playback as ended");
|
|
320
323
|
this.markEnded();
|
|
321
324
|
}
|
|
@@ -557,6 +560,12 @@ class StreamingAudioPlayer {
|
|
|
557
560
|
onEnded(callback) {
|
|
558
561
|
this.onEndedCallback = callback;
|
|
559
562
|
}
|
|
563
|
+
/**
|
|
564
|
+
* 设置音频缓冲卡顿回调:缓冲耗尽自动暂停时 stalled=true,有数据自动恢复时 stalled=false。
|
|
565
|
+
*/
|
|
566
|
+
onAudioStall(callback) {
|
|
567
|
+
this.onAudioStallCallback = callback;
|
|
568
|
+
}
|
|
560
569
|
/**
|
|
561
570
|
* Check if playing
|
|
562
571
|
*/
|
|
@@ -46,6 +46,11 @@ export declare class AvatarController {
|
|
|
46
46
|
private isFallbackMode;
|
|
47
47
|
private frameStarvationEvents;
|
|
48
48
|
private isFrameStarved;
|
|
49
|
+
private audioStallEvents;
|
|
50
|
+
private isAudioBufferStalled;
|
|
51
|
+
private playbackEndReason;
|
|
52
|
+
private receivedAudioBytes;
|
|
53
|
+
private receivedAnimationFrames;
|
|
49
54
|
/**
|
|
50
55
|
* Whether this round's final animation batch (ServerResponseAnimation.end) has arrived.
|
|
51
56
|
* Frame starvation is only possible BEFORE this — once all frames are in, any remaining
|
|
@@ -11739,7 +11739,7 @@ class ClockSync {
|
|
|
11739
11739
|
this.monoAtCalibrate = best.monoAtCalibrate;
|
|
11740
11740
|
this.calibrated = true;
|
|
11741
11741
|
if (firstCalibration) this.fireReady();
|
|
11742
|
-
|
|
11742
|
+
logMetric("time_calibrated", "info", {
|
|
11743
11743
|
has_server: true,
|
|
11744
11744
|
rtt_ms: Math.round(best.rttNet)
|
|
11745
11745
|
});
|
|
@@ -11754,7 +11754,7 @@ class ClockSync {
|
|
|
11754
11754
|
this.serverBase = null;
|
|
11755
11755
|
this.calibrated = true;
|
|
11756
11756
|
this.fireReady();
|
|
11757
|
-
|
|
11757
|
+
logMetric("time_calibrated", "info", { has_server: false });
|
|
11758
11758
|
} finally {
|
|
11759
11759
|
this.syncing = false;
|
|
11760
11760
|
}
|
|
@@ -11811,6 +11811,14 @@ class ClockSync {
|
|
|
11811
11811
|
}
|
|
11812
11812
|
}
|
|
11813
11813
|
const clockSync = new ClockSync();
|
|
11814
|
+
const FILTERED_HOSTNAMES = ["localhost", "127.0.0.1", "0.0.0.0"];
|
|
11815
|
+
function shouldFilterHostname() {
|
|
11816
|
+
if (typeof window === "undefined") {
|
|
11817
|
+
return false;
|
|
11818
|
+
}
|
|
11819
|
+
const hostname = window.location.hostname;
|
|
11820
|
+
return FILTERED_HOSTNAMES.includes(hostname);
|
|
11821
|
+
}
|
|
11814
11822
|
const MARK_KEY = "__mark";
|
|
11815
11823
|
function monoTimestamp() {
|
|
11816
11824
|
return { [MARK_KEY]: "mono", value: performance.now() };
|
|
@@ -11825,14 +11833,34 @@ function resolveMark(mark) {
|
|
|
11825
11833
|
}
|
|
11826
11834
|
}
|
|
11827
11835
|
function resolveMarks(contents) {
|
|
11828
|
-
|
|
11829
|
-
|
|
11830
|
-
|
|
11831
|
-
|
|
11832
|
-
|
|
11836
|
+
return resolveValue(contents);
|
|
11837
|
+
}
|
|
11838
|
+
function resolveValue(value) {
|
|
11839
|
+
if (isMark(value)) return resolveMark(value);
|
|
11840
|
+
if (Array.isArray(value)) {
|
|
11841
|
+
let out = null;
|
|
11842
|
+
for (let i2 = 0; i2 < value.length; i2++) {
|
|
11843
|
+
const resolved = resolveValue(value[i2]);
|
|
11844
|
+
if (resolved !== value[i2]) {
|
|
11845
|
+
if (!out) out = [...value];
|
|
11846
|
+
out[i2] = resolved;
|
|
11847
|
+
}
|
|
11833
11848
|
}
|
|
11849
|
+
return out ?? value;
|
|
11834
11850
|
}
|
|
11835
|
-
|
|
11851
|
+
if (typeof value === "object" && value !== null) {
|
|
11852
|
+
const obj = value;
|
|
11853
|
+
let out = null;
|
|
11854
|
+
for (const [k2, v2] of Object.entries(obj)) {
|
|
11855
|
+
const resolved = resolveValue(v2);
|
|
11856
|
+
if (resolved !== v2) {
|
|
11857
|
+
if (!out) out = { ...obj };
|
|
11858
|
+
out[k2] = resolved;
|
|
11859
|
+
}
|
|
11860
|
+
}
|
|
11861
|
+
return out ?? value;
|
|
11862
|
+
}
|
|
11863
|
+
return value;
|
|
11836
11864
|
}
|
|
11837
11865
|
const OTEL_LOGGER_NAME = "spatius-avatarkit";
|
|
11838
11866
|
let sdkVersion$1 = "1.0.0";
|
|
@@ -11954,6 +11982,7 @@ function flushEventQueue$1() {
|
|
|
11954
11982
|
eventQueue$1.length = 0;
|
|
11955
11983
|
}
|
|
11956
11984
|
function trackEventOtel(event, level = "info", contents = {}) {
|
|
11985
|
+
if (shouldFilterHostname()) return;
|
|
11957
11986
|
if (!clockSync.isReady()) {
|
|
11958
11987
|
preCalibrationQueue.push({ event, level, contents, enqueueMono: performance.now() });
|
|
11959
11988
|
return;
|
|
@@ -12095,14 +12124,6 @@ function isPostHogEnabled() {
|
|
|
12095
12124
|
const SDK_POSTHOG_INSTANCE_NAME = "spatius-posthog";
|
|
12096
12125
|
let sdkPosthogInstance = null;
|
|
12097
12126
|
const eventQueue = [];
|
|
12098
|
-
const FILTERED_HOSTNAMES = ["localhost", "127.0.0.1", "0.0.0.0"];
|
|
12099
|
-
function shouldFilterHostname() {
|
|
12100
|
-
if (typeof window === "undefined") {
|
|
12101
|
-
return false;
|
|
12102
|
-
}
|
|
12103
|
-
const hostname = window.location.hostname;
|
|
12104
|
-
return FILTERED_HOSTNAMES.includes(hostname);
|
|
12105
|
-
}
|
|
12106
12127
|
function getCommonFields() {
|
|
12107
12128
|
const logContext = idManager.getLogContext();
|
|
12108
12129
|
return {
|
|
@@ -12300,28 +12321,39 @@ function cleanupPostHog() {
|
|
|
12300
12321
|
sdkPosthogInstance = null;
|
|
12301
12322
|
}
|
|
12302
12323
|
}
|
|
12303
|
-
|
|
12324
|
+
const TELEMETRY_LOG_METHOD = {
|
|
12325
|
+
debug: "log",
|
|
12326
|
+
info: "log",
|
|
12327
|
+
warning: "warn",
|
|
12328
|
+
error: "error"
|
|
12329
|
+
};
|
|
12330
|
+
function buildTelemetryContext(event, level, contents) {
|
|
12304
12331
|
const sessionToken = idManager.getSessionToken() ?? "";
|
|
12305
12332
|
const context2 = {
|
|
12306
12333
|
session_token_suffix: sessionToken ? sessionToken.slice(-8) : "",
|
|
12307
12334
|
...contents
|
|
12308
12335
|
};
|
|
12309
|
-
const
|
|
12310
|
-
debug: "log",
|
|
12311
|
-
info: "log",
|
|
12312
|
-
warning: "warn",
|
|
12313
|
-
error: "error"
|
|
12314
|
-
};
|
|
12315
|
-
const logMethod = logLevels[level] ?? "log";
|
|
12336
|
+
const logMethod = TELEMETRY_LOG_METHOD[level] ?? "log";
|
|
12316
12337
|
const propsDescription = Object.entries(contents).map(([k2, v2]) => `${k2}=${v2}`).join(", ");
|
|
12317
12338
|
logger[logMethod](`[Telemetry] ${event} [${propsDescription}]`);
|
|
12318
|
-
|
|
12339
|
+
return context2;
|
|
12340
|
+
}
|
|
12341
|
+
function emitToOtel(event, level, context2) {
|
|
12319
12342
|
try {
|
|
12320
12343
|
trackEventOtel(event, level, context2);
|
|
12321
12344
|
} catch (error) {
|
|
12322
12345
|
logger.warn("[OTel] Dual-send failed:", error instanceof Error ? error.message : String(error));
|
|
12323
12346
|
}
|
|
12324
12347
|
}
|
|
12348
|
+
function logEvent(event, level = "info", contents = {}) {
|
|
12349
|
+
const context2 = buildTelemetryContext(event, level, contents);
|
|
12350
|
+
trackEvent(event, level, context2);
|
|
12351
|
+
emitToOtel(event, level, context2);
|
|
12352
|
+
}
|
|
12353
|
+
function logMetric(event, level = "info", contents = {}) {
|
|
12354
|
+
const context2 = buildTelemetryContext(event, level, contents);
|
|
12355
|
+
emitToOtel(event, level, context2);
|
|
12356
|
+
}
|
|
12325
12357
|
const _LogSink = class _LogSink {
|
|
12326
12358
|
constructor() {
|
|
12327
12359
|
__publicField(this, "db", null);
|
|
@@ -12823,6 +12855,8 @@ const _AnimationPlayer = class _AnimationPlayer {
|
|
|
12823
12855
|
__publicField(this, "_isPlaying", false);
|
|
12824
12856
|
__publicField(this, "fps", APP_CONFIG.animation.fps);
|
|
12825
12857
|
__publicField(this, "onEndedCallback");
|
|
12858
|
+
// 音频缓冲卡顿回调(透传给 streamingPlayer;控制器注册,用于并入 playback_quality)
|
|
12859
|
+
__publicField(this, "onAudioStallCallback");
|
|
12826
12860
|
__publicField(this, "useStreaming", false);
|
|
12827
12861
|
}
|
|
12828
12862
|
/**
|
|
@@ -12906,7 +12940,7 @@ const _AnimationPlayer = class _AnimationPlayer {
|
|
|
12906
12940
|
if (this.streamingPlayer) {
|
|
12907
12941
|
return;
|
|
12908
12942
|
}
|
|
12909
|
-
const { StreamingAudioPlayer } = await import("./StreamingAudioPlayer-
|
|
12943
|
+
const { StreamingAudioPlayer } = await import("./StreamingAudioPlayer-CF0Yrsmz.js");
|
|
12910
12944
|
const { AvatarSDK: AvatarSDK2 } = await Promise.resolve().then(() => AvatarSDK$1);
|
|
12911
12945
|
const audioFormat = AvatarSDK2.getAudioFormat();
|
|
12912
12946
|
this.streamingPlayer = new StreamingAudioPlayer({
|
|
@@ -12945,6 +12979,22 @@ const _AnimationPlayer = class _AnimationPlayer {
|
|
|
12945
12979
|
this._isPlaying = false;
|
|
12946
12980
|
(_a = this.onEndedCallback) == null ? void 0 : _a.call(this);
|
|
12947
12981
|
});
|
|
12982
|
+
this.streamingPlayer.onAudioStall((stalled) => {
|
|
12983
|
+
var _a;
|
|
12984
|
+
(_a = this.onAudioStallCallback) == null ? void 0 : _a.call(this, stalled);
|
|
12985
|
+
});
|
|
12986
|
+
}
|
|
12987
|
+
/**
|
|
12988
|
+
* 注册音频缓冲卡顿回调,透传给底层 streamingPlayer(若已就绪则立即接上)。
|
|
12989
|
+
* @internal
|
|
12990
|
+
*/
|
|
12991
|
+
onAudioStall(callback) {
|
|
12992
|
+
var _a;
|
|
12993
|
+
this.onAudioStallCallback = callback;
|
|
12994
|
+
(_a = this.streamingPlayer) == null ? void 0 : _a.onAudioStall((stalled) => {
|
|
12995
|
+
var _a2;
|
|
12996
|
+
(_a2 = this.onAudioStallCallback) == null ? void 0 : _a2.call(this, stalled);
|
|
12997
|
+
});
|
|
12948
12998
|
}
|
|
12949
12999
|
setupEventListeners() {
|
|
12950
13000
|
if (!this.audio)
|
|
@@ -15030,7 +15080,7 @@ class AvatarSDK {
|
|
|
15030
15080
|
__publicField(AvatarSDK, "_initializationState", "uninitialized");
|
|
15031
15081
|
__publicField(AvatarSDK, "_initializingPromise", null);
|
|
15032
15082
|
__publicField(AvatarSDK, "_configuration", null);
|
|
15033
|
-
__publicField(AvatarSDK, "_version", "1.3.1-beta.
|
|
15083
|
+
__publicField(AvatarSDK, "_version", "1.3.1-beta.3");
|
|
15034
15084
|
__publicField(AvatarSDK, "_avatarCore", null);
|
|
15035
15085
|
__publicField(AvatarSDK, "_cachedDeviceScore", null);
|
|
15036
15086
|
__publicField(AvatarSDK, "_rendererBackend", null);
|
|
@@ -15671,7 +15721,6 @@ class NetworkLayer {
|
|
|
15671
15721
|
}
|
|
15672
15722
|
if (this.isFallbackMode) {
|
|
15673
15723
|
if (this.currentConversationId) {
|
|
15674
|
-
logger.warn("[NetworkLayer] Fallback mode: skipping WebSocket send, triggering fallback");
|
|
15675
15724
|
this.dataController.yieldKeyframes([], this.currentConversationId);
|
|
15676
15725
|
}
|
|
15677
15726
|
return;
|
|
@@ -15899,26 +15948,24 @@ class NetworkLayer {
|
|
|
15899
15948
|
this.audioMetrics = this.createAudioMetrics();
|
|
15900
15949
|
}
|
|
15901
15950
|
/**
|
|
15902
|
-
*
|
|
15903
|
-
* 整轮播放结束(进入 idle)时由 AvatarController
|
|
15951
|
+
* 取本轮端到端延迟打点(tap_N=第N秒音频送入时刻,anim_N=第N组动画到达时刻)。
|
|
15952
|
+
* 整轮播放结束(进入 idle)时由 AvatarController 读取,并入 playback_quality 的
|
|
15953
|
+
* latency 成员统一上报(不再单发 driving_service_latency)。
|
|
15954
|
+
* 无打点数据(如未走 direct 驱动)时返回 null,playback_quality 不带 latency。
|
|
15904
15955
|
* @internal
|
|
15905
15956
|
*/
|
|
15906
|
-
|
|
15907
|
-
|
|
15908
|
-
if (!conversationId) return;
|
|
15957
|
+
getDrivingServiceLatency() {
|
|
15958
|
+
if (!this.currentConversationId) return null;
|
|
15909
15959
|
const metrics = this.audioMetrics;
|
|
15910
|
-
if (metrics.taps.size === 0 || metrics.anims.size === 0) return;
|
|
15911
|
-
const props = {
|
|
15912
|
-
conversation_id: conversationId,
|
|
15913
|
-
dsm: "direct"
|
|
15914
|
-
};
|
|
15960
|
+
if (metrics.taps.size === 0 || metrics.anims.size === 0) return null;
|
|
15961
|
+
const props = {};
|
|
15915
15962
|
for (const [n2, t2] of metrics.taps) {
|
|
15916
15963
|
props[`tap_${n2}`] = t2;
|
|
15917
15964
|
}
|
|
15918
15965
|
for (const [n2, t2] of metrics.anims) {
|
|
15919
15966
|
props[`anim_${n2}`] = t2;
|
|
15920
15967
|
}
|
|
15921
|
-
|
|
15968
|
+
return props;
|
|
15922
15969
|
}
|
|
15923
15970
|
/**
|
|
15924
15971
|
* Start driving service heartbeat check (check connection status every 2 minutes)
|
|
@@ -16215,8 +16262,23 @@ class AvatarController {
|
|
|
16215
16262
|
__publicField(this, "isFallbackMode", false);
|
|
16216
16263
|
// 降级模式标志
|
|
16217
16264
|
// ========== Frame Starvation Tracking ==========
|
|
16265
|
+
// enterMono:进入卡顿那刻的单调读数(performance.now()),恢复时用来算 durationMs。
|
|
16266
|
+
// durationMs:这次卡顿持续多久(ms),帧恢复时回填;卡到播放结束都没恢复则为 null。
|
|
16267
|
+
// startTs:卡顿开始的时间线时刻(clockSync.timelineNow(),优先后台、退本地,与 tap/anim 同轴)。
|
|
16268
|
+
// framesAtEnter / framesAtRecover:卡顿开始/恢复时本轮已收到的服务端动画帧数
|
|
16269
|
+
// (差值 = 卡顿期间到帧量,区分"服务端没发"与"发了但不够消费")。未恢复则 framesAtRecover=null。
|
|
16218
16270
|
__publicField(this, "frameStarvationEvents", []);
|
|
16219
16271
|
__publicField(this, "isFrameStarved", false);
|
|
16272
|
+
// ========== Audio Buffer Stall Tracking ==========
|
|
16273
|
+
// 音频缓冲卡顿(缓冲耗尽自动暂停、有数据自动恢复),与帧饥饿对称记录。字段同义。
|
|
16274
|
+
__publicField(this, "audioStallEvents", []);
|
|
16275
|
+
__publicField(this, "isAudioBufferStalled", false);
|
|
16276
|
+
// 本轮播放的结束原因(供 playback_quality 上报):completed=正常播完,interrupted=中途打断。
|
|
16277
|
+
__publicField(this, "playbackEndReason", "completed");
|
|
16278
|
+
// 本轮已接收的音频总字节数(两模式统一累计)。总时长 = /audioBytesPerSecond。每轮重置。
|
|
16279
|
+
__publicField(this, "receivedAudioBytes", 0);
|
|
16280
|
+
// 本轮已接收的服务端下发动画帧总数(不含本地起播过渡帧)。每轮重置。
|
|
16281
|
+
__publicField(this, "receivedAnimationFrames", 0);
|
|
16220
16282
|
/**
|
|
16221
16283
|
* Whether this round's final animation batch (ServerResponseAnimation.end) has arrived.
|
|
16222
16284
|
* Frame starvation is only possible BEFORE this — once all frames are in, any remaining
|
|
@@ -16397,42 +16459,100 @@ class AvatarController {
|
|
|
16397
16459
|
/**
|
|
16398
16460
|
* Notify conversation state change and handle telemetry
|
|
16399
16461
|
* @internal
|
|
16400
|
-
* @param isFallback - true when entering fallback mode (suppresses playback stats)
|
|
16401
16462
|
*/
|
|
16402
|
-
notifyConversationState(avatarState
|
|
16403
|
-
var _a
|
|
16463
|
+
notifyConversationState(avatarState) {
|
|
16464
|
+
var _a;
|
|
16404
16465
|
switch (avatarState) {
|
|
16405
16466
|
case AvatarState.playing:
|
|
16406
16467
|
this.frameRateMonitor.resetPlaybackStats();
|
|
16407
|
-
this.shouldReportPlaybackStats =
|
|
16468
|
+
this.shouldReportPlaybackStats = true;
|
|
16408
16469
|
break;
|
|
16409
16470
|
case AvatarState.paused:
|
|
16410
16471
|
this.shouldReportPlaybackStats = false;
|
|
16411
16472
|
break;
|
|
16412
16473
|
case AvatarState.idle:
|
|
16413
|
-
(_a = this.networkLayer) == null ? void 0 : _a.reportDrivingServiceLatency();
|
|
16414
16474
|
if (this.shouldReportPlaybackStats) {
|
|
16415
|
-
|
|
16416
|
-
const bypassSampling = typeof window !== "undefined" && new URLSearchParams(window.location.search).get("integration") === "1";
|
|
16417
|
-
if (stats.durationMs >= 2e3 && stats.frameCount > 0 && (bypassSampling || Math.random() < 0.3)) {
|
|
16418
|
-
const props = {
|
|
16419
|
-
avg_fps: Number(stats.avgFps.toFixed(1)),
|
|
16420
|
-
frame_count: stats.frameCount,
|
|
16421
|
-
duration_ms: stats.durationMs,
|
|
16422
|
-
jank_ratio: Number(stats.jankRatio.toFixed(2)),
|
|
16423
|
-
fps_cv: Number(stats.fpsCv.toFixed(3)),
|
|
16424
|
-
...this._getDeviceScoreProps()
|
|
16425
|
-
};
|
|
16426
|
-
if (AvatarSDK.getRendererBackend()) {
|
|
16427
|
-
props.renderer_backend = AvatarSDK.getRendererBackend();
|
|
16428
|
-
}
|
|
16429
|
-
logEvent("avatar_playback_fps", "info", props);
|
|
16430
|
-
}
|
|
16475
|
+
this.reportPlaybackQuality();
|
|
16431
16476
|
}
|
|
16432
16477
|
this.shouldReportPlaybackStats = false;
|
|
16433
16478
|
break;
|
|
16434
16479
|
}
|
|
16435
|
-
(
|
|
16480
|
+
(_a = this.onConversationState) == null ? void 0 : _a.call(this, this.mapToConversationState(avatarState));
|
|
16481
|
+
}
|
|
16482
|
+
/**
|
|
16483
|
+
* 上报本轮播放质量(playback_quality)。每轮播放结束都发一次(不采样、不设时长门槛),
|
|
16484
|
+
* 把帧率与卡顿两块合并为嵌套成员。end_reason 区分完整播完 / 中途打断。
|
|
16485
|
+
* 有卡顿时用 warning,否则 info(保留卡顿的告警语义)。
|
|
16486
|
+
* @internal
|
|
16487
|
+
*/
|
|
16488
|
+
reportPlaybackQuality() {
|
|
16489
|
+
var _a, _b;
|
|
16490
|
+
const stats = this.frameRateMonitor.getPlaybackStats();
|
|
16491
|
+
const settled = this.frameStarvationEvents.map((e2) => e2.durationMs).filter((d2) => d2 !== null);
|
|
16492
|
+
const totalStallMs = settled.reduce((a2, b2) => a2 + b2, 0);
|
|
16493
|
+
const stall = {
|
|
16494
|
+
count: this.frameStarvationEvents.length,
|
|
16495
|
+
max_ms: settled.length > 0 ? Math.max(...settled) : 0,
|
|
16496
|
+
total_ms: totalStallMs,
|
|
16497
|
+
// 卡顿率 = 卡顿总时长 / 已播放时长(fps.duration_ms;打断轮即播到打断那刻,上限音频时长)
|
|
16498
|
+
ratio: stats.durationMs > 0 ? Number((totalStallMs / stats.durationMs).toFixed(4)) : 0,
|
|
16499
|
+
req_end: this.frameStarvationEvents.some((e2) => e2.reqEnd),
|
|
16500
|
+
// 每次卡顿明细:开始时间戳 / 卡顿时的音频时刻(ms) / 等了多久(null=没恢复) / 卡顿开始与恢复时的动画帧数
|
|
16501
|
+
events: this.frameStarvationEvents.map((e2) => ({
|
|
16502
|
+
start_ts: Math.round(e2.startTs),
|
|
16503
|
+
audio_time_ms: Math.round(e2.audioTime * 1e3),
|
|
16504
|
+
duration_ms: e2.durationMs,
|
|
16505
|
+
frames_at_enter: e2.framesAtEnter,
|
|
16506
|
+
frames_at_recover: e2.framesAtRecover
|
|
16507
|
+
}))
|
|
16508
|
+
};
|
|
16509
|
+
const audioSettled = this.audioStallEvents.map((e2) => e2.durationMs).filter((d2) => d2 !== null);
|
|
16510
|
+
const audioStall = {
|
|
16511
|
+
count: this.audioStallEvents.length,
|
|
16512
|
+
max_ms: audioSettled.length > 0 ? Math.max(...audioSettled) : 0,
|
|
16513
|
+
total_ms: audioSettled.reduce((a2, b2) => a2 + b2, 0),
|
|
16514
|
+
events: this.audioStallEvents.map((e2) => ({
|
|
16515
|
+
start_ts: Math.round(e2.startTs),
|
|
16516
|
+
duration_ms: e2.durationMs,
|
|
16517
|
+
frames_at_enter: e2.framesAtEnter,
|
|
16518
|
+
frames_at_recover: e2.framesAtRecover
|
|
16519
|
+
}))
|
|
16520
|
+
};
|
|
16521
|
+
const fps = {
|
|
16522
|
+
avg_fps: Number(stats.avgFps.toFixed(1)),
|
|
16523
|
+
frame_count: stats.frameCount,
|
|
16524
|
+
duration_ms: stats.durationMs,
|
|
16525
|
+
jank_ratio: Number(stats.jankRatio.toFixed(2)),
|
|
16526
|
+
fps_cv: Number(stats.fpsCv.toFixed(3)),
|
|
16527
|
+
...this._getDeviceScoreProps()
|
|
16528
|
+
};
|
|
16529
|
+
if (AvatarSDK.getRendererBackend()) {
|
|
16530
|
+
fps.renderer_backend = AvatarSDK.getRendererBackend();
|
|
16531
|
+
}
|
|
16532
|
+
const latency = ((_a = this.networkLayer) == null ? void 0 : _a.getDrivingServiceLatency()) ?? null;
|
|
16533
|
+
logMetric("playback_quality", stall.count > 0 || audioStall.count > 0 ? "warning" : "info", {
|
|
16534
|
+
avatar_id: this.avatar.id,
|
|
16535
|
+
// backend 模式无 networkLayer,conversation_id 存于本地 currentConversationId;
|
|
16536
|
+
// 对齐 playback_ended 的兜底顺序,避免 backend 上报 conversation_id=undefined。
|
|
16537
|
+
conversation_id: this.currentConversationId || ((_b = this.networkLayer) == null ? void 0 : _b.getCurrentConversationId()) || void 0,
|
|
16538
|
+
end_reason: this.playbackEndReason,
|
|
16539
|
+
// 是否降级模式(audio-only 无动画帧):为 true 时 fps/animation_frame_count 空是预期
|
|
16540
|
+
is_fallback: this.isFallbackMode,
|
|
16541
|
+
// 本轮接收情况:总音频时长(ms) / 是否收到最后音频(end=true) /
|
|
16542
|
+
// 服务端下发总动画帧数(不含过渡帧) / 是否收到最后动画包(end=true)
|
|
16543
|
+
audio_duration_ms: Math.round(this.receivedAudioBytes / this.audioBytesPerSecond * 1e3),
|
|
16544
|
+
audio_ended: this.reqEnd,
|
|
16545
|
+
animation_frame_count: this.receivedAnimationFrames,
|
|
16546
|
+
animation_ended: this.animationEnded,
|
|
16547
|
+
fps: JSON.stringify(fps),
|
|
16548
|
+
stall: JSON.stringify(stall),
|
|
16549
|
+
audio_stall: JSON.stringify(audioStall),
|
|
16550
|
+
...latency ? { latency: JSON.stringify(resolveMarks(latency)) } : {}
|
|
16551
|
+
});
|
|
16552
|
+
this.frameStarvationEvents = [];
|
|
16553
|
+
this.isFrameStarved = false;
|
|
16554
|
+
this.audioStallEvents = [];
|
|
16555
|
+
this.isAudioBufferStalled = false;
|
|
16436
16556
|
}
|
|
16437
16557
|
/**
|
|
16438
16558
|
* Get animation player instance
|
|
@@ -16552,6 +16672,7 @@ class AvatarController {
|
|
|
16552
16672
|
if (this.reqEnd && this.isPlaying && networkConversationId) {
|
|
16553
16673
|
this.interrupt();
|
|
16554
16674
|
}
|
|
16675
|
+
this.receivedAudioBytes += audioData.byteLength;
|
|
16555
16676
|
this.addAudioChunkToBuffer(new Uint8Array(audioData), end);
|
|
16556
16677
|
this.networkLayer.sendAudioData(audioData, end);
|
|
16557
16678
|
if (end) {
|
|
@@ -16651,6 +16772,7 @@ class AvatarController {
|
|
|
16651
16772
|
if (isLast) {
|
|
16652
16773
|
this.reqEnd = true;
|
|
16653
16774
|
}
|
|
16775
|
+
this.receivedAudioBytes += data.length;
|
|
16654
16776
|
if (this.playbackMode === DrivingServiceMode.backend) {
|
|
16655
16777
|
const metrics = this.hostModeMetrics;
|
|
16656
16778
|
if (metrics.startTimestamp === 0) {
|
|
@@ -16779,6 +16901,7 @@ class AvatarController {
|
|
|
16779
16901
|
return;
|
|
16780
16902
|
}
|
|
16781
16903
|
const flameKeyframes = keyframes;
|
|
16904
|
+
this.receivedAnimationFrames += flameKeyframes.length;
|
|
16782
16905
|
if (this.currentKeyframes.length === 0) {
|
|
16783
16906
|
this.currentKeyframes = flameKeyframes;
|
|
16784
16907
|
if (this.playbackMode === DrivingServiceMode.backend && !this.hostModeMetrics.didRecvFirstFlame) {
|
|
@@ -16919,6 +17042,8 @@ class AvatarController {
|
|
|
16919
17042
|
this.lastSyncLogTime = 0;
|
|
16920
17043
|
this.lastOutOfBoundsState = false;
|
|
16921
17044
|
this.animationEnded = false;
|
|
17045
|
+
this.receivedAudioBytes = 0;
|
|
17046
|
+
this.receivedAnimationFrames = 0;
|
|
16922
17047
|
if (this.isAudioStalledForStarvation) {
|
|
16923
17048
|
this.isAudioStalledForStarvation = false;
|
|
16924
17049
|
(_a = this.onPlaybackStall) == null ? void 0 : _a.call(this, false);
|
|
@@ -17111,6 +17236,9 @@ class AvatarController {
|
|
|
17111
17236
|
await this.animationPlayer.prepareStreamingPlayer(() => {
|
|
17112
17237
|
this.onPlaybackEnded();
|
|
17113
17238
|
});
|
|
17239
|
+
this.animationPlayer.onAudioStall((stalled) => {
|
|
17240
|
+
this.trackAudioBufferStall(stalled);
|
|
17241
|
+
});
|
|
17114
17242
|
if (!this.isFallbackMode) {
|
|
17115
17243
|
this.emit("startRendering");
|
|
17116
17244
|
}
|
|
@@ -17128,11 +17256,12 @@ class AvatarController {
|
|
|
17128
17256
|
this.pendingAudioChunks = [];
|
|
17129
17257
|
this.isPlaying = true;
|
|
17130
17258
|
this.currentState = AvatarState.playing;
|
|
17131
|
-
this.notifyConversationState(AvatarState.playing
|
|
17259
|
+
this.notifyConversationState(AvatarState.playing);
|
|
17132
17260
|
this.isStartingPlayback = false;
|
|
17133
17261
|
logEvent("playback_started", "info", {
|
|
17134
17262
|
avatar_id: this.avatar.id,
|
|
17135
|
-
|
|
17263
|
+
// backend 模式无 networkLayer,取本地 currentConversationId 兜底(同 playback_ended/quality)
|
|
17264
|
+
conversation_id: this.currentConversationId || ((_a = this.networkLayer) == null ? void 0 : _a.getCurrentConversationId()) || void 0
|
|
17136
17265
|
});
|
|
17137
17266
|
} catch (error) {
|
|
17138
17267
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -17201,7 +17330,7 @@ class AvatarController {
|
|
|
17201
17330
|
state.reported = true;
|
|
17202
17331
|
logEvent("playback_stuck_after_transition", "error", {
|
|
17203
17332
|
avatar_id: this.avatar.id,
|
|
17204
|
-
conversation_id: ((_e2 = this.networkLayer) == null ? void 0 : _e2.getCurrentConversationId()) || void 0,
|
|
17333
|
+
conversation_id: this.currentConversationId || ((_e2 = this.networkLayer) == null ? void 0 : _e2.getCurrentConversationId()) || void 0,
|
|
17205
17334
|
// 诊断信息(包含 audio_context_state 用于诊断,但不作为触发条件)
|
|
17206
17335
|
audio_context_state: audioContextState,
|
|
17207
17336
|
audio_time: audioTime,
|
|
@@ -17271,13 +17400,28 @@ class AvatarController {
|
|
|
17271
17400
|
const totalKeyframes = this.currentKeyframes.length + this.keyframesOffset;
|
|
17272
17401
|
const frameDiff = frameIndex - totalKeyframes;
|
|
17273
17402
|
if (!(this.reqEnd && frameDiff <= 1)) {
|
|
17274
|
-
this.frameStarvationEvents.push({
|
|
17403
|
+
this.frameStarvationEvents.push({
|
|
17404
|
+
audioTime,
|
|
17405
|
+
reqEnd: this.reqEnd,
|
|
17406
|
+
enterMono: performance.now(),
|
|
17407
|
+
durationMs: null,
|
|
17408
|
+
startTs: clockSync.timelineNow(),
|
|
17409
|
+
framesAtEnter: this.receivedAnimationFrames,
|
|
17410
|
+
framesAtRecover: null
|
|
17411
|
+
});
|
|
17275
17412
|
}
|
|
17276
17413
|
}
|
|
17277
17414
|
if (!this.animationEnded) {
|
|
17278
17415
|
this.pauseAudioForStarvation();
|
|
17279
17416
|
}
|
|
17280
17417
|
} else {
|
|
17418
|
+
if (this.isFrameStarved) {
|
|
17419
|
+
const last = this.frameStarvationEvents[this.frameStarvationEvents.length - 1];
|
|
17420
|
+
if (last && last.durationMs === null) {
|
|
17421
|
+
last.durationMs = Math.round(performance.now() - last.enterMono);
|
|
17422
|
+
last.framesAtRecover = this.receivedAnimationFrames;
|
|
17423
|
+
}
|
|
17424
|
+
}
|
|
17281
17425
|
this.isFrameStarved = false;
|
|
17282
17426
|
if (isOutOfBounds !== this.lastOutOfBoundsState) {
|
|
17283
17427
|
this.lastOutOfBoundsState = isOutOfBounds;
|
|
@@ -17285,6 +17429,33 @@ class AvatarController {
|
|
|
17285
17429
|
this.resumeAudioFromStarvation();
|
|
17286
17430
|
}
|
|
17287
17431
|
}
|
|
17432
|
+
/**
|
|
17433
|
+
* 记录音频缓冲卡顿(缓冲耗尽自动暂停 / 有数据自动恢复),并入 playback_quality.audio_stall。
|
|
17434
|
+
* 与 trackFrameStarvation 对称:进入 push 事件、恢复回填时长与动画帧数。
|
|
17435
|
+
* @param stalled true=进入卡顿,false=恢复
|
|
17436
|
+
* @internal
|
|
17437
|
+
*/
|
|
17438
|
+
trackAudioBufferStall(stalled) {
|
|
17439
|
+
if (stalled) {
|
|
17440
|
+
if (this.isAudioBufferStalled) return;
|
|
17441
|
+
this.isAudioBufferStalled = true;
|
|
17442
|
+
this.audioStallEvents.push({
|
|
17443
|
+
enterMono: performance.now(),
|
|
17444
|
+
durationMs: null,
|
|
17445
|
+
startTs: clockSync.timelineNow(),
|
|
17446
|
+
framesAtEnter: this.receivedAnimationFrames,
|
|
17447
|
+
framesAtRecover: null
|
|
17448
|
+
});
|
|
17449
|
+
} else {
|
|
17450
|
+
if (!this.isAudioBufferStalled) return;
|
|
17451
|
+
this.isAudioBufferStalled = false;
|
|
17452
|
+
const last = this.audioStallEvents[this.audioStallEvents.length - 1];
|
|
17453
|
+
if (last && last.durationMs === null) {
|
|
17454
|
+
last.durationMs = Math.round(performance.now() - last.enterMono);
|
|
17455
|
+
last.framesAtRecover = this.receivedAnimationFrames;
|
|
17456
|
+
}
|
|
17457
|
+
}
|
|
17458
|
+
}
|
|
17288
17459
|
/**
|
|
17289
17460
|
* strictSync: pause audio on frame starvation, waiting for new frames.
|
|
17290
17461
|
*
|
|
@@ -17360,7 +17531,6 @@ class AvatarController {
|
|
|
17360
17531
|
if (this.isFallbackMode) {
|
|
17361
17532
|
return;
|
|
17362
17533
|
}
|
|
17363
|
-
logger.warn("[AvatarController] Enabling fallback mode");
|
|
17364
17534
|
this.isFallbackMode = true;
|
|
17365
17535
|
this.resumeAudioFromStarvation();
|
|
17366
17536
|
logEvent("fallback_mode_entered", "warning", {
|
|
@@ -17394,26 +17564,15 @@ class AvatarController {
|
|
|
17394
17564
|
* @internal
|
|
17395
17565
|
*/
|
|
17396
17566
|
onPlaybackEnded() {
|
|
17397
|
-
var _a
|
|
17398
|
-
if (this.frameStarvationEvents.length > 0) {
|
|
17399
|
-
const hasReqEnd = this.frameStarvationEvents.some((e2) => e2.reqEnd);
|
|
17400
|
-
logEvent("frame_starvation", "warning", {
|
|
17401
|
-
avatar_id: this.avatar.id,
|
|
17402
|
-
conversation_id: ((_a = this.networkLayer) == null ? void 0 : _a.getCurrentConversationId()) || void 0,
|
|
17403
|
-
starvation_count: this.frameStarvationEvents.length,
|
|
17404
|
-
starvation_times: this.frameStarvationEvents.map((e2) => Number(e2.audioTime.toFixed(3))),
|
|
17405
|
-
...hasReqEnd ? { req_end: true } : {}
|
|
17406
|
-
});
|
|
17407
|
-
}
|
|
17408
|
-
this.frameStarvationEvents = [];
|
|
17409
|
-
this.isFrameStarved = false;
|
|
17567
|
+
var _a;
|
|
17410
17568
|
logEvent("playback_ended", "info", {
|
|
17411
17569
|
avatar_id: this.avatar.id,
|
|
17412
|
-
conversation_id: this.currentConversationId || ((
|
|
17570
|
+
conversation_id: this.currentConversationId || ((_a = this.networkLayer) == null ? void 0 : _a.getCurrentConversationId()) || void 0,
|
|
17413
17571
|
source: this.isFallbackMode ? "fallback" : "streaming"
|
|
17414
17572
|
});
|
|
17415
17573
|
this.isPlaying = false;
|
|
17416
17574
|
this.currentState = AvatarState.idle;
|
|
17575
|
+
this.playbackEndReason = "completed";
|
|
17417
17576
|
this.notifyConversationState(AvatarState.idle);
|
|
17418
17577
|
this.emit("stopRendering");
|
|
17419
17578
|
this.emit("stopFallback");
|
|
@@ -17431,6 +17590,7 @@ class AvatarController {
|
|
|
17431
17590
|
this.emit("stopRendering");
|
|
17432
17591
|
this.isPlaying = false;
|
|
17433
17592
|
this.currentState = AvatarState.idle;
|
|
17593
|
+
this.playbackEndReason = "interrupted";
|
|
17434
17594
|
this.notifyConversationState(AvatarState.idle);
|
|
17435
17595
|
}
|
|
17436
17596
|
/**
|
|
@@ -17808,7 +17968,7 @@ class AvatarDownloader {
|
|
|
17808
17968
|
if (cached) {
|
|
17809
17969
|
const duration = Date.now() - startTime;
|
|
17810
17970
|
logger.log(`✅ Unified template loaded from cache (${(cached.byteLength / 1024 / 1024).toFixed(1)} MB)`);
|
|
17811
|
-
|
|
17971
|
+
logMetric("template_resources_load_measure", "info", {
|
|
17812
17972
|
duration,
|
|
17813
17973
|
file_count: 1,
|
|
17814
17974
|
cache_hit: true,
|
|
@@ -17837,7 +17997,7 @@ class AvatarDownloader {
|
|
|
17837
17997
|
logger.warn(`[loadUnifiedTemplate] Failed to cache:`, err);
|
|
17838
17998
|
});
|
|
17839
17999
|
const duration = Date.now() - startTime;
|
|
17840
|
-
|
|
18000
|
+
logMetric("template_resources_load_measure", "info", {
|
|
17841
18001
|
duration,
|
|
17842
18002
|
file_count: 1,
|
|
17843
18003
|
cache_hit: false,
|
|
@@ -17990,7 +18150,7 @@ class AvatarDownloader {
|
|
|
17990
18150
|
const totalSize = Object.values(characterData).reduce((sum, buffer) => {
|
|
17991
18151
|
return sum + (buffer ? buffer.byteLength : 0);
|
|
17992
18152
|
}, 0);
|
|
17993
|
-
|
|
18153
|
+
logMetric("download_avatar_assets_latency", "info", {
|
|
17994
18154
|
resolution: "default",
|
|
17995
18155
|
// 目前写死 default
|
|
17996
18156
|
avatar_id: characterMeta.characterId ?? "unknown",
|
|
@@ -18184,7 +18344,7 @@ class AvatarDownloader {
|
|
|
18184
18344
|
);
|
|
18185
18345
|
}
|
|
18186
18346
|
const duration = Date.now() - startTime;
|
|
18187
|
-
|
|
18347
|
+
logMetric("fetch_avatar_metadata_latency", "info", {
|
|
18188
18348
|
avatar_id: characterId,
|
|
18189
18349
|
duration,
|
|
18190
18350
|
trace_id: traceId
|
|
@@ -18426,7 +18586,7 @@ const _AvatarManager = class _AvatarManager {
|
|
|
18426
18586
|
this.avatarCache.set(id, avatar);
|
|
18427
18587
|
logger.log("[AvatarManager] Avatar loaded successfully");
|
|
18428
18588
|
const totalDuration = Date.now() - startTime;
|
|
18429
|
-
|
|
18589
|
+
logMetric("fetch_avatar_latency", "info", {
|
|
18430
18590
|
resolution: "default",
|
|
18431
18591
|
// 目前写死 default
|
|
18432
18592
|
avatar_id: id,
|
package/dist/index.js
CHANGED
package/package.json
CHANGED