@xmov/avatar 2.0.0-beta.11 → 2.0.0-beta.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/baseRender/AvatarRenderer.d.ts +0 -1
- package/dist/control/RenderScheduler.d.ts +8 -1
- package/dist/index.cjs +1 -1
- package/dist/index.cjs.map +1 -1
- package/dist/index.modern.js +1 -1
- package/dist/index.modern.js.map +1 -1
- package/dist/index.module.js +1 -1
- package/dist/index.module.js.map +1 -1
- package/dist/index.umd.js +1 -1
- package/dist/index.umd.js.map +1 -1
- package/dist/modules/ResourceManager.d.ts +2 -1
- package/package.json +4 -2
- package/src/baseRender/AudioRenderer.ts +16 -5
- package/src/baseRender/AvatarRenderer.ts +11 -20
- package/src/baseRender/MSEAudioPlayer.ts +18 -7
- package/src/control/RenderScheduler.ts +59 -8
- package/src/index.ts +26 -7
- package/src/modules/ResourceManager.ts +19 -6
- package/src/modules/decoder.ts +34 -1
- package/src/view/DebugOverlay.ts +9 -4
|
@@ -3,7 +3,7 @@ import XmovAvatar from "../index";
|
|
|
3
3
|
/**
|
|
4
4
|
* 资源处理模块,加载并存储资源
|
|
5
5
|
*/
|
|
6
|
-
type TOptions = Pick<IAvatarOptions, "appId" | "appSecret" | "gatewayServer" | 'cacheServer' | "sdkInstance" | "config" | "headers" | "gateway_type" | "tag" | "asr_id" | "llm_id" | "session_speak_req_id"> & {
|
|
6
|
+
type TOptions = Pick<IAvatarOptions, "appId" | "appSecret" | "gatewayServer" | 'cacheServer' | "sdkInstance" | "config" | "headers" | "gateway_type" | "tag" | "asr_id" | "llm_id" | "session_speak_req_id" | "audioOnlyMode"> & {
|
|
7
7
|
sessionRequestData?: Record<string, unknown>;
|
|
8
8
|
onNetworkInfo(quality: INetworkInfo): void;
|
|
9
9
|
onStartSessionWarning?: (message: Object) => void;
|
|
@@ -169,6 +169,7 @@ export default class ResourceManager {
|
|
|
169
169
|
private isProcessingVideo;
|
|
170
170
|
private cacheServer?;
|
|
171
171
|
private startSessionTimer;
|
|
172
|
+
private audioOnlyMode;
|
|
172
173
|
private downloadingVideos;
|
|
173
174
|
first_load: boolean;
|
|
174
175
|
constructor(options: TOptions);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@xmov/avatar",
|
|
3
|
-
"version": "2.0.0-beta.
|
|
3
|
+
"version": "2.0.0-beta.13",
|
|
4
4
|
"description": "魔珐数字人 JS-SDK",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"source": "src/index.ts",
|
|
@@ -28,7 +28,9 @@
|
|
|
28
28
|
"scripts": {
|
|
29
29
|
"build": "microbundle --workers --define ENV=production,VERSION=$npm_package_version --external none --tsconfig tsconfig.json && pnpm run build:agent && node upload-sdk.js build",
|
|
30
30
|
"build:agent": "microbundle --entry src/agent/index.ts --output dist/agent --format modern,esm,cjs,umd --external @xmov/avatar --alias ../index=@xmov/avatar --globals @xmov/avatar=XmovAvatar --name XingyunAvatarAgentModule --tsconfig tsconfig.agent.json && node scripts/build-agent-entry.mjs",
|
|
31
|
-
"sdk": "
|
|
31
|
+
"sdk": "pnpm --filter @xmov/avatar run --parallel --no-bail \"/^sdk:.+$/\"",
|
|
32
|
+
"sdk:core": "microbundle watch --workers --define ENV=development,VERSION=$npm_package_version --external none --tsconfig tsconfig.json",
|
|
33
|
+
"sdk:agent": "node scripts/build-agent-entry.mjs && microbundle watch --entry src/agent/index.ts --output dist/agent --format modern,esm,cjs,umd --external @xmov/avatar --alias ../index=@xmov/avatar --globals @xmov/avatar=XmovAvatar --name XingyunAvatarAgentModule --tsconfig tsconfig.agent.json",
|
|
32
34
|
"app": "cd app/ && npm run dev",
|
|
33
35
|
"app:lan": "pnpm --dir app dev:lan",
|
|
34
36
|
"test:agent": "node scripts/run-agent-tests.mjs",
|
|
@@ -225,19 +225,25 @@ export default class AudioRender {
|
|
|
225
225
|
pause() {
|
|
226
226
|
this.isPlaying = false;
|
|
227
227
|
this.options.dataCacheQueue._clearAudio(this.speech_id);
|
|
228
|
+
this.lastFrameIndex = -1;
|
|
228
229
|
this.firstFrameIndex = -1;
|
|
230
|
+
this.cacheFirstFrameIndex = -1;
|
|
231
|
+
this.cacheAudioData = [];
|
|
232
|
+
this.mseCacheAudioData = [];
|
|
233
|
+
this.oldSpeechId = -1;
|
|
229
234
|
// 停止 PCM 播放
|
|
230
235
|
this.audio?.stop();
|
|
231
|
-
|
|
236
|
+
|
|
232
237
|
// 停止 WebM 播放
|
|
233
238
|
this.mseAudioPlayer?.stop(this.addNewAudioSegment.bind(this));
|
|
234
239
|
this.speech_id = -1;
|
|
235
240
|
}
|
|
236
241
|
|
|
237
242
|
stop(speech_id: number) {
|
|
238
|
-
//
|
|
239
|
-
|
|
240
|
-
|
|
243
|
+
// speech_id === -1 表示强制清理(用于 pauseRender / destroy / 断网重连场景),不受 oldSpeechId 限制
|
|
244
|
+
// 否则如果 oldSpeechId >= speech_id 直接返回,避免重复处理
|
|
245
|
+
if (speech_id !== -1 && this.oldSpeechId >= speech_id) return;
|
|
246
|
+
|
|
241
247
|
this.isPlaying = false;
|
|
242
248
|
this.options.dataCacheQueue._clearAudio(speech_id);
|
|
243
249
|
this.firstFrameIndex = -1;
|
|
@@ -255,7 +261,12 @@ export default class AudioRender {
|
|
|
255
261
|
}
|
|
256
262
|
this.oldSpeechId = speech_id;
|
|
257
263
|
this.speech_id = -1;
|
|
258
|
-
if (
|
|
264
|
+
if (speech_id === -1) {
|
|
265
|
+
// 强制清理:清空所有缓存数据,不推入音频流(用于 pauseRender / destroy / 断网重连)
|
|
266
|
+
this.cacheFirstFrameIndex = -1;
|
|
267
|
+
this.cacheAudioData = [];
|
|
268
|
+
this.mseCacheAudioData = [];
|
|
269
|
+
} else if (this.cacheAudioData.length > 0) {
|
|
259
270
|
this.firstFrameIndex = this.cacheFirstFrameIndex;
|
|
260
271
|
this.cacheFirstFrameIndex = -1;
|
|
261
272
|
// 新的缓存下来的数据,推入音频流
|
|
@@ -19,16 +19,11 @@ type Option = {
|
|
|
19
19
|
resourceManager: ResourceManager;
|
|
20
20
|
saveAndDownload: SaveAndDownload;
|
|
21
21
|
forceSyncDecoder: () => void;
|
|
22
|
-
reportMessage: (message: {
|
|
23
|
-
code: EErrorCode;
|
|
24
|
-
message: string;
|
|
25
|
-
e?: object;
|
|
26
|
-
}) => void;
|
|
22
|
+
reportMessage: (message: { code: EErrorCode; message: string; e?: object }) => void;
|
|
27
23
|
onDownloadProgress: (progress: number) => void;
|
|
28
24
|
onStateChange: (state: string) => void;
|
|
29
25
|
onRenderChange: (state: RenderState) => void;
|
|
30
26
|
sendVideoInfo: (info: { name: string; body_id: number; id: number }) => void;
|
|
31
|
-
onError: (error: any) => void;
|
|
32
27
|
sendSdkPoint: (point: string, data: any) => void;
|
|
33
28
|
};
|
|
34
29
|
export default class AvatarRender {
|
|
@@ -989,13 +984,13 @@ export default class AvatarRender {
|
|
|
989
984
|
} else {
|
|
990
985
|
if(this.lastRealFaceFrame === -1) {
|
|
991
986
|
faceFrame = curRealFaceData;
|
|
992
|
-
// this.options.
|
|
987
|
+
// this.options.reportMessage({
|
|
993
988
|
// code: EErrorCode.RENDER_FACE_ERROR,
|
|
994
989
|
// message: `第${frameIndex}帧 实时面部数据为空,原始数据渲染`,
|
|
995
990
|
// e: JSON.stringify({ bodyFrame, faceFrame }),
|
|
996
991
|
// });
|
|
997
992
|
} else {
|
|
998
|
-
// this.options.
|
|
993
|
+
// this.options.reportMessage({
|
|
999
994
|
// code: EErrorCode.RENDER_FACE_ERROR,
|
|
1000
995
|
// message: `第${frameIndex}帧 实时面部数据为空,插值渲染`,
|
|
1001
996
|
// e: JSON.stringify({ bodyFrame, faceFrame }),
|
|
@@ -1036,30 +1031,26 @@ export default class AvatarRender {
|
|
|
1036
1031
|
// 判断bodyFrame和faceFrame是否为空
|
|
1037
1032
|
if (!bodyFrame) {
|
|
1038
1033
|
this.allLostFrameCount++;
|
|
1039
|
-
this.options.
|
|
1034
|
+
this.options.reportMessage({
|
|
1040
1035
|
code: EErrorCode.RENDER_BODY_ERROR,
|
|
1041
1036
|
message: `Error: 第${frameIndex}帧 bodyFrame为空`,
|
|
1042
|
-
e:
|
|
1037
|
+
e: { bodyFrame, faceFrame },
|
|
1043
1038
|
});
|
|
1044
1039
|
} else if (!faceFrame) {
|
|
1045
1040
|
this.allLostFaceFrameCount++;
|
|
1046
|
-
this.options.
|
|
1041
|
+
this.options.reportMessage({
|
|
1047
1042
|
code: EErrorCode.RENDER_FACE_ERROR,
|
|
1048
1043
|
message: `Error: 第${frameIndex}帧 faceFrame为空`,
|
|
1049
|
-
e:
|
|
1044
|
+
e: { bodyFrame, faceFrame },
|
|
1050
1045
|
});
|
|
1051
1046
|
}
|
|
1052
1047
|
if(this.allLostFrameCount === this.maxLostFrameCount) {
|
|
1053
1048
|
this.options.reportMessage({
|
|
1054
1049
|
code: EErrorCode.RENDER_DATA_MISSING,
|
|
1055
|
-
message: `连续${this.maxLostFrameCount}帧
|
|
1050
|
+
message: `连续${this.maxLostFrameCount}帧 身体渲染数据缺失`,
|
|
1051
|
+
e: { bodyFrame, faceFrame },
|
|
1056
1052
|
});
|
|
1057
1053
|
// 丢失过多帧,达到最大允许丢失帧数后,触发解码器对齐到最新的一帧
|
|
1058
|
-
this.options.onError({
|
|
1059
|
-
code: EErrorCode.RENDER_DATA_MISSING,
|
|
1060
|
-
message: `Error: 连续${this.maxLostFrameCount}帧 身体渲染数据缺失`,
|
|
1061
|
-
e: JSON.stringify({ bodyFrame, faceFrame }),
|
|
1062
|
-
});
|
|
1063
1054
|
this.forceSyncDecoder();
|
|
1064
1055
|
this.allLostFrameCount = 0;
|
|
1065
1056
|
}
|
|
@@ -1180,10 +1171,10 @@ export default class AvatarRender {
|
|
|
1180
1171
|
rendered = true;
|
|
1181
1172
|
} catch (error) {
|
|
1182
1173
|
(window as any).avatarSDKLogger.error(this.TAG, "渲染帧失败:", error);
|
|
1183
|
-
this.options.
|
|
1174
|
+
this.options.reportMessage({
|
|
1184
1175
|
code: EErrorCode.WebGL_RENDER_ERROR,
|
|
1185
1176
|
message: `Error: 第${frameIndex}帧 webgl渲染异常`,
|
|
1186
|
-
e:
|
|
1177
|
+
e: { bodyFrame, faceFrame, error },
|
|
1187
1178
|
});
|
|
1188
1179
|
} finally {
|
|
1189
1180
|
// 渲染完成后再关闭VideoFrame
|
|
@@ -184,14 +184,25 @@ export default class MediaSourceAudioPlayer {
|
|
|
184
184
|
* 获取缓冲范围
|
|
185
185
|
*/
|
|
186
186
|
private getBufferedRanges(): string {
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
187
|
+
try {
|
|
188
|
+
// stop/destroy 流程中浏览器会先移除 SourceBuffer 再置空引用,
|
|
189
|
+
// 此时访问 .buffered 会抛 InvalidStateError,需要 readyState 校验 + 兜底
|
|
190
|
+
if (
|
|
191
|
+
this.sourceBuffer &&
|
|
192
|
+
this.mediaSource?.readyState === "open" &&
|
|
193
|
+
this.sourceBuffer.buffered.length > 0
|
|
194
|
+
) {
|
|
195
|
+
const ranges: string[] = [];
|
|
196
|
+
for (let i = 0; i < this.sourceBuffer.buffered.length; i++) {
|
|
197
|
+
const start = this.sourceBuffer.buffered.start(i).toFixed(2);
|
|
198
|
+
const end = this.sourceBuffer.buffered.end(i).toFixed(2);
|
|
199
|
+
ranges.push(`[${start}-${end}]`);
|
|
200
|
+
}
|
|
201
|
+
return ranges.join(", ");
|
|
193
202
|
}
|
|
194
|
-
|
|
203
|
+
} catch (e) {
|
|
204
|
+
// SourceBuffer 已被移除(stop/destroy 竞态),视为无缓冲数据
|
|
205
|
+
return "-";
|
|
195
206
|
}
|
|
196
207
|
return "-";
|
|
197
208
|
}
|
|
@@ -158,9 +158,6 @@ export default class RenderScheduler {
|
|
|
158
158
|
this.renderState = state;
|
|
159
159
|
},
|
|
160
160
|
sendVideoInfo: config.sendVideoInfo,
|
|
161
|
-
onError: (error: any) => {
|
|
162
|
-
this.sdk.onMessage(error);
|
|
163
|
-
},
|
|
164
161
|
});
|
|
165
162
|
this.uiRenderer = new UIRenderer({
|
|
166
163
|
sdk: this.sdk,
|
|
@@ -341,9 +338,10 @@ export default class RenderScheduler {
|
|
|
341
338
|
} else {
|
|
342
339
|
nowFaceData.push(faceData[i]);
|
|
343
340
|
}
|
|
344
|
-
nowFaceData.length && this.dataCacheQueue._updateFacial(nowFaceData);
|
|
345
|
-
realFaceData.length && this.dataCacheQueue._updateRealFacial(realFaceData);
|
|
346
341
|
}
|
|
342
|
+
// 分类完成后一次性入队,避免循环内对累加数组重复 push 导致队列数据重复
|
|
343
|
+
nowFaceData.length && this.dataCacheQueue._updateFacial(nowFaceData);
|
|
344
|
+
realFaceData.length && this.dataCacheQueue._updateRealFacial(realFaceData);
|
|
347
345
|
break;
|
|
348
346
|
case EFrameDataType.AUDIO:
|
|
349
347
|
const audioData = data as IRawAudioFrameData[];
|
|
@@ -411,6 +409,11 @@ export default class RenderScheduler {
|
|
|
411
409
|
clearTimeout(this._speedResetTimer);
|
|
412
410
|
}
|
|
413
411
|
|
|
412
|
+
// 播放加速大于4,直接丢弃
|
|
413
|
+
if (playbackSpeed > 4) {
|
|
414
|
+
return;
|
|
415
|
+
}
|
|
416
|
+
|
|
414
417
|
// 8. 应用速度
|
|
415
418
|
this.audioRenderer.setSpeed(playbackSpeed);
|
|
416
419
|
|
|
@@ -490,7 +493,34 @@ export default class RenderScheduler {
|
|
|
490
493
|
code: EErrorCode.EVENT_TRIGGERED,
|
|
491
494
|
message: 'WARNING: 有新的speak_start事件,但是上一轮仍未结束,触发兜底结束',
|
|
492
495
|
});
|
|
493
|
-
|
|
496
|
+
const currentSpeakStart = d.flatMap(item => item.e ?? []).find(ev => ev.type === 'speak_start');
|
|
497
|
+
const newSpeakId = currentSpeakStart?.speech_id ?? 0;
|
|
498
|
+
const playingSpeechId = this.audioRenderer.speech_id;
|
|
499
|
+
// 仅当正在播放的确实是上一轮(sid 小于新一轮)时才打断旧轮次。
|
|
500
|
+
// 若事件晚于音频到达(playingSpeechId 已是新一轮 sid),此时播放的就是新一轮,
|
|
501
|
+
// 不能用 interrupt()(其目标取 audioRenderer.speech_id),否则会误清新一轮的音频队列、
|
|
502
|
+
// 字幕事件,并因 lastSpeechId/oldSpeechId 更新导致后续音视频帧全部被丢弃,
|
|
503
|
+
// 表现为"有口型、无字幕、无音频"。
|
|
504
|
+
if (playingSpeechId !== -1 && playingSpeechId < newSpeakId) {
|
|
505
|
+
this.interrupt('new_speak_start', playingSpeechId);
|
|
506
|
+
// stop() 内部有 oldSpeechId >= speech_id 的防重入守卫:当 speech_id 停留在
|
|
507
|
+
// 异常残留值(如初始值 0,长会话中 isPlaying 卡死为 true)时 stop 会被拦截,
|
|
508
|
+
// isPlaying/speech_id 无法复位,导致新一轮音频在 _updateAudio 中永远走缓存分支
|
|
509
|
+
// 不播放(日志表现:缓存数据长度持续增长且无"开始播放")。
|
|
510
|
+
// 这里检测到 stop 未生效(speech_id 未复位为 -1)时,用 pause() 强制复位,
|
|
511
|
+
// pause 会 flush 已缓存音频并把 speech_id/oldSpeechId 重置,新一轮可正常起播。
|
|
512
|
+
if (this.audioRenderer.speech_id !== -1) {
|
|
513
|
+
this.audioRenderer.pause();
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
// 新 speak_start 已到来,立即复位 interrupt 并更新轮次状态,
|
|
517
|
+
// 避免依赖 UIRenderer 在 sf 帧处理事件时(sf==当前帧且事件在 compose 后被处理)被漏处理,
|
|
518
|
+
// 导致新轮次实际说话期间仍被强制渲染为闭嘴表情
|
|
519
|
+
if (currentSpeakStart) {
|
|
520
|
+
this.currentSpeechId = currentSpeakStart.speech_id ?? 0;
|
|
521
|
+
this.avatarRenderer.setInterrupt(false);
|
|
522
|
+
this.uiRenderer.lastSpeechId = currentSpeakStart.speech_id ?? 0;
|
|
523
|
+
}
|
|
494
524
|
}
|
|
495
525
|
|
|
496
526
|
// 2. 处理过期speak_start
|
|
@@ -830,6 +860,14 @@ export default class RenderScheduler {
|
|
|
830
860
|
if (frame) {
|
|
831
861
|
this.decoder._offLineMode(this.resourceManager._getOfflineIdle(), frame);
|
|
832
862
|
}
|
|
863
|
+
// 重置所有 speechId 相关状态,避免联网后新语音被过滤
|
|
864
|
+
// - lastSpeechId: 用于过滤旧 sid 的音频数据(L444)
|
|
865
|
+
// - currentSpeechId: 用于 speak_end 时判断是否 setInterrupt
|
|
866
|
+
// - uiRenderer.lastSpeechId: UIRenderer 会过滤 sid < lastSpeechId 的事件(含 speak_start)
|
|
867
|
+
// 若不重置,联网后新语音的 speak_start 会被过滤,导致字幕正常但无音频
|
|
868
|
+
this.lastSpeechId = -1;
|
|
869
|
+
this.currentSpeechId = -1;
|
|
870
|
+
this.uiRenderer.lastSpeechId = -1;
|
|
833
871
|
}
|
|
834
872
|
_offlineRun() {
|
|
835
873
|
this.decoder._offlineRun()
|
|
@@ -844,6 +882,16 @@ export default class RenderScheduler {
|
|
|
844
882
|
this.audioRenderer.resume();
|
|
845
883
|
}
|
|
846
884
|
|
|
885
|
+
/**
|
|
886
|
+
* app 切后台时暂停音频并清空对齐状态(复用断网离线时的 pause())
|
|
887
|
+
* 不能保留缓存续播:Android 后台视频帧号持续递增而音频停留在暂停位置,
|
|
888
|
+
* 恢复后音画基准脱节;且后台期间可能下发新 speak 数据,旧缓存与新 sf 冲突。
|
|
889
|
+
* 恢复后依赖新下发音频的 sf 作为 firstFrameIndex 重新对齐
|
|
890
|
+
*/
|
|
891
|
+
pauseForBackground() {
|
|
892
|
+
this.audioRenderer.pause();
|
|
893
|
+
}
|
|
894
|
+
|
|
847
895
|
/**
|
|
848
896
|
* 设置数字人canvas的显隐状态
|
|
849
897
|
* @param visible 是否可见
|
|
@@ -859,10 +907,13 @@ export default class RenderScheduler {
|
|
|
859
907
|
this.avatarRenderer.setWalkConfig(walkConfig);
|
|
860
908
|
}
|
|
861
909
|
|
|
862
|
-
|
|
910
|
+
// targetSpeechId: 显式指定要打断的轮次 sid。缺省时取 audioRenderer.speech_id(当前播放轮次)。
|
|
911
|
+
// 当事件晚于音频到达时,audioRenderer.speech_id 可能已经切换到新一轮,此时必须显式传旧 sid,
|
|
912
|
+
// 避免误清新一轮的音频与字幕。
|
|
913
|
+
interrupt(type?: string, targetSpeechId?: number): number{
|
|
863
914
|
const start = Date.now();
|
|
864
915
|
// 插入中断字幕和语音结束事件
|
|
865
|
-
const speech_id = this.audioRenderer.speech_id;
|
|
916
|
+
const speech_id = targetSpeechId ?? this.audioRenderer.speech_id;
|
|
866
917
|
if(type === "in_offline_mode") {
|
|
867
918
|
this.lastSpeechId = -1;
|
|
868
919
|
} else {
|
package/src/index.ts
CHANGED
|
@@ -270,6 +270,7 @@ export default class XmovAvatar {
|
|
|
270
270
|
gatewayServer,
|
|
271
271
|
cacheServer,
|
|
272
272
|
config:_config,
|
|
273
|
+
audioOnlyMode: options.audioOnlyMode || false,
|
|
273
274
|
onNetworkInfo(networkInfo) {
|
|
274
275
|
self.networkInfo = networkInfo;
|
|
275
276
|
options.onNetworkInfo?.(networkInfo);
|
|
@@ -1039,12 +1040,28 @@ export default class XmovAvatar {
|
|
|
1039
1040
|
|
|
1040
1041
|
visibilitychange() {
|
|
1041
1042
|
if (document.hidden) {
|
|
1042
|
-
//
|
|
1043
|
-
//
|
|
1044
|
-
//
|
|
1043
|
+
// 进入后台:暂停渲染循环和解码器(省电)、清空音频与表情数据
|
|
1044
|
+
// 仅渲染中暂停(resumeRender 要求 paused 状态才能恢复),
|
|
1045
|
+
// 其他状态(init/stopped/隐身 paused)只停止音频,避免强制暂停后无法恢复
|
|
1046
|
+
const rs = this.renderScheduler?.getRenderState();
|
|
1047
|
+
if (rs === RenderState.rendering || rs === RenderState.resumed) {
|
|
1048
|
+
this.renderScheduler?.pauseRender();
|
|
1049
|
+
} else {
|
|
1050
|
+
this.renderScheduler?.pauseForBackground();
|
|
1051
|
+
}
|
|
1045
1052
|
} else {
|
|
1046
|
-
|
|
1047
|
-
|
|
1053
|
+
// 回到前台:若处于暂停态则恢复渲染(内部已 forceSyncDecoder 对齐最新帧)
|
|
1054
|
+
if (this.renderScheduler?.getRenderState() === RenderState.paused) {
|
|
1055
|
+
this.renderScheduler?.resumeRender();
|
|
1056
|
+
} else if (this.ttsa?.getStatus()) {
|
|
1057
|
+
// 未暂停但连接正常:兜底同步解帧队列
|
|
1058
|
+
this.renderScheduler?.forceSyncDecoder();
|
|
1059
|
+
}
|
|
1060
|
+
// WebSocket 在后台断开且已进入离线模式,网络可用则主动触发重连,
|
|
1061
|
+
// 否则用户切回前台续播时无响应且无任何报错
|
|
1062
|
+
if (!this.ttsa?.getStatus() && this.status === AvatarStatus.offline && NetworkMonitor.ONLINE) {
|
|
1063
|
+
(window as any).avatarSDKLogger.warn(this.TAG, "切回前台发现连接已断开,主动触发重连");
|
|
1064
|
+
this.onlineMode();
|
|
1048
1065
|
}
|
|
1049
1066
|
// tab 切换回来时,恢复音频捕获的 AudioContext
|
|
1050
1067
|
if (this.captureAudioCtx && this.captureAudioCtx.state === 'suspended') {
|
|
@@ -1350,7 +1367,7 @@ export default class XmovAvatar {
|
|
|
1350
1367
|
// 薄客户端模式:通知 server 端销毁,然后清理本地资源
|
|
1351
1368
|
if (this._isClientMode) {
|
|
1352
1369
|
this._apiForwarder?.destroy(stop_reason);
|
|
1353
|
-
this.destroyClient();
|
|
1370
|
+
await this.destroyClient();
|
|
1354
1371
|
this.onStatusChange(AvatarStatus.close);
|
|
1355
1372
|
return;
|
|
1356
1373
|
}
|
|
@@ -1358,8 +1375,8 @@ export default class XmovAvatar {
|
|
|
1358
1375
|
this.ttsa?.sendSdkPoint('close_session', {
|
|
1359
1376
|
reason: stop_reason,
|
|
1360
1377
|
});
|
|
1361
|
-
this.destroyClient();
|
|
1362
1378
|
const res = await this.resourceManager.stopSession(stop_reason);
|
|
1379
|
+
await this.destroyClient();
|
|
1363
1380
|
if(res) {
|
|
1364
1381
|
this.onStatusChange(AvatarStatus.close);
|
|
1365
1382
|
}
|
|
@@ -1480,6 +1497,8 @@ export default class XmovAvatar {
|
|
|
1480
1497
|
});
|
|
1481
1498
|
this.incompleteInitializationCleanupPromise = (async () => {
|
|
1482
1499
|
try {
|
|
1500
|
+
// 先等待房间释放完成,再清理本地资源,避免房间泄漏
|
|
1501
|
+
await stopSessionPromise;
|
|
1483
1502
|
await this.destroyClient();
|
|
1484
1503
|
} catch (error) {
|
|
1485
1504
|
(window as any).avatarSDKLogger.warn(this.TAG, '清理初始化未完成的本地资源时出错', error);
|
|
@@ -17,7 +17,7 @@ import CacheManager from './cache-manager';
|
|
|
17
17
|
type TOptions = Pick<
|
|
18
18
|
IAvatarOptions,
|
|
19
19
|
"appId" | "appSecret" | "gatewayServer" | 'cacheServer' | "sdkInstance" | "config" | "headers" | "gateway_type" | "tag" |
|
|
20
|
-
"asr_id" | "llm_id" | "session_speak_req_id"
|
|
20
|
+
"asr_id" | "llm_id" | "session_speak_req_id" | "audioOnlyMode"
|
|
21
21
|
> & {
|
|
22
22
|
sessionRequestData?: Record<string, unknown>
|
|
23
23
|
onNetworkInfo(quality: INetworkInfo): void
|
|
@@ -205,12 +205,14 @@ export default class ResourceManager {
|
|
|
205
205
|
private isProcessingVideo = false; // 防止在视频处理过程中清理缓存
|
|
206
206
|
private cacheServer?: CacheManager;
|
|
207
207
|
private startSessionTimer: any = null;
|
|
208
|
+
private audioOnlyMode: boolean = false;
|
|
208
209
|
|
|
209
210
|
// 新增:正在下载的视频跟踪
|
|
210
211
|
private downloadingVideos = new Map<string, Promise<ArrayBuffer | undefined>>();
|
|
211
212
|
first_load = true;
|
|
212
213
|
constructor(options: TOptions) {
|
|
213
214
|
this.options = options;
|
|
215
|
+
this.audioOnlyMode = options.audioOnlyMode || false;
|
|
214
216
|
this.sdk = options.sdkInstance as XmovAvatar;
|
|
215
217
|
this.mouthShapeLib = {
|
|
216
218
|
char_info: null,
|
|
@@ -303,9 +305,15 @@ export default class ResourceManager {
|
|
|
303
305
|
}, 100);
|
|
304
306
|
// http 请求算 10 进度
|
|
305
307
|
const result = await this.startSession();
|
|
306
|
-
|
|
308
|
+
const res = result as unknown as ISessionResponse;
|
|
309
|
+
// 先记录 session_id,确保后续 stopSession 能正确释放房间
|
|
310
|
+
if (res?.session_id) {
|
|
311
|
+
this.session_id = res.session_id;
|
|
312
|
+
}
|
|
313
|
+
// 如果在 startSession 过程中被销毁了,释放刚创建的房间并返回,避免房间泄漏
|
|
307
314
|
if (this.destroyed) {
|
|
308
315
|
(window as any).avatarSDKLogger.warn(this.TAG, "sdk destroyed during load, aborting after startSession");
|
|
316
|
+
await this.stopSession("destroy_during_init");
|
|
309
317
|
return null;
|
|
310
318
|
}
|
|
311
319
|
if (!result) {
|
|
@@ -318,15 +326,15 @@ export default class ResourceManager {
|
|
|
318
326
|
this.startSessionTimer = null;
|
|
319
327
|
this.progress = 15;
|
|
320
328
|
onDownloadProgress?.(this.progress);
|
|
321
|
-
const res = result as unknown as ISessionResponse;
|
|
322
|
-
this.session_id = res?.session_id;
|
|
323
329
|
this.resource_pack = res.resource_pack;
|
|
324
330
|
this.config = res.config as any;
|
|
325
331
|
this.offlineIdle = res.resource_pack?.offline_idle || []
|
|
326
332
|
if (this.resource_pack) {
|
|
327
|
-
|
|
333
|
+
if(!this.audioOnlyMode) {
|
|
334
|
+
await this.loadMouthShapeLib(onDownloadProgress);
|
|
335
|
+
}
|
|
328
336
|
this.getBackgroundImage();
|
|
329
|
-
if (!this.mouthShapeLib?.char_info && res?.resource_pack?.face_ani_char_data) {
|
|
337
|
+
if (!this.mouthShapeLib?.char_info && res?.resource_pack?.face_ani_char_data && !this.audioOnlyMode) {
|
|
330
338
|
return null
|
|
331
339
|
}
|
|
332
340
|
}
|
|
@@ -838,6 +846,11 @@ export default class ResourceManager {
|
|
|
838
846
|
}
|
|
839
847
|
const res = result as unknown as ISessionResponse;
|
|
840
848
|
this.session_id = res?.session_id;
|
|
849
|
+
// 重连过程中被销毁,释放刚创建的房间,避免房间泄漏
|
|
850
|
+
if (this.destroyed) {
|
|
851
|
+
await this.stopSession("destroy_during_reload");
|
|
852
|
+
return null;
|
|
853
|
+
}
|
|
841
854
|
this.resource_pack = res.resource_pack;
|
|
842
855
|
this.config = res.config as any;
|
|
843
856
|
this.offlineIdle = res.resource_pack?.offline_idle || []
|
package/src/modules/decoder.ts
CHANGED
|
@@ -279,12 +279,32 @@ export default class ParallelDecoder {
|
|
|
279
279
|
|
|
280
280
|
_startWorker(file: bodyFile) {
|
|
281
281
|
const taskId = this.currentTaskId;
|
|
282
|
-
|
|
282
|
+
let worker: Worker;
|
|
283
|
+
try {
|
|
284
|
+
worker = new Worker(workerURL);
|
|
285
|
+
} catch (err) {
|
|
286
|
+
this.reportMessage({
|
|
287
|
+
code: EErrorCode.INIT_WORKER_ERROR,
|
|
288
|
+
message: '初始化视频抽帧Worker失败',
|
|
289
|
+
e: err as object,
|
|
290
|
+
});
|
|
291
|
+
return;
|
|
292
|
+
}
|
|
283
293
|
const id = `${file.body_id ?? file.id ?? 0}_${file.name}`;
|
|
284
294
|
const data = file.data;
|
|
285
295
|
const dataCopy = new ArrayBuffer(data.byteLength);
|
|
286
296
|
new Uint8Array(dataCopy).set(new Uint8Array(data));
|
|
287
297
|
|
|
298
|
+
// 监听 Worker 运行时错误(脚本加载/解析失败等)
|
|
299
|
+
worker.addEventListener("error", (err) => {
|
|
300
|
+
if (taskId !== this.currentTaskId) return;
|
|
301
|
+
this.reportMessage({
|
|
302
|
+
code: EErrorCode.INIT_WORKER_ERROR,
|
|
303
|
+
message: '视频抽帧Worker运行时错误',
|
|
304
|
+
e: err as object,
|
|
305
|
+
});
|
|
306
|
+
});
|
|
307
|
+
|
|
288
308
|
const onMessage = (e: MessageEvent) => {
|
|
289
309
|
if (taskId !== this.currentTaskId) return; // 只处理当前任务
|
|
290
310
|
if (e.data.type === "frame") {
|
|
@@ -335,6 +355,19 @@ export default class ParallelDecoder {
|
|
|
335
355
|
}
|
|
336
356
|
this._tryStartNext();
|
|
337
357
|
} else if (e.data.type === "error") {
|
|
358
|
+
// 根据 Worker 上报的 error 内容区分错误码
|
|
359
|
+
// Worker 中有两处 postMessage({ type: 'error' }):
|
|
360
|
+
// 1. VideoDecoder error 回调 → 视频抽帧错误 (VIDEO_FRAME_EXTRACT_ERROR)
|
|
361
|
+
// 2. 解码配置不支持 → 视频流处理错误 (PROCESS_VIDEO_STREAM_ERROR)
|
|
362
|
+
const errorMsg = e.data.error ?? '';
|
|
363
|
+
const code = errorMsg.includes('不支持')
|
|
364
|
+
? EErrorCode.PROCESS_VIDEO_STREAM_ERROR
|
|
365
|
+
: EErrorCode.VIDEO_FRAME_EXTRACT_ERROR;
|
|
366
|
+
this.reportMessage({
|
|
367
|
+
code,
|
|
368
|
+
message: errorMsg || '视频解码错误',
|
|
369
|
+
e: { fileName: file.name, error: errorMsg } as object,
|
|
370
|
+
});
|
|
338
371
|
const task = this.tasks.get(id);
|
|
339
372
|
if (task) {
|
|
340
373
|
task.status = "error";
|
package/src/view/DebugOverlay.ts
CHANGED
|
@@ -71,12 +71,17 @@ export class DebugOverlay {
|
|
|
71
71
|
}
|
|
72
72
|
|
|
73
73
|
public show(): void {
|
|
74
|
-
if (this.container) {
|
|
74
|
+
if (!this.container) {
|
|
75
|
+
this.createOverlay();
|
|
76
|
+
}
|
|
77
|
+
if(this.container) {
|
|
75
78
|
this.container.style.display = "block";
|
|
76
|
-
return;
|
|
77
79
|
}
|
|
78
|
-
|
|
79
|
-
|
|
80
|
+
// 重启定时器:hide() 会清除 updateInterval,再次 show 时需要重建
|
|
81
|
+
// 否则界面显示但内容永不刷新
|
|
82
|
+
if (!this.updateInterval) {
|
|
83
|
+
this.updateInterval = window.setInterval(() => this.updateTextOnly(), 200);
|
|
84
|
+
}
|
|
80
85
|
this.updateTextOnly();
|
|
81
86
|
}
|
|
82
87
|
|