@xmov/avatar 2.0.1 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/agent/cdn-entry.d.ts +1 -0
- package/dist/baseRender/AvatarRenderer.d.ts +0 -1
- package/dist/control/RenderScheduler.d.ts +17 -1
- package/dist/control/ttsa.d.ts +8 -1
- package/dist/index.cjs +1 -1
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +5 -1
- package/dist/index.modern.js +1 -1
- package/dist/index.modern.js.map +1 -1
- package/dist/index.module.js +1 -1
- package/dist/index.module.js.map +1 -1
- package/dist/index.umd.js +1 -1
- package/dist/index.umd.js.map +1 -1
- package/dist/modules/ResourceManager.d.ts +2 -1
- package/dist/utils/request.d.ts +1 -0
- package/package.json +9 -4
- package/src/agent/cdn-entry.ts +15 -0
- package/src/baseRender/AudioRenderer.ts +16 -5
- package/src/baseRender/AvatarRenderer.ts +11 -20
- package/src/baseRender/MSEAudioPlayer.ts +18 -7
- package/src/control/RenderScheduler.ts +74 -8
- package/src/control/ttsa.ts +47 -11
- package/src/index.ts +66 -45
- package/src/modules/ResourceManager.ts +19 -6
- package/src/modules/decoder.ts +34 -1
- package/src/utils/capability-checker.ts +95 -21
- package/src/utils/request.ts +14 -2
- package/src/view/DebugOverlay.ts +9 -4
|
@@ -3,7 +3,7 @@ import XmovAvatar from "../index";
|
|
|
3
3
|
/**
|
|
4
4
|
* 资源处理模块,加载并存储资源
|
|
5
5
|
*/
|
|
6
|
-
type TOptions = Pick<IAvatarOptions, "appId" | "appSecret" | "gatewayServer" | 'cacheServer' | "sdkInstance" | "config" | "headers" | "gateway_type" | "tag" | "asr_id" | "llm_id" | "session_speak_req_id"> & {
|
|
6
|
+
type TOptions = Pick<IAvatarOptions, "appId" | "appSecret" | "gatewayServer" | 'cacheServer' | "sdkInstance" | "config" | "headers" | "gateway_type" | "tag" | "asr_id" | "llm_id" | "session_speak_req_id" | "audioOnlyMode"> & {
|
|
7
7
|
sessionRequestData?: Record<string, unknown>;
|
|
8
8
|
onNetworkInfo(quality: INetworkInfo): void;
|
|
9
9
|
onStartSessionWarning?: (message: Object) => void;
|
|
@@ -169,6 +169,7 @@ export default class ResourceManager {
|
|
|
169
169
|
private isProcessingVideo;
|
|
170
170
|
private cacheServer?;
|
|
171
171
|
private startSessionTimer;
|
|
172
|
+
private audioOnlyMode;
|
|
172
173
|
private downloadingVideos;
|
|
173
174
|
first_load: boolean;
|
|
174
175
|
constructor(options: TOptions);
|
package/dist/utils/request.d.ts
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@xmov/avatar",
|
|
3
|
-
"version": "2.0
|
|
3
|
+
"version": "2.1.0",
|
|
4
4
|
"description": "魔珐数字人 JS-SDK",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"source": "src/index.ts",
|
|
@@ -26,15 +26,20 @@
|
|
|
26
26
|
"src/"
|
|
27
27
|
],
|
|
28
28
|
"scripts": {
|
|
29
|
-
"build": "
|
|
29
|
+
"build": "pnpm run build:core && pnpm run build:agent && pnpm run build:e2e:cdn && pnpm run test:e2e:cdn && node upload-sdk.js build",
|
|
30
|
+
"build:core": "microbundle --workers --define ENV=production,VERSION=$npm_package_version --external none --tsconfig tsconfig.json",
|
|
30
31
|
"build:agent": "microbundle --entry src/agent/index.ts --output dist/agent --format modern,esm,cjs,umd --external @xmov/avatar --alias ../index=@xmov/avatar --globals @xmov/avatar=XmovAvatar --name XingyunAvatarAgentModule --tsconfig tsconfig.agent.json && node scripts/build-agent-entry.mjs",
|
|
31
|
-
"
|
|
32
|
+
"build:e2e:cdn": "microbundle --entry src/agent/cdn-entry.ts --output cdn-dist/xmovAvatar_e2e.js --format iife --external none --workers --define ENV=production,VERSION=$npm_package_version --tsconfig tsconfig.agent.json --no-pkg-main --no-sourcemap",
|
|
33
|
+
"sdk": "pnpm --filter @xmov/avatar run --parallel --no-bail \"/^sdk:.+$/\"",
|
|
34
|
+
"sdk:core": "microbundle watch --workers --define ENV=development,VERSION=$npm_package_version --external none --tsconfig tsconfig.json",
|
|
35
|
+
"sdk:agent": "node scripts/build-agent-entry.mjs && microbundle watch --entry src/agent/index.ts --output dist/agent --format modern,esm,cjs,umd --external @xmov/avatar --alias ../index=@xmov/avatar --globals @xmov/avatar=XmovAvatar --name XingyunAvatarAgentModule --tsconfig tsconfig.agent.json",
|
|
32
36
|
"app": "cd app/ && npm run dev",
|
|
33
37
|
"app:lan": "pnpm --dir app dev:lan",
|
|
34
38
|
"test:agent": "node scripts/run-agent-tests.mjs",
|
|
35
39
|
"test:agent:exports": "pnpm run build:agent && node scripts/test-agent-package-exports.mjs",
|
|
36
40
|
"test:agent:browser": "playwright test",
|
|
37
|
-
"
|
|
41
|
+
"test:e2e:cdn": "node scripts/test-e2e-cdn-build.mjs",
|
|
42
|
+
"lite": "node replaceString.js && pnpm run build:core && node upload-sdk.js lite",
|
|
38
43
|
"proto": "pbjs -t static-module -w closure -o src/proto/face_data_pb.js src/proto/face_data.proto",
|
|
39
44
|
"commitlint": "commitlint --edit",
|
|
40
45
|
"prepare": "husky"
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import XingyunAvatarAgent, {
|
|
2
|
+
AgentE2EClient,
|
|
3
|
+
DEFAULT_AGENT_AUDIO,
|
|
4
|
+
MicrophoneController,
|
|
5
|
+
} from "./index";
|
|
6
|
+
|
|
7
|
+
Object.assign(XingyunAvatarAgent, {
|
|
8
|
+
AgentE2EClient,
|
|
9
|
+
DEFAULT_AGENT_AUDIO,
|
|
10
|
+
MicrophoneController,
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
(globalThis as typeof globalThis & {
|
|
14
|
+
XingyunAvatarAgent: typeof XingyunAvatarAgent;
|
|
15
|
+
}).XingyunAvatarAgent = XingyunAvatarAgent;
|
|
@@ -225,19 +225,25 @@ export default class AudioRender {
|
|
|
225
225
|
pause() {
|
|
226
226
|
this.isPlaying = false;
|
|
227
227
|
this.options.dataCacheQueue._clearAudio(this.speech_id);
|
|
228
|
+
this.lastFrameIndex = -1;
|
|
228
229
|
this.firstFrameIndex = -1;
|
|
230
|
+
this.cacheFirstFrameIndex = -1;
|
|
231
|
+
this.cacheAudioData = [];
|
|
232
|
+
this.mseCacheAudioData = [];
|
|
233
|
+
this.oldSpeechId = -1;
|
|
229
234
|
// 停止 PCM 播放
|
|
230
235
|
this.audio?.stop();
|
|
231
|
-
|
|
236
|
+
|
|
232
237
|
// 停止 WebM 播放
|
|
233
238
|
this.mseAudioPlayer?.stop(this.addNewAudioSegment.bind(this));
|
|
234
239
|
this.speech_id = -1;
|
|
235
240
|
}
|
|
236
241
|
|
|
237
242
|
stop(speech_id: number) {
|
|
238
|
-
//
|
|
239
|
-
|
|
240
|
-
|
|
243
|
+
// speech_id === -1 表示强制清理(用于 pauseRender / destroy / 断网重连场景),不受 oldSpeechId 限制
|
|
244
|
+
// 否则如果 oldSpeechId >= speech_id 直接返回,避免重复处理
|
|
245
|
+
if (speech_id !== -1 && this.oldSpeechId >= speech_id) return;
|
|
246
|
+
|
|
241
247
|
this.isPlaying = false;
|
|
242
248
|
this.options.dataCacheQueue._clearAudio(speech_id);
|
|
243
249
|
this.firstFrameIndex = -1;
|
|
@@ -255,7 +261,12 @@ export default class AudioRender {
|
|
|
255
261
|
}
|
|
256
262
|
this.oldSpeechId = speech_id;
|
|
257
263
|
this.speech_id = -1;
|
|
258
|
-
if (
|
|
264
|
+
if (speech_id === -1) {
|
|
265
|
+
// 强制清理:清空所有缓存数据,不推入音频流(用于 pauseRender / destroy / 断网重连)
|
|
266
|
+
this.cacheFirstFrameIndex = -1;
|
|
267
|
+
this.cacheAudioData = [];
|
|
268
|
+
this.mseCacheAudioData = [];
|
|
269
|
+
} else if (this.cacheAudioData.length > 0) {
|
|
259
270
|
this.firstFrameIndex = this.cacheFirstFrameIndex;
|
|
260
271
|
this.cacheFirstFrameIndex = -1;
|
|
261
272
|
// 新的缓存下来的数据,推入音频流
|
|
@@ -19,16 +19,11 @@ type Option = {
|
|
|
19
19
|
resourceManager: ResourceManager;
|
|
20
20
|
saveAndDownload: SaveAndDownload;
|
|
21
21
|
forceSyncDecoder: () => void;
|
|
22
|
-
reportMessage: (message: {
|
|
23
|
-
code: EErrorCode;
|
|
24
|
-
message: string;
|
|
25
|
-
e?: object;
|
|
26
|
-
}) => void;
|
|
22
|
+
reportMessage: (message: { code: EErrorCode; message: string; e?: object }) => void;
|
|
27
23
|
onDownloadProgress: (progress: number) => void;
|
|
28
24
|
onStateChange: (state: string) => void;
|
|
29
25
|
onRenderChange: (state: RenderState) => void;
|
|
30
26
|
sendVideoInfo: (info: { name: string; body_id: number; id: number }) => void;
|
|
31
|
-
onError: (error: any) => void;
|
|
32
27
|
sendSdkPoint: (point: string, data: any) => void;
|
|
33
28
|
};
|
|
34
29
|
export default class AvatarRender {
|
|
@@ -989,13 +984,13 @@ export default class AvatarRender {
|
|
|
989
984
|
} else {
|
|
990
985
|
if(this.lastRealFaceFrame === -1) {
|
|
991
986
|
faceFrame = curRealFaceData;
|
|
992
|
-
// this.options.
|
|
987
|
+
// this.options.reportMessage({
|
|
993
988
|
// code: EErrorCode.RENDER_FACE_ERROR,
|
|
994
989
|
// message: `第${frameIndex}帧 实时面部数据为空,原始数据渲染`,
|
|
995
990
|
// e: JSON.stringify({ bodyFrame, faceFrame }),
|
|
996
991
|
// });
|
|
997
992
|
} else {
|
|
998
|
-
// this.options.
|
|
993
|
+
// this.options.reportMessage({
|
|
999
994
|
// code: EErrorCode.RENDER_FACE_ERROR,
|
|
1000
995
|
// message: `第${frameIndex}帧 实时面部数据为空,插值渲染`,
|
|
1001
996
|
// e: JSON.stringify({ bodyFrame, faceFrame }),
|
|
@@ -1036,30 +1031,26 @@ export default class AvatarRender {
|
|
|
1036
1031
|
// 判断bodyFrame和faceFrame是否为空
|
|
1037
1032
|
if (!bodyFrame) {
|
|
1038
1033
|
this.allLostFrameCount++;
|
|
1039
|
-
this.options.
|
|
1034
|
+
this.options.reportMessage({
|
|
1040
1035
|
code: EErrorCode.RENDER_BODY_ERROR,
|
|
1041
1036
|
message: `Error: 第${frameIndex}帧 bodyFrame为空`,
|
|
1042
|
-
e:
|
|
1037
|
+
e: { bodyFrame, faceFrame },
|
|
1043
1038
|
});
|
|
1044
1039
|
} else if (!faceFrame) {
|
|
1045
1040
|
this.allLostFaceFrameCount++;
|
|
1046
|
-
this.options.
|
|
1041
|
+
this.options.reportMessage({
|
|
1047
1042
|
code: EErrorCode.RENDER_FACE_ERROR,
|
|
1048
1043
|
message: `Error: 第${frameIndex}帧 faceFrame为空`,
|
|
1049
|
-
e:
|
|
1044
|
+
e: { bodyFrame, faceFrame },
|
|
1050
1045
|
});
|
|
1051
1046
|
}
|
|
1052
1047
|
if(this.allLostFrameCount === this.maxLostFrameCount) {
|
|
1053
1048
|
this.options.reportMessage({
|
|
1054
1049
|
code: EErrorCode.RENDER_DATA_MISSING,
|
|
1055
|
-
message: `连续${this.maxLostFrameCount}帧
|
|
1050
|
+
message: `连续${this.maxLostFrameCount}帧 身体渲染数据缺失`,
|
|
1051
|
+
e: { bodyFrame, faceFrame },
|
|
1056
1052
|
});
|
|
1057
1053
|
// 丢失过多帧,达到最大允许丢失帧数后,触发解码器对齐到最新的一帧
|
|
1058
|
-
this.options.onError({
|
|
1059
|
-
code: EErrorCode.RENDER_DATA_MISSING,
|
|
1060
|
-
message: `Error: 连续${this.maxLostFrameCount}帧 身体渲染数据缺失`,
|
|
1061
|
-
e: JSON.stringify({ bodyFrame, faceFrame }),
|
|
1062
|
-
});
|
|
1063
1054
|
this.forceSyncDecoder();
|
|
1064
1055
|
this.allLostFrameCount = 0;
|
|
1065
1056
|
}
|
|
@@ -1180,10 +1171,10 @@ export default class AvatarRender {
|
|
|
1180
1171
|
rendered = true;
|
|
1181
1172
|
} catch (error) {
|
|
1182
1173
|
(window as any).avatarSDKLogger.error(this.TAG, "渲染帧失败:", error);
|
|
1183
|
-
this.options.
|
|
1174
|
+
this.options.reportMessage({
|
|
1184
1175
|
code: EErrorCode.WebGL_RENDER_ERROR,
|
|
1185
1176
|
message: `Error: 第${frameIndex}帧 webgl渲染异常`,
|
|
1186
|
-
e:
|
|
1177
|
+
e: { bodyFrame, faceFrame, error },
|
|
1187
1178
|
});
|
|
1188
1179
|
} finally {
|
|
1189
1180
|
// 渲染完成后再关闭VideoFrame
|
|
@@ -184,14 +184,25 @@ export default class MediaSourceAudioPlayer {
|
|
|
184
184
|
* 获取缓冲范围
|
|
185
185
|
*/
|
|
186
186
|
private getBufferedRanges(): string {
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
187
|
+
try {
|
|
188
|
+
// stop/destroy 流程中浏览器会先移除 SourceBuffer 再置空引用,
|
|
189
|
+
// 此时访问 .buffered 会抛 InvalidStateError,需要 readyState 校验 + 兜底
|
|
190
|
+
if (
|
|
191
|
+
this.sourceBuffer &&
|
|
192
|
+
this.mediaSource?.readyState === "open" &&
|
|
193
|
+
this.sourceBuffer.buffered.length > 0
|
|
194
|
+
) {
|
|
195
|
+
const ranges: string[] = [];
|
|
196
|
+
for (let i = 0; i < this.sourceBuffer.buffered.length; i++) {
|
|
197
|
+
const start = this.sourceBuffer.buffered.start(i).toFixed(2);
|
|
198
|
+
const end = this.sourceBuffer.buffered.end(i).toFixed(2);
|
|
199
|
+
ranges.push(`[${start}-${end}]`);
|
|
200
|
+
}
|
|
201
|
+
return ranges.join(", ");
|
|
193
202
|
}
|
|
194
|
-
|
|
203
|
+
} catch (e) {
|
|
204
|
+
// SourceBuffer 已被移除(stop/destroy 竞态),视为无缓冲数据
|
|
205
|
+
return "-";
|
|
195
206
|
}
|
|
196
207
|
return "-";
|
|
197
208
|
}
|
|
@@ -158,9 +158,6 @@ export default class RenderScheduler {
|
|
|
158
158
|
this.renderState = state;
|
|
159
159
|
},
|
|
160
160
|
sendVideoInfo: config.sendVideoInfo,
|
|
161
|
-
onError: (error: any) => {
|
|
162
|
-
this.sdk.onMessage(error);
|
|
163
|
-
},
|
|
164
161
|
});
|
|
165
162
|
this.uiRenderer = new UIRenderer({
|
|
166
163
|
sdk: this.sdk,
|
|
@@ -341,9 +338,10 @@ export default class RenderScheduler {
|
|
|
341
338
|
} else {
|
|
342
339
|
nowFaceData.push(faceData[i]);
|
|
343
340
|
}
|
|
344
|
-
nowFaceData.length && this.dataCacheQueue._updateFacial(nowFaceData);
|
|
345
|
-
realFaceData.length && this.dataCacheQueue._updateRealFacial(realFaceData);
|
|
346
341
|
}
|
|
342
|
+
// 分类完成后一次性入队,避免循环内对累加数组重复 push 导致队列数据重复
|
|
343
|
+
nowFaceData.length && this.dataCacheQueue._updateFacial(nowFaceData);
|
|
344
|
+
realFaceData.length && this.dataCacheQueue._updateRealFacial(realFaceData);
|
|
347
345
|
break;
|
|
348
346
|
case EFrameDataType.AUDIO:
|
|
349
347
|
const audioData = data as IRawAudioFrameData[];
|
|
@@ -411,6 +409,11 @@ export default class RenderScheduler {
|
|
|
411
409
|
clearTimeout(this._speedResetTimer);
|
|
412
410
|
}
|
|
413
411
|
|
|
412
|
+
// 播放加速大于4,直接丢弃
|
|
413
|
+
if (playbackSpeed > 4) {
|
|
414
|
+
return;
|
|
415
|
+
}
|
|
416
|
+
|
|
414
417
|
// 8. 应用速度
|
|
415
418
|
this.audioRenderer.setSpeed(playbackSpeed);
|
|
416
419
|
|
|
@@ -490,7 +493,34 @@ export default class RenderScheduler {
|
|
|
490
493
|
code: EErrorCode.EVENT_TRIGGERED,
|
|
491
494
|
message: 'WARNING: 有新的speak_start事件,但是上一轮仍未结束,触发兜底结束',
|
|
492
495
|
});
|
|
493
|
-
|
|
496
|
+
const currentSpeakStart = d.flatMap(item => item.e ?? []).find(ev => ev.type === 'speak_start');
|
|
497
|
+
const newSpeakId = currentSpeakStart?.speech_id ?? 0;
|
|
498
|
+
const playingSpeechId = this.audioRenderer.speech_id;
|
|
499
|
+
// 仅当正在播放的确实是上一轮(sid 小于新一轮)时才打断旧轮次。
|
|
500
|
+
// 若事件晚于音频到达(playingSpeechId 已是新一轮 sid),此时播放的就是新一轮,
|
|
501
|
+
// 不能用 interrupt()(其目标取 audioRenderer.speech_id),否则会误清新一轮的音频队列、
|
|
502
|
+
// 字幕事件,并因 lastSpeechId/oldSpeechId 更新导致后续音视频帧全部被丢弃,
|
|
503
|
+
// 表现为"有口型、无字幕、无音频"。
|
|
504
|
+
if (playingSpeechId !== -1 && playingSpeechId < newSpeakId) {
|
|
505
|
+
this.interrupt('new_speak_start', playingSpeechId);
|
|
506
|
+
// stop() 内部有 oldSpeechId >= speech_id 的防重入守卫:当 speech_id 停留在
|
|
507
|
+
// 异常残留值(如初始值 0,长会话中 isPlaying 卡死为 true)时 stop 会被拦截,
|
|
508
|
+
// isPlaying/speech_id 无法复位,导致新一轮音频在 _updateAudio 中永远走缓存分支
|
|
509
|
+
// 不播放(日志表现:缓存数据长度持续增长且无"开始播放")。
|
|
510
|
+
// 这里检测到 stop 未生效(speech_id 未复位为 -1)时,用 pause() 强制复位,
|
|
511
|
+
// pause 会 flush 已缓存音频并把 speech_id/oldSpeechId 重置,新一轮可正常起播。
|
|
512
|
+
if (this.audioRenderer.speech_id !== -1) {
|
|
513
|
+
this.audioRenderer.pause();
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
// 新 speak_start 已到来,立即复位 interrupt 并更新轮次状态,
|
|
517
|
+
// 避免依赖 UIRenderer 在 sf 帧处理事件时(sf==当前帧且事件在 compose 后被处理)被漏处理,
|
|
518
|
+
// 导致新轮次实际说话期间仍被强制渲染为闭嘴表情
|
|
519
|
+
if (currentSpeakStart) {
|
|
520
|
+
this.currentSpeechId = currentSpeakStart.speech_id ?? 0;
|
|
521
|
+
this.avatarRenderer.setInterrupt(false);
|
|
522
|
+
this.uiRenderer.lastSpeechId = currentSpeakStart.speech_id ?? 0;
|
|
523
|
+
}
|
|
494
524
|
}
|
|
495
525
|
|
|
496
526
|
// 2. 处理过期speak_start
|
|
@@ -810,6 +840,21 @@ export default class RenderScheduler {
|
|
|
810
840
|
}])
|
|
811
841
|
}
|
|
812
842
|
|
|
843
|
+
/**
|
|
844
|
+
* 重连成功恢复(含窗口内 socket.io 自动重连恢复、窗口外重开房间):
|
|
845
|
+
* 数据缓存在断网进入离线时已清空(_offlineMode/_offlineRun),此处仅恢复播放状态:
|
|
846
|
+
* - 标记帧号对齐:首个 BODY/AUDIO 数据到达时按 sf 重设帧时钟
|
|
847
|
+
* - 恢复音频播放
|
|
848
|
+
* - 解码器退出离线模式(重置 _offlineLastFrame、清空离线队列)
|
|
849
|
+
* 避免离线 idle 播放推进的帧号领先服务端续推数据导致人物卡死、音画不同步
|
|
850
|
+
*/
|
|
851
|
+
markReconnectRecover() {
|
|
852
|
+
// 恢复音频播放
|
|
853
|
+
this.audioRenderer.resume();
|
|
854
|
+
// 解码器退出离线模式(重置 _offlineLastFrame、清空离线队列)
|
|
855
|
+
this.decoder._reload();
|
|
856
|
+
}
|
|
857
|
+
|
|
813
858
|
_offlineMode() {
|
|
814
859
|
this.dataCacheQueue._clearEvent();
|
|
815
860
|
const currentFrame = this.getCurrentFrameIndex() ?? 0;
|
|
@@ -830,6 +875,14 @@ export default class RenderScheduler {
|
|
|
830
875
|
if (frame) {
|
|
831
876
|
this.decoder._offLineMode(this.resourceManager._getOfflineIdle(), frame);
|
|
832
877
|
}
|
|
878
|
+
// 重置所有 speechId 相关状态,避免联网后新语音被过滤
|
|
879
|
+
// - lastSpeechId: 用于过滤旧 sid 的音频数据(L444)
|
|
880
|
+
// - currentSpeechId: 用于 speak_end 时判断是否 setInterrupt
|
|
881
|
+
// - uiRenderer.lastSpeechId: UIRenderer 会过滤 sid < lastSpeechId 的事件(含 speak_start)
|
|
882
|
+
// 若不重置,联网后新语音的 speak_start 会被过滤,导致字幕正常但无音频
|
|
883
|
+
this.lastSpeechId = -1;
|
|
884
|
+
this.currentSpeechId = -1;
|
|
885
|
+
this.uiRenderer.lastSpeechId = -1;
|
|
833
886
|
}
|
|
834
887
|
_offlineRun() {
|
|
835
888
|
this.decoder._offlineRun()
|
|
@@ -844,6 +897,16 @@ export default class RenderScheduler {
|
|
|
844
897
|
this.audioRenderer.resume();
|
|
845
898
|
}
|
|
846
899
|
|
|
900
|
+
/**
|
|
901
|
+
* app 切后台时暂停音频并清空对齐状态(复用断网离线时的 pause())
|
|
902
|
+
* 不能保留缓存续播:Android 后台视频帧号持续递增而音频停留在暂停位置,
|
|
903
|
+
* 恢复后音画基准脱节;且后台期间可能下发新 speak 数据,旧缓存与新 sf 冲突。
|
|
904
|
+
* 恢复后依赖新下发音频的 sf 作为 firstFrameIndex 重新对齐
|
|
905
|
+
*/
|
|
906
|
+
pauseForBackground() {
|
|
907
|
+
this.audioRenderer.pause();
|
|
908
|
+
}
|
|
909
|
+
|
|
847
910
|
/**
|
|
848
911
|
* 设置数字人canvas的显隐状态
|
|
849
912
|
* @param visible 是否可见
|
|
@@ -859,10 +922,13 @@ export default class RenderScheduler {
|
|
|
859
922
|
this.avatarRenderer.setWalkConfig(walkConfig);
|
|
860
923
|
}
|
|
861
924
|
|
|
862
|
-
|
|
925
|
+
// targetSpeechId: 显式指定要打断的轮次 sid。缺省时取 audioRenderer.speech_id(当前播放轮次)。
|
|
926
|
+
// 当事件晚于音频到达时,audioRenderer.speech_id 可能已经切换到新一轮,此时必须显式传旧 sid,
|
|
927
|
+
// 避免误清新一轮的音频与字幕。
|
|
928
|
+
interrupt(type?: string, targetSpeechId?: number): number{
|
|
863
929
|
const start = Date.now();
|
|
864
930
|
// 插入中断字幕和语音结束事件
|
|
865
|
-
const speech_id = this.audioRenderer.speech_id;
|
|
931
|
+
const speech_id = targetSpeechId ?? this.audioRenderer.speech_id;
|
|
866
932
|
if(type === "in_offline_mode") {
|
|
867
933
|
this.lastSpeechId = -1;
|
|
868
934
|
} else {
|
package/src/control/ttsa.ts
CHANGED
|
@@ -55,6 +55,7 @@ interface ITtsaOptions {
|
|
|
55
55
|
enterOfflineMode: () => void;
|
|
56
56
|
reStartSDK: () => void;
|
|
57
57
|
reconnect_client_timeout: number;
|
|
58
|
+
reconnect_timeout: number;
|
|
58
59
|
sendVoiceEnd: () => void;
|
|
59
60
|
getCurrentFrameIndex: () => number;
|
|
60
61
|
}
|
|
@@ -77,7 +78,9 @@ export default class Ttsa {
|
|
|
77
78
|
private enterOfflineMode: () => void;
|
|
78
79
|
private reStartSDK: () => void;
|
|
79
80
|
private reconnect_client_timeout: number;
|
|
80
|
-
private
|
|
81
|
+
private reconnect_timeout: number;
|
|
82
|
+
private reconnectTimer = -1; // 重开房间定时器(reconnect_client_timeout 后 stopSession + reStartSDK)
|
|
83
|
+
private socketReconnectTimer = -1; // 停止 socket.io 自动重连定时器(reconnect_timeout 后旧房间已失效)
|
|
81
84
|
private wsConnectTimer = 0;
|
|
82
85
|
private enterOfflineFlag = false;
|
|
83
86
|
private sendVoiceEnd: () => void;
|
|
@@ -94,13 +97,26 @@ export default class Ttsa {
|
|
|
94
97
|
private readonly HEARTBEAT_INTERVAL = 2000;
|
|
95
98
|
|
|
96
99
|
/**
|
|
97
|
-
* 清除WebSocket
|
|
100
|
+
* 清除WebSocket重连定时器(含停止 socket.io 自动重连的定时器)
|
|
98
101
|
*/
|
|
99
102
|
public clearReconnectTimer(): void {
|
|
100
103
|
if (this.reconnectTimer !== -1) {
|
|
101
104
|
clearTimeout(this.reconnectTimer);
|
|
102
105
|
this.reconnectTimer = -1;
|
|
103
106
|
}
|
|
107
|
+
// 一并清理停止 socket.io 自动重连的定时器:网络恢复后交由 socket.io 自行重连,
|
|
108
|
+
// 不再强制断开(下次 disconnect 会重新按窗口设置)
|
|
109
|
+
this.clearSocketReconnectTimer();
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* 清除停止 socket.io 自动重连的定时器
|
|
114
|
+
*/
|
|
115
|
+
private clearSocketReconnectTimer(): void {
|
|
116
|
+
if (this.socketReconnectTimer !== -1) {
|
|
117
|
+
clearTimeout(this.socketReconnectTimer);
|
|
118
|
+
this.socketReconnectTimer = -1;
|
|
119
|
+
}
|
|
104
120
|
}
|
|
105
121
|
|
|
106
122
|
/**
|
|
@@ -146,6 +162,7 @@ export default class Ttsa {
|
|
|
146
162
|
this.enterOfflineMode = options.enterOfflineMode;
|
|
147
163
|
this.reStartSDK = options.reStartSDK;
|
|
148
164
|
this.reconnect_client_timeout = options.reconnect_client_timeout;
|
|
165
|
+
this.reconnect_timeout = options.reconnect_timeout;
|
|
149
166
|
this.sendVoiceEnd = options.sendVoiceEnd;
|
|
150
167
|
this.getCurrentFrameIndex = options.getCurrentFrameIndex;
|
|
151
168
|
|
|
@@ -154,6 +171,9 @@ export default class Ttsa {
|
|
|
154
171
|
token: this.token,
|
|
155
172
|
},
|
|
156
173
|
transports: ['websocket'],
|
|
174
|
+
// 自动重连不限制次数:在 reconnect_timeout(服务端断开旧房间的窗口)内持续尝试,
|
|
175
|
+
// 失效房间必然重连失败;超过窗口由 disconnect 处理器中的停止定时器断开,
|
|
176
|
+
// 超过 reconnect_client_timeout 由重开房间定时器兜底(stopSession + reStartSDK)
|
|
157
177
|
reconnection: true,
|
|
158
178
|
reconnectionAttempts: Infinity,
|
|
159
179
|
reconnectionDelay: 1000,
|
|
@@ -167,17 +187,21 @@ export default class Ttsa {
|
|
|
167
187
|
this.initWsTimeoutTimer();
|
|
168
188
|
// 启动客户端心跳
|
|
169
189
|
this.startHeartbeat();
|
|
190
|
+
// 重置离线处理标志:重连成功后允许下次断网正常进入离线/重连流程
|
|
191
|
+
this.enterOfflineFlag = false;
|
|
170
192
|
(window as any).performanceTracker.markEnd(performanceConstant.ttsa_connect);
|
|
171
193
|
const payload = {
|
|
172
194
|
room: this.room,
|
|
173
195
|
client_type: "web",
|
|
174
196
|
invisible_mode: this.sdk.getPendingInvisibleMode(),
|
|
175
197
|
};
|
|
176
|
-
// TODO: 临时处理,待后端修复
|
|
177
|
-
if(this.sdk.getStatus()=== AvatarStatus.offline) {
|
|
178
|
-
this.sdk.onStatusChange(AvatarStatus.online);
|
|
179
|
-
}
|
|
180
198
|
this.send("enter_room", payload);
|
|
199
|
+
// 窗口内自动重连恢复:服务端可能不重发 first_start_timestamp,
|
|
200
|
+
// 若仅依赖 reloadSuccess 补发 online/恢复播放,状态会卡在离线导致重连后卡死。
|
|
201
|
+
// 因此重连成功即恢复(仅初始化完成且此前已离线时;初次启动 status 为 -1 不受影响)
|
|
202
|
+
if (this.sdk.getStatus() === AvatarStatus.offline && this.sdk.getIsInitialized()) {
|
|
203
|
+
this.reloadSuccess();
|
|
204
|
+
}
|
|
181
205
|
});
|
|
182
206
|
ws.on("first_start_timestamp", (e: any) => {
|
|
183
207
|
(window as any).avatarSDKLogger.log(this.TAG, "first_start_timestamp", e);
|
|
@@ -339,11 +363,23 @@ export default class Ttsa {
|
|
|
339
363
|
// 无网断开则进入离线模式
|
|
340
364
|
this.enterOfflineMode();
|
|
341
365
|
}
|
|
342
|
-
//
|
|
343
|
-
//
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
366
|
+
// 重连窗口策略(不再依赖 navigator.onLine,socket 断开即启动):
|
|
367
|
+
// ① socketReconnectTimer(reconnect_timeout):窗口内 socket.io 自动重连;
|
|
368
|
+
// 服务端会在 reconnect_timeout 后断开旧房间,因此窗口结束即停止无谓的自动重连
|
|
369
|
+
// ② reconnectTimer(reconnect_client_timeout):仍未能重连则停止旧连接并重开房间
|
|
370
|
+
// 均仅首次设置,避免 socket.io 重连失败反复触发 disconnect 重置定时器导致永不执行;
|
|
371
|
+
// 重连成功时 connect 处理器、网络恢复时 NetworkMonitor -> clearReconnectTimer 都会取消
|
|
372
|
+
if (this.socketReconnectTimer === -1 && this.reconnect_timeout > 0) {
|
|
373
|
+
this.socketReconnectTimer = window.setTimeout(() => {
|
|
374
|
+
try {
|
|
375
|
+
ws.disconnect();
|
|
376
|
+
} catch (error) {
|
|
377
|
+
(window as any).avatarSDKLogger.warn(this.TAG, "disconnect error", error);
|
|
378
|
+
}
|
|
379
|
+
this.socketReconnectTimer = -1;
|
|
380
|
+
}, this.reconnect_timeout * 1000);
|
|
381
|
+
}
|
|
382
|
+
if (this.reconnectTimer === -1) {
|
|
347
383
|
this.reconnectTimer = window.setTimeout(async () => {
|
|
348
384
|
try {
|
|
349
385
|
ws.disconnect();
|