@xmov/avatar 2.4.0 → 2.5.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. package/README.md +34 -6
  2. package/package-dist/agent/avatar.cjs +1 -0
  3. package/package-dist/agent/avatar.modern.js +1 -0
  4. package/package-dist/agent/avatar.module.js +1 -0
  5. package/package-dist/agent/avatar.umd.js +1 -0
  6. package/package-dist/agent/index.d.ts +42 -0
  7. package/{dist/agent/types.d.ts → package-dist/agent/public-types.d.ts} +115 -116
  8. package/package-dist/index.cjs +3 -0
  9. package/package-dist/index.d.ts +34 -0
  10. package/package-dist/index.modern.js +1 -0
  11. package/package-dist/index.module.js +1 -0
  12. package/package-dist/index.umd.js +14 -0
  13. package/package-dist/public-types.d.ts +205 -0
  14. package/package.json +41 -33
  15. package/dist/agent/avatar.cjs +0 -1
  16. package/dist/agent/avatar.modern.js +0 -1
  17. package/dist/agent/avatar.module.js +0 -1
  18. package/dist/agent/avatar.umd.js +0 -1
  19. package/dist/agent/index.d.ts +0 -137
  20. package/dist/baseRender/AudioRenderer.d.ts +0 -46
  21. package/dist/baseRender/AudioWorklet.d.ts +0 -78
  22. package/dist/baseRender/AvatarRenderer.d.ts +0 -225
  23. package/dist/baseRender/MSEAudioPlayer.d.ts +0 -51
  24. package/dist/baseRender/UIRenderer.d.ts +0 -39
  25. package/dist/baseRender/pcm-audio-processor.d.ts +0 -63
  26. package/dist/control/APIForwarder.d.ts +0 -23
  27. package/dist/control/DataCacheQueue.d.ts +0 -69
  28. package/dist/control/EventDispatcher.d.ts +0 -35
  29. package/dist/control/RenderScheduler.d.ts +0 -171
  30. package/dist/control/SaveAndDownload.d.ts +0 -14
  31. package/dist/control/StreamingClient.d.ts +0 -33
  32. package/dist/control/VideoPlayer.d.ts +0 -47
  33. package/dist/control/ttsa.d.ts +0 -125
  34. package/dist/encoding/media-recorder-encoder.d.ts +0 -33
  35. package/dist/encoding/pcm-webm-encoder.d.ts +0 -13
  36. package/dist/encoding/webcodec-opus-encoder.d.ts +0 -41
  37. package/dist/encoding/webm-muxer.d.ts +0 -90
  38. package/dist/index.cjs +0 -2
  39. package/dist/index.cjs.map +0 -1
  40. package/dist/index.d.ts +0 -251
  41. package/dist/index.modern.js +0 -2
  42. package/dist/index.modern.js.map +0 -1
  43. package/dist/index.module.js +0 -2
  44. package/dist/index.module.js.map +0 -1
  45. package/dist/index.umd.js +0 -2
  46. package/dist/index.umd.js.map +0 -1
  47. package/dist/modules/Composition.d.ts +0 -28
  48. package/dist/modules/ResourceManager.d.ts +0 -299
  49. package/dist/modules/TrackRenderer/base-track.d.ts +0 -4
  50. package/dist/modules/TrackRenderer/index.d.ts +0 -12
  51. package/dist/modules/TrackRenderer/render-implements.d.ts +0 -19
  52. package/dist/modules/TrackRenderer/track-pic.d.ts +0 -8
  53. package/dist/modules/TrackRenderer/track-subtitle.d.ts +0 -8
  54. package/dist/modules/cache-manager.d.ts +0 -5
  55. package/dist/modules/decoder.d.ts +0 -84
  56. package/dist/modules/error-handle.d.ts +0 -17
  57. package/dist/modules/network.d.ts +0 -21
  58. package/dist/modules/sentry-reporter.d.ts +0 -58
  59. package/dist/proto/face_data_pb.d.ts +0 -0
  60. package/dist/types/capability.d.ts +0 -15
  61. package/dist/types/error.d.ts +0 -70
  62. package/dist/types/event.d.ts +0 -25
  63. package/dist/types/frame-data.d.ts +0 -107
  64. package/dist/types/index.d.ts +0 -196
  65. package/dist/types/render.d.ts +0 -18
  66. package/dist/utils/DataInterface.d.ts +0 -113
  67. package/dist/utils/GLDevice.d.ts +0 -46
  68. package/dist/utils/GLPipeline.d.ts +0 -84
  69. package/dist/utils/GLPipelineDebugTools.d.ts +0 -5
  70. package/dist/utils/Math.d.ts +0 -21
  71. package/dist/utils/__tests__/capability-checker.test.d.ts +0 -1
  72. package/dist/utils/audio-session.d.ts +0 -11
  73. package/dist/utils/audio.d.ts +0 -1
  74. package/dist/utils/capability-checker.d.ts +0 -28
  75. package/dist/utils/encodeToken.d.ts +0 -14
  76. package/dist/utils/face.d.ts +0 -1
  77. package/dist/utils/float32-decoder.d.ts +0 -26
  78. package/dist/utils/fpsTracker.d.ts +0 -27
  79. package/dist/utils/index.d.ts +0 -22
  80. package/dist/utils/logger.d.ts +0 -9
  81. package/dist/utils/media-recorder-timestamp.d.ts +0 -15
  82. package/dist/utils/pcmAnalyzer.d.ts +0 -40
  83. package/dist/utils/perfermance.d.ts +0 -15
  84. package/dist/utils/request.d.ts +0 -20
  85. package/dist/utils/requestAnimateFrames.d.ts +0 -31
  86. package/dist/utils/time.d.ts +0 -9
  87. package/dist/utils/video-probe-asset.d.ts +0 -28
  88. package/dist/view/DebugOverlay.d.ts +0 -54
  89. package/dist/worker/streaming-video.d.ts +0 -1
  90. package/src/baseRender/AudioRenderer.ts +0 -328
  91. package/src/baseRender/AudioWorklet.ts +0 -776
  92. package/src/baseRender/AvatarRenderer.ts +0 -1550
  93. package/src/baseRender/MSEAudioPlayer.ts +0 -254
  94. package/src/baseRender/UIRenderer.ts +0 -195
  95. package/src/baseRender/pcm-audio-processor.js +0 -267
  96. package/src/control/APIForwarder.ts +0 -67
  97. package/src/control/DataCacheQueue.ts +0 -372
  98. package/src/control/EventDispatcher.ts +0 -106
  99. package/src/control/RenderScheduler.ts +0 -1000
  100. package/src/control/SaveAndDownload.js +0 -77
  101. package/src/control/StreamingClient.ts +0 -138
  102. package/src/control/VideoPlayer.ts +0 -306
  103. package/src/control/ttsa.ts +0 -754
  104. package/src/encoding/media-recorder-encoder.ts +0 -175
  105. package/src/encoding/pcm-webm-encoder.ts +0 -72
  106. package/src/encoding/webcodec-opus-encoder.ts +0 -258
  107. package/src/encoding/webm-muxer.ts +0 -337
  108. package/src/global.d.ts +0 -112
  109. package/src/index.ts +0 -2053
  110. package/src/modules/Composition.ts +0 -79
  111. package/src/modules/ResourceManager.ts +0 -926
  112. package/src/modules/TrackRenderer/base-track.ts +0 -7
  113. package/src/modules/TrackRenderer/index.ts +0 -40
  114. package/src/modules/TrackRenderer/render-implements.ts +0 -269
  115. package/src/modules/TrackRenderer/track-pic.ts +0 -19
  116. package/src/modules/TrackRenderer/track-subtitle.ts +0 -16
  117. package/src/modules/cache-manager.ts +0 -10
  118. package/src/modules/decoder.ts +0 -548
  119. package/src/modules/error-handle.ts +0 -20
  120. package/src/modules/network.ts +0 -71
  121. package/src/modules/sentry-reporter.ts +0 -171
  122. package/src/proto/face_data.proto +0 -45
  123. package/src/proto/face_data_pb.js +0 -1297
  124. package/src/proto/protobuf.min.js +0 -8
  125. package/src/types/capability.ts +0 -24
  126. package/src/types/error.ts +0 -211
  127. package/src/types/event.ts +0 -38
  128. package/src/types/frame-data.ts +0 -121
  129. package/src/types/index.ts +0 -220
  130. package/src/types/render.ts +0 -18
  131. package/src/utils/DataInterface.ts +0 -746
  132. package/src/utils/GLDevice.ts +0 -286
  133. package/src/utils/GLPipeline.ts +0 -1293
  134. package/src/utils/GLPipelineDebugTools.ts +0 -174
  135. package/src/utils/Math.ts +0 -266
  136. package/src/utils/__tests__/capability-checker.test.ts +0 -159
  137. package/src/utils/audio-session.ts +0 -144
  138. package/src/utils/audio.ts +0 -44
  139. package/src/utils/blueimp-md5.d.ts +0 -3
  140. package/src/utils/capability-checker.ts +0 -762
  141. package/src/utils/encodeToken.ts +0 -115
  142. package/src/utils/face.ts +0 -29
  143. package/src/utils/float32-decoder.js +0 -130
  144. package/src/utils/fpsTracker.ts +0 -100
  145. package/src/utils/index.ts +0 -182
  146. package/src/utils/logger.js +0 -36
  147. package/src/utils/media-recorder-timestamp.ts +0 -79
  148. package/src/utils/pcmAnalyzer.ts +0 -154
  149. package/src/utils/perfermance.ts +0 -141
  150. package/src/utils/request.ts +0 -107
  151. package/src/utils/requestAnimateFrames.ts +0 -195
  152. package/src/utils/time.ts +0 -32
  153. package/src/utils/video-probe-asset.ts +0 -99
  154. package/src/view/DebugOverlay.ts +0 -409
  155. package/src/worker/streaming-video.ts +0 -4647
  156. /package/{dist → package-dist}/agent/index.cjs +0 -0
  157. /package/{dist → package-dist}/agent/index.umd.js +0 -0
package/src/index.ts DELETED
@@ -1,2053 +0,0 @@
1
- import { AvatarStatus, RenderState, IAvatarOptions, IInitParams, InitModel, Layout, WalkConfig } from "./types/index";
2
- import { EErrorCode } from "./types/error";
3
- import { EFrameDataType, IRawFrameData, StateChangeInfo, SYNC_FRAME_DATA } from "./types/frame-data";
4
- import RenderScheduler from "./control/RenderScheduler";
5
- import ResourceManager, { ISessionResponse, TDownloadProgress } from "./modules/ResourceManager";
6
- import WidgetDefaultRender from "./modules/TrackRenderer/render-implements";
7
- import Ttsa from "./control/ttsa";
8
- import { DebugOverlay } from "./view/DebugOverlay";
9
- import NetworkMonitor from './modules/network';
10
- import Errors from './modules/error-handle';
11
- import { SentryReporter } from './modules/sentry-reporter';
12
-
13
- import { performanceConstant } from "./utils/perfermance";
14
- import './utils/logger.js';
15
- import { IBRAnimationGeneratorCharInfo_NN, unpackIBRAnimation, formatMJT } from './utils/DataInterface'
16
- import { getVertices, getWavefrontObjFromVertices, getPCATextures } from './utils/GLPipelineDebugTools'
17
- import { GLDevice } from './utils/GLDevice'
18
- import { GLPipeline } from './utils/GLPipeline'
19
- import { getDeviceInfo, isharmony, isSupportMES, validateScaleWithDecimal } from "./utils";
20
- import { StreamingClient } from './control/StreamingClient';
21
- import { VideoPlayer } from './control/VideoPlayer';
22
- import { EventDispatcher } from './control/EventDispatcher';
23
- import { APIForwarder } from './control/APIForwarder';
24
- import { checkCapability } from './utils/capability-checker';
25
- import { FPSStats } from "./utils/fpsTracker";
26
- import { PcmWebMEncoder } from './encoding/pcm-webm-encoder';
27
- import { MediaRecorderEncoder } from './encoding/media-recorder-encoder';
28
-
29
- type InternalAvatarOptions = IAvatarOptions & {
30
- sessionRequestData?: Record<string, unknown>
31
- }
32
-
33
- export default class XmovAvatar {
34
- //static STATUS: AvatarStatus | -1 = -1;
35
-
36
- private options!: IAvatarOptions;
37
-
38
- private TAG = "[XMOV AVATAR]";
39
- private el: HTMLElement | null = null;
40
-
41
- // 实例状态,每个 XmovAvatar 实例都有自己独立的状态
42
- private status: AvatarStatus | -1 = -1;
43
-
44
- // 使用 ! 操作符告诉 TypeScript 这些属性会在构造函数中被初始化
45
- private resourceManager!: ResourceManager;
46
- private renderScheduler!: RenderScheduler;
47
- private networkMonitor: NetworkMonitor | null = null;
48
- private ttsa: Ttsa | null = null;
49
-
50
- private debugOverlay: DebugOverlay | null = null;
51
- private sentryReporter: SentryReporter | null = null;
52
- /** 首帧 span 句柄(init 时启动,首次 BODY 帧到达时结束) */
53
- private firstFrameSpan: { end(): void } | null = null;
54
-
55
- private _offlineTimer = -1
56
- private readonly _offlineInterval = 300
57
- private _env = "production"
58
- // 可以与隐身模式做联动,
59
- private avatarCanvasVisible: boolean = true; // 数字人canvas显隐状态,默认为true
60
- private pendingInvisibleMode: boolean = false; // 待处理的隐身模式(在init时设置,在start时应用)
61
- private isInitialized: boolean = false; // 初始化是否完成(progress === 100)
62
- public onStateChange: (state: string) => void = () => {};
63
-
64
- public onDownloadProgress: TDownloadProgress | null = null;
65
-
66
- static IBRAnimationGeneratorCharInfo_NN = IBRAnimationGeneratorCharInfo_NN;
67
- static unpackIBRAnimation = unpackIBRAnimation;
68
- static formatMJT = formatMJT;
69
- static getVertices = getVertices;
70
- static getPCATextures = getPCATextures;
71
- static getWavefrontObjFromVertices = getWavefrontObjFromVertices;
72
- static GLDevice = GLDevice;
73
- static GLPipeline = GLPipeline;
74
- static checkCapability = checkCapability;
75
- private boundVisibilityChange: () => void;
76
- private replayData: any = null;
77
- private startConnectTime: number = 0;
78
- private connectSuccessTime: number = 0;
79
- private sessionResTime: number = 0;
80
- private charbinResTime: number = 0;
81
- private isfirstReceive: boolean = true;
82
- private enableClientInterrupt: boolean = false; // 是否开启客户端打断
83
- private retryCount = 1;
84
- private retryTimer = -1;
85
- private maxRetryCount = 9;
86
- private retryRound = 1; // 重试轮次
87
- private maxRetryRound = 3; // 最大轮次
88
- private isRetrying = false; // 重试中锁,避免并发调用
89
- private isStartRetry = false;
90
- private destroyed = false;
91
- private incompleteInitializationCleanupPromise: Promise<void> | null = null;
92
-
93
- // 音频捕获相关(用于 ASR 回声消除)
94
- private captureAudioCtx: AudioContext | null = null;
95
- private captureWorklet: AudioWorkletNode | null = null;
96
- private captureSource: MediaElementAudioSourceNode | null = null;
97
- private captureSetupPromise: Promise<void> | null = null;
98
- private captureLastSpeechId: number = -2;
99
- private restartPlaybackCaptureImpl: (() => void) | null = null;
100
- private captureEncodeQueue: Promise<void> = Promise.resolve();
101
- private captureGeneration = 0;
102
-
103
- // PCM 捕获相关(raw_audio=true 模式)
104
- private pcmCaptureLastSpeechId: number = -2;
105
- private pcmCaptureFirstChunk: boolean = true;
106
- private pcmWebMEncoder: PcmWebMEncoder | null = null;
107
- private mediaRecorderEncoder: MediaRecorderEncoder | null = null;
108
- private pcmCaptureMode: 'webcodec' | 'mediarecorder' | 'none' = 'none';
109
- private pcmEncodeId: number = 0; // 用于 reset 竞态检测,递增标识当前编码会话
110
-
111
- private networkInfo: any = {
112
- rtt: 0,
113
- downlink: 0.0,
114
- networkStatus: false,
115
- };
116
-
117
- private mediaRecorder: MediaRecorder | null = null;
118
- private recordedChunks: Blob[] = [];
119
- private stream: MediaStream | null = null;
120
-
121
- // 薄客户端模式相关字段
122
- private _isClientMode: boolean = false;
123
- private _streamingClient: StreamingClient | null = null;
124
- private _videoPlayer: VideoPlayer | null = null;
125
- private _eventDispatcher: EventDispatcher | null = null;
126
- private _apiForwarder: APIForwarder | null = null;
127
- private _clientOnDownloadProgress: ((progress: number) => void) | null = null;
128
- private _pendingInit: boolean = false;
129
- private _pendingInitConfig: any = null;
130
- private _clientSessionId: string = '';
131
-
132
- constructor(options: IAvatarOptions) {
133
- (window as any).avatarSDKLogger.setEnabled(options.enableLogger);
134
- this.options = options;
135
- this._env = options.env || "production"
136
- // 提前创建 Sentry:无网络 / 容器不存在 / 能力检查失败等错误都要经 onMessage 上报,
137
- // 而 onMessage 的上报条件是 sentryReporter 已存在,晚建会让这些早期错误全部丢失
138
- this.initSentryReporter();
139
- this.enableClientInterrupt = options.enableClientInterrupt || false;
140
- this.boundVisibilityChange = this.visibilitychange.bind(this);
141
-
142
- // 薄客户端模式:跳过所有重量级初始化,仅准备容器
143
- if (options.renderMode === 'client') {
144
- this._isClientMode = true;
145
- this.initializeClient(options);
146
- return;
147
- }
148
-
149
- // 完整渲染模式(原有逻辑)
150
- // initializeSDK 是 fire-and-forget:内部同步抛错(如 layout.scale 校验)与未捕获的异步失败
151
- // 都不会走 onMessage,这里兜底上报,避免业务既拿不到错误回调、监控也收不到
152
- void this.initializeSDK(options).catch((e) => {
153
- this.reportInitializationFailure(
154
- `SDK 初始化失败: ${e?.message || e}`,
155
- e
156
- );
157
- });
158
- this.networkMonitor = new NetworkMonitor({
159
- offlineCallback: (message) => {
160
- this.onMessage({
161
- code: EErrorCode.NETWORK_DOWN,
162
- message: message || '网络断开',
163
- });
164
- console.log('网络断开(network off)', message);
165
-
166
- // 如果初始化未完成(progress < 100),清除所有已初始化的资源并断开连接
167
- if (!this.isInitialized) {
168
- (window as any).avatarSDKLogger.warn(this.TAG, '初始化未完成时断网,清除所有资源并断开连接');
169
- void this.cleanupBeforeInitComplete("network_offline_before_init");
170
- return;
171
- }
172
-
173
- if(this.status !== AvatarStatus.offline) {
174
- // 无网断开则进入离线模式
175
- this.offlineHandle();
176
- }
177
- },
178
- onlineCallback: (message) => {
179
- this.onMessage({
180
- code: EErrorCode.NETWORK_UP,
181
- message: message || '网络恢复',
182
- });
183
-
184
- // 如果初始化未完成(progress < 100),清除所有已初始化的资源并断开连接
185
- if (!this.isInitialized) {
186
- (window as any).avatarSDKLogger.warn(this.TAG, '网络恢复但初始化未完成,清除所有资源并断开连接');
187
- void this.cleanupBeforeInitComplete("network_online_before_init");
188
- return;
189
- }
190
-
191
- // 网络恢复时,取消WebSocket的reconnectTimer,避免与网络恢复的重连逻辑冲突
192
- this.ttsa?.clearReconnectTimer();
193
-
194
- if(this.isStartRetry) {
195
- this.triggerReconnect();
196
- }
197
- // this.onStatusChange(AvatarStatus.network_on)
198
- // this._reload()
199
- // this.ttsa?.connect()
200
- },
201
- // retryCallback: (count: number) => {
202
- // this.onMessage({
203
- // code: EErrorCode.NETWORK_RETRY,
204
- // message: '网络重试中',
205
- // });
206
- // this.ttsa?.connect();
207
- // return this.resourceManager._startSession();
208
- // },
209
- })
210
- }
211
-
212
- /**
213
- * 创建 Sentry 错误上报实例(编译时常量 ENABLE_SENTRY + 运行时 sentry.enabled 双重控制)。
214
- * 放在构造函数内完成:onMessage 的早期分支(无网络 / 容器不存在 / 能力检查失败)都发生在其后。
215
- */
216
- private initSentryReporter(): void {
217
- if (!ENABLE_SENTRY || !this.options.sentry?.enabled) return;
218
- try {
219
- this.sentryReporter = new SentryReporter(SENTRY_DSN, {
220
- tracesSampleRate: this.options.sentry?.tracesSampleRate,
221
- });
222
- this.sentryReporter.setTag('app_id', this.options.appId || '');
223
- this.sentryReporter.setTag('env', this._env);
224
- } catch (e) {
225
- (window as any).avatarSDKLogger.warn(this.TAG, 'Sentry 初始化失败', e);
226
- }
227
- }
228
-
229
- /**
230
- * 释放 Sentry 客户端(内部会 flush 未发送的事件)。
231
- * 用于「实例确定不可用」的提前退出路径,避免业务反复重试创建时残留多个 client。
232
- */
233
- private disposeSentryReporter(): void {
234
- if (!this.sentryReporter) return;
235
- const reporter = this.sentryReporter;
236
- this.sentryReporter = null;
237
- void reporter.destroy().catch(() => {});
238
- }
239
-
240
- public getStatus() {
241
- return this.status
242
- }
243
-
244
- /**
245
- * 初始化是否完成(progress === 100)
246
- * 供 TTSA 重连恢复时判断是否可提前恢复在线状态
247
- */
248
- public getIsInitialized() {
249
- return this.isInitialized
250
- }
251
-
252
- public get businessENV() {
253
- return this._env
254
- }
255
-
256
- public getUniqueSpeakId() {
257
- return this.ttsa?.getUniqueSpeakId() || `0-${this.getSessionId()}`
258
-
259
- }
260
-
261
- private async initializeSDK(options: IAvatarOptions) {
262
- if (!navigator.onLine) {
263
- this.onMessage({
264
- code: EErrorCode.NETWORK_BREAK,
265
- message: '没有网络,请联网后重试',
266
- });
267
- // 无网络直接退出,实例不会进入可用状态,释放 Sentry(内部会 flush 刚上报的事件)
268
- this.disposeSentryReporter();
269
- return
270
- }
271
- const self = this;
272
- const internalOptions = options as InternalAvatarOptions;
273
- const { containerId, container, appId, appSecret, gatewayServer, cacheServer, config } = options;
274
-
275
- const {scale = 1} = config?.layout?.avatar ?? {}
276
- if(config?.layout?.avatar?.scale){
277
- config.layout.avatar.scale = validateScaleWithDecimal(scale, "initializeSDK") as number | string;
278
- }
279
- const _config = { ...(config || {}), raw_audio: !isSupportMES() };
280
-
281
- if(container && container instanceof HTMLElement){
282
- this.el = container as HTMLElement;
283
- } else {
284
- const el = document.querySelector(containerId) as HTMLElement;
285
- if (!el) {
286
- this.onMessage({
287
- code: EErrorCode.CONTAINER_NOT_FOUND,
288
- message: `containerId: ${containerId} 不存在`,
289
- })
290
- // 容器不存在,实例不会进入可用状态,释放 Sentry(内部会 flush 刚上报的事件)
291
- this.disposeSentryReporter();
292
- return;
293
- }
294
- this.el = el;
295
- }
296
- const bgContainer = document.createElement("div");
297
- bgContainer.setAttribute("id", "avatar-bg-container");
298
- bgContainer.setAttribute("style", "position:absolute;top:0;left:0;width:100%;height:100%;z-index:0;opacity:0;");
299
- this.el.appendChild(bgContainer);
300
- const style = this.el.getAttribute("style");
301
- if (style) {
302
- this.el.setAttribute(
303
- "style",
304
- `position:relative;overflow:hidden;${style}`
305
- );
306
- } else {
307
- this.el.setAttribute(
308
- "style",
309
- `position:relative;overflow:hidden;`
310
- );
311
- }
312
- this.resourceManager = new ResourceManager({
313
- sdkInstance: this,
314
- headers: options.headers,
315
- tag: options.tag,
316
- gateway_type: options.gateway_type || 'default',
317
- asr_id: options.asr_id,
318
- llm_id: options.llm_id,
319
- session_speak_req_id: options.session_speak_req_id,
320
- sessionRequestData: internalOptions.sessionRequestData,
321
- appId,
322
- appSecret,
323
- gatewayServer,
324
- cacheServer,
325
- config:_config,
326
- audioOnlyMode: options.audioOnlyMode || false,
327
- onNetworkInfo(networkInfo) {
328
- self.networkInfo = networkInfo;
329
- options.onNetworkInfo?.(networkInfo);
330
- },
331
- onStartSessionWarning: (message: Object) => {
332
- this.options.onStartSessionWarning?.(message);
333
- },
334
- });
335
-
336
- this.renderScheduler = new RenderScheduler({
337
- sdkInstance: this,
338
- container: this.el,
339
- resourceManager: this.resourceManager,
340
- enableDebugger: options.enableDebugger,
341
- hardwareAcceleration: options.hardwareAcceleration || "default",
342
- enableClientInterrupt: this.enableClientInterrupt,
343
- audioOnlyMode: options.audioOnlyMode || false,
344
- onDownloadProgress: (progress: number) => {
345
- this.onDownloadProgress?.(progress);
346
- this.el?.setAttribute("style", this.el?.style.cssText + `opacity:${progress === 100 ? '1' : '0'};`);
347
- // 标记初始化完成
348
- this.isInitialized = progress === 100;
349
- if(progress === 100) {
350
- // 初始化完成:若 start 时因未初始化完成而未上线(状态守卫),此处补发 online
351
- if (!this.pendingInvisibleMode && this.status !== AvatarStatus.online && this.status !== AvatarStatus.invisible) {
352
- this.onStatusChange(AvatarStatus.online);
353
- }
354
- this.resourceManager._setOfflineCache();
355
- this.ttsa?.sendSdkPoint('loadvideo_decode_time', {
356
- timestamp: Date.now(),
357
- });
358
- setTimeout(() => {
359
- this.ttsa?.sendDeviceInfo({
360
- ...getDeviceInfo(),
361
- ...(window as any).performanceTracker.getFirstInfo(),
362
- // @ts-ignore
363
- sdk_version: VERSION,
364
- resolution: this.resourceManager.getConfig()?.resolution,
365
- download_speed: self.networkInfo.downlink || 0,
366
- })
367
- }, 20);
368
- // 初始化音频捕获(MSE 模式下,用于 ASR 回声消除)
369
- void this.enableAudioPlaybackCapture();
370
- }
371
- },
372
- onStateChange: (state: string) => {
373
- if((this.getStatus() === AvatarStatus.offline && state !== 'idle') || state === "") {
374
- return
375
- }
376
- options.onStateChange?.(state);
377
- },
378
- onRenderChange: (state: RenderState, oldState?: RenderState) => {
379
- options.onRenderChange?.(state);
380
- // 在如果数字人在恢复渲染态一会和数字人在线态的时候,数字人进入渲染中了,都应该认为是可见状态
381
- if(state === RenderState.rendering && (oldState === RenderState.resumed || this.getStatus() === AvatarStatus.online) && this.isInitialized) {
382
- (window as any).avatarSDKLogger.log(this.TAG, 'Status:===== 渲染中,状态: rendering', this.getSessionId());
383
- this.onStatusChange(AvatarStatus.visible)
384
- }
385
- },
386
- onVoiceStateChange: (state: string, duration?: number, client_speak_id?: string | number) => {
387
- options.onVoiceStateChange?.(state, duration, client_speak_id);
388
- },
389
- onWalkStateChange: (state: string) => {
390
- options.onWalkStateChange?.(state);
391
- },
392
- sendVideoInfo: (info: {name: string, body_id: number, id: number}) => {
393
- this.debugOverlay?.setVideoInfo(info);
394
- },
395
- onSpeakStateChange: (state: string, client_speak_id: string | number) => {
396
- options.onSpeakStateChange?.(state, client_speak_id);
397
- },
398
- setAudioInfo: (info: {
399
- sf: number;
400
- ef: number;
401
- ad: Uint8Array;
402
- }) => {
403
- this.debugOverlay?.setAudioInfo(info);
404
- },
405
- setEventData: (info: {
406
- sf: number;
407
- ef: number;
408
- event: Array<any>;
409
- }) => {
410
- this.debugOverlay?.setEventData(info);
411
- },
412
- renderFrameCallback: (frame: number) => {
413
- this.renderFrameCallback(frame);
414
- this.handleReplaySend(frame);
415
- },
416
- reportMessage: (message: {
417
- code: EErrorCode;
418
- message: string;
419
- e?: object;
420
- }) => {
421
- this.onMessage(message);
422
- },
423
- sendSdkPoint: (type: string, data: any, extra?: any) => {
424
- this.ttsa?.sendSdkPoint(type, data, extra);
425
- },
426
- sendMessage: (type: string, payload: any) => {
427
- this.ttsa?.send(type, payload);
428
- },
429
- onFPSUpdate: (stats: FPSStats) => {
430
- options.onFPSUpdate?.(stats);
431
- },
432
- onFrameDataUpdate: (data) => {
433
- options.onFrameDataUpdate?.(data);
434
- },
435
- });
436
- options.onStateRenderChange && (window as any).performanceTracker.setOnStateRenderChange(options.onStateRenderChange);
437
-
438
-
439
- if(config?.walk_config){
440
- this.changeWalkConfig(config.walk_config)
441
- }
442
- if(config?.layout) {
443
- this.changeLayout(config.layout);
444
- }
445
- WidgetDefaultRender.CUSTOM_WIDGET = options.onWidgetEvent;
446
- WidgetDefaultRender.PROXY_WIDGET = options.proxyWidget || {};
447
- }
448
-
449
- /**
450
- * 薄客户端模式初始化:创建容器、StreamingClient、VideoPlayer、EventDispatcher、APIForwarder
451
- */
452
- private initializeClient(options: IAvatarOptions) {
453
- const { containerId, container } = options;
454
-
455
- // 准备容器
456
- if (container && container instanceof HTMLElement) {
457
- this.el = container;
458
- } else {
459
- const el = document.querySelector(containerId) as HTMLElement;
460
- if (!el) {
461
- // 走 onMessage 统一出口(而非直接调 options.onMessage),否则该错误不会进 Sentry
462
- this.onMessage({ code: EErrorCode.CONTAINER_NOT_FOUND, message: `containerId: ${containerId} 不存在` });
463
- // 容器不存在,实例不会进入可用状态,释放 Sentry(内部会 flush 刚上报的事件)
464
- this.disposeSentryReporter();
465
- return;
466
- }
467
- this.el = el;
468
- }
469
- const style = this.el.getAttribute("style");
470
- if (style) {
471
- this.el.setAttribute("style", `position:relative;overflow:hidden;${style}`);
472
- } else {
473
- this.el.setAttribute("style", "position:relative;overflow:hidden;");
474
- }
475
-
476
- // 创建 EventDispatcher
477
- this._eventDispatcher = new EventDispatcher({
478
- onStateChange: options.onStateChange,
479
- onSpeakStateChange: options.onSpeakStateChange,
480
- onVoiceStateChange: options.onVoiceStateChange,
481
- onStatusChange: options.onStatusChange,
482
- onRenderChange: options.onRenderChange,
483
- onWalkStateChange: options.onWalkStateChange,
484
- onWidgetEvent: options.onWidgetEvent,
485
- onMessage: options.onMessage,
486
- onNetworkInfo: options.onNetworkInfo,
487
- onFPSUpdate: options.onFPSUpdate,
488
- onFrameDataUpdate: options.onFrameDataUpdate,
489
- onDownloadProgress: (progress: number) => {
490
- this._clientOnDownloadProgress?.(progress);
491
- if (progress >= 100 && !this.isInitialized) {
492
- this.isInitialized = true;
493
- }
494
- },
495
- onSessionId: (sessionId: string) => {
496
- this._clientSessionId = sessionId;
497
- (window as any).avatarSDKLogger.log(this.TAG, '[client] 收到 session_id:', sessionId);
498
- },
499
- });
500
-
501
- const streamingUrl = options.streamingServer || `ws://${location.hostname}:8080`;
502
-
503
- // 创建 StreamingClient
504
- this._streamingClient = new StreamingClient({
505
- url: streamingUrl,
506
- onReady: () => {
507
- (window as any).avatarSDKLogger.log(this.TAG, '[client] 双方就绪,等待 Server 推流...');
508
- // 如果 init() 在 WebSocket 连接前被调用,在此补发
509
- if (this._pendingInit) {
510
- this._pendingInit = false;
511
- const config = this._pendingInitConfig;
512
- this._pendingInitConfig = null;
513
- (window as any).avatarSDKLogger.log(this.TAG, '[client] 补发 init 命令');
514
- this._apiForwarder?.init(config);
515
- }
516
- },
517
- onOffer: (sdp: string) => {
518
- this._videoPlayer?.handleOffer(sdp);
519
- },
520
- onIce: (candidate: RTCIceCandidateInit) => {
521
- this._videoPlayer?.addIceCandidate(candidate);
522
- },
523
- onEventProxy: (eventType: string, data: any, serverFrame?: number) => {
524
- this._eventDispatcher?.dispatch(eventType, data, serverFrame);
525
- },
526
- onError: (error: string) => {
527
- this.options.onMessage?.({ code: EErrorCode.NETWORK_DOWN, message: error } as any);
528
- },
529
- onDisconnect: () => {
530
- this.onStatusChange(AvatarStatus.offline);
531
- },
532
- });
533
-
534
- // 创建 VideoPlayer
535
- this._videoPlayer = new VideoPlayer({
536
- container: this.el,
537
- streamingClient: this._streamingClient,
538
- useAlphaMask: options.useAlphaMask,
539
- onConnected: () => {
540
- (window as any).avatarSDKLogger.log(this.TAG, '[client] WebRTC 已连接');
541
- this.onStatusChange(AvatarStatus.online);
542
- },
543
- onDisconnected: () => {
544
- this.onStatusChange(AvatarStatus.offline);
545
- },
546
- });
547
-
548
- // 创建 APIForwarder
549
- this._apiForwarder = new APIForwarder(this._streamingClient);
550
-
551
- // 连接信令
552
- this._streamingClient.connect();
553
-
554
- window.addEventListener('beforeunload', this.handleBeforeUnload);
555
- }
556
-
557
- // async preCache() {
558
- // return this.resourceManager.preCache(this.options.appId, this.options.appSecret);
559
- // }
560
-
561
- private renderFrameCallback(frame: number) {
562
- // 间隔24帧执行一次
563
- if(!this.ttsa?.getStatus() && ![AvatarStatus.offline].includes(this.status as AvatarStatus) && frame % 24 === 0) {
564
- this.offlineHandle()
565
- }
566
- }
567
-
568
- async init(params: IInitParams): Promise<void> {
569
- await this.initializeSession(params);
570
- }
571
-
572
- protected async initializeSession(params: IInitParams): Promise<ISessionResponse | null | undefined> {
573
- // 浏览器能力检查(hardwareAcceleration 需与生产解码配置保持一致)
574
- const capabilityReport = await checkCapability(this.options.audioOnlyMode, this.options.hardwareAcceleration);
575
- if (!capabilityReport.supported) {
576
- const fatalItems = capabilityReport.items.filter(i => i.severity === 'fatal' && !i.supported);
577
- const errorMsg = `浏览器能力检查失败: ${fatalItems.map(i => i.reason).join('; ')}`;
578
- this.onMessage({
579
- code: EErrorCode.INIT_ERROR,
580
- message: errorMsg,
581
- });
582
- return;
583
- }
584
-
585
- this.destroyed = false;
586
- this.isInitialized = false; // 重置初始化状态
587
- this.incompleteInitializationCleanupPromise = null;
588
- this.startConnectTime = Date.now();
589
-
590
- // 薄客户端模式:保存进度回调,通知 server 端开始初始化,等待 server 的 download_progress 事件
591
- if (this._isClientMode) {
592
- this._clientOnDownloadProgress = params.onDownloadProgress || null;
593
- // 构建发给 server 的配置(排除客户端专用字段)
594
- const clientConfig = {
595
- appId: this.options.appId,
596
- appSecret: this.options.appSecret,
597
- gatewayServer: this.options.gatewayServer,
598
- cacheServer: this.options.cacheServer,
599
- headers: this.options.headers,
600
- tag: this.options.tag,
601
- gateway_type: this.options.gateway_type,
602
- env: this.options.env,
603
- config: this.options.config,
604
- hardwareAcceleration: this.options.hardwareAcceleration,
605
- enableDebugger: this.options.enableDebugger,
606
- initModel: params.initModel,
607
- scaleDownBy: this.options.scaleDownBy || 1,
608
- useAlphaMask: this.options.useAlphaMask,
609
- };
610
- // 发送 init 命令通知 server 端初始化数字人
611
- // 如果 WebSocket 尚未连接,暂存等 onReady 时补发
612
- if (this._streamingClient?.isConnected()) {
613
- this._apiForwarder?.init(clientConfig);
614
- } else {
615
- this._pendingInit = true;
616
- this._pendingInitConfig = clientConfig;
617
- (window as any).avatarSDKLogger.log(this.TAG, '[client] WebSocket 未连接,init 命令暂存');
618
- }
619
- return;
620
- }
621
-
622
- // Sentry 实例已在构造函数中提前创建(见 initSentryReporter),
623
- // 这样无网络 / 容器不存在 / 能力检查失败等更早的错误也能上报
624
- // sdk.init:初始化总耗时(含资源加载、连接就绪)
625
- const initSpan = this.sentryReporter?.startInactiveSpan('sdk.init', { op: 'init' });
626
- // sdk.first_frame:从初始化到首个 body 帧到达(handleMessage 首次 BODY 帧时结束)
627
- this.firstFrameSpan = this.sentryReporter?.startInactiveSpan('sdk.first_frame', { op: 'render' }) ?? null;
628
-
629
- const { onDownloadProgress, initModel } = params;
630
- this.onDownloadProgress = (progress: number) => {
631
- if(progress === 15) {
632
- this.sessionResTime = Date.now();
633
- }
634
- if(progress === 80) {
635
- this.charbinResTime = Date.now();
636
- }
637
- onDownloadProgress(progress);
638
- };
639
-
640
- // 如果初始化时指定了隐身模式,先标记待处理,等start时再真正应用
641
- if(initModel === InitModel.invisible) {
642
- this.pendingInvisibleMode = true;
643
- }
644
-
645
- // 加载资源(sdk.resource_load:资源包下载/解析耗时)
646
- (window as any).performanceTracker.markStart(performanceConstant.load_resource);
647
- (window as any).performanceTracker.markStart(performanceConstant.first_avatar_render);
648
- (window as any).performanceTracker.markStart(performanceConstant.first_webgl_render);
649
- const downloadProgress = this.onDownloadProgress;
650
- const sessionInfo = this.sentryReporter
651
- ? await this.sentryReporter.startSpan('sdk.resource_load', { op: 'resource.load' }, () =>
652
- this.resourceManager.load(downloadProgress.bind(this))
653
- )
654
- : await this.resourceManager.load(downloadProgress.bind(this));
655
- // 如果在load过程中被销毁了,直接返回,避免继续初始化
656
- if(this.destroyed) {
657
- initSpan?.end();
658
- (window as any).avatarSDKLogger.warn(this.TAG, "sdk destroyed during load, aborting after resourceManager load");
659
- return
660
- };
661
- if(!sessionInfo?.socket_io_url || !sessionInfo?.token) {
662
- initSpan?.end();
663
- (window as any).avatarSDKLogger.error(this.TAG, "init error, socket_io_url or token is empty reload");
664
- return
665
- };
666
- this.connectSuccessTime = Date.now();
667
- (window as any).performanceTracker.markEnd(performanceConstant.load_resource);
668
- this.debugOverlay = new DebugOverlay(this, sessionInfo);
669
-
670
- this.ttsa = this.connectTtsa(sessionInfo)
671
- this.renderScheduler.init();
672
- // 同步初始canvas显隐状态
673
- this.renderScheduler.setAvatarCanvasVisible(this.avatarCanvasVisible);
674
- // 设置上报使用ttsa
675
- (window as any).performanceTracker.setReportFunc(this.ttsa);
676
-
677
- window.addEventListener('beforeunload', this.handleBeforeUnload);
678
- document.addEventListener('visibilitychange', this.boundVisibilityChange);
679
- initSpan?.end();
680
- return sessionInfo;
681
- }
682
-
683
- /**
684
- * 初始化音频捕获(MSE 模式)
685
- * 通过 AudioContext.createMediaElementSource 接管 <audio> 元素输出,
686
- * 按 1600 samples 分桶后使用 WebCodecs 编码为 opus (WebM 容器),
687
- * 并回调给业务侧用于 ASR 回声消除(AEC)。
688
- *
689
- * 架构:
690
- * - AudioWorklet 负责 100ms 分桶和静音检测
691
- * - WebCodecs AudioEncoder + WebMMuxer 负责 opus/WebM 编码
692
- * - 每次 speak 会话的首段音频自动包含 WebM 容器头(isFirstChunk=true)
693
- *
694
- * 注意:
695
- * - createMediaElementSource 对同一个 audio 元素只能调用一次
696
- * - 接管后必须手动 connect 到 destination,否则音频不会播放
697
- * - MSE 播放器 stop 时只销毁 MediaSource,audio 元素不变,捕获绑定不受影响
698
- */
699
- protected setAudioPlaybackDataHandler(handler?: IAvatarOptions["onAudioPlaybackData"]) {
700
- this.options.onAudioPlaybackData = handler;
701
- }
702
-
703
- protected restartAudioPlaybackCapture() {
704
- this.restartPlaybackCaptureImpl?.();
705
- }
706
-
707
- protected enableAudioPlaybackCapture() {
708
- if (!this.options.onAudioPlaybackData) {
709
- return Promise.resolve();
710
- }
711
- if (!this.captureSetupPromise) {
712
- if (this.resourceManager?.config.raw_audio) {
713
- // PCM 模式:通过 capture worklet 捕获并编码
714
- this.captureSetupPromise = this.setupPCMCapture().finally(() => {
715
- this.captureSetupPromise = null;
716
- });
717
- } else {
718
- // MSE/WebM 模式:通过 AudioWorklet 分桶并用 WebCodecs 编码
719
- this.captureSetupPromise = this.setupAudioCapture().finally(() => {
720
- this.captureSetupPromise = null;
721
- });
722
- }
723
- }
724
- return this.captureSetupPromise;
725
- }
726
-
727
- private async setupAudioCapture() {
728
- const audioEl = document.getElementById('xmov-avatar-audio') as HTMLAudioElement;
729
- if (!audioEl || !this.options.onAudioPlaybackData) return;
730
- if (this.captureAudioCtx || this.captureSource) return;
731
-
732
- // 检查 AudioWorkletNode 支持
733
- if (typeof (window as any).AudioWorkletNode === 'undefined') {
734
- this.onMessage({
735
- code: EErrorCode.INIT_ERROR,
736
- message: '当前浏览器不支持 AudioWorkletNode,无法启用音频捕获(ASR 回声消除)',
737
- });
738
- return;
739
- }
740
-
741
- try {
742
- if (!await PcmWebMEncoder.isSupported()) {
743
- this.onMessage({
744
- code: EErrorCode.INIT_ERROR,
745
- message: '当前浏览器不支持 WebCodecs AudioEncoder Opus,无法启用 100ms 双路音频',
746
- });
747
- return;
748
- }
749
- this.pcmWebMEncoder = new PcmWebMEncoder();
750
- await this.pcmWebMEncoder.init();
751
-
752
- // 创建 16kHz 采样率的 AudioContext(farend 协议要求)
753
- this.captureAudioCtx = new AudioContext({ sampleRate: 16000 });
754
- if (this.captureAudioCtx.sampleRate !== 16000) {
755
- throw new Error(`浏览器无法创建 16000 Hz AudioContext: ${this.captureAudioCtx.sampleRate}`);
756
- }
757
- this.captureSource = this.captureAudioCtx.createMediaElementSource(audioEl);
758
-
759
- // 路由到扬声器(必须连接,否则无声)
760
- this.captureSource.connect(this.captureAudioCtx.destination);
761
-
762
- let isRecording = false;
763
- let isFirstEmit = true;
764
- let captureStartedAt = 0;
765
- let captureSequence = 0;
766
- const startCapture = (sessionId: number) => {
767
- const generation = ++this.captureGeneration;
768
- isRecording = true;
769
- isFirstEmit = true;
770
- captureStartedAt = Date.now() - 100;
771
- captureSequence = 0;
772
- this.captureLastSpeechId = sessionId;
773
- this.captureEncodeQueue = this.captureEncodeQueue.then(() => {
774
- if (generation === this.captureGeneration) {
775
- this.pcmWebMEncoder?.reset();
776
- }
777
- });
778
- };
779
-
780
- const stopCapture = () => {
781
- const generation = ++this.captureGeneration;
782
- isRecording = false;
783
- this.captureLastSpeechId = -2;
784
- this.captureEncodeQueue = this.captureEncodeQueue.then(() => {
785
- if (generation === this.captureGeneration) {
786
- this.pcmWebMEncoder?.reset();
787
- }
788
- });
789
- };
790
-
791
- this.restartPlaybackCaptureImpl = () => {
792
- if (isRecording) {
793
- startCapture(this.renderScheduler.getSpeechId());
794
- }
795
- };
796
-
797
- // 创建 AudioWorkletProcessor 脚本(在音频线程中运行)
798
- // 缓冲 1600 个采样点(100ms @ 16kHz),直接回传 S16LE PCM。
799
- const processorCode = `
800
- class AudioCaptureProcessor extends AudioWorkletProcessor {
801
- constructor() {
802
- super();
803
- this.buffer = new Float32Array(1600); // 100ms @ 16kHz
804
- this.bufferIndex = 0;
805
- }
806
-
807
- process(inputs, outputs) {
808
- for (const output of outputs) {
809
- for (const channel of output) channel.fill(0);
810
- }
811
- const input = inputs[0];
812
- if (input && input.length > 0 && input[0] && input[0].length > 0) {
813
- const frameCount = input[0].length;
814
-
815
- for (let i = 0; i < frameCount; i++) {
816
- let sample = 0;
817
- for (let channel = 0; channel < input.length; channel++) {
818
- sample += input[channel][i] || 0;
819
- }
820
- this.buffer[this.bufferIndex++] = sample / input.length;
821
-
822
- // 缓冲区满 1600 采样点时,检测静音并发送到主线程
823
- if (this.bufferIndex >= 1600) {
824
- // 检测是否全是静音(全零或极低值)
825
- let hasAudio = false;
826
- for (let j = 0; j < 1600; j++) {
827
- if (Math.abs(this.buffer[j]) > 0.0001) {
828
- hasAudio = true;
829
- break;
830
- }
831
- }
832
-
833
- const pcm = new Int16Array(1600);
834
- for (let j = 0; j < 1600; j++) {
835
- const value = Math.max(-1, Math.min(1, this.buffer[j]));
836
- pcm[j] = value < 0 ? value * 0x8000 : value * 0x7fff;
837
- }
838
- this.port.postMessage(
839
- { hasAudio: hasAudio, pcmData: pcm.buffer },
840
- [pcm.buffer]
841
- );
842
- this.bufferIndex = 0;
843
- }
844
- }
845
- }
846
- return true;
847
- }
848
- }
849
- registerProcessor('audio-capture-processor', AudioCaptureProcessor);
850
- `;
851
-
852
- // 通过 Blob URL 加载 processor
853
- const blob = new Blob([processorCode], { type: 'application/javascript' });
854
- const processorUrl = URL.createObjectURL(blob);
855
- await this.captureAudioCtx.audioWorklet.addModule(processorUrl);
856
- URL.revokeObjectURL(processorUrl);
857
-
858
- // 创建 AudioWorkletNode(仅用于静音检测,不连接 destination)
859
- this.captureWorklet = new AudioWorkletNode(
860
- this.captureAudioCtx,
861
- 'audio-capture-processor'
862
- );
863
- this.captureSource.connect(this.captureWorklet);
864
- this.captureWorklet.connect(this.captureAudioCtx.destination);
865
-
866
- this.captureEncodeQueue = Promise.resolve();
867
- this.captureWorklet.port.onmessage = (e) => {
868
- if (!this.options.onAudioPlaybackData) return;
869
-
870
- const { hasAudio, pcmData } = e.data as { hasAudio: boolean; pcmData: ArrayBuffer };
871
- const speechId = this.renderScheduler.getSpeechId();
872
-
873
- if (hasAudio && !isRecording) {
874
- startCapture(speechId);
875
- } else if (hasAudio && isRecording && speechId !== this.captureLastSpeechId) {
876
- startCapture(speechId);
877
- } else if (!hasAudio && isRecording) {
878
- stopCapture();
879
- }
880
- if (!hasAudio || !isRecording || !(pcmData instanceof ArrayBuffer)) return;
881
-
882
- const generation = this.captureGeneration;
883
- const firstChunk = isFirstEmit;
884
- isFirstEmit = false;
885
- const blockStartedAt = captureStartedAt + captureSequence * 100;
886
- captureSequence += 1;
887
- this.captureEncodeQueue = this.captureEncodeQueue
888
- .then(async () => {
889
- const buffer = await this.pcmWebMEncoder?.encode(new Int16Array(pcmData));
890
- if (!buffer || buffer.byteLength === 0 || generation !== this.captureGeneration) return;
891
- this.options.onAudioPlaybackData?.({
892
- data: buffer,
893
- codec: 'opus',
894
- isFirstChunk: firstChunk,
895
- sampleRate: 16000,
896
- samples: 1600,
897
- timestamp: blockStartedAt,
898
- speech_id: speechId,
899
- });
900
- })
901
- .catch((error) => {
902
- (window as any).avatarSDKLogger?.error(
903
- this.TAG,
904
- '[AudioCapture] PCM/WebCodecs 编码失败:',
905
- error
906
- );
907
- });
908
- };
909
-
910
- // 处理 AudioContext 挂起(浏览器自动播放策略)
911
- if (this.captureAudioCtx.state === 'suspended') {
912
- await this.captureAudioCtx.resume();
913
- }
914
- if (this.captureAudioCtx.state !== 'running') {
915
- throw new Error('AudioContext 未运行,无法启动 farend PCM 捕获');
916
- }
917
-
918
- (window as any).avatarSDKLogger?.log(
919
- this.TAG,
920
- '[AudioCapture] 初始化成功,采样率:',
921
- this.captureAudioCtx.sampleRate,
922
- 'Hz,opus 编码 (WebM 容器)'
923
- );
924
- } catch (e) {
925
- this.captureGeneration++;
926
- this.restartPlaybackCaptureImpl = null;
927
- if (this.captureWorklet) {
928
- try {
929
- this.captureWorklet.port.onmessage = null;
930
- this.captureWorklet.disconnect();
931
- } catch {}
932
- this.captureWorklet = null;
933
- }
934
- try {
935
- this.captureSource?.disconnect();
936
- } catch {}
937
- this.captureSource = null;
938
- if (this.captureAudioCtx && this.captureAudioCtx.state !== 'closed') {
939
- await this.captureAudioCtx.close().catch(() => undefined);
940
- }
941
- this.captureAudioCtx = null;
942
- void this.pcmWebMEncoder?.destroy();
943
- this.pcmWebMEncoder = null;
944
- (window as any).avatarSDKLogger?.error(
945
- this.TAG,
946
- '[AudioCapture] 初始化失败:',
947
- e
948
- );
949
- }
950
- }
951
-
952
- /**
953
- * 初始化 PCM 模式音频捕获(raw_audio=true 时使用)
954
- * 通过 PCMAudioPlayer 的 capture worklet 拦截音频输出,
955
- * 经 WebCodecs 或 MediaRecorder 编码为 WebM/Opus 后回调给业务侧
956
- */
957
- private async setupPCMCapture() {
958
- if (!this.options.onAudioPlaybackData) return;
959
-
960
- const audioPlayer = this.renderScheduler.getAudioRenderer()?.getAudioPlayer();
961
- if (!audioPlayer) {
962
- (window as any).avatarSDKLogger?.error(
963
- this.TAG, '[PCMCapture] PCMAudioPlayer 未就绪'
964
- );
965
- return;
966
- }
967
-
968
- // 运行时检测编码能力,选择编码路径
969
- const webCodecsSupported = await PcmWebMEncoder.isSupported();
970
- const mediaRecorderSupported = MediaRecorderEncoder.isSupported();
971
-
972
- if (!webCodecsSupported && !mediaRecorderSupported) {
973
- (window as any).avatarSDKLogger?.warn(
974
- this.TAG, '[PCMCapture] 当前浏览器不支持 Opus 编码(WebCodecs 和 MediaRecorder 均不可用),跳过 PCM 捕获'
975
- );
976
- this.onMessage({
977
- code: EErrorCode.INIT_ERROR,
978
- message: '当前浏览器不支持 Opus 编码,PCM 模式下无法启用音频捕获(ASR 回声消除)',
979
- });
980
- return;
981
- }
982
-
983
- try {
984
- if (webCodecsSupported) {
985
- // 首选路径:WebCodecs AudioEncoder
986
- this.pcmWebMEncoder = new PcmWebMEncoder();
987
- await this.pcmWebMEncoder.init();
988
- this.pcmCaptureMode = 'webcodec';
989
- } else {
990
- // 降级路径:MediaRecorder
991
- this.mediaRecorderEncoder = new MediaRecorderEncoder();
992
- await this.mediaRecorderEncoder.init();
993
- this.pcmCaptureMode = 'mediarecorder';
994
- }
995
-
996
- this.restartPlaybackCaptureImpl = () => {
997
- this.pcmCaptureFirstChunk = true;
998
- this.pcmCaptureLastSpeechId = this.renderScheduler.getSpeechId();
999
- this.pcmEncodeId++;
1000
- this.pcmWebMEncoder?.reset();
1001
- this.mediaRecorderEncoder?.reset();
1002
- audioPlayer.resetCaptureBuffer();
1003
- };
1004
-
1005
- await audioPlayer.attachCapture(async ({ pcmData, hasAudio }) => {
1006
- try {
1007
- if (!this.options.onAudioPlaybackData) return;
1008
- const blockStartedAt = Date.now() - 100;
1009
-
1010
- // 音频未播放时丢弃数据(speak_end 后 capture worklet 可能还有尾巴数据)
1011
- if (!audioPlayer.isPlaying) {
1012
- audioPlayer.resetCaptureBuffer();
1013
- return;
1014
- }
1015
-
1016
- const speechId = this.renderScheduler.getSpeechId();
1017
-
1018
- // 会话变化时重置
1019
- if (speechId !== this.pcmCaptureLastSpeechId) {
1020
- this.pcmCaptureFirstChunk = true;
1021
- this.pcmCaptureLastSpeechId = speechId;
1022
- this.pcmEncodeId++; // 递增编码会话 ID
1023
- this.pcmWebMEncoder?.reset();
1024
- this.mediaRecorderEncoder?.reset();
1025
- // 清空 capture worklet 内部缓冲区,避免打断后旧音频数据混入新 session
1026
- audioPlayer.resetCaptureBuffer();
1027
- }
1028
-
1029
- // 静音时不发送
1030
- if (!hasAudio) return;
1031
-
1032
- // 记录当前编码会话 ID(用于竞态检测)
1033
- const currentEncodeId = this.pcmEncodeId;
1034
-
1035
- let webmData: ArrayBuffer | null = null;
1036
- const startTime = performance.now();
1037
-
1038
- if (this.pcmCaptureMode === 'webcodec' && this.pcmWebMEncoder) {
1039
- // WebCodecs 路径:PCM → Opus → WebM
1040
- const pcm = new Int16Array(pcmData);
1041
- webmData = await this.pcmWebMEncoder.encode(pcm);
1042
-
1043
- // 竞态检测:如果 reset 已被调用,丢弃这个结果
1044
- if (currentEncodeId !== this.pcmEncodeId) {
1045
- (window as any).avatarSDKLogger?.log(
1046
- this.TAG, '[PCMCapture] 丢弃过期编码数据(编码会话已重置)'
1047
- );
1048
- return;
1049
- }
1050
-
1051
- if (webmData.byteLength === 0) return;
1052
-
1053
- // 性能监控
1054
- const encodeTime = performance.now() - startTime;
1055
- if (encodeTime > 10) {
1056
- (window as any).avatarSDKLogger?.warn(
1057
- this.TAG, `[PCMCapture] WebCodecs 编码耗时过长: ${encodeTime.toFixed(2)}ms`
1058
- );
1059
- }
1060
- } else if (this.pcmCaptureMode === 'mediarecorder' && this.mediaRecorderEncoder) {
1061
- // MediaRecorder 降级路径
1062
- webmData = await this.mediaRecorderEncoder.feedPCM(pcmData);
1063
-
1064
- // 竞态检测
1065
- if (currentEncodeId !== this.pcmEncodeId) {
1066
- (window as any).avatarSDKLogger?.log(
1067
- this.TAG, '[PCMCapture] 丢弃过期编码数据(编码会话已重置)'
1068
- );
1069
- return;
1070
- }
1071
-
1072
- if (!webmData) return; // 首次启动无数据
1073
-
1074
- // 性能监控
1075
- const encodeTime = performance.now() - startTime;
1076
- if (encodeTime > 10) {
1077
- (window as any).avatarSDKLogger?.warn(
1078
- this.TAG, `[PCMCapture] MediaRecorder 编码耗时过长: ${encodeTime.toFixed(2)}ms`
1079
- );
1080
- }
1081
- }
1082
-
1083
- if (!webmData) return;
1084
-
1085
- const firstChunk = this.pcmCaptureFirstChunk;
1086
- this.pcmCaptureFirstChunk = false;
1087
-
1088
- // 回调(格式与 MSE 模式完全一致)
1089
- this.options.onAudioPlaybackData({
1090
- data: webmData,
1091
- codec: 'opus',
1092
- isFirstChunk: firstChunk,
1093
- sampleRate: 16000,
1094
- samples: 1600,
1095
- timestamp: blockStartedAt,
1096
- speech_id: speechId,
1097
- });
1098
- } catch (error) {
1099
- (window as any).avatarSDKLogger?.error(
1100
- this.TAG, '[PCMCapture] 捕获回调错误:', error
1101
- );
1102
- }
1103
- });
1104
-
1105
- (window as any).avatarSDKLogger?.log(
1106
- this.TAG, `[PCMCapture] PCM 捕获初始化成功,编码路径: ${this.pcmCaptureMode}`
1107
- );
1108
- } catch (e) {
1109
- (window as any).avatarSDKLogger?.error(
1110
- this.TAG, '[PCMCapture] PCM 捕获初始化失败:', e
1111
- );
1112
- void this.pcmWebMEncoder?.destroy();
1113
- this.pcmWebMEncoder = null;
1114
- this.mediaRecorderEncoder?.destroy();
1115
- this.mediaRecorderEncoder = null;
1116
- this.pcmCaptureMode = 'none';
1117
- }
1118
- }
1119
-
1120
- visibilitychange() {
1121
- if (document.hidden) {
1122
- this.setVolume(0);
1123
- } else {
1124
- if (this.ttsa?.getStatus()) {
1125
- // 连接正常:兜底同步解帧队列
1126
- this.renderScheduler?.forceSyncDecoder();
1127
- this.setVolume(1);
1128
- }
1129
- // WebSocket 在后台断开且已进入离线模式,网络可用则主动触发重连,
1130
- // 否则用户切回前台续播时无响应且无任何报错
1131
- if (!this.ttsa?.getStatus() && this.status === AvatarStatus.offline && NetworkMonitor.ONLINE) {
1132
- (window as any).avatarSDKLogger.warn(this.TAG, "切回前台发现连接已断开,主动触发重连");
1133
- this.onlineMode();
1134
- this.setVolume(1);
1135
- }
1136
- // tab 切换回来时,恢复音频捕获的 AudioContext
1137
- if (this.captureAudioCtx && this.captureAudioCtx.state === 'suspended') {
1138
- this.captureAudioCtx.resume().catch(() => {});
1139
- }
1140
- }
1141
- }
1142
-
1143
- setReplayData(data: any) {
1144
- this.replayData = data;
1145
- }
1146
-
1147
- handleReplaySend(frame: number) {
1148
- if(!this.replayData) {
1149
- return;
1150
- }
1151
- const frameReplayData = this.replayData.filter((item: any) => (Number(item.client_frame_number) === frame && item.event_type !== "sdk_burial_point"));
1152
- for(const item of frameReplayData) {
1153
- if(item.event_type === 'state_change') {
1154
- this.ttsa?.stateChange(item.event_data.state, item.event_data.params);
1155
- }else if(item.event_type === 'send_text') {
1156
- this.speak(item.event_data.ssml, item.event_data.is_start, item.event_data.is_end, item.event_data.extra);
1157
-
1158
- }else {
1159
- (window as any).avatarSDKLogger.log(this.TAG, `[handleReplaySend] 未知事件类型: ${item}`);
1160
- }
1161
- }
1162
- }
1163
-
1164
- // 初始化 ttsa
1165
- private connectTtsa(sessionInfo: ISessionResponse) {
1166
- (window as any).performanceTracker.markStart(performanceConstant.ttsa_connect);
1167
- (window as any).performanceTracker.markStart(performanceConstant.ttsa_ready);
1168
- return new Ttsa({
1169
- sdkInstance: this,
1170
- url: sessionInfo.socket_io_url,
1171
- path: sessionInfo?.socket_io_path || "",
1172
- room: sessionInfo.room,
1173
- session_id: sessionInfo.session_id,
1174
- token: sessionInfo.token,
1175
- framedata_proto_version: this.resourceManager.getConfig().framedata_proto_version,
1176
- reconnect_client_timeout: sessionInfo?.reconnect_client_timeout || 0,
1177
- reconnect_timeout: sessionInfo?.reconnect_timeout || 0,
1178
- appInfo: {
1179
- ...this.resourceManager.getAppInfo(),
1180
- env: this._env
1181
- },
1182
- onReady: () => {
1183
- this.ttsa?.sendSdkPoint('connect_sdk', {
1184
- timestamp: this.startConnectTime,
1185
- });
1186
- // 不再强行上报 85:进度以真实资源下载为准,避免重连时 UI 进度被错误拉低
1187
- this.ttsa?.sendSdkPoint('connect_sdk_success', {
1188
- timestamp: this.connectSuccessTime,
1189
- });
1190
- this.ttsa?.sendSdkPoint('session_processing_time', {
1191
- timestamp: this.sessionResTime,
1192
- });
1193
- this.ttsa?.sendSdkPoint('Char_bin_download_time', {
1194
- timestamp: this.charbinResTime,
1195
- });
1196
- this.ttsa?.sendSdkPoint('socket_connect_time', {
1197
- timestamp: Date.now(),
1198
- });
1199
- this.onTtsaReady(sessionInfo)
1200
- this.ttsa?.syncTimestamp();
1201
- },
1202
- handleMessage: (type: EFrameDataType, data: IRawFrameData[] | SYNC_FRAME_DATA) => {
1203
- if(this.isfirstReceive && type === EFrameDataType.BODY) {
1204
- this.ttsa?.sendSdkPoint('body_completed_first_time', {
1205
- timestamp: Date.now(),
1206
- });
1207
- this.isfirstReceive = false;
1208
- // 结束 sdk.first_frame span(首帧数据到达)
1209
- this.firstFrameSpan?.end();
1210
- this.firstFrameSpan = null;
1211
- }
1212
- // 处理下行数据
1213
- this.renderScheduler.handleData(type, data);
1214
- },
1215
- handleAAFrame: (data: any) => {
1216
- this.options.onAAFrameHandle?.(data);
1217
- },
1218
- runStartFrameIndex: (client_time: number) => {
1219
- this.renderScheduler.runStartFrameIndex();
1220
- },
1221
- ttsaStateChangeHandle: (state: StateChangeInfo) => {
1222
- this.renderScheduler.ttsaStateChangeHandle(state);
1223
- },
1224
- reloadSuccess: () => {
1225
- this.reloadSuccess()
1226
- },
1227
- enterOfflineMode: () => {
1228
- this.offlineHandle()
1229
- },
1230
- reStartSDK: () => {
1231
- this.reStartSession()
1232
- },
1233
- sendVoiceEnd: () => {
1234
- this.renderScheduler?.sendVoiceEnd();
1235
- },
1236
- getCurrentFrameIndex: () => {
1237
- return this.renderScheduler?.getCurrentFrameIndex()??0;
1238
- }
1239
- });
1240
- }
1241
-
1242
- protected start() {
1243
- this.renderScheduler.render();
1244
- this.ttsa?.start();
1245
- // 先发布在线状态,再启动渲染:
1246
- // 避免先 render() 触发 rendering -> visible(5) 后再发 online(0),造成 0/5 状态横跳
1247
- if (this.pendingInvisibleMode) {
1248
- this.pendingInvisibleMode = false;
1249
- this.setInvisibleMode()
1250
- } else if (this.isInitialized) {
1251
- // 状态守卫:仅在初始化完成(progress === 100)时上线,
1252
- // 避免重连重建房间过程中(资源未下载完)反复触发 online -> visible 状态横跳
1253
- this.onStatusChange(AvatarStatus.online);
1254
- }
1255
- }
1256
-
1257
- protected onTtsaReady(_sessionInfo?: ISessionResponse) {
1258
- this.start();
1259
- }
1260
-
1261
- private async _reload() {
1262
- const res = await this.resourceManager._reload();
1263
- if (res && !this.destroyed) {
1264
- await this.onSessionReloaded(res);
1265
- if (this.destroyed) {
1266
- return null;
1267
- }
1268
- this.ttsa = this.connectTtsa(res);
1269
- this.ttsa._setResumeInfoCallback(
1270
- this.resourceManager._getSessionId(),
1271
- this.renderScheduler._getResumeInfo.bind(this.renderScheduler)
1272
- )
1273
- }
1274
- return res;
1275
- }
1276
-
1277
- protected async onSessionReloaded(_sessionInfo: ISessionResponse): Promise<void> {}
1278
-
1279
- protected onSessionReloadExhausted(_error?: unknown): void {}
1280
-
1281
- protected async restartSessionForTransport(reason: string): Promise<void> {
1282
- this.renderScheduler.interrupt("in_offline_mode");
1283
- await this.resourceManager.stopSession(reason);
1284
- if (this.destroyed) {
1285
- return;
1286
- }
1287
- this.ttsa?.close();
1288
- if (this.status !== AvatarStatus.offline) {
1289
- this.offlineHandle();
1290
- }
1291
- this.reStartSession();
1292
- }
1293
-
1294
- protected cancelSessionRestart(): void {
1295
- this.resetRetryState();
1296
- this.clearAllRetryTimers();
1297
- }
1298
-
1299
- reloadSuccess() {
1300
- // 重连成功恢复:恢复音频、重置解码器离线状态、清空旧数据缓存,并按首个数据 sf 对齐帧号,
1301
- // 解决窗口内自动重连恢复后人物卡死(帧号错位数据被过滤)、无声音/音画不同步的问题
1302
- this.renderScheduler?.markReconnectRecover();
1303
- // 状态守卫:仅初始化完成时上线;未完成时由 onDownloadProgress(100) 补发,
1304
- // 避免重连重建房间过程中初始化未完成却提前进入 online
1305
- if (this.isInitialized) {
1306
- this.onStatusChange(AvatarStatus.online);
1307
- }
1308
- setTimeout(() => {
1309
- window.clearInterval(this._offlineTimer)
1310
- }, 1000)
1311
- }
1312
-
1313
- protected async stop() {
1314
- await this.renderScheduler.stop();
1315
- }
1316
-
1317
- private handleBeforeUnload = () => {
1318
- this.destroy("beforeunload");
1319
- };
1320
-
1321
- public isDestroyed() {
1322
- return this.destroyed;
1323
- }
1324
-
1325
- public async destroyClient() {
1326
- // 增加重复销毁检查,避免重复销毁导致的异常
1327
- if (this.destroyed) {
1328
- (window as any).avatarSDKLogger.warn(this.TAG, "SDK 已销毁,忽略重复销毁请求");
1329
- return;
1330
- }
1331
- this.destroyed = true;
1332
-
1333
- // 薄客户端模式:清理薄客户端相关资源
1334
- if (this._isClientMode) {
1335
- if (this._streamingClient) {
1336
- this._streamingClient.destroy();
1337
- this._streamingClient = null;
1338
- }
1339
- if (this._videoPlayer) {
1340
- this._videoPlayer.destroy();
1341
- this._videoPlayer = null;
1342
- }
1343
- this._eventDispatcher = null;
1344
- this._apiForwarder = null;
1345
- this.el = null;
1346
- window.removeEventListener("beforeunload", this.handleBeforeUnload);
1347
- (window as any).avatarSDKLogger.log(this.TAG, "薄客户端 SDK 已销毁");
1348
- return;
1349
- }
1350
-
1351
- this.cancelSessionRestart();
1352
-
1353
- // 清理离线模式定时器
1354
- if (this._offlineTimer !== -1) {
1355
- clearInterval(this._offlineTimer);
1356
- this._offlineTimer = -1;
1357
- }
1358
-
1359
- // 停止TTSA连接
1360
- if (this.ttsa) {
1361
- this.ttsa.close();
1362
- this.ttsa = null;
1363
- }
1364
-
1365
- // 停止渲染调度器
1366
- if (this.renderScheduler) {
1367
- this.renderScheduler.destroy();
1368
- }
1369
-
1370
- // 清理资源管理器(包括缓存)
1371
- if (this.resourceManager) {
1372
- this.resourceManager.destroy();
1373
- }
1374
-
1375
- // 清理调试浮层
1376
- if (this.debugOverlay) {
1377
- this.debugOverlay.destroy();
1378
- this.debugOverlay = null;
1379
- }
1380
- // 主动断开需要销毁网络监听相关事件
1381
- // 增加状态检查和异常捕获,避免重复销毁 networkMonitor
1382
- if(this.networkMonitor) {
1383
- try {
1384
- this.networkMonitor.destroy();
1385
- this.networkMonitor = null; // 标记已销毁
1386
- } catch (e) {
1387
- (window as any).avatarSDKLogger.warn(this.TAG, '销毁 networkMonitor 时出错', e);
1388
- }
1389
- }
1390
-
1391
- // 清理音频捕获资源
1392
- this.pcmEncodeId++;
1393
- this.captureGeneration++;
1394
- this.restartPlaybackCaptureImpl = null;
1395
- if (this.captureWorklet) {
1396
- try {
1397
- this.captureWorklet.port.onmessage = null;
1398
- this.captureWorklet.disconnect();
1399
- } catch {}
1400
- this.captureWorklet = null;
1401
- }
1402
- if (this.captureSource) {
1403
- try {
1404
- this.captureSource.disconnect();
1405
- } catch {}
1406
- this.captureSource = null;
1407
- }
1408
- if (this.captureAudioCtx) {
1409
- try {
1410
- this.captureAudioCtx.close();
1411
- } catch {}
1412
- this.captureAudioCtx = null;
1413
- }
1414
-
1415
- // 清理 PCM 捕获资源
1416
- const audioPlayer = this.renderScheduler?.getAudioRenderer()?.getAudioPlayer();
1417
- if (audioPlayer) {
1418
- audioPlayer.detachCapture();
1419
- }
1420
- void this.pcmWebMEncoder?.destroy();
1421
- this.pcmWebMEncoder = null;
1422
- this.mediaRecorderEncoder?.destroy();
1423
- this.mediaRecorderEncoder = null;
1424
- this.pcmCaptureLastSpeechId = -2;
1425
- this.pcmCaptureFirstChunk = true;
1426
- this.pcmCaptureMode = 'none';
1427
-
1428
- // 清理DOM元素
1429
- this.el = null;
1430
- this.startConnectTime = 0;
1431
- this.connectSuccessTime = 0;
1432
- this.sessionResTime = 0;
1433
- this.charbinResTime = 0;
1434
- this.isfirstReceive = true;
1435
- this.networkInfo = {
1436
- rtt: 0,
1437
- downlink: 0.0,
1438
- networkStatus: false,
1439
- };
1440
-
1441
- window.removeEventListener("beforeunload", this.handleBeforeUnload);
1442
- document.removeEventListener("visibilitychange", this.boundVisibilityChange);
1443
-
1444
- // 清理 Sentry 客户端(内部会 flush 未发送的事件)
1445
- this.disposeSentryReporter();
1446
-
1447
- (window as any).avatarSDKLogger.log(this.TAG, "SDK 已销毁");
1448
- }
1449
-
1450
- public async destroy(stop_reason: string = "user"): Promise<void> {
1451
- this.replayData = null;
1452
-
1453
- // 薄客户端模式:通知 server 端销毁,然后清理本地资源
1454
- if (this._isClientMode) {
1455
- await this.disconnectClientForDestroy(stop_reason);
1456
- return;
1457
- }
1458
-
1459
- this.ttsa?.sendSdkPoint('close_session', {
1460
- reason: stop_reason,
1461
- });
1462
- const res = await this.resourceManager.stopSession(stop_reason);
1463
- await this.destroyClient();
1464
- if(res) {
1465
- this.onStatusChange(AvatarStatus.close);
1466
- }
1467
- return res
1468
- }
1469
-
1470
- protected async disconnectClientForDestroy(stopReason: string): Promise<void> {
1471
- this.replayData = null;
1472
- try {
1473
- if (this._isClientMode) {
1474
- this._apiForwarder?.destroy(stopReason);
1475
- } else if (!this.ttsa?.hasConnected) {
1476
- // Initialization can own a session/request before TTSA first connects.
1477
- // Mark an in-flight connection as quitting before waiting for DELETE.
1478
- await this.ttsa?.quit(stopReason);
1479
- await this.resourceManager.stopSession(stopReason);
1480
- } else {
1481
- this.ttsa.sendSdkPoint('close_session', {
1482
- reason: stopReason,
1483
- });
1484
- await this.ttsa.quit(stopReason);
1485
- }
1486
- } catch (error) {
1487
- (window as any).avatarSDKLogger.warn(this.TAG, "TTSA client_quit 未确认,继续释放本地资源", error);
1488
- } finally {
1489
- await this.destroyClient();
1490
- this.onStatusChange(AvatarStatus.close);
1491
- }
1492
- }
1493
-
1494
- idle() {
1495
- if (this._isClientMode) { this._apiForwarder?.idle(); return; }
1496
- if(this.enableClientInterrupt) {
1497
- this.renderScheduler.interrupt("idle");
1498
- }
1499
- this.ttsa?.idle();
1500
- }
1501
-
1502
- listen() {
1503
- if (this._isClientMode) { this._apiForwarder?.listen(); return; }
1504
- if(this.enableClientInterrupt) {
1505
- this.renderScheduler.interrupt("listen");
1506
- }
1507
- this.ttsa?.listen();
1508
- }
1509
-
1510
- think() {
1511
- if (this._isClientMode) { this._apiForwarder?.think(); return; }
1512
- if(this.enableClientInterrupt) {
1513
- this.renderScheduler.interrupt("think");
1514
- }
1515
- this.ttsa?.think();
1516
- }
1517
- interactiveidle() {
1518
- if (this._isClientMode) { this._apiForwarder?.interactiveidle(); return; }
1519
- if(this.enableClientInterrupt) {
1520
- this.renderScheduler.interrupt("interactiveidle");
1521
- }
1522
- this.ttsa?.interactiveidle();
1523
- }
1524
-
1525
- speak(ssml: string, is_start: boolean = true, is_end: boolean = true, extra = {client_speak_id: '', client_frame: 0}): string | undefined{
1526
- if (this._isClientMode) { this._apiForwarder?.speak(ssml, is_start, is_end, extra); return; }
1527
- if(this.sentryReporter) {
1528
- return this.sentryReporter.startSpan('sdk.speak', {
1529
- op: 'speak',
1530
- data: { client_speak_id: extra.client_speak_id },
1531
- }, () => this.speakInternal(ssml, is_start, is_end, extra));
1532
- }
1533
- return this.speakInternal(ssml, is_start, is_end, extra);
1534
- }
1535
-
1536
- private speakInternal(ssml: string, is_start: boolean = true, is_end: boolean = true, extra = {client_speak_id: '', client_frame: 0}): string | undefined{
1537
- if(this.enableClientInterrupt && is_start) {
1538
- this.renderScheduler.interrupt("speak");
1539
- }
1540
- extra.client_frame = this.renderScheduler?.getCurrentFrameIndex()??0;
1541
- const uniqueSpeakId = this.ttsa?.sendText(ssml, is_start, is_end, extra);
1542
- this.renderScheduler.resume();
1543
- return uniqueSpeakId;
1544
- }
1545
-
1546
- setOffsetFrame(offsetFrame: number) {
1547
- this.renderScheduler.setOffsetFrame(offsetFrame);
1548
- }
1549
-
1550
- // 主动同步时间戳
1551
- syncTimestamp() {
1552
- this.ttsa?.sendSyncTimestamp(this.renderScheduler?.getCurrentFrameIndex()??0);
1553
- }
1554
-
1555
- /**
1556
- * 通知后端进入隐身模式
1557
- */
1558
- private notifyEnterInvisibleMode(): void {
1559
- this.ttsa?.enterInvisibleMode();
1560
- }
1561
-
1562
- /**
1563
- * 通知后端退出隐身模式
1564
- */
1565
- private notifyExitInvisibleMode(): void {
1566
- this.ttsa?.exitInvisibleMode();
1567
- }
1568
-
1569
- setVolume(volume: number) {
1570
- if (this._isClientMode) { this._apiForwarder?.setVolume(volume); return; }
1571
- this.renderScheduler.setVolume(volume);
1572
- }
1573
-
1574
- getSessionId() {
1575
- if (this._isClientMode) {
1576
- return this._clientSessionId;
1577
- }
1578
- return this.resourceManager?.session_id;
1579
- }
1580
-
1581
- // 显示调试浮层
1582
- public showDebugInfo() {
1583
- if (!this.debugOverlay) {
1584
- return;
1585
- }
1586
- this.debugOverlay?.show();
1587
- }
1588
-
1589
- // 隐藏调试浮层
1590
- public hideDebugInfo() {
1591
- this.debugOverlay?.hide();
1592
- }
1593
-
1594
- // 改变数字人canvas的显隐状态
1595
- public changeAvatarVisible(visible: boolean) {
1596
- this.avatarCanvasVisible = visible;
1597
- if (this._isClientMode) { this._apiForwarder?.changeAvatarVisible(visible); return; }
1598
- this.renderScheduler?.setAvatarCanvasVisible(visible);
1599
- }
1600
-
1601
- /**
1602
- * 在初始化未完成时清理所有已初始化的资源
1603
- * @param reason 清理原因
1604
- */
1605
- protected cleanupBeforeInitComplete(reason: string): Promise<void> {
1606
- if (this.incompleteInitializationCleanupPromise) {
1607
- return this.incompleteInitializationCleanupPromise;
1608
- }
1609
-
1610
- (window as any).avatarSDKLogger.warn(this.TAG, `清理初始化未完成的资源,原因: ${reason}`);
1611
- this.isInitialized = false;
1612
- const stopSessionPromise = this.resourceManager?.stopSession(reason);
1613
- void stopSessionPromise?.catch((error) => {
1614
- (window as any).avatarSDKLogger.warn(this.TAG, '停止初始化未完成的会话时出错', error);
1615
- });
1616
- this.incompleteInitializationCleanupPromise = (async () => {
1617
- try {
1618
- // 先等待房间释放完成,再清理本地资源,避免房间泄漏
1619
- await stopSessionPromise;
1620
- await this.destroyClient();
1621
- } catch (error) {
1622
- (window as any).avatarSDKLogger.warn(this.TAG, '清理初始化未完成的本地资源时出错', error);
1623
- }
1624
- })();
1625
- return this.incompleteInitializationCleanupPromise;
1626
- }
1627
-
1628
- protected reportInitializationFailure(message: string, cause?: unknown) {
1629
- this.onMessage({
1630
- code: EErrorCode.INIT_ERROR,
1631
- message,
1632
- e: cause as any,
1633
- });
1634
- }
1635
-
1636
- private offlineHandle() {
1637
- this.onStatusChange(AvatarStatus.offline)
1638
- this.renderScheduler._offlineMode()
1639
- this._offlineTimer = window.setInterval(() => {
1640
- if(this.status !== AvatarStatus.offline) {
1641
- clearInterval(this._offlineTimer);
1642
- this._offlineTimer = -1;
1643
- return
1644
- }
1645
- this.renderScheduler._offlineRun()
1646
- }, this._offlineInterval)
1647
- }
1648
-
1649
- public async stopSessionFromSocket(reason: string) {
1650
- await this.resourceManager.stopSession(reason)
1651
- }
1652
-
1653
- offlineMode() {
1654
- // this.networkMonitor.setState(false)
1655
- this.renderScheduler.interrupt("in_offline_mode")
1656
- this.resourceManager.stopSession("offline_mode")
1657
- this.ttsa?.close()
1658
- this.offlineHandle()
1659
- }
1660
-
1661
- onlineMode() {
1662
- if (!NetworkMonitor.ONLINE) {
1663
- return
1664
- }
1665
- // this.networkMonitor.setState(true)
1666
- this._reload()
1667
- }
1668
-
1669
- onMessage(params: Parameters<typeof Errors>[0]) {
1670
- // 添加 session_id
1671
- try {
1672
- if (typeof params.e !== 'object' || params.e === null) {
1673
- params.e = { msg: params.e };
1674
- }
1675
- params.session_id = this.getSessionId();
1676
- } catch (e) {
1677
- (window as any).avatarSDKLogger.warn(this.TAG, '获取 session_id 时出错', e);
1678
- }
1679
- const e = Errors(params)
1680
- if(params.code !== EErrorCode.RENDER_BODY_ERROR && params.code !== EErrorCode.RENDER_FACE_ERROR && params.code !== EErrorCode.BODY_DATA_EXPIRED) {
1681
- this.options.onMessage(e)
1682
- // Sentry 错误上报(仅公有云构建时生效)
1683
- if (ENABLE_SENTRY && this.sentryReporter) {
1684
- try {
1685
- this.sentryReporter.captureError(e);
1686
- } catch (err) {
1687
- (window as any).avatarSDKLogger.warn(this.TAG, 'Sentry 上报失败', err);
1688
- }
1689
- }
1690
- }
1691
-
1692
- this.ttsa?.send("client_error", params);
1693
- this.debugOverlay?.addError(e)
1694
- }
1695
-
1696
- onStatusChange(status: AvatarStatus) {
1697
- if(status === this.status) {
1698
- return
1699
- }
1700
- this.status = status
1701
- this.options.onStatusChange?.(status)
1702
- }
1703
-
1704
- /**
1705
- * 切换隐身模式(暂停/恢复音视频实时渲染)
1706
- */
1707
- public switchInvisibleMode(): void {
1708
- if (this._isClientMode) { this._apiForwarder?.switchInvisibleMode(); return; }
1709
- if (!this.renderScheduler) {
1710
- (window as any).avatarSDKLogger?.warn(
1711
- this.TAG,
1712
- 'RenderScheduler 未初始化,无法切换隐身模式'
1713
- );
1714
- return;
1715
- }
1716
-
1717
- // 如果当前数字人状态是在线 或者 visible,切换到暂停时
1718
- if (this.status === AvatarStatus.visible || this.status === AvatarStatus.online ) {
1719
- this.setInvisibleMode()
1720
- } else {
1721
- this.notifyExitInvisibleMode();
1722
- this.renderScheduler.switchInvisibleMode();
1723
- this.listen();
1724
- }
1725
- }
1726
- private setInvisibleMode() {
1727
- // 先调用 interactiveidle()
1728
- this.interactiveidle();
1729
- this.renderScheduler.switchInvisibleMode();
1730
- this.notifyEnterInvisibleMode();
1731
- this.onStatusChange(AvatarStatus.invisible);
1732
- }
1733
- public getPendingInvisibleMode() {
1734
- return this.pendingInvisibleMode || false;
1735
- }
1736
- /**
1737
- * 获取当前渲染状态
1738
- */
1739
- public getRenderState(): RenderState {
1740
- if (!this.renderScheduler) {
1741
- return RenderState.init;
1742
- }
1743
- return this.renderScheduler.getRenderState();
1744
- }
1745
-
1746
- changeLayout(layout: Layout) {
1747
- if (this._isClientMode) { this._apiForwarder?.changeLayout(layout); return; }
1748
- const {scale = 1} = layout.avatar || {}
1749
- layout.avatar.scale = validateScaleWithDecimal(scale, "changeLayout") as number | string;
1750
- this.ttsa?.changeLayout(layout);
1751
- this.renderScheduler.setCharacterCanvasLayout(layout);
1752
- }
1753
-
1754
- changeWalkConfig(walkConfig: WalkConfig) {
1755
- if (this._isClientMode) { this._apiForwarder?.changeWalkConfig(walkConfig); return; }
1756
- this.ttsa?.changeWalkConfig(walkConfig)
1757
- this.renderScheduler.setWalkConfig(walkConfig);
1758
- }
1759
-
1760
- interrupt(type: string):number {
1761
- if (this._isClientMode) { this._apiForwarder?.interrupt(type); return 0; }
1762
- return this.renderScheduler.interrupt(type)
1763
- }
1764
-
1765
- private triggerReconnect() {
1766
- // 网络恢复触发:无论是否正在退避重试,都取消挂起的重试定时器并立即重连(重开房间),
1767
- // 避免网络恢复后仍需等待当前退避窗口(最长 32s)才重连
1768
- this.clearAllRetryTimers();
1769
- this.isRetrying = false;
1770
- this.retryCount = 1;
1771
- this.retryRound = 1;
1772
- this.reStartSession();
1773
- }
1774
-
1775
- reStartSession() {
1776
- if (this.isRetrying || this.retryRound > this.maxRetryRound) return;
1777
-
1778
- this.clearAllRetryTimers();
1779
-
1780
- // 标记重试状态
1781
- this.isStartRetry = true;
1782
- this.isRetrying = true;
1783
-
1784
- // 使用非递归方式实现重试逻辑
1785
- this._retryImpl();
1786
- }
1787
-
1788
- private async _retryImpl() {
1789
- if (this.retryRound > this.maxRetryRound || !this.isRetrying) {
1790
- return;
1791
- }
1792
-
1793
- let delay = 1000;
1794
- if (this.retryCount > 1) {
1795
- delay = Math.pow(2, this.retryCount) * 1000;
1796
- }
1797
-
1798
- // 使用setTimeout代替递归,避免调用栈溢出;记录句柄以便网络恢复时取消挂起的退避重试
1799
- this.retryTimer = window.setTimeout(async () => {
1800
- this.retryTimer = -1;
1801
- if (!this.isRetrying || this.destroyed) {
1802
- return;
1803
- }
1804
- try {
1805
- // sdk.reconnect:单次重连耗时,带轮次/次数 tag
1806
- const result = this.sentryReporter
1807
- ? await this.sentryReporter.startSpan('sdk.reconnect', {
1808
- op: 'reconnect',
1809
- data: { round: this.retryRound, count: this.retryCount },
1810
- }, () => this._reload())
1811
- : await this._reload();
1812
- if (!this.isRetrying || this.destroyed) {
1813
- return;
1814
- }
1815
- if (result === null) {
1816
- // 重试失败:判断本轮是否达上限
1817
- if (this.retryCount >= this.maxRetryCount) {
1818
- // 本轮次数用尽,判断是否达最大轮次
1819
- if (this.retryRound >= this.maxRetryRound) {
1820
- // 总次数用尽(3*9=27次),停止重试
1821
- (window as any).avatarSDKLogger.warn(
1822
- this.TAG,
1823
- `重试达最大轮次(${this.maxRetryRound})+每轮最大次数(${this.maxRetryCount}),总次数=${this.maxRetryRound*this.maxRetryCount},停止重试`
1824
- );
1825
- this.onSessionReloadExhausted();
1826
- this.resetRetryState(); // 完全重置状态
1827
- return;
1828
- } else {
1829
- // 未达最大轮次:重置本轮计数,轮次+1
1830
- (window as any).avatarSDKLogger.warn(
1831
- this.TAG,
1832
- `第${this.retryRound}轮重试次数达上限(${this.maxRetryCount}),进入第${this.retryRound+1}轮重试`
1833
- );
1834
- this.retryCount = 1; // 重置本轮计数
1835
- this.retryRound += 1; // 轮次+1
1836
- }
1837
- } else {
1838
- // 本轮未达上限:计数+1
1839
- this.retryCount += 1;
1840
- }
1841
- // 继续重试
1842
- this._retryImpl();
1843
- } else {
1844
- // 重试成功:完全重置所有状态
1845
- this.renderScheduler.stopAudio(-1);
1846
- (window as any).avatarSDKLogger.log(
1847
- this.TAG,
1848
- `第${this.retryRound}轮第${this.retryCount}次重试成功,停止重试`
1849
- );
1850
- this.resetRetryState();
1851
- }
1852
- } catch (error) {
1853
- // 异常处理:逻辑同失败场景
1854
- (window as any).avatarSDKLogger.error(this.TAG, "重试过程异常", error);
1855
- if (this.retryCount >= this.maxRetryCount) {
1856
- if (this.retryRound >= this.maxRetryRound) {
1857
- (window as any).avatarSDKLogger.warn(
1858
- this.TAG,
1859
- `重试异常且达总次数上限(${this.maxRetryRound*this.maxRetryCount}),停止重试`
1860
- );
1861
- this.onSessionReloadExhausted(error);
1862
- this.resetRetryState();
1863
- return;
1864
- } else {
1865
- this.retryCount = 1;
1866
- this.retryRound += 1;
1867
- }
1868
- } else {
1869
- this.retryCount += 1;
1870
- }
1871
- // 仅当未达最大轮次时,继续重试
1872
- if (this.retryRound <= this.maxRetryRound && this.isRetrying) {
1873
- this._retryImpl();
1874
- } else {
1875
- this.resetRetryState();
1876
- }
1877
- }
1878
- }, delay);
1879
- }
1880
- private clearAllRetryTimers() {
1881
- // 清理重试定时器(含退避重试挂起的定时器)
1882
- if (this.retryTimer !== -1) {
1883
- clearTimeout(this.retryTimer);
1884
- this.retryTimer = -1;
1885
- }
1886
- }
1887
-
1888
- // 统一重置重试状态
1889
- private resetRetryState() {
1890
- this.isRetrying = false;
1891
- this.isStartRetry = false;
1892
- this.retryCount = 1; // 重置每轮计数
1893
- this.retryRound = 1; // 重置轮次(关键)
1894
- }
1895
-
1896
- public seekToFrame(frame: number) {
1897
- this.renderScheduler.seekToFrame(frame);
1898
- this.ttsa?.seekToFrame(frame);
1899
- }
1900
-
1901
- public async startRecoder() {
1902
- try {
1903
- const canvas = this.el?.querySelector('canvas') as HTMLCanvasElement;
1904
- const audio = this.el?.querySelector('audio') as HTMLAudioElement;
1905
-
1906
- if (!canvas || !audio) {
1907
- console.error('找不到 canvas 或 audio 元素');
1908
- return;
1909
- }
1910
-
1911
- // 画面流
1912
- const canvasStream = canvas.captureStream(60);
1913
-
1914
- // 获取音频流:
1915
- // 如果 setupAudioCapture 已调用 createMediaElementSource,audio 元素的输出
1916
- // 已被劫持到 Web Audio API,captureStream() 会拿到静音流。
1917
- // 此时复用已有的 captureSource 节点,追加 MediaStreamDestination 获取音频。
1918
- let audioTracks: MediaStreamTrack[] = [];
1919
-
1920
- if (this.captureSource && this.captureAudioCtx) {
1921
- // 复用 setupAudioCapture 的音频图,追加录制专用输出节点
1922
- const recorderDestination = this.captureAudioCtx.createMediaStreamDestination();
1923
- this.captureSource.connect(recorderDestination);
1924
- audioTracks = recorderDestination.stream.getAudioTracks();
1925
- } else if (typeof (audio as any).captureStream === 'function') {
1926
- // audio 未被劫持,直接用 captureStream
1927
- audioTracks = (audio as any).captureStream().getAudioTracks();
1928
- } else if (typeof (audio as any).mozCaptureStream === 'function') {
1929
- audioTracks = (audio as any).mozCaptureStream().getAudioTracks();
1930
- } else {
1931
- (window as any).avatarSDKLogger.error('❌ 当前浏览器不支持 captureStream,无法录制音频');
1932
- return;
1933
- }
1934
-
1935
- // 合并流
1936
- this.stream = new MediaStream([
1937
- ...canvasStream.getVideoTracks(),
1938
- ...audioTracks
1939
- ]);
1940
-
1941
- // 清空上一次数据
1942
- this.recordedChunks = [];
1943
-
1944
- // 创建录制器
1945
- this.mediaRecorder = new MediaRecorder(this.stream, {
1946
- mimeType: 'video/mp4',
1947
- videoBitsPerSecond: 12 * 1024 * 1024, // 12Mbps 超清
1948
- });
1949
-
1950
- // 数据收集
1951
- this.mediaRecorder.ondataavailable = (e) => {
1952
- if (e.data && e.data.size > 0) {
1953
- this.recordedChunks.push(e.data);
1954
- }
1955
- };
1956
-
1957
- // 开始录制
1958
- this.mediaRecorder.start(100); // 100ms 收集一次,避免丢数据
1959
-
1960
- (window as any).avatarSDKLogger.log('✅ 开始录制');
1961
- } catch (err) {
1962
- (window as any).avatarSDKLogger.error('❌ 录制失败:', err);
1963
- }
1964
- }
1965
-
1966
- public stopRecoder() {
1967
- if (!this.mediaRecorder) return;
1968
- this.mediaRecorder.stop();
1969
- this.stream?.getTracks().forEach(track => track.stop());
1970
- (window as any).avatarSDKLogger.log('⏹ 停止录制');
1971
- }
1972
-
1973
- public exportVideo() {
1974
- // 关键:检查有没有数据
1975
- if (!this.recordedChunks.length) {
1976
- (window as any).avatarSDKLogger.error('没有录制到数据');
1977
- return;
1978
- }
1979
-
1980
- const blob = new Blob(this.recordedChunks, { type: 'video/mp4' });
1981
- const url = URL.createObjectURL(blob);
1982
-
1983
- const a = document.createElement('a');
1984
- a.href = url;
1985
- a.download = `录制_${new Date().getTime()}.mp4`;
1986
- a.click();
1987
- (window as any).avatarSDKLogger.log('✅ 下载成功,大小:', blob.size);
1988
- }
1989
-
1990
- public getOptions() {
1991
- return this.options;
1992
- }
1993
-
1994
- // ── 测试用方法:模拟 WebGL 上下文丢失 ─────────────────────────────
1995
-
1996
- /**
1997
- * [测试用] 模拟 WebGL 上下文丢失
1998
- * 用于测试分阶段恢复策略是否正常工作
1999
- *
2000
- * @param restoreAfterMs 自动恢复上下文的延迟时间(毫秒),默认 5000ms
2001
- * @param forceFail 强制恢复失败(用于测试 medium/full 阶段的升级逻辑)
2002
- *
2003
- * @example
2004
- * // 在浏览器控制台执行:
2005
- * avatar.simulateContextLoss(5000); // 模拟丢失,5秒后自动恢复
2006
- * avatar.simulateContextLoss(0, true); // 模拟丢失,不自动恢复(测试升级逻辑)
2007
- */
2008
- public simulateContextLoss(restoreAfterMs: number = 5000, forceFail: boolean = false): void {
2009
- if (!this.renderScheduler) {
2010
- (window as any).avatarSDKLogger.warn("[Test] renderScheduler not initialized");
2011
- return;
2012
- }
2013
- this.renderScheduler.simulateContextLoss(restoreAfterMs, forceFail);
2014
- }
2015
-
2016
- /**
2017
- * [测试用] 获取当前 WebGL 上下文恢复状态
2018
- */
2019
- public getRecoveryStatus() {
2020
- if (!this.renderScheduler) {
2021
- return null;
2022
- }
2023
- return this.renderScheduler.getRecoveryStatus();
2024
- }
2025
-
2026
- /**
2027
- * [测试用] 批量测试分阶段恢复
2028
- * 依次触发多次上下文丢失,观察各阶段的恢复情况
2029
- */
2030
- public async testStagedRecovery(): Promise<void> {
2031
- if (!this.renderScheduler) {
2032
- (window as any).avatarSDKLogger.warn("[Test] renderScheduler not initialized");
2033
- return;
2034
- }
2035
- return this.renderScheduler.testStagedRecovery();
2036
- }
2037
-
2038
- /**
2039
- * 手动重启 WebGL:创建新 canvas + 新 GLDevice + 新 GLPipeline
2040
- * 用于分阶段恢复全部失败后的手动恢复
2041
- * @returns 是否成功
2042
- *
2043
- * @example
2044
- * const ok = avatar.restartWebGL();
2045
- */
2046
- public restartWebGL(): boolean {
2047
- if (!this.renderScheduler) {
2048
- (window as any).avatarSDKLogger.warn("[restartWebGL] renderScheduler not initialized");
2049
- return false;
2050
- }
2051
- return this.renderScheduler.restartWebGL();
2052
- }
2053
- }