@xmov/avatar 2.4.0 → 2.5.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. package/README.md +34 -6
  2. package/package-dist/agent/avatar.cjs +1 -0
  3. package/package-dist/agent/avatar.modern.js +1 -0
  4. package/package-dist/agent/avatar.module.js +1 -0
  5. package/package-dist/agent/avatar.umd.js +1 -0
  6. package/package-dist/agent/index.d.ts +42 -0
  7. package/{dist/agent/types.d.ts → package-dist/agent/public-types.d.ts} +115 -116
  8. package/package-dist/index.cjs +3 -0
  9. package/package-dist/index.d.ts +34 -0
  10. package/package-dist/index.modern.js +1 -0
  11. package/package-dist/index.module.js +1 -0
  12. package/package-dist/index.umd.js +14 -0
  13. package/package-dist/public-types.d.ts +205 -0
  14. package/package.json +41 -33
  15. package/dist/agent/avatar.cjs +0 -1
  16. package/dist/agent/avatar.modern.js +0 -1
  17. package/dist/agent/avatar.module.js +0 -1
  18. package/dist/agent/avatar.umd.js +0 -1
  19. package/dist/agent/index.d.ts +0 -137
  20. package/dist/baseRender/AudioRenderer.d.ts +0 -46
  21. package/dist/baseRender/AudioWorklet.d.ts +0 -78
  22. package/dist/baseRender/AvatarRenderer.d.ts +0 -225
  23. package/dist/baseRender/MSEAudioPlayer.d.ts +0 -51
  24. package/dist/baseRender/UIRenderer.d.ts +0 -39
  25. package/dist/baseRender/pcm-audio-processor.d.ts +0 -63
  26. package/dist/control/APIForwarder.d.ts +0 -23
  27. package/dist/control/DataCacheQueue.d.ts +0 -69
  28. package/dist/control/EventDispatcher.d.ts +0 -35
  29. package/dist/control/RenderScheduler.d.ts +0 -171
  30. package/dist/control/SaveAndDownload.d.ts +0 -14
  31. package/dist/control/StreamingClient.d.ts +0 -33
  32. package/dist/control/VideoPlayer.d.ts +0 -47
  33. package/dist/control/ttsa.d.ts +0 -125
  34. package/dist/encoding/media-recorder-encoder.d.ts +0 -33
  35. package/dist/encoding/pcm-webm-encoder.d.ts +0 -13
  36. package/dist/encoding/webcodec-opus-encoder.d.ts +0 -41
  37. package/dist/encoding/webm-muxer.d.ts +0 -90
  38. package/dist/index.cjs +0 -2
  39. package/dist/index.cjs.map +0 -1
  40. package/dist/index.d.ts +0 -251
  41. package/dist/index.modern.js +0 -2
  42. package/dist/index.modern.js.map +0 -1
  43. package/dist/index.module.js +0 -2
  44. package/dist/index.module.js.map +0 -1
  45. package/dist/index.umd.js +0 -2
  46. package/dist/index.umd.js.map +0 -1
  47. package/dist/modules/Composition.d.ts +0 -28
  48. package/dist/modules/ResourceManager.d.ts +0 -299
  49. package/dist/modules/TrackRenderer/base-track.d.ts +0 -4
  50. package/dist/modules/TrackRenderer/index.d.ts +0 -12
  51. package/dist/modules/TrackRenderer/render-implements.d.ts +0 -19
  52. package/dist/modules/TrackRenderer/track-pic.d.ts +0 -8
  53. package/dist/modules/TrackRenderer/track-subtitle.d.ts +0 -8
  54. package/dist/modules/cache-manager.d.ts +0 -5
  55. package/dist/modules/decoder.d.ts +0 -84
  56. package/dist/modules/error-handle.d.ts +0 -17
  57. package/dist/modules/network.d.ts +0 -21
  58. package/dist/modules/sentry-reporter.d.ts +0 -58
  59. package/dist/proto/face_data_pb.d.ts +0 -0
  60. package/dist/types/capability.d.ts +0 -15
  61. package/dist/types/error.d.ts +0 -70
  62. package/dist/types/event.d.ts +0 -25
  63. package/dist/types/frame-data.d.ts +0 -107
  64. package/dist/types/index.d.ts +0 -196
  65. package/dist/types/render.d.ts +0 -18
  66. package/dist/utils/DataInterface.d.ts +0 -113
  67. package/dist/utils/GLDevice.d.ts +0 -46
  68. package/dist/utils/GLPipeline.d.ts +0 -84
  69. package/dist/utils/GLPipelineDebugTools.d.ts +0 -5
  70. package/dist/utils/Math.d.ts +0 -21
  71. package/dist/utils/__tests__/capability-checker.test.d.ts +0 -1
  72. package/dist/utils/audio-session.d.ts +0 -11
  73. package/dist/utils/audio.d.ts +0 -1
  74. package/dist/utils/capability-checker.d.ts +0 -28
  75. package/dist/utils/encodeToken.d.ts +0 -14
  76. package/dist/utils/face.d.ts +0 -1
  77. package/dist/utils/float32-decoder.d.ts +0 -26
  78. package/dist/utils/fpsTracker.d.ts +0 -27
  79. package/dist/utils/index.d.ts +0 -22
  80. package/dist/utils/logger.d.ts +0 -9
  81. package/dist/utils/media-recorder-timestamp.d.ts +0 -15
  82. package/dist/utils/pcmAnalyzer.d.ts +0 -40
  83. package/dist/utils/perfermance.d.ts +0 -15
  84. package/dist/utils/request.d.ts +0 -20
  85. package/dist/utils/requestAnimateFrames.d.ts +0 -31
  86. package/dist/utils/time.d.ts +0 -9
  87. package/dist/utils/video-probe-asset.d.ts +0 -28
  88. package/dist/view/DebugOverlay.d.ts +0 -54
  89. package/dist/worker/streaming-video.d.ts +0 -1
  90. package/src/baseRender/AudioRenderer.ts +0 -328
  91. package/src/baseRender/AudioWorklet.ts +0 -776
  92. package/src/baseRender/AvatarRenderer.ts +0 -1550
  93. package/src/baseRender/MSEAudioPlayer.ts +0 -254
  94. package/src/baseRender/UIRenderer.ts +0 -195
  95. package/src/baseRender/pcm-audio-processor.js +0 -267
  96. package/src/control/APIForwarder.ts +0 -67
  97. package/src/control/DataCacheQueue.ts +0 -372
  98. package/src/control/EventDispatcher.ts +0 -106
  99. package/src/control/RenderScheduler.ts +0 -1000
  100. package/src/control/SaveAndDownload.js +0 -77
  101. package/src/control/StreamingClient.ts +0 -138
  102. package/src/control/VideoPlayer.ts +0 -306
  103. package/src/control/ttsa.ts +0 -754
  104. package/src/encoding/media-recorder-encoder.ts +0 -175
  105. package/src/encoding/pcm-webm-encoder.ts +0 -72
  106. package/src/encoding/webcodec-opus-encoder.ts +0 -258
  107. package/src/encoding/webm-muxer.ts +0 -337
  108. package/src/global.d.ts +0 -112
  109. package/src/index.ts +0 -2053
  110. package/src/modules/Composition.ts +0 -79
  111. package/src/modules/ResourceManager.ts +0 -926
  112. package/src/modules/TrackRenderer/base-track.ts +0 -7
  113. package/src/modules/TrackRenderer/index.ts +0 -40
  114. package/src/modules/TrackRenderer/render-implements.ts +0 -269
  115. package/src/modules/TrackRenderer/track-pic.ts +0 -19
  116. package/src/modules/TrackRenderer/track-subtitle.ts +0 -16
  117. package/src/modules/cache-manager.ts +0 -10
  118. package/src/modules/decoder.ts +0 -548
  119. package/src/modules/error-handle.ts +0 -20
  120. package/src/modules/network.ts +0 -71
  121. package/src/modules/sentry-reporter.ts +0 -171
  122. package/src/proto/face_data.proto +0 -45
  123. package/src/proto/face_data_pb.js +0 -1297
  124. package/src/proto/protobuf.min.js +0 -8
  125. package/src/types/capability.ts +0 -24
  126. package/src/types/error.ts +0 -211
  127. package/src/types/event.ts +0 -38
  128. package/src/types/frame-data.ts +0 -121
  129. package/src/types/index.ts +0 -220
  130. package/src/types/render.ts +0 -18
  131. package/src/utils/DataInterface.ts +0 -746
  132. package/src/utils/GLDevice.ts +0 -286
  133. package/src/utils/GLPipeline.ts +0 -1293
  134. package/src/utils/GLPipelineDebugTools.ts +0 -174
  135. package/src/utils/Math.ts +0 -266
  136. package/src/utils/__tests__/capability-checker.test.ts +0 -159
  137. package/src/utils/audio-session.ts +0 -144
  138. package/src/utils/audio.ts +0 -44
  139. package/src/utils/blueimp-md5.d.ts +0 -3
  140. package/src/utils/capability-checker.ts +0 -762
  141. package/src/utils/encodeToken.ts +0 -115
  142. package/src/utils/face.ts +0 -29
  143. package/src/utils/float32-decoder.js +0 -130
  144. package/src/utils/fpsTracker.ts +0 -100
  145. package/src/utils/index.ts +0 -182
  146. package/src/utils/logger.js +0 -36
  147. package/src/utils/media-recorder-timestamp.ts +0 -79
  148. package/src/utils/pcmAnalyzer.ts +0 -154
  149. package/src/utils/perfermance.ts +0 -141
  150. package/src/utils/request.ts +0 -107
  151. package/src/utils/requestAnimateFrames.ts +0 -195
  152. package/src/utils/time.ts +0 -32
  153. package/src/utils/video-probe-asset.ts +0 -99
  154. package/src/view/DebugOverlay.ts +0 -409
  155. package/src/worker/streaming-video.ts +0 -4647
  156. /package/{dist → package-dist}/agent/index.cjs +0 -0
  157. /package/{dist → package-dist}/agent/index.umd.js +0 -0
@@ -1,926 +0,0 @@
1
- import Pako from 'pako'
2
- import { IAvatarOptions, INetworkInfo, Layout, WalkConfig } from "../types/index";
3
- import request, { XMLRequest } from "../utils/request";
4
- import {
5
- IBRAnimationGeneratorCharInfo_NN,
6
- } from "../utils/DataInterface";
7
- import XmovAvatar from "../index";
8
- import { EErrorCode } from "../types/error";
9
- import { headersNeedSign, generateRequestId } from "../utils/encodeToken";
10
- // import { isDEV } from "../utils";
11
- import CacheManager from './cache-manager';
12
-
13
- /**
14
- * 资源处理模块,加载并存储资源
15
- */
16
-
17
- type TOptions = Pick<
18
- IAvatarOptions,
19
- "appId" | "appSecret" | "gatewayServer" | 'cacheServer' | "sdkInstance" | "config" | "headers" | "gateway_type" | "tag" |
20
- "asr_id" | "llm_id" | "session_speak_req_id" | "audioOnlyMode"
21
- > & {
22
- sessionRequestData?: Record<string, unknown>
23
- onNetworkInfo(quality: INetworkInfo): void
24
- onStartSessionWarning?: (message: Object) => void
25
- };
26
- export type TDownloadProgress = (progress: number) => void;
27
- export interface IOfflineIdle {
28
- asf: number
29
- aef: number
30
- n: string
31
- }
32
- export interface IResumeParams {
33
- client_frame: number
34
- current_ani: string
35
- current_ani_frame: number
36
- next_state: string
37
- }
38
- export interface ISessionResponse {
39
- session_id: string;
40
- room: string;
41
- token: string;
42
- socket_io_url: string;
43
- socket_io_path?: string;
44
- e2e_resp?: {
45
- ws_url?: string;
46
- e2e_token?: string;
47
- monitor_ws_url?: string;
48
- features?: {
49
- speech_frontend?: {
50
- enable_aec?: boolean;
51
- [key: string]: unknown;
52
- };
53
- [key: string]: unknown;
54
- };
55
- };
56
- features?: {
57
- speech_frontend?: {
58
- enable_aec?: boolean;
59
- [key: string]: unknown;
60
- };
61
- [key: string]: unknown;
62
- };
63
- session_start_time: string;
64
- resource_pack: {
65
- enable_robot_expression: boolean;
66
- body_data_dir: string;
67
- face_ani_char_data: string;
68
- face_ani_preload_data: string;
69
- offline_idle: IOfflineIdle[],
70
- interpolate_joints: [number, number][]
71
- };
72
- reconnect_timeout: number;
73
- reconnect_client_timeout: number;
74
- config: {
75
- background_img: string;
76
- frame_rate: number;
77
- look_name: string;
78
- tts_vcn_id: string;
79
- sta_face_id: string;
80
- mp_service_id: string;
81
- raw_audio: boolean;
82
- resolution: {
83
- width: number;
84
- height: number;
85
- },
86
- init_events: [
87
- {
88
- type: string;
89
- data: {
90
- image: string;
91
- axis_id: number;
92
- }
93
- }
94
- ],
95
- fx?: {
96
- transform?: {
97
- offsetX: number;
98
- offsetY: number;
99
- scaleX: number;
100
- scaleY: number;
101
- };
102
- // 阴影效果
103
- shadow?: {
104
- // [x, y, z],阴影投射的基准点
105
- refPos: number[];
106
- // 光源位置
107
- light: number[];
108
- // 光源类型:"point"(点光源)| "directional"(平行光)
109
- lightType: string;
110
- // 阴影模糊半径(越大越柔和)
111
- blurRadius: number;
112
- // 阴影颜色
113
- tint: number[];
114
- // 阴影强度(0.0 = 无阴影,1.0 = 最深)
115
- intensity: number;
116
- };
117
- };
118
- };
119
- }
120
- export default class ResourceManager {
121
- private TAG = "[ResourceManager]";
122
- options: TOptions;
123
- sdk: XmovAvatar;
124
- session_id?: string;
125
- /** 单次 init 生命周期内的稳定 request_id,由 generateRequestId 生成;session_id 未返回时也能用于 stopSession 兜底销毁 */
126
- session_request_id?: string;
127
- mouthShapeLib: any;
128
- offlineIdle: IOfflineIdle[] = []
129
- offlineCache = new Map<string, {
130
- data: ArrayBuffer;
131
- }>();
132
- networkInfos: INetworkInfo[] = [];
133
- private bg: HTMLImageElement | null = null;
134
- private destroyed = false;
135
- public resource_pack: {
136
- enable_robot_expression: boolean;
137
- body_data_dir: string;
138
- face_ani_char_data: string;
139
- face_ani_preload_data: string;
140
- blendshape_map?: number[][];
141
- interpolate_joints: [number, number][]
142
- }
143
- progress = 0; // start session 以后之后资源下载进度
144
-
145
- config: {
146
- background_img: string;
147
- frame_rate: number;
148
- look_name: string;
149
- tts_vcn_id: string;
150
- sta_face_id: string;
151
- mp_service_id: string;
152
- raw_audio: boolean;
153
- resolution: {
154
- width: number;
155
- height: number;
156
- },
157
- init_events: [
158
- {
159
- type: string;
160
- "x_location"?: number;//控件锚点信息,控件左上角在屏幕上的横向位置。范围0-1
161
- "y_location"?: number;//控件锚点信息,控件左上角在屏幕上的纵向位置。范围0-1
162
- width?: number;//控件锚点信息,控件在屏幕上的宽度。范围0-1
163
- height: number;//控件锚点信息,控件在屏幕上的高度。范围0-1
164
- speech_id: number;
165
- data?: {
166
- image: string;
167
- axis_id: number;
168
- }
169
- }
170
- ],
171
- framedata_proto_version: number;
172
- layout: Layout;
173
- walk_config: WalkConfig;
174
- fx?: {
175
- transform?: {
176
- offsetX: number;
177
- offsetY: number;
178
- scaleX: number;
179
- scaleY: number;
180
- };
181
- // 阴影效果
182
- shadow?: {
183
- // [x, y, z],阴影投射的基准点
184
- refPos: number[];
185
- // 光源位置
186
- light: number[];
187
- // 光源类型:"point"(点光源)| "directional"(平行光)
188
- lightType: string;
189
- // 阴影模糊半径(越大越柔和)
190
- blurRadius: number;
191
- // 阴影颜色
192
- tint: number[];
193
- // 阴影强度(0.0 = 无阴影,1.0 = 最深)
194
- intensity: number;
195
- };
196
- };
197
- };
198
-
199
- // 视频缓存相关
200
- private videoCache = new Map<string, {
201
- data: ArrayBuffer;
202
- lastAccessTime: number;
203
- accessCount: number;
204
- }>();
205
- private maxCacheSize = 10 * 1024 * 1024; // 5MB 默认缓存大小
206
- private maxCacheEntries = 20; // 最大缓存条目数
207
- private cacheCleanupInterval: any = null;
208
- private isProcessingVideo = false; // 防止在视频处理过程中清理缓存
209
- private cacheServer?: CacheManager;
210
- private startSessionTimer: any = null;
211
- private audioOnlyMode: boolean = false;
212
-
213
- // 新增:正在下载的视频跟踪
214
- private downloadingVideos = new Map<string, Promise<ArrayBuffer | undefined>>();
215
- first_load = true;
216
- constructor(options: TOptions) {
217
- this.options = options;
218
- this.audioOnlyMode = options.audioOnlyMode || false;
219
- this.sdk = options.sdkInstance as XmovAvatar;
220
- this.mouthShapeLib = {
221
- char_info: null,
222
- };
223
- this.config = options.config || {};
224
- this.resource_pack = {
225
- enable_robot_expression: false,
226
- body_data_dir: "",
227
- face_ani_char_data: "",
228
- face_ani_preload_data: "",
229
- interpolate_joints: [[4, 3]]
230
- };
231
-
232
- if (options.cacheServer) {
233
- this.cacheServer = new CacheManager(options.cacheServer);
234
- }
235
- // 启动缓存清理定时器
236
- this.startCacheCleanup();
237
- }
238
-
239
- public getAppInfo() {
240
- return {
241
- appId: this.options.appId
242
- }
243
- }
244
-
245
- /**
246
- * 加载背景图片
247
- * @returns Promise<void>
248
- */
249
- getBackgroundImage(): Promise<void> {
250
- return new Promise((resolve, reject) => {
251
- try {
252
- const img = new Image();
253
- // 设置跨域属性
254
- img.crossOrigin = "anonymous";
255
- img.src = this.config?.background_img || "";
256
- if (!this.config?.background_img) return;
257
- img.onload = () => {
258
- this.bg = img;
259
- resolve();
260
- };
261
-
262
- img.onerror = (error) => {
263
- this.sdk.onMessage({
264
- code: EErrorCode.BACKGROUND_IMAGE_LOAD_ERROR,
265
- message: `Error: 背景图片加载失败`,
266
- e: { error, img },
267
- });
268
- this.bg = null;
269
- resolve();
270
- };
271
- } catch (error) {
272
- }
273
- });
274
- }
275
-
276
- getIsEnableRobotExpression() {
277
- return this.resource_pack?.enable_robot_expression
278
- }
279
-
280
- /**
281
- * 获取背景图片
282
- * @returns HTMLImageElement | null
283
- */
284
- getBackgroundImageElement(): HTMLImageElement | null {
285
- return this.bg;
286
- }
287
-
288
- getConfig() {
289
- return this.config;
290
- }
291
-
292
- async load(onDownloadProgress?: TDownloadProgress) {
293
- this.destroyed = false; // 重置销毁状态,允许重新加载
294
- onDownloadProgress?.(this.progress);
295
- if (this.startSessionTimer) {
296
- clearInterval(this.startSessionTimer);
297
- this.startSessionTimer = null;
298
- }
299
-
300
- this.startSessionTimer = setInterval(() => {
301
- if (this.destroyed || this.progress >= 14) {
302
- clearInterval(this.startSessionTimer);
303
- this.startSessionTimer = null;
304
- return;
305
- }
306
- this.progress += 1;
307
- onDownloadProgress?.(this.progress);
308
- }, 100);
309
- // http 请求算 10 进度
310
- this.session_request_id = generateRequestId(this.options.appId);
311
- const result = await this.startSession();
312
- const res = result as unknown as ISessionResponse;
313
- // 先记录 session_id,确保后续 stopSession 能正确释放房间
314
- if (res?.session_id) {
315
- this.session_id = res.session_id;
316
- }
317
- // 如果在 startSession 过程中被销毁了,释放刚创建的房间并返回,避免房间泄漏
318
- if (this.destroyed) {
319
- (window as any).avatarSDKLogger.warn(this.TAG, "sdk destroyed during load, aborting after startSession");
320
- await this.stopSession("destroy_during_init");
321
- return null;
322
- }
323
- if (!result) {
324
- clearInterval(this.startSessionTimer);
325
- this.startSessionTimer = null;
326
- return null;
327
- }
328
-
329
- clearInterval(this.startSessionTimer);
330
- this.startSessionTimer = null;
331
- this.progress = 15;
332
- onDownloadProgress?.(this.progress);
333
- this.resource_pack = res.resource_pack;
334
- this.config = res.config as any;
335
- this.offlineIdle = res.resource_pack?.offline_idle || []
336
- if (this.resource_pack) {
337
- if(!this.audioOnlyMode) {
338
- await this.loadMouthShapeLib(onDownloadProgress);
339
- }
340
- this.getBackgroundImage();
341
- if (!this.mouthShapeLib?.char_info && res?.resource_pack?.face_ani_char_data && !this.audioOnlyMode) {
342
- return null
343
- }
344
- }
345
- return res;
346
- }
347
-
348
- async startSession() {
349
- try {
350
- const result = await this._startSession();
351
- if (result?.verify_res) {
352
- this.options.onStartSessionWarning?.(result.verify_res);
353
- }
354
- return result;
355
- } catch (error) {
356
- return null;
357
- }
358
- }
359
- _startSession() {
360
- if (!navigator.onLine) {
361
- this.sdk.onMessage({
362
- code: EErrorCode.NETWORK_BREAK,
363
- message: '没有网络,请联网后重试~',
364
- });
365
- return null;
366
- }
367
-
368
- const url = `${this.options.gatewayServer}`;
369
-
370
-
371
- const data = {
372
- request_id: this.session_request_id,
373
- tag: this.options.tag,
374
- gateway_type: this.options.gateway_type,
375
- ...(this.options.asr_id !== undefined ? { asr_id: this.options.asr_id } : {}),
376
- ...(this.options.llm_id !== undefined ? { llm_id: this.options.llm_id } : {}),
377
- ...(this.options.session_speak_req_id !== undefined
378
- ? { session_speak_req_id: this.options.session_speak_req_id }
379
- : {}),
380
- ...(this.options.sessionRequestData ?? {}),
381
- config: {
382
- ...this.config,
383
- framedata_proto_version: 2,
384
- walk_version: 3,
385
- }
386
- }
387
- const { headers, data: signedData } = headersNeedSign(this.options.appId, this.options.appSecret, "POST", url, data);
388
- return request(url, { method: "POST", data: signedData, headers: { ...headers, ...this.options.headers } })
389
- .then(response => response.json())
390
- .then(res => {
391
- if (!res?.data?.resource_pack) {
392
- this.sdk.onMessage({
393
- code: res?.error_code,
394
- message: `Error: ${res?.error_code}, ${res?.error_reason}`,
395
- });
396
- return null;
397
- }
398
- return res?.data;
399
- }).catch(error => {
400
- this.sdk.onMessage({
401
- code: EErrorCode.START_SESSION_ERROR,
402
- message: `Error: 启动会话失败`,
403
- e: error,
404
- });
405
- return null;
406
- });
407
- }
408
-
409
- async stopSession(stop_reason: string) {
410
- try {
411
- const url = `${this.options.gatewayServer}`;
412
- // session_id 尚未返回时(如 init 未完成就 destroy),只要有 request_id 仍发请求,
413
- // 让服务端用 request_id 匹配并销毁房间占用,避免 init 失败后的资源泄漏。
414
- if (!this.session_id && !this.session_request_id) {
415
- return;
416
- }
417
- const data: Record<string, any> = {
418
- stop_reason,
419
- };
420
- if (this.session_id) {
421
- data.session_id = this.session_id;
422
- }
423
- if (this.session_request_id) {
424
- data.request_id = this.session_request_id;
425
- }
426
- const { headers, data: signedData } = headersNeedSign(this.options.appId, this.options.appSecret, "DELETE", url, data);
427
- const response = await request(url, { method: "DELETE", data: signedData, headers: { ...headers, ...this.options.headers } });
428
- return await response.json();
429
- } catch (error) {
430
- this.sdk.onMessage({
431
- code: EErrorCode.STOP_SESSION_ERROR,
432
- message: `Error: 停止会话失败`,
433
- e: error,
434
- });
435
- }
436
- }
437
-
438
- /**
439
- * 加载表情数据
440
- */
441
- async loadMouthShapeLib(onDownloadProgress?: TDownloadProgress) {
442
- try {
443
- if (!this.resource_pack.face_ani_char_data) {
444
- this.progress += 60;
445
- onDownloadProgress?.(this.progress);
446
- return;
447
- }
448
- // 优先加载 .gz,失败则回退 bin
449
- let char_data: ArrayBuffer | null = null;
450
- let gzError: any = null;
451
- let finalUrl = '';
452
- let fileSize = 0;
453
- let startTime = 0;
454
- let preTime = 0;
455
- let preLoaded = 0;
456
-
457
- const createProgressHandler = () => (progress: number, e?: ProgressEvent) => {
458
- if (!e) return;
459
- const now = performance.now();
460
- const deltaTime = now - preTime;
461
- const deltaLoaded = e.loaded - preLoaded;
462
-
463
- const speedMbps = (deltaTime > 0 && deltaLoaded > 0)
464
- ? (deltaLoaded * 8) / (deltaTime / 1000) / 1000000
465
- : 0;
466
-
467
- this.options.onNetworkInfo?.({
468
- rtt: deltaTime,
469
- downlink: speedMbps,
470
- networkStatus: navigator.onLine,
471
- });
472
-
473
- preLoaded = e.loaded;
474
- preTime = now;
475
- onDownloadProgress?.(this.progress + progress * 0.65);
476
- };
477
-
478
- try {
479
- const gzUrl = `${this.resource_pack.face_ani_char_data}.gz`;
480
- finalUrl = gzUrl;
481
- startTime = performance.now();
482
- preTime = startTime;
483
- preLoaded = 0;
484
-
485
- const [, gz_data] = await XMLRequest({
486
- url: gzUrl,
487
- onProgress: createProgressHandler(),
488
- });
489
-
490
- const decompressed = Pako.ungzip(new Uint8Array(gz_data as ArrayBuffer));
491
- const ab = new ArrayBuffer(decompressed.byteLength);
492
- new Uint8Array(ab).set(decompressed);
493
- char_data = ab;
494
- } catch (e) {
495
- gzError = e;
496
- char_data = null;
497
- }
498
-
499
- if (!char_data) {
500
- try {
501
- const binUrl = this.resource_pack.face_ani_char_data;
502
- finalUrl = binUrl;
503
- startTime = performance.now();
504
- preTime = startTime;
505
- preLoaded = 0;
506
-
507
- const [, bin_data] = await XMLRequest({
508
- url: binUrl,
509
- onProgress: createProgressHandler(),
510
- });
511
-
512
- char_data = bin_data as ArrayBuffer;
513
- } catch (e) {
514
- if (this.first_load) {
515
- this.first_load = false;
516
- this.stopSession("char_bin_load_error");
517
- }
518
- this.sdk.onMessage?.({
519
- code: EErrorCode.FACE_BIN_LOAD_ERROR,
520
- message: "表情数据加载失败(.gz 和 bin 均失败)",
521
- e: { gzError, binError: e },
522
- });
523
- return;
524
- }
525
- }
526
-
527
- const totalCostMs = performance.now() - startTime;
528
- const totalSpeedMbps = fileSize > 0 && totalCostMs > 0
529
- ? (fileSize * 8) / (totalCostMs / 1000) / 1000000
530
- : 0;
531
-
532
- this.options.onNetworkInfo?.({
533
- rtt: totalCostMs,
534
- downlink: totalSpeedMbps,
535
- networkStatus: navigator.onLine,
536
- });
537
-
538
- this.mouthShapeLib = {
539
- char_info: new IBRAnimationGeneratorCharInfo_NN(char_data, {
540
- blendshapeMap: this.resource_pack.blendshape_map || [],
541
- }),
542
- };
543
- this.progress += 65;
544
- onDownloadProgress?.(this.progress);
545
- } catch (error) {
546
- if (this.first_load) {
547
- this.first_load = false;
548
- this.stopSession("char_bin_load_error");
549
- }
550
- this.sdk.onMessage?.({
551
- code: EErrorCode.FACE_BIN_LOAD_ERROR,
552
- message: "表情数据加载失败,请检查文件是否正确",
553
- e: error,
554
- });
555
- }
556
- }
557
-
558
- getMouthShapeLib() {
559
- return this.mouthShapeLib;
560
- }
561
-
562
- getVideoUrl(name: string) {
563
- if (!this.resource_pack.body_data_dir) {
564
- return "";
565
- }
566
- if (!name) {
567
- if (this.first_load) {
568
- this.first_load = false;
569
- this.stopSession("no_video_name_error");
570
- }
571
- this.sdk.onMessage({
572
- code: EErrorCode.INVALID_BODY_NAME,
573
- message: `Error: 视频地址获取失败`,
574
- e: name,
575
- });
576
- return "";
577
- }
578
- const url = `${this.resource_pack.body_data_dir}${name}.mp4`
579
- if (this.cacheServer) {
580
- return this.cacheServer?.getVideo(url);
581
- }
582
- return url
583
- }
584
-
585
- /**
586
- * 预加载视频
587
- * @param name 视频名称
588
- * @returns Promise<ArrayBuffer>
589
- */
590
- async preloadVideo(name: string) {
591
- if (this.videoCache.has(name)) return;
592
- return this.loadVideo(name);
593
- }
594
- /**
595
- * 加载视频
596
- * @param name 视频名称
597
- * @returns Promise<ArrayBuffer>
598
- */
599
- async loadVideo(name: string) {
600
- // 检查缓存
601
- const cacheEntry = this.videoCache.get(name) || this.offlineCache.get(name);
602
- if (cacheEntry) {
603
- if (cacheEntry.data) {
604
- this.updateCacheAccess(name);
605
- return cacheEntry.data;
606
- } else {
607
- // 缓存条目存在但数据为空,删除这个无效条目
608
- (window as any).avatarSDKLogger.warn(this.TAG, `发现无效缓存条目: ${name},正在删除`);
609
- this.videoCache.delete(name);
610
- }
611
- }
612
-
613
- // 检查是否正在下载
614
- if (this.downloadingVideos.has(name)) {
615
- try {
616
- const result = await this.downloadingVideos.get(name);
617
- return result;
618
- } catch (error) {
619
- // 如果下载失败,从下载列表中移除
620
- this.downloadingVideos.delete(name);
621
- (window as any).avatarSDKLogger.warn(this.TAG, `${name} 视频下载失败,从下载列表移除`);
622
- throw error;
623
- }
624
- }
625
-
626
- // 开始下载
627
- this.isProcessingVideo = true;
628
- const downloadPromise = this.downloadVideo(name);
629
- this.downloadingVideos.set(name, downloadPromise);
630
- try {
631
- const result = await downloadPromise;
632
- return result;
633
- } finally {
634
- // 下载完成后从下载列表中移除
635
- this.downloadingVideos.delete(name);
636
- this.isProcessingVideo = false;
637
- }
638
- }
639
-
640
- /**
641
- * 实际下载视频的方法
642
- * @param name 视频名称
643
- * @returns Promise<ArrayBuffer>
644
- */
645
- private async downloadVideo(name: string): Promise<ArrayBuffer | undefined> {
646
- const url = this.getVideoUrl(name);
647
- if (!url) {
648
- return;
649
- }
650
- try {
651
- const { response, arrayBuffer } = await this._fetchVideo(url);
652
- if (!response.ok) {
653
- if (this.first_load) {
654
- this.first_load = false;
655
- this.stopSession("load_video_error");
656
- }
657
- this.sdk.onMessage({
658
- code: EErrorCode.VIDEO_DOWNLOAD_ERROR,
659
- message: `Error: ${name} 视频下载失败`,
660
- e: { name, url, res: response },
661
- });
662
- return;
663
- }
664
-
665
- // 将下载的数据添加到缓存
666
- this.addToCache(name, arrayBuffer);
667
-
668
- return arrayBuffer;
669
- } catch (error) {
670
- this.sdk.onMessage({
671
- code: EErrorCode.VIDEO_DOWNLOAD_ERROR,
672
- message: `Error: ${name} 视频下载失败`,
673
- e: { name, url, error },
674
- });
675
- return;
676
- }
677
- }
678
-
679
- private async _fetchVideo(url: string) {
680
- if (this.networkInfos.length >= 4) {
681
- const avgRtt = this.networkInfos.reduce((sum, curr) => sum + curr.rtt, 0) / this.networkInfos.length;
682
- const avgDownlink = this.networkInfos.reduce((sum, curr) => sum + curr.downlink, 0) / this.networkInfos.length;
683
- this.options.onNetworkInfo?.({
684
- rtt: avgRtt,
685
- downlink: avgDownlink,
686
- networkStatus: navigator.onLine,
687
- })
688
- this.networkInfos.length = 0
689
- }
690
- const startTime = performance.now();
691
- const response = await fetch(url)
692
- const totalSize = parseFloat(response.headers.get('Content-Length') || '0');
693
- const endTime = performance.now();
694
- const arrayBuffer = await response.arrayBuffer()
695
- const rtt = endTime - startTime; // 毫秒
696
- const downlink = (totalSize * 8) / (rtt / 1000 * 1000000); // Mbps
697
-
698
- const networkInfo = {
699
- rtt,
700
- downlink,
701
- networkStatus: navigator.onLine,
702
- };
703
- this.networkInfos.push(networkInfo)
704
- return {
705
- response,
706
- arrayBuffer,
707
- }
708
- }
709
-
710
- /**
711
- * 添加数据到缓存
712
- */
713
- private addToCache(key: string, data: ArrayBuffer) {
714
- const now = Date.now();
715
-
716
- // 检查缓存大小限制
717
- let totalSize = 0;
718
- for (const value of this.videoCache.values()) {
719
- totalSize += value.data.byteLength;
720
- }
721
-
722
- // 如果添加这个数据会超过限制,先清理一些缓存
723
- if (totalSize + data.byteLength > this.maxCacheSize || this.videoCache.size >= this.maxCacheEntries) {
724
- this.cleanupCache();
725
-
726
- // 重新计算大小
727
- totalSize = 0;
728
- for (const value of this.videoCache.values()) {
729
- totalSize += value.data.byteLength;
730
- }
731
-
732
- // 如果仍然超过限制,且单个文件太大,不缓存这个数据
733
- if (totalSize + data.byteLength > this.maxCacheSize && data.byteLength > this.maxCacheSize * 0.5) {
734
- (window as any).avatarSDKLogger.warn(this.TAG, `视频文件过大(${(data.byteLength / 1024 / 1024).toFixed(2)}MB),跳过缓存: ${key}`);
735
- return;
736
- }
737
-
738
- // 如果缓存空间仍然不足,清理更多缓存
739
- if (totalSize + data.byteLength > this.maxCacheSize) {
740
- // 强制清理到50%容量
741
- const targetSize = this.maxCacheSize * 0.5;
742
- const sortedEntries = Array.from(this.videoCache.entries())
743
- .sort((a, b) => a[1].lastAccessTime - b[1].lastAccessTime);
744
-
745
- for (const [cacheKey, cacheValue] of sortedEntries) {
746
- if (totalSize <= targetSize) break;
747
- this.videoCache.delete(cacheKey);
748
- totalSize -= cacheValue.data.byteLength;
749
- }
750
- }
751
- }
752
-
753
- // 添加到缓存
754
- this.videoCache.set(key, {
755
- data: data,
756
- lastAccessTime: now,
757
- accessCount: 1
758
- });
759
- }
760
-
761
- /**
762
- * 启动缓存清理定时器
763
- */
764
- private startCacheCleanup() {
765
- // 每5分钟清理一次缓存
766
- this.cacheCleanupInterval = setInterval(() => {
767
- this.cleanupCache();
768
- }, 5 * 60 * 1000);
769
- }
770
-
771
- /**
772
- * 清理缓存
773
- */
774
- private cleanupCache() {
775
- // 如果正在处理视频,跳过清理
776
- if (this.isProcessingVideo) {
777
- return;
778
- }
779
-
780
- const now = Date.now();
781
- const inactiveThreshold = 10 * 60 * 1000; // 10分钟不活跃的缓存将被清理
782
-
783
- // 按最后访问时间排序
784
- const sortedEntries = Array.from(this.videoCache.entries())
785
- .sort((a, b) => a[1].lastAccessTime - b[1].lastAccessTime);
786
-
787
- let totalSize = 0;
788
- const entriesToRemove: string[] = [];
789
-
790
- // 计算当前缓存总大小
791
- for (const [_, value] of this.videoCache.entries()) {
792
- totalSize += value.data.byteLength;
793
- }
794
-
795
-
796
- // 清理不活跃的缓存(但保留最近访问的)
797
- for (const [key, value] of sortedEntries) {
798
- if (now - value.lastAccessTime > inactiveThreshold) {
799
- entriesToRemove.push(key);
800
- totalSize -= value.data.byteLength;
801
- }
802
- }
803
-
804
- // 如果缓存仍然过大,清理最旧的条目(但至少保留2个条目)
805
- if (totalSize > this.maxCacheSize || this.videoCache.size > this.maxCacheEntries) {
806
- for (const [key, value] of sortedEntries) {
807
- if (!entriesToRemove.includes(key)) {
808
- entriesToRemove.push(key);
809
- totalSize -= value.data.byteLength;
810
-
811
- // 确保至少保留2个缓存条目,防止完全清空
812
- if ((this.videoCache.size - entriesToRemove.length <= 2) ||
813
- (totalSize <= this.maxCacheSize * 0.8 && this.videoCache.size - entriesToRemove.length <= this.maxCacheEntries * 0.8)) {
814
- break;
815
- }
816
- }
817
- }
818
- }
819
-
820
- // 执行清理
821
- for (const key of entriesToRemove) {
822
- this.videoCache.delete(key);
823
- }
824
- }
825
-
826
- /**
827
- * 更新缓存访问信息
828
- */
829
- private updateCacheAccess(key: string) {
830
- const cacheEntry = this.videoCache.get(key);
831
- if (cacheEntry) {
832
- cacheEntry.lastAccessTime = Date.now();
833
- cacheEntry.accessCount++;
834
- }
835
- }
836
-
837
- /**
838
- * 清空所有缓存
839
- */
840
- clearAllCache() {
841
- this.videoCache.clear();
842
- }
843
-
844
- /** 获取离线 idle */
845
- _getOfflineIdle() {
846
- return this.offlineIdle
847
- }
848
- /** 获取上一次 session_id */
849
- _getSessionId() {
850
- return this.session_id || ''
851
- }
852
-
853
- /** 离线重连 */
854
- async _reload() {
855
- const result = await this.startSession();
856
- if (!result) {
857
- return null;
858
- }
859
- const res = result as unknown as ISessionResponse;
860
- this.session_id = res?.session_id;
861
- // 重连过程中被销毁,释放刚创建的房间,避免房间泄漏
862
- if (this.destroyed) {
863
- await this.stopSession("destroy_during_reload");
864
- return null;
865
- }
866
- this.resource_pack = res.resource_pack;
867
- this.config = res.config as any;
868
- this.offlineIdle = res.resource_pack?.offline_idle || []
869
- if (this.resource_pack) {
870
- this.getBackgroundImage();
871
- }
872
- return res;
873
- }
874
-
875
- public async _setOfflineCache() {
876
- if (!this.offlineIdle || this.offlineIdle.length === 0) {
877
- return
878
- }
879
- this.offlineCache.clear()
880
- const names = this.offlineIdle.map($i => $i.n)
881
- for (const name of names) {
882
- try {
883
- await this._runOfflineCache(name)
884
- } catch (error) {
885
- }
886
- }
887
- }
888
- private async _runOfflineCache(name: string) {
889
- const url = this.getVideoUrl(name)
890
- if (!url) {
891
- return;
892
- }
893
- return this._fetchVideo(url)
894
- .then(({ arrayBuffer }) => {
895
- this.offlineCache.set(name, {
896
- data: arrayBuffer,
897
- })
898
- })
899
- }
900
-
901
- /**
902
- * 销毁资源管理器
903
- */
904
- destroy() {
905
- this.destroyed = true;
906
- // 清理定时器
907
- if (this.cacheCleanupInterval) {
908
- clearInterval(this.cacheCleanupInterval);
909
- this.cacheCleanupInterval = null;
910
- }
911
-
912
- // 清理缓存
913
- this.clearAllCache();
914
-
915
- // 清理下载跟踪
916
- this.downloadingVideos.clear();
917
-
918
- // 清理背景图片
919
- if (this.bg) {
920
- this.bg = null;
921
- }
922
- this.progress = 0
923
- this.session_id = undefined;
924
- this.session_request_id = undefined;
925
- }
926
- }