@xmov/avatar 2.0.1 → 2.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/README.md +3 -1
  2. package/dist/agent/avatar.cjs +1 -2
  3. package/dist/agent/avatar.modern.js +1 -2
  4. package/dist/agent/avatar.module.js +1 -2
  5. package/dist/agent/avatar.umd.js +1 -2
  6. package/dist/agent/index.cjs +1 -4
  7. package/dist/agent/index.d.ts +12 -5
  8. package/dist/agent/index.umd.js +3 -5
  9. package/dist/agent/types.d.ts +5 -43
  10. package/dist/baseRender/AvatarRenderer.d.ts +0 -1
  11. package/dist/control/RenderScheduler.d.ts +17 -1
  12. package/dist/control/ttsa.d.ts +8 -1
  13. package/dist/index.cjs +1 -1
  14. package/dist/index.cjs.map +1 -1
  15. package/dist/index.d.ts +5 -1
  16. package/dist/index.modern.js +1 -1
  17. package/dist/index.modern.js.map +1 -1
  18. package/dist/index.module.js +1 -1
  19. package/dist/index.module.js.map +1 -1
  20. package/dist/index.umd.js +1 -1
  21. package/dist/index.umd.js.map +1 -1
  22. package/dist/modules/ResourceManager.d.ts +2 -1
  23. package/dist/utils/request.d.ts +1 -0
  24. package/package.json +29 -7
  25. package/src/baseRender/AudioRenderer.ts +16 -5
  26. package/src/baseRender/AvatarRenderer.ts +11 -20
  27. package/src/baseRender/MSEAudioPlayer.ts +18 -7
  28. package/src/control/RenderScheduler.ts +74 -8
  29. package/src/control/ttsa.ts +47 -11
  30. package/src/index.ts +66 -45
  31. package/src/modules/ResourceManager.ts +19 -6
  32. package/src/modules/decoder.ts +34 -1
  33. package/src/utils/capability-checker.ts +95 -21
  34. package/src/utils/request.ts +14 -2
  35. package/src/view/DebugOverlay.ts +9 -4
  36. package/dist/agent/__tests__/agent.test.d.ts +0 -1
  37. package/dist/agent/audio-debug.d.ts +0 -3
  38. package/dist/agent/audio-uplink.d.ts +0 -62
  39. package/dist/agent/avatar.cjs.map +0 -1
  40. package/dist/agent/avatar.modern.js.map +0 -1
  41. package/dist/agent/avatar.module.js.map +0 -1
  42. package/dist/agent/avatar.umd.js.map +0 -1
  43. package/dist/agent/e2e-client.d.ts +0 -32
  44. package/dist/agent/fixed-audio-track.d.ts +0 -48
  45. package/dist/agent/microphone.d.ts +0 -49
  46. package/src/agent/__tests__/agent.test.ts +0 -3787
  47. package/src/agent/audio-debug.ts +0 -36
  48. package/src/agent/audio-uplink.ts +0 -392
  49. package/src/agent/e2e-client.ts +0 -215
  50. package/src/agent/fixed-audio-track.ts +0 -547
  51. package/src/agent/index.ts +0 -1393
  52. package/src/agent/microphone.ts +0 -534
  53. package/src/agent/types.ts +0 -197
package/src/index.ts CHANGED
@@ -104,7 +104,6 @@ export default class XmovAvatar {
104
104
  private pcmCaptureMode: 'webcodec' | 'mediarecorder' | 'none' = 'none';
105
105
  private pcmEncodeId: number = 0; // 用于 reset 竞态检测,递增标识当前编码会话
106
106
 
107
- private reconnectDebounceTimer = -1; // 重连防抖定时器
108
107
  private networkInfo: any = {
109
108
  rtt: 0,
110
109
  downlink: 0.0,
@@ -200,6 +199,14 @@ export default class XmovAvatar {
200
199
  return this.status
201
200
  }
202
201
 
202
+ /**
203
+ * 初始化是否完成(progress === 100)
204
+ * 供 TTSA 重连恢复时判断是否可提前恢复在线状态
205
+ */
206
+ public getIsInitialized() {
207
+ return this.isInitialized
208
+ }
209
+
203
210
  public get businessENV() {
204
211
  return this._env
205
212
  }
@@ -270,6 +277,7 @@ export default class XmovAvatar {
270
277
  gatewayServer,
271
278
  cacheServer,
272
279
  config:_config,
280
+ audioOnlyMode: options.audioOnlyMode || false,
273
281
  onNetworkInfo(networkInfo) {
274
282
  self.networkInfo = networkInfo;
275
283
  options.onNetworkInfo?.(networkInfo);
@@ -293,6 +301,10 @@ export default class XmovAvatar {
293
301
  // 标记初始化完成
294
302
  this.isInitialized = progress === 100;
295
303
  if(progress === 100) {
304
+ // 初始化完成:若 start 时因未初始化完成而未上线(状态守卫),此处补发 online
305
+ if (!this.pendingInvisibleMode && this.status !== AvatarStatus.online && this.status !== AvatarStatus.invisible) {
306
+ this.onStatusChange(AvatarStatus.online);
307
+ }
296
308
  this.resourceManager._setOfflineCache();
297
309
  this.ttsa?.sendSdkPoint('loadvideo_decode_time', {
298
310
  timestamp: Date.now(),
@@ -1039,12 +1051,28 @@ export default class XmovAvatar {
1039
1051
 
1040
1052
  visibilitychange() {
1041
1053
  if (document.hidden) {
1042
- // this.offlineMode()
1043
- // tab 回来时,强制同步解帧队列
1044
- // this.renderScheduler.forceSyncDecoder();
1054
+ // 进入后台:暂停渲染循环和解码器(省电)、清空音频与表情数据
1055
+ // 仅渲染中暂停(resumeRender 要求 paused 状态才能恢复),
1056
+ // 其他状态(init/stopped/隐身 paused)只停止音频,避免强制暂停后无法恢复
1057
+ const rs = this.renderScheduler?.getRenderState();
1058
+ if (rs === RenderState.rendering || rs === RenderState.resumed) {
1059
+ this.renderScheduler?.pauseRender();
1060
+ } else {
1061
+ this.renderScheduler?.pauseForBackground();
1062
+ }
1045
1063
  } else {
1046
- if (this.ttsa?.getStatus()) {
1047
- this.renderScheduler.forceSyncDecoder();
1064
+ // 回到前台:若处于暂停态则恢复渲染(内部已 forceSyncDecoder 对齐最新帧)
1065
+ if (this.renderScheduler?.getRenderState() === RenderState.paused) {
1066
+ this.renderScheduler?.resumeRender();
1067
+ } else if (this.ttsa?.getStatus()) {
1068
+ // 未暂停但连接正常:兜底同步解帧队列
1069
+ this.renderScheduler?.forceSyncDecoder();
1070
+ }
1071
+ // WebSocket 在后台断开且已进入离线模式,网络可用则主动触发重连,
1072
+ // 否则用户切回前台续播时无响应且无任何报错
1073
+ if (!this.ttsa?.getStatus() && this.status === AvatarStatus.offline && NetworkMonitor.ONLINE) {
1074
+ (window as any).avatarSDKLogger.warn(this.TAG, "切回前台发现连接已断开,主动触发重连");
1075
+ this.onlineMode();
1048
1076
  }
1049
1077
  // tab 切换回来时,恢复音频捕获的 AudioContext
1050
1078
  if (this.captureAudioCtx && this.captureAudioCtx.state === 'suspended') {
@@ -1086,6 +1114,7 @@ export default class XmovAvatar {
1086
1114
  token: sessionInfo.token,
1087
1115
  framedata_proto_version: this.resourceManager.getConfig().framedata_proto_version,
1088
1116
  reconnect_client_timeout: sessionInfo?.reconnect_client_timeout || 0,
1117
+ reconnect_timeout: sessionInfo?.reconnect_timeout || 0,
1089
1118
  appInfo: {
1090
1119
  ...this.resourceManager.getAppInfo(),
1091
1120
  env: this._env
@@ -1094,7 +1123,7 @@ export default class XmovAvatar {
1094
1123
  this.ttsa?.sendSdkPoint('connect_sdk', {
1095
1124
  timestamp: this.startConnectTime,
1096
1125
  });
1097
- this.onDownloadProgress?.(85);
1126
+ // 不再强行上报 85:进度以真实资源下载为准,避免重连时 UI 进度被错误拉低
1098
1127
  this.ttsa?.sendSdkPoint('connect_sdk_success', {
1099
1128
  timestamp: this.connectSuccessTime,
1100
1129
  });
@@ -1148,16 +1177,18 @@ export default class XmovAvatar {
1148
1177
  }
1149
1178
 
1150
1179
  protected start() {
1151
- this.renderScheduler.render();
1152
- this.ttsa?.start();
1153
-
1154
- // 如果初始化时设置了隐身模式,则立即进入隐身状态
1180
+ // 先发布在线状态,再启动渲染:
1181
+ // 避免先 render() 触发 rendering -> visible(5) 后再发 online(0),造成 0/5 状态横跳
1155
1182
  if (this.pendingInvisibleMode) {
1156
1183
  this.pendingInvisibleMode = false;
1157
1184
  this.setInvisibleMode()
1158
- } else {
1185
+ } else if (this.isInitialized) {
1186
+ // 状态守卫:仅在初始化完成(progress === 100)时上线,
1187
+ // 避免重连重建房间过程中(资源未下载完)反复触发 online -> visible 状态横跳
1159
1188
  this.onStatusChange(AvatarStatus.online);
1160
1189
  }
1190
+ this.renderScheduler.render();
1191
+ this.ttsa?.start();
1161
1192
 
1162
1193
  }
1163
1194
 
@@ -1204,7 +1235,14 @@ export default class XmovAvatar {
1204
1235
  }
1205
1236
 
1206
1237
  reloadSuccess() {
1207
- this.onStatusChange(AvatarStatus.online);
1238
+ // 重连成功恢复:恢复音频、重置解码器离线状态、清空旧数据缓存,并按首个数据 sf 对齐帧号,
1239
+ // 解决窗口内自动重连恢复后人物卡死(帧号错位数据被过滤)、无声音/音画不同步的问题
1240
+ this.renderScheduler?.markReconnectRecover();
1241
+ // 状态守卫:仅初始化完成时上线;未完成时由 onDownloadProgress(100) 补发,
1242
+ // 避免重连重建房间过程中初始化未完成却提前进入 online
1243
+ if (this.isInitialized) {
1244
+ this.onStatusChange(AvatarStatus.online);
1245
+ }
1208
1246
  setTimeout(() => {
1209
1247
  window.clearInterval(this._offlineTimer)
1210
1248
  }, 1000)
@@ -1350,7 +1388,7 @@ export default class XmovAvatar {
1350
1388
  // 薄客户端模式:通知 server 端销毁,然后清理本地资源
1351
1389
  if (this._isClientMode) {
1352
1390
  this._apiForwarder?.destroy(stop_reason);
1353
- this.destroyClient();
1391
+ await this.destroyClient();
1354
1392
  this.onStatusChange(AvatarStatus.close);
1355
1393
  return;
1356
1394
  }
@@ -1358,8 +1396,8 @@ export default class XmovAvatar {
1358
1396
  this.ttsa?.sendSdkPoint('close_session', {
1359
1397
  reason: stop_reason,
1360
1398
  });
1361
- this.destroyClient();
1362
1399
  const res = await this.resourceManager.stopSession(stop_reason);
1400
+ await this.destroyClient();
1363
1401
  if(res) {
1364
1402
  this.onStatusChange(AvatarStatus.close);
1365
1403
  }
@@ -1480,6 +1518,8 @@ export default class XmovAvatar {
1480
1518
  });
1481
1519
  this.incompleteInitializationCleanupPromise = (async () => {
1482
1520
  try {
1521
+ // 先等待房间释放完成,再清理本地资源,避免房间泄漏
1522
+ await stopSessionPromise;
1483
1523
  await this.destroyClient();
1484
1524
  } catch (error) {
1485
1525
  (window as any).avatarSDKLogger.warn(this.TAG, '清理初始化未完成的本地资源时出错', error);
@@ -1617,28 +1657,13 @@ export default class XmovAvatar {
1617
1657
  }
1618
1658
 
1619
1659
  private triggerReconnect() {
1620
- // 如果已经在重连中,直接返回,避免重复触发
1621
- if (this.isRetrying) {
1622
- (window as any).avatarSDKLogger.warn(this.TAG, "正在重连中,跳过本次触发");
1623
- return;
1624
- }
1625
-
1626
- if (this.reconnectDebounceTimer !== -1) {
1627
- clearTimeout(this.reconnectDebounceTimer);
1628
- this.reconnectDebounceTimer = -1;
1629
- }
1630
- if (this.retryTimer !== -1) {
1631
- clearTimeout(this.retryTimer);
1632
- this.retryTimer = -1;
1633
- }
1634
-
1635
- this.reconnectDebounceTimer = window.setTimeout(() => {
1636
- this.isRetrying = false;
1637
- this.retryCount = 1;
1638
- this.retryRound = 1;
1639
- this.reStartSession();
1640
- this.reconnectDebounceTimer = -1;
1641
- }, 100);
1660
+ // 网络恢复触发:无论是否正在退避重试,都取消挂起的重试定时器并立即重连(重开房间),
1661
+ // 避免网络恢复后仍需等待当前退避窗口(最长 32s)才重连
1662
+ this.clearAllRetryTimers();
1663
+ this.isRetrying = false;
1664
+ this.retryCount = 1;
1665
+ this.retryRound = 1;
1666
+ this.reStartSession();
1642
1667
  }
1643
1668
 
1644
1669
  reStartSession() {
@@ -1664,8 +1689,9 @@ export default class XmovAvatar {
1664
1689
  delay = Math.pow(2, this.retryCount) * 1000;
1665
1690
  }
1666
1691
 
1667
- // 使用setTimeout代替递归,避免调用栈溢出
1668
- setTimeout(async () => {
1692
+ // 使用setTimeout代替递归,避免调用栈溢出;记录句柄以便网络恢复时取消挂起的退避重试
1693
+ this.retryTimer = window.setTimeout(async () => {
1694
+ this.retryTimer = -1;
1669
1695
  if (!this.isRetrying || this.destroyed) {
1670
1696
  return;
1671
1697
  }
@@ -1740,12 +1766,7 @@ export default class XmovAvatar {
1740
1766
  }, delay);
1741
1767
  }
1742
1768
  private clearAllRetryTimers() {
1743
- // 清理重连防抖定时器
1744
- if (this.reconnectDebounceTimer !== -1) {
1745
- clearTimeout(this.reconnectDebounceTimer);
1746
- this.reconnectDebounceTimer = -1;
1747
- }
1748
- // 清理重试定时器
1769
+ // 清理重试定时器(含退避重试挂起的定时器)
1749
1770
  if (this.retryTimer !== -1) {
1750
1771
  clearTimeout(this.retryTimer);
1751
1772
  this.retryTimer = -1;
@@ -17,7 +17,7 @@ import CacheManager from './cache-manager';
17
17
  type TOptions = Pick<
18
18
  IAvatarOptions,
19
19
  "appId" | "appSecret" | "gatewayServer" | 'cacheServer' | "sdkInstance" | "config" | "headers" | "gateway_type" | "tag" |
20
- "asr_id" | "llm_id" | "session_speak_req_id"
20
+ "asr_id" | "llm_id" | "session_speak_req_id" | "audioOnlyMode"
21
21
  > & {
22
22
  sessionRequestData?: Record<string, unknown>
23
23
  onNetworkInfo(quality: INetworkInfo): void
@@ -205,12 +205,14 @@ export default class ResourceManager {
205
205
  private isProcessingVideo = false; // 防止在视频处理过程中清理缓存
206
206
  private cacheServer?: CacheManager;
207
207
  private startSessionTimer: any = null;
208
+ private audioOnlyMode: boolean = false;
208
209
 
209
210
  // 新增:正在下载的视频跟踪
210
211
  private downloadingVideos = new Map<string, Promise<ArrayBuffer | undefined>>();
211
212
  first_load = true;
212
213
  constructor(options: TOptions) {
213
214
  this.options = options;
215
+ this.audioOnlyMode = options.audioOnlyMode || false;
214
216
  this.sdk = options.sdkInstance as XmovAvatar;
215
217
  this.mouthShapeLib = {
216
218
  char_info: null,
@@ -303,9 +305,15 @@ export default class ResourceManager {
303
305
  }, 100);
304
306
  // http 请求算 10 进度
305
307
  const result = await this.startSession();
306
- // 如果在 startSession 过程中被销毁了,直接返回,避免继续加载资源
308
+ const res = result as unknown as ISessionResponse;
309
+ // 先记录 session_id,确保后续 stopSession 能正确释放房间
310
+ if (res?.session_id) {
311
+ this.session_id = res.session_id;
312
+ }
313
+ // 如果在 startSession 过程中被销毁了,释放刚创建的房间并返回,避免房间泄漏
307
314
  if (this.destroyed) {
308
315
  (window as any).avatarSDKLogger.warn(this.TAG, "sdk destroyed during load, aborting after startSession");
316
+ await this.stopSession("destroy_during_init");
309
317
  return null;
310
318
  }
311
319
  if (!result) {
@@ -318,15 +326,15 @@ export default class ResourceManager {
318
326
  this.startSessionTimer = null;
319
327
  this.progress = 15;
320
328
  onDownloadProgress?.(this.progress);
321
- const res = result as unknown as ISessionResponse;
322
- this.session_id = res?.session_id;
323
329
  this.resource_pack = res.resource_pack;
324
330
  this.config = res.config as any;
325
331
  this.offlineIdle = res.resource_pack?.offline_idle || []
326
332
  if (this.resource_pack) {
327
- await this.loadMouthShapeLib(onDownloadProgress);
333
+ if(!this.audioOnlyMode) {
334
+ await this.loadMouthShapeLib(onDownloadProgress);
335
+ }
328
336
  this.getBackgroundImage();
329
- if (!this.mouthShapeLib?.char_info && res?.resource_pack?.face_ani_char_data) {
337
+ if (!this.mouthShapeLib?.char_info && res?.resource_pack?.face_ani_char_data && !this.audioOnlyMode) {
330
338
  return null
331
339
  }
332
340
  }
@@ -838,6 +846,11 @@ export default class ResourceManager {
838
846
  }
839
847
  const res = result as unknown as ISessionResponse;
840
848
  this.session_id = res?.session_id;
849
+ // 重连过程中被销毁,释放刚创建的房间,避免房间泄漏
850
+ if (this.destroyed) {
851
+ await this.stopSession("destroy_during_reload");
852
+ return null;
853
+ }
841
854
  this.resource_pack = res.resource_pack;
842
855
  this.config = res.config as any;
843
856
  this.offlineIdle = res.resource_pack?.offline_idle || []
@@ -279,12 +279,32 @@ export default class ParallelDecoder {
279
279
 
280
280
  _startWorker(file: bodyFile) {
281
281
  const taskId = this.currentTaskId;
282
- const worker = new Worker(workerURL);
282
+ let worker: Worker;
283
+ try {
284
+ worker = new Worker(workerURL);
285
+ } catch (err) {
286
+ this.reportMessage({
287
+ code: EErrorCode.INIT_WORKER_ERROR,
288
+ message: '初始化视频抽帧Worker失败',
289
+ e: err as object,
290
+ });
291
+ return;
292
+ }
283
293
  const id = `${file.body_id ?? file.id ?? 0}_${file.name}`;
284
294
  const data = file.data;
285
295
  const dataCopy = new ArrayBuffer(data.byteLength);
286
296
  new Uint8Array(dataCopy).set(new Uint8Array(data));
287
297
 
298
+ // 监听 Worker 运行时错误(脚本加载/解析失败等)
299
+ worker.addEventListener("error", (err) => {
300
+ if (taskId !== this.currentTaskId) return;
301
+ this.reportMessage({
302
+ code: EErrorCode.INIT_WORKER_ERROR,
303
+ message: '视频抽帧Worker运行时错误',
304
+ e: err as object,
305
+ });
306
+ });
307
+
288
308
  const onMessage = (e: MessageEvent) => {
289
309
  if (taskId !== this.currentTaskId) return; // 只处理当前任务
290
310
  if (e.data.type === "frame") {
@@ -335,6 +355,19 @@ export default class ParallelDecoder {
335
355
  }
336
356
  this._tryStartNext();
337
357
  } else if (e.data.type === "error") {
358
+ // 根据 Worker 上报的 error 内容区分错误码
359
+ // Worker 中有两处 postMessage({ type: 'error' }):
360
+ // 1. VideoDecoder error 回调 → 视频抽帧错误 (VIDEO_FRAME_EXTRACT_ERROR)
361
+ // 2. 解码配置不支持 → 视频流处理错误 (PROCESS_VIDEO_STREAM_ERROR)
362
+ const errorMsg = e.data.error ?? '';
363
+ const code = errorMsg.includes('不支持')
364
+ ? EErrorCode.PROCESS_VIDEO_STREAM_ERROR
365
+ : EErrorCode.VIDEO_FRAME_EXTRACT_ERROR;
366
+ this.reportMessage({
367
+ code,
368
+ message: errorMsg || '视频解码错误',
369
+ e: { fileName: file.name, error: errorMsg } as object,
370
+ });
338
371
  const task = this.tasks.get(id);
339
372
  if (task) {
340
373
  task.status = "error";
@@ -58,11 +58,11 @@ function checkWebGL2Sync(): ICapabilityCheckItem {
58
58
 
59
59
  /**
60
60
  * 同步检查音频系统(PCM 或 MSE 任一可用即可)
61
- *
61
+ *
62
62
  * SDK 有两种音频播放器:
63
63
  * 1. PCM 播放器:AudioWorklet + AudioContext(raw_audio = true)
64
64
  * 2. MSE 播放器:MediaSource + HTMLAudioElement(raw_audio = false)
65
- *
65
+ *
66
66
  * 两者二选一,任一可用则 SDK 音频能力可用
67
67
  */
68
68
  function checkAudioSystemSync(): ICapabilityCheckItem {
@@ -110,17 +110,27 @@ function checkAudioSystemSync(): ICapabilityCheckItem {
110
110
 
111
111
  /**
112
112
  * 同步检查 WebCodecs API 是否存在(不探测具体编解码器)
113
+ *
114
+ * 与 streaming-video.ts 解码策略一致标注平台解码路径:
115
+ * - Android(含 OpenHarmony/ArkWeb):prefer-software 软解码
116
+ * - 其余环境:默认配置(硬解码)
117
+ * 实际解码配置支持度由异步 checkWebCodecsDetailed 探测
113
118
  */
114
119
  function checkWebCodecsSync(): ICapabilityCheckItem {
115
120
  const hasVideoDecoder = typeof (window as any).VideoDecoder !== 'undefined';
116
- const supported = hasVideoDecoder;
121
+ const userAgent = (navigator.userAgent || '').toLowerCase();
122
+ const isAndroid = /android/.test(userAgent) || /openharmony|arkweb/.test(userAgent);
117
123
 
118
124
  return {
119
125
  name: 'webcodecs',
120
- supported,
126
+ supported: hasVideoDecoder,
121
127
  severity: 'fatal',
122
- reason: supported ? undefined : '浏览器不支持 WebCodecs VideoDecoder API',
123
- detail: { hasVideoDecoder },
128
+ reason: hasVideoDecoder ? undefined : '浏览器不支持 WebCodecs VideoDecoder API',
129
+ detail: {
130
+ hasVideoDecoder,
131
+ isAndroid,
132
+ hardwareAcceleration: isAndroid ? 'prefer-software' : 'default',
133
+ },
124
134
  };
125
135
  }
126
136
 
@@ -139,14 +149,14 @@ function checkWorkerSync(): ICapabilityCheckItem {
139
149
 
140
150
  /**
141
151
  * 同步检查所有浏览器能力(用于 constructor 快速校验)
142
- *
152
+ *
143
153
  * 检查项:
144
154
  * - secureContext:安全上下文(HTTPS / localhost)
145
155
  * - webgl2:WebGL2 渲染能力
146
156
  * - audio:音频播放能力(PCM 或 MSE 任一可用即可)
147
157
  * - webcodecs:WebCodecs API 存在性
148
158
  * - worker:Web Worker 支持
149
- *
159
+ *
150
160
  * 音频检查逻辑:SDK 有两种音频播放器(PCM 和 MSE),只需其中一种可用即可
151
161
  * - PCM 播放器依赖:AudioContext + AudioWorkletNode
152
162
  * - MSE 播放器依赖:MediaSource
@@ -186,13 +196,17 @@ export function checkCapabilitySync(audioOnlyMode = false): ICapabilityCheckRepo
186
196
 
187
197
  /**
188
198
  * 异步检查 WebCodecs 详细编解码器支持
199
+ *
200
+ * 与 streaming-video.ts 解码策略一致:
201
+ * - Android(含 OpenHarmony/ArkWeb):主路径为 prefer-software 软解码,默认配置兜底
202
+ * - 其余环境:主路径为默认配置(硬解码)
203
+ * 逐 codec 探测两种配置的支持度,并按平台主路径判定 SDK 可用性
189
204
  */
190
205
  async function checkWebCodecsDetailed(): Promise<ICapabilityCheckItem> {
191
206
  try {
192
207
  const hasVideoDecoder = typeof (window as any).VideoDecoder !== 'undefined';
193
- const supported = hasVideoDecoder;
194
208
 
195
- if (!supported) {
209
+ if (!hasVideoDecoder) {
196
210
  return {
197
211
  name: 'webcodecs',
198
212
  supported: false,
@@ -202,29 +216,89 @@ async function checkWebCodecsDetailed(): Promise<ICapabilityCheckItem> {
202
216
  };
203
217
  }
204
218
 
219
+ // 平台判定与 streaming-video.ts 保持一致
220
+ const userAgent = (navigator.userAgent || '').toLowerCase();
221
+ const isAndroid = /android/.test(userAgent) || /openharmony|arkweb/.test(userAgent);
222
+
223
+ // 探测配置补全尺寸字段:与业务 isConfigSupported(config) 一致(纯 codec 在部分浏览器被拒)
224
+ const probeW = 2160;
225
+ const probeH = 1920;
226
+
205
227
  // 探测具体编解码器支持
206
- let codecSupport: Record<string, boolean> = {};
228
+ // 注意:WebCodecs codec 字段是纯编解码器字符串(如 avc1.640032),
229
+ // 不能用 video/mp4; codecs="..." 的 MIME 形式(会导致 isConfigSupported 抛异常)
230
+ let codecSupport: Record<string, { default: boolean; software: boolean | null; error?: string }> = {};
207
231
  const testCodecs = [
208
- 'video/mp4; codecs="avc1.42E01E"', // H.264 Baseline
209
- 'video/mp4; codecs="avc1.4D001F"', // H.264 Main
210
- 'video/mp4; codecs="avc1.64001F"', // H.264 High
232
+ 'avc1.42E01E', // H.264 Baseline
233
+ 'avc1.4D001F', // H.264 Main
234
+ 'avc1.64001F', // H.264 High
211
235
  ];
236
+
237
+ // 主路径(平台默认解码配置)与兜底(默认配置)任一支持即判定可用
238
+ let primarySupported = false;
239
+ let fallbackSupported = false;
240
+ // 探测过程是否出现异常(区别于"确实不支持"):异常时降级 warn,避免误杀可用环境
241
+ let probeFailed = false;
212
242
  for (const codec of testCodecs) {
213
243
  try {
214
- const result = await (window as any).VideoDecoder.isConfigSupported({
244
+ // 默认配置(浏览器默认/硬解),业务兜底路径
245
+ const defaultResult = await (window as any).VideoDecoder.isConfigSupported({
215
246
  codec,
247
+ codedWidth: probeW,
248
+ codedHeight: probeH,
216
249
  });
217
- codecSupport[codec] = result.supported;
250
+ const defaultOk = !!defaultResult?.supported;
251
+ if (defaultOk) {
252
+ fallbackSupported = true;
253
+ }
254
+
255
+ // Android 主路径:prefer-software 软解码(与 streaming-video.ts 一致)
256
+ let softwareOk: boolean | null = null;
257
+ if (isAndroid) {
258
+ try {
259
+ const softwareResult = await (window as any).VideoDecoder.isConfigSupported({
260
+ codec,
261
+ codedWidth: probeW,
262
+ codedHeight: probeH,
263
+ hardwareAcceleration: 'prefer-software',
264
+ });
265
+ softwareOk = !!softwareResult?.supported;
266
+ } catch {
267
+ softwareOk = false;
268
+ }
269
+ if (softwareOk) {
270
+ primarySupported = true;
271
+ }
272
+ } else if (defaultOk) {
273
+ primarySupported = true;
274
+ }
275
+
276
+ codecSupport[codec] = { default: defaultOk, software: softwareOk };
218
277
  } catch {
219
- codecSupport[codec] = false;
278
+ // isConfigSupported 抛异常:探测方式受限(如浏览器实现差异),非"确实不支持"
279
+ probeFailed = true;
280
+ codecSupport[codec] = { default: false, software: null, error: 'probe-failed' };
220
281
  }
221
282
  }
222
283
 
284
+ const supported = primarySupported || fallbackSupported;
285
+ // 探测异常且无任何配置支持:降级 warn(运行时业务用真实 config 的 isConfigSupported 为准)
286
+ const severity = probeFailed && !supported ? 'warn' : 'fatal';
223
287
  return {
224
288
  name: 'webcodecs',
225
- supported: true,
226
- severity: 'fatal',
227
- detail: { hasVideoDecoder, codecSupport },
289
+ supported,
290
+ severity,
291
+ reason: supported
292
+ ? undefined
293
+ : severity === 'warn'
294
+ ? 'WebCodecs 探测异常(运行时将以实际视频配置二次确认)'
295
+ : '浏览器不支持 H.264 解码(默认/软解配置均不可用)',
296
+ detail: {
297
+ hasVideoDecoder,
298
+ isAndroid,
299
+ hardwareAcceleration: isAndroid ? 'prefer-software' : 'default',
300
+ codecSupport,
301
+ },
228
302
  };
229
303
  } catch (e) {
230
304
  return {
@@ -332,7 +406,7 @@ async function checkWorkerDetailed(): Promise<ICapabilityCheckItem> {
332
406
 
333
407
  /**
334
408
  * 异步检查所有浏览器能力(详细版,用于 init 阶段详细探测)
335
- *
409
+ *
336
410
  * 包含更详细的编解码器探测和 Worker 执行测试
337
411
  */
338
412
  export async function checkCapability(audioOnlyMode = false): Promise<ICapabilityCheckReport> {
@@ -70,20 +70,32 @@ export function XMLRequest(options: {
70
70
  cache?: {
71
71
  enable: boolean
72
72
  }
73
+ timeout?: number
73
74
  onProgress: (progress: number, e: ProgressEvent) => void
74
75
  }): Promise<[boolean, ArrayBuffer | string]> {
75
76
  return new Promise((resolve, reject) => {
76
77
  const xhr = new XMLHttpRequest();
77
78
  xhr.open('GET', options.url);
78
79
  xhr.responseType = 'arraybuffer';
80
+ // 超时兜底:断网/请求挂起时不再永久等待(默认 60s,资源下载可能超时失败)
81
+ xhr.timeout = options.timeout ?? 60000;
79
82
  xhr.onload = function() {
80
83
  if (xhr.readyState === 4 && xhr.status === 200) {
81
84
  const arrayBuffer = xhr.response;
82
85
  resolve([false, arrayBuffer]);
83
- } else if (xhr.status !== 200) {
84
- reject([true, xhr.statusText]);
86
+ } else {
87
+ reject([true, `HTTP ${xhr.status} ${xhr.statusText}`]);
85
88
  }
86
89
  };
90
+ xhr.onerror = function() {
91
+ reject([true, 'network error']);
92
+ };
93
+ xhr.ontimeout = function() {
94
+ reject([true, `timeout after ${xhr.timeout}ms`]);
95
+ };
96
+ xhr.onabort = function() {
97
+ reject([true, 'aborted']);
98
+ };
87
99
  xhr.onprogress = (e) => {
88
100
  if (e.lengthComputable) {
89
101
  const percent = Math.round((e.loaded / e.total) * 100);
@@ -71,12 +71,17 @@ export class DebugOverlay {
71
71
  }
72
72
 
73
73
  public show(): void {
74
- if (this.container) {
74
+ if (!this.container) {
75
+ this.createOverlay();
76
+ }
77
+ if(this.container) {
75
78
  this.container.style.display = "block";
76
- return;
77
79
  }
78
- this.createOverlay();
79
- this.updateInterval = window.setInterval(() => this.updateTextOnly(), 200);
80
+ // 重启定时器:hide() 会清除 updateInterval,再次 show 时需要重建
81
+ // 否则界面显示但内容永不刷新
82
+ if (!this.updateInterval) {
83
+ this.updateInterval = window.setInterval(() => this.updateTextOnly(), 200);
84
+ }
80
85
  this.updateTextOnly();
81
86
  }
82
87
 
@@ -1 +0,0 @@
1
- export {};
@@ -1,3 +0,0 @@
1
- type AudioChunkDebugMetadata = Record<string, unknown>;
2
- export declare function logAudioChunkContent(label: string, chunk: Blob | ArrayBuffer | ArrayBufferView, metadata?: AudioChunkDebugMetadata, previewBytes?: number): Promise<void>;
3
- export {};
@@ -1,62 +0,0 @@
1
- export declare const AU_STREAM_NEAREND: 1;
2
- export declare const AU_STREAM_FAREND: 2;
3
- export type AuStreamId = typeof AU_STREAM_NEAREND | typeof AU_STREAM_FAREND;
4
- export type AuPayload = Blob | ArrayBuffer | ArrayBufferView;
5
- export interface AgentAudioBlock {
6
- data: AuPayload;
7
- /** Epoch timestamp for the beginning of this 100 ms media block. */
8
- timestamp: number;
9
- /** Whether this block starts a new WebM container stream. */
10
- isFirstChunk?: boolean;
11
- }
12
- interface AgentAudioUplinkOptions {
13
- sendBinary(frame: ArrayBuffer): void;
14
- onError?(error: Error): void;
15
- }
16
- export declare function packAuAudioFrame(streamId: AuStreamId, tsMs: number, payload: ArrayBuffer | ArrayBufferView): ArrayBuffer;
17
- /**
18
- * Owns the connection-level AU timeline and serializes nearend/farend writes.
19
- * Nearend/farend are matched by 100 ms bucket. When independent capture clocks
20
- * straddle a bucket boundary, an aged nearend may consume farend from
21
- * the immediately preceding bucket. This keeps farend-before-nearend wire
22
- * order without uploading farend-only buckets or dropping WebM continuations.
23
- */
24
- export declare class AgentAudioUplink {
25
- private readonly options;
26
- private connectionStartedAt;
27
- private connectionGeneration;
28
- private sendChain;
29
- private failed;
30
- private readonly pendingFarend;
31
- private readonly pendingNearend;
32
- private readonly closedNearendBuckets;
33
- private readonly lastTimestampByStream;
34
- private readonly lastSentTimestampByStream;
35
- private readonly pendingWebmPrefixByStream;
36
- private farendStreamState;
37
- constructor(options: AgentAudioUplinkOptions);
38
- reset(connectionStartedAt?: number | null): void;
39
- startInput(): void;
40
- clearFarend(): void;
41
- offerFarend(block: AgentAudioBlock): void;
42
- offerNearend(block: AgentAudioBlock): void;
43
- drain(): Promise<void>;
44
- private normalizeBlock;
45
- private enqueue;
46
- /**
47
- * Chromium may emit the first WebM byte (0x1a) as its own MediaRecorder
48
- * Blob and put the remaining EBML magic in the next Blob. A one-byte AU
49
- * payload is not a valid timed audio block, so keep an incomplete WebM
50
- * magic prefix and prepend it to the next block from the same stream.
51
- */
52
- private coalesceSplitWebmMagic;
53
- private flushOlderNearend;
54
- private flushExpiredNearend;
55
- private flushBucket;
56
- private takeNearend;
57
- private takeFarendThrough;
58
- private abandonFarendStream;
59
- private prune;
60
- }
61
- export declare const AU_AUDIO_BLOCK_MS = 100;
62
- export {};