@xmov/avatar 2.0.0-alpha.38

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (160) hide show
  1. package/README.md +139 -0
  2. package/dist/agent/__tests__/agent.test.d.ts +1 -0
  3. package/dist/agent/audio-debug.d.ts +3 -0
  4. package/dist/agent/audio-uplink.d.ts +62 -0
  5. package/dist/agent/avatar.cjs +2 -0
  6. package/dist/agent/avatar.cjs.map +1 -0
  7. package/dist/agent/avatar.modern.js +2 -0
  8. package/dist/agent/avatar.modern.js.map +1 -0
  9. package/dist/agent/avatar.module.js +2 -0
  10. package/dist/agent/avatar.module.js.map +1 -0
  11. package/dist/agent/avatar.umd.js +2 -0
  12. package/dist/agent/avatar.umd.js.map +1 -0
  13. package/dist/agent/e2e-client.d.ts +32 -0
  14. package/dist/agent/fixed-audio-track.d.ts +48 -0
  15. package/dist/agent/index.cjs +5 -0
  16. package/dist/agent/index.d.ts +104 -0
  17. package/dist/agent/index.umd.js +10 -0
  18. package/dist/agent/microphone.d.ts +49 -0
  19. package/dist/agent/types.d.ts +153 -0
  20. package/dist/baseRender/AudioRenderer.d.ts +46 -0
  21. package/dist/baseRender/AudioWorklet.d.ts +78 -0
  22. package/dist/baseRender/AvatarRenderer.d.ts +226 -0
  23. package/dist/baseRender/MSEAudioPlayer.d.ts +51 -0
  24. package/dist/baseRender/UIRenderer.d.ts +39 -0
  25. package/dist/baseRender/pcm-audio-processor.d.ts +63 -0
  26. package/dist/control/APIForwarder.d.ts +23 -0
  27. package/dist/control/DataCacheQueue.d.ts +69 -0
  28. package/dist/control/EventDispatcher.d.ts +35 -0
  29. package/dist/control/RenderScheduler.d.ts +162 -0
  30. package/dist/control/SaveAndDownload.d.ts +14 -0
  31. package/dist/control/StreamingClient.d.ts +33 -0
  32. package/dist/control/VideoPlayer.d.ts +47 -0
  33. package/dist/control/ttsa.d.ts +111 -0
  34. package/dist/encoding/media-recorder-encoder.d.ts +33 -0
  35. package/dist/encoding/pcm-webm-encoder.d.ts +13 -0
  36. package/dist/encoding/webcodec-opus-encoder.d.ts +41 -0
  37. package/dist/encoding/webm-muxer.d.ts +90 -0
  38. package/dist/index.cjs +2 -0
  39. package/dist/index.cjs.map +1 -0
  40. package/dist/index.d.ts +231 -0
  41. package/dist/index.modern.js +2 -0
  42. package/dist/index.modern.js.map +1 -0
  43. package/dist/index.module.js +2 -0
  44. package/dist/index.module.js.map +1 -0
  45. package/dist/index.umd.js +2 -0
  46. package/dist/index.umd.js.map +1 -0
  47. package/dist/modules/Composition.d.ts +28 -0
  48. package/dist/modules/ResourceManager.d.ts +295 -0
  49. package/dist/modules/TrackRenderer/base-track.d.ts +4 -0
  50. package/dist/modules/TrackRenderer/index.d.ts +12 -0
  51. package/dist/modules/TrackRenderer/render-implements.d.ts +19 -0
  52. package/dist/modules/TrackRenderer/track-pic.d.ts +8 -0
  53. package/dist/modules/TrackRenderer/track-subtitle.d.ts +8 -0
  54. package/dist/modules/cache-manager.d.ts +5 -0
  55. package/dist/modules/decoder.d.ts +84 -0
  56. package/dist/modules/error-handle.d.ts +17 -0
  57. package/dist/modules/network.d.ts +21 -0
  58. package/dist/proto/face_data_pb.d.ts +0 -0
  59. package/dist/types/capability.d.ts +15 -0
  60. package/dist/types/error.d.ts +70 -0
  61. package/dist/types/event.d.ts +25 -0
  62. package/dist/types/frame-data.d.ts +107 -0
  63. package/dist/types/index.d.ts +187 -0
  64. package/dist/types/render.d.ts +18 -0
  65. package/dist/utils/DataInterface.d.ts +113 -0
  66. package/dist/utils/GLDevice.d.ts +46 -0
  67. package/dist/utils/GLPipeline.d.ts +84 -0
  68. package/dist/utils/GLPipelineDebugTools.d.ts +5 -0
  69. package/dist/utils/Math.d.ts +21 -0
  70. package/dist/utils/__tests__/capability-checker.test.d.ts +1 -0
  71. package/dist/utils/audio-session.d.ts +11 -0
  72. package/dist/utils/audio.d.ts +1 -0
  73. package/dist/utils/capability-checker.d.ts +23 -0
  74. package/dist/utils/encodeToken.d.ts +7 -0
  75. package/dist/utils/face.d.ts +1 -0
  76. package/dist/utils/float32-decoder.d.ts +26 -0
  77. package/dist/utils/fpsTracker.d.ts +27 -0
  78. package/dist/utils/index.d.ts +22 -0
  79. package/dist/utils/logger.d.ts +9 -0
  80. package/dist/utils/media-recorder-timestamp.d.ts +15 -0
  81. package/dist/utils/pcmAnalyzer.d.ts +40 -0
  82. package/dist/utils/perfermance.d.ts +15 -0
  83. package/dist/utils/request.d.ts +19 -0
  84. package/dist/utils/requestAnimateFrames.d.ts +31 -0
  85. package/dist/utils/time.d.ts +9 -0
  86. package/dist/view/DebugOverlay.d.ts +54 -0
  87. package/dist/worker/streaming-video.d.ts +1 -0
  88. package/package.json +82 -0
  89. package/src/agent/__tests__/agent.test.ts +3787 -0
  90. package/src/agent/audio-debug.ts +36 -0
  91. package/src/agent/audio-uplink.ts +392 -0
  92. package/src/agent/e2e-client.ts +215 -0
  93. package/src/agent/fixed-audio-track.ts +547 -0
  94. package/src/agent/index.ts +1393 -0
  95. package/src/agent/microphone.ts +534 -0
  96. package/src/agent/types.ts +197 -0
  97. package/src/baseRender/AudioRenderer.ts +317 -0
  98. package/src/baseRender/AudioWorklet.ts +776 -0
  99. package/src/baseRender/AvatarRenderer.ts +1559 -0
  100. package/src/baseRender/MSEAudioPlayer.ts +243 -0
  101. package/src/baseRender/UIRenderer.ts +195 -0
  102. package/src/baseRender/pcm-audio-processor.js +267 -0
  103. package/src/control/APIForwarder.ts +67 -0
  104. package/src/control/DataCacheQueue.ts +372 -0
  105. package/src/control/EventDispatcher.ts +106 -0
  106. package/src/control/RenderScheduler.ts +945 -0
  107. package/src/control/SaveAndDownload.js +77 -0
  108. package/src/control/StreamingClient.ts +138 -0
  109. package/src/control/VideoPlayer.ts +306 -0
  110. package/src/control/ttsa.ts +676 -0
  111. package/src/encoding/media-recorder-encoder.ts +175 -0
  112. package/src/encoding/pcm-webm-encoder.ts +72 -0
  113. package/src/encoding/webcodec-opus-encoder.ts +258 -0
  114. package/src/encoding/webm-muxer.ts +337 -0
  115. package/src/global.d.ts +105 -0
  116. package/src/index.ts +1920 -0
  117. package/src/modules/Composition.ts +79 -0
  118. package/src/modules/ResourceManager.ts +899 -0
  119. package/src/modules/TrackRenderer/base-track.ts +7 -0
  120. package/src/modules/TrackRenderer/index.ts +40 -0
  121. package/src/modules/TrackRenderer/render-implements.ts +269 -0
  122. package/src/modules/TrackRenderer/track-pic.ts +19 -0
  123. package/src/modules/TrackRenderer/track-subtitle.ts +16 -0
  124. package/src/modules/cache-manager.ts +10 -0
  125. package/src/modules/decoder.ts +515 -0
  126. package/src/modules/error-handle.ts +20 -0
  127. package/src/modules/network.ts +71 -0
  128. package/src/proto/face_data.proto +45 -0
  129. package/src/proto/face_data_pb.js +1297 -0
  130. package/src/proto/protobuf.min.js +8 -0
  131. package/src/types/capability.ts +23 -0
  132. package/src/types/error.ts +211 -0
  133. package/src/types/event.ts +38 -0
  134. package/src/types/frame-data.ts +121 -0
  135. package/src/types/index.ts +207 -0
  136. package/src/types/render.ts +18 -0
  137. package/src/utils/DataInterface.ts +746 -0
  138. package/src/utils/GLDevice.ts +286 -0
  139. package/src/utils/GLPipeline.ts +1293 -0
  140. package/src/utils/GLPipelineDebugTools.ts +174 -0
  141. package/src/utils/Math.ts +266 -0
  142. package/src/utils/__tests__/capability-checker.test.ts +159 -0
  143. package/src/utils/audio-session.ts +144 -0
  144. package/src/utils/audio.ts +44 -0
  145. package/src/utils/blueimp-md5.d.ts +3 -0
  146. package/src/utils/capability-checker.ts +379 -0
  147. package/src/utils/encodeToken.ts +100 -0
  148. package/src/utils/face.ts +29 -0
  149. package/src/utils/float32-decoder.js +130 -0
  150. package/src/utils/fpsTracker.ts +100 -0
  151. package/src/utils/index.ts +182 -0
  152. package/src/utils/logger.js +36 -0
  153. package/src/utils/media-recorder-timestamp.ts +79 -0
  154. package/src/utils/pcmAnalyzer.ts +154 -0
  155. package/src/utils/perfermance.ts +141 -0
  156. package/src/utils/request.ts +95 -0
  157. package/src/utils/requestAnimateFrames.ts +195 -0
  158. package/src/utils/time.ts +32 -0
  159. package/src/view/DebugOverlay.ts +404 -0
  160. package/src/worker/streaming-video.ts +4647 -0
@@ -0,0 +1,776 @@
1
+ import {
2
+ configureWebAudioPlaybackSession,
3
+ subscribeWebAudioPlaybackSessionRestored,
4
+ } from "../utils/audio-session";
5
+
6
+ const pcm_audio_script = `// pcm-audio-processor.js - 运行在音频线程的Worklet处理器
7
+ class PCMAudioProcessor extends AudioWorkletProcessor {
8
+ // 音量配置(与主线程保持一致)
9
+ static get parameterDescriptors() {
10
+ return [
11
+ {
12
+ name: 'volume',
13
+ defaultValue: 1.0,
14
+ minValue: 0.0,
15
+ maxValue: 1.0,
16
+ automationRate: 'a-rate' // 支持音频速率自动化
17
+ }
18
+ ];
19
+ }
20
+
21
+ constructor() {
22
+ super();
23
+ this.config = {
24
+ sampleRate: 24000, // 固定24kHz采样率
25
+ bytesPerSample: 2, // S16LE=2字节/采样点
26
+ };
27
+ this.pcmCache = new Uint8Array(0); // Worklet内的PCM缓存
28
+ this.isActive = false; // 播放音频状态(控制是否生成声音)
29
+ this.isDestroyed = false;
30
+ this.fadeState = {
31
+ // 淡入淡出状态(避免片段切换爆音)
32
+ phase: 'idle', // idle/fadeIn/fadeOut/steady
33
+ currentGain: 0,
34
+ fadeTime: 0.01, // 淡入淡出时间(秒,建议最小0.01s避免爆音)
35
+ fadeSamples: Math.floor(0.01 * 24000), // 淡入淡出样本数(24000Hz下为240样本)
36
+ };
37
+ this.currentSegment = null; // 当前处理的音频段(float32数组)
38
+ this.segmentOffset = 0; // 当前音频段的播放偏移量
39
+
40
+ // 监听主线程消息(接收PCM数据、状态控制等)
41
+ this.port.onmessage = (e) => this.handleMainThreadMessage(e.data);
42
+
43
+ // 初始化淡入淡出样本数
44
+ this.fadeState.fadeSamples = Math.floor(this.fadeState.fadeTime * this.config.sampleRate);
45
+ }
46
+
47
+ /**
48
+ * 处理主线程发来的消息(PCM数据/状态控制等)
49
+ * @param {Object} data - 消息数据
50
+ * @param {string} data.type - 消息类型:'pcm'/'start'/'pause'/'stop'/'config'/'destroy'/'volume'
51
+ * @param {Uint8Array} [data.pcmData] - PCM数据(仅type='pcm'时存在)
52
+ * @param {Object} [data.config] - 配置更新(仅type='config'时存在)
53
+ * @param {number} [data.volume] - 音量值(仅type='volume'时存在)
54
+ */
55
+ handleMainThreadMessage(data) {
56
+ switch (data.type) {
57
+ // 接收主线程下发的PCM流数据
58
+ case 'pcm':
59
+ if (data.pcmData instanceof Uint8Array && data.pcmData.length > 0) {
60
+ // 确保PCM数据字节数为2的整数倍(S16LE要求)
61
+ const validLength = data.pcmData.length - (data.pcmData.length % this.config.bytesPerSample);
62
+ if (validLength <= 0) break;
63
+
64
+ // 合并到Worklet缓存
65
+ const newCache = new Uint8Array(this.pcmCache.length + validLength);
66
+ newCache.set(this.pcmCache, 0);
67
+ newCache.set(data.pcmData.subarray(0, validLength), this.pcmCache.length);
68
+ this.pcmCache = newCache;
69
+
70
+ // 向主线程发送缓存状态
71
+ this.port.postMessage({
72
+ type: 'cacheStatus',
73
+ size: this.pcmCache.length,
74
+ sampleCount: this.pcmCache.length / this.config.bytesPerSample
75
+ });
76
+ }
77
+ break;
78
+
79
+ // 启动播放(继续音频生成)
80
+ case 'start':
81
+ this.isActive = true;
82
+ break;
83
+
84
+ // 暂停播放(停止音频生成,保留缓存)
85
+ case 'pause':
86
+ this.isActive = false;
87
+ this.resetSegmentState();
88
+ break;
89
+
90
+ // 停止播放(清空缓存+重置状态)
91
+ case 'stop':
92
+ this.isActive = false;
93
+ this.pcmCache = new Uint8Array(0);
94
+ this.resetSegmentState();
95
+ break;
96
+
97
+ // 更新配置(如采样率/淡入淡出时间)
98
+ case 'config':
99
+ if (data.config.sampleRate) {
100
+ this.config.sampleRate = data.config.sampleRate;
101
+ // 重新计算淡入淡出样本数
102
+ this.fadeState.fadeSamples = Math.floor(this.fadeState.fadeTime * this.config.sampleRate);
103
+ }
104
+ if (data.config.fadeTime) {
105
+ this.fadeState.fadeTime = data.config.fadeTime;
106
+ this.fadeState.fadeSamples = Math.floor(this.fadeState.fadeTime * this.config.sampleRate);
107
+ }
108
+ break;
109
+
110
+ // 单独更新音量(可选,也可通过参数自动化控制)
111
+ case 'volume':
112
+ this.volume = Number(data.volume);
113
+ // 通知主线程同步AudioWorkletNode的parameters
114
+ this.port.postMessage({
115
+ type: 'syncVolume',
116
+ volume: this.volume
117
+ });
118
+ break;
119
+
120
+ // 单独更新播放速率(可选,也可通过参数自动化控制)
121
+ case 'speed':
122
+ this.speed = Number(data.speed);
123
+ // 通知主线程同步AudioWorkletNode的parameters
124
+ this.port.postMessage({
125
+ type: 'syncSpeed',
126
+ speed: this.speed
127
+ });
128
+ break;
129
+
130
+ // 销毁处理器
131
+ case 'destroy':
132
+ this.isDestroyed = true;
133
+ this.isActive = false;
134
+ this.pcmCache = new Uint8Array(0);
135
+ this.resetSegmentState();
136
+ this.port.postMessage({ type: 'destroyConfirmed' });
137
+ break;
138
+
139
+ default:
140
+ console.warn('[PCMAudioProcessor] 未知消息类型:', data.type);
141
+ }
142
+ }
143
+
144
+ /**
145
+ * 重置当前音频段状态(暂停/停止时调用)
146
+ */
147
+ resetSegmentState() {
148
+ this.currentSegment = null;
149
+ this.segmentOffset = 0;
150
+ this.fadeState.phase = 'idle';
151
+ this.fadeState.currentGain = 0;
152
+ }
153
+
154
+ /**
155
+ * S16LE Uint8Array 转 Float32Array(-1~1范围)
156
+ * @param {Uint8Array} pcmData - PCM数据
157
+ * @returns {Float32Array} 转换后的音频样本
158
+ */
159
+ pcmToFloat32(pcmData) {
160
+ const sampleCount = pcmData.length / this.config.bytesPerSample;
161
+ const floatData = new Float32Array(sampleCount);
162
+ const dataView = new DataView(pcmData.buffer);
163
+
164
+ for (let i = 0; i < sampleCount; i++) {
165
+ // S16LE格式:小端序解析,范围-32768~32767 归一化到-1~1
166
+ floatData[i] = dataView.getInt16(i * this.config.bytesPerSample, true) / 32768;
167
+ }
168
+ return floatData;
169
+ }
170
+
171
+ /**
172
+ * 计算当前样本的增益(淡入淡出)
173
+ * @param {number} segmentLength - 当前音频段的总样本数
174
+ * @returns {number} 增益值(0~1)
175
+ */
176
+ calculateGain(segmentLength) {
177
+ const { phase, fadeSamples } = this.fadeState;
178
+ const { segmentOffset } = this;
179
+
180
+ // 边界保护:防止无效值导致的NaN
181
+ if (typeof segmentOffset !== 'number' || typeof segmentLength !== 'number' || segmentLength <= 0) {
182
+ return 0;
183
+ }
184
+
185
+ switch (phase) {
186
+ // 淡入阶段:从0线性增加到1
187
+ case 'fadeIn':
188
+ if (segmentOffset >= fadeSamples) {
189
+ this.fadeState.phase = 'steady';
190
+ return 1;
191
+ }
192
+ return segmentOffset / fadeSamples;
193
+
194
+ // 稳定阶段:增益保持1
195
+ case 'steady':
196
+ // 检测是否进入淡出阶段(接近段尾)
197
+ if (segmentOffset >= segmentLength - fadeSamples) {
198
+ this.fadeState.phase = 'fadeOut';
199
+ }
200
+ return 1;
201
+
202
+ // 淡出阶段:从1线性降低到0.0001(避免分音)
203
+ case 'fadeOut':
204
+ const fadeStart = Math.max(segmentLength - fadeSamples, 0);
205
+ const fadeProgress = (segmentOffset - fadeStart) / fadeSamples;
206
+ return Math.max(1 - fadeProgress, 0.0001);
207
+
208
+ // 空闲阶段:增益0
209
+ case 'idle':
210
+ default:
211
+ return 0;
212
+ }
213
+ }
214
+
215
+ /**
216
+ * AudioWorklet核心方法:实时生成音频样本(音频线程调用)
217
+ * @param {Float32Array[][]} inputs - 输入样本
218
+ * @param {Float32Array[][]} outputs - 输出样本(单声道)
219
+ * @param {Object} parameters - 音频参数
220
+ * @returns {boolean} 是否继续运行
221
+ */
222
+ process(inputs, outputs, parameters) {
223
+ // 销毁状态:终止处理
224
+ if (this.isDestroyed) {
225
+ if (outputs.length > 0 && outputs[0].length > 0) {
226
+ outputs[0][0].fill(0);
227
+ }
228
+ return false;
229
+ }
230
+
231
+ // 未激活或无输出:返回静音
232
+ if (!this.isActive || outputs.length === 0 || outputs[0].length === 0) {
233
+ outputs[0][0].fill(0);
234
+ return true;
235
+ }
236
+
237
+ const outputBuffer = outputs[0][0];
238
+ const bufferLength = outputBuffer.length;
239
+ // 获取音量参数(支持自动化)
240
+ const volume = parameters.volume.length > 1
241
+ ? parameters.volume // 自动化模式:每个样本点可能有不同值
242
+ : parameters.volume[0]; // 控制模式:整个缓冲区使用相同值
243
+
244
+ // 生成输出样本
245
+ for (let i = 0; i < bufferLength; i++) {
246
+ // 如当前无音频段且缓存有数据,创建新段
247
+ if (!this.currentSegment && this.pcmCache.length > 0) {
248
+ // 使用缓存中的所有数据创建新段
249
+ this.currentSegment = this.pcmToFloat32(this.pcmCache);
250
+ // 清空缓存(已处理)
251
+ this.pcmCache = new Uint8Array(0);
252
+ this.segmentOffset = 0;
253
+ // 根据前一段状态决定是否淡入
254
+ this.fadeState.phase = this.fadeState.phase === 'fadeOut' ? 'steady' : 'fadeIn';
255
+ }
256
+
257
+ // 生成当前样本
258
+ let sample = 0;
259
+ if (this.currentSegment) {
260
+ const segmentLength = this.currentSegment.length;
261
+ // 确保偏移量有效
262
+ if (this.segmentOffset < segmentLength) {
263
+ const gain = this.calculateGain(segmentLength);
264
+ // 应用淡入淡出增益和音量控制
265
+ sample = this.currentSegment[this.segmentOffset] * gain * volume;
266
+ this.segmentOffset++;
267
+ } else {
268
+ // 当前段处理完毕,重置
269
+ this.currentSegment = null;
270
+ this.segmentOffset = 0;
271
+ this.fadeState.phase = 'idle';
272
+ }
273
+ }
274
+
275
+ outputBuffer[i] = sample;
276
+ }
277
+
278
+ return true;
279
+ }
280
+ }
281
+
282
+ // 注册Worklet处理器
283
+ registerProcessor('pcm-audio-processor', PCMAudioProcessor);
284
+ `
285
+ const pcm_capture_script = `// pcm-capture-processor.js - 运行在音频线程的捕获处理器
286
+ class PCMCaptureProcessor extends AudioWorkletProcessor {
287
+ constructor() {
288
+ super();
289
+ this.inputSampleRate = 24000; // PCM 播放器采样率
290
+ this.outputSampleRate = 16000; // Opus 编码要求采样率
291
+ this.bufferSize = 2400; // 100ms @ 24kHz
292
+ this.buffer = new Float32Array(this.bufferSize);
293
+ this.bufferIndex = 0;
294
+
295
+ // 监听主线程的 reset 消息,清空内部缓冲区(打断/会话切换时使用)
296
+ this.port.onmessage = (e) => {
297
+ if (e.data && e.data.type === 'reset') {
298
+ this.bufferIndex = 0;
299
+ this.buffer.fill(0);
300
+ }
301
+ };
302
+ }
303
+
304
+ process(inputs) {
305
+ const input = inputs[0];
306
+ if (!input || !input[0] || input[0].length === 0) return true;
307
+
308
+ const chunk = input[0];
309
+ for (let i = 0; i < chunk.length; i++) {
310
+ this.buffer[this.bufferIndex++] = chunk[i];
311
+
312
+ if (this.bufferIndex >= this.bufferSize) {
313
+ // 重采样 24kHz → 16kHz
314
+ const resampled = this.resample(this.buffer);
315
+ // Float32 → S16LE
316
+ const pcmBuffer = this.float32ToS16LE(resampled);
317
+ // 静音检测
318
+ const hasAudio = this.detectAudio(this.buffer);
319
+ // 回传主线程(Transferable,避免拷贝)
320
+ this.port.postMessage(
321
+ { pcmData: pcmBuffer.buffer, hasAudio: hasAudio },
322
+ [pcmBuffer.buffer]
323
+ );
324
+ this.bufferIndex = 0;
325
+ }
326
+ }
327
+ return true;
328
+ }
329
+
330
+ // 线性插值重采样 24kHz → 16kHz (3:2 比率)
331
+ resample(input) {
332
+ const ratio = this.outputSampleRate / this.inputSampleRate; // 2/3
333
+ const outputLength = Math.floor(input.length * ratio); // 1600
334
+ const output = new Float32Array(outputLength);
335
+ for (let i = 0; i < outputLength; i++) {
336
+ const srcIndex = i / ratio;
337
+ const srcIndexFloor = Math.floor(srcIndex);
338
+ const frac = srcIndex - srcIndexFloor;
339
+ const s1 = input[srcIndexFloor] || 0;
340
+ const s2 = input[srcIndexFloor + 1] || 0;
341
+ output[i] = s1 + (s2 - s1) * frac;
342
+ }
343
+ return output;
344
+ }
345
+
346
+ // Float32 → S16LE Int16Array
347
+ float32ToS16LE(float32) {
348
+ const int16 = new Int16Array(float32.length);
349
+ for (let i = 0; i < float32.length; i++) {
350
+ let s = Math.max(-1, Math.min(1, float32[i]));
351
+ int16[i] = s < 0 ? s * 0x8000 : s * 0x7FFF;
352
+ }
353
+ return int16;
354
+ }
355
+
356
+ // 静音检测
357
+ detectAudio(buffer) {
358
+ for (let i = 0; i < buffer.length; i++) {
359
+ if (Math.abs(buffer[i]) > 0.0001) return true;
360
+ }
361
+ return false;
362
+ }
363
+ }
364
+ registerProcessor('pcm-capture-processor', PCMCaptureProcessor);
365
+ `;
366
+ const workletBlob = new Blob([pcm_audio_script], { type: 'application/javascript' });
367
+ const workletUrl = URL.createObjectURL(workletBlob);
368
+ const captureWorkletBlob = new Blob([pcm_capture_script], { type: 'application/javascript' });
369
+ const captureWorkletUrl = URL.createObjectURL(captureWorkletBlob);
370
+ /**
371
+ * Web Audio Worklet版PCM流式播放器(24kHz采样率、S16LE单声道)
372
+ * 核心:无定时器,依赖AudioWorklet的process方法实时处理流式数据
373
+ */
374
+ export default class PCMAudioPlayer {
375
+ // 常量配置(与Worklet保持一致)
376
+ readonly SAMPLE_RATE: number = 24000;
377
+ readonly FRAME_ALIGNMENT: number = 2000; // 帧对齐字节数
378
+ readonly WORKLET_SCRIPT_URL: string = workletUrl; // 改为本地引入,内网客户无法使用远程资源
379
+ // readonly WORKLET_SCRIPT_URL: string = 'http://localhost:5173/pcm-audio-processor.js'; // Worklet脚本路径(需与HTML同域)
380
+
381
+ // 实例状态与资源
382
+ audioCtx: AudioContext | null = null;
383
+ audioWorkletNode: AudioWorkletNode | null = null; // Worklet节点(主线程与Worklet通信桥梁)
384
+ isInitialized: boolean = false; // 初始化状态(Worklet加载完成)
385
+ isPlaying: boolean = false; // 播放状态标记
386
+ private destroyResolve: (() => void) | null = null;
387
+ private unsubscribePlaybackSessionRestored: (() => void) | null = null;
388
+
389
+ // 音频捕获相关
390
+ private captureWorklet: AudioWorkletNode | null = null;
391
+ private captureCallback: ((data: { pcmData: ArrayBuffer; hasAudio: boolean }) => void) | null = null;
392
+
393
+ /**
394
+ * 初始化(加载Worklet + 创建AudioContext + 连接节点)
395
+ * @returns {Promise<void>} 初始化结果(需用户交互中调用)
396
+ */
397
+ async init(): Promise<void> {
398
+ if (this.isInitialized) return;
399
+
400
+ // iOS WebKit defaults Web Audio to an ambient session, which follows the
401
+ // Ring/Silent switch. Playback sessions remain audible in silent mode.
402
+ configureWebAudioPlaybackSession();
403
+
404
+ // 1. 浏览器兼容性处理
405
+ const AudioContextConstructor =
406
+ window.AudioContext || (window as any).webkitAudioContext;
407
+ if (!AudioContextConstructor || !window.AudioWorklet) {
408
+ throw new Error("当前浏览器不支持Web Audio Worklet,无法实现流式播放");
409
+ }
410
+
411
+ // 2. 创建AudioContext(24kHz采样率)
412
+ this.audioCtx = new AudioContextConstructor({
413
+ sampleRate: this.SAMPLE_RATE,
414
+ });
415
+ const logger = (window as any).avatarSDKLogger || console;
416
+
417
+ try {
418
+ // 3. 加载Worklet脚本(必须先加载,再创建节点)
419
+ await this.audioCtx.audioWorklet.addModule(this.WORKLET_SCRIPT_URL);
420
+ logger.log(
421
+ `[PCMAudioPlayer] Worklet脚本加载完成: ${this.WORKLET_SCRIPT_URL}`
422
+ );
423
+
424
+ // 4. 创建Worklet节点(指定处理器名称,与Worklet中registerProcessor一致)
425
+ this.audioWorkletNode = new AudioWorkletNode(
426
+ this.audioCtx,
427
+ "pcm-audio-processor" // 必须与Worklet中注册的名称一致
428
+ );
429
+
430
+ // 5. 连接音频链路:WorkletNode → 扬声器(destination)
431
+ this.audioWorkletNode.connect(this.audioCtx.destination);
432
+ logger.log("[PCMAudioPlayer] Worklet节点已连接到扬声器");
433
+
434
+ // 6. 监听Worklet发来的消息(如缓存状态、错误)
435
+ this.audioWorkletNode.port.onmessage = (e) => {
436
+ switch (e.data.type) {
437
+ case "cacheStatus":
438
+ logger.log(`[PCMAudioPlayer] Worklet缓存状态: ${e.data.size}字节`);
439
+ break;
440
+ case "error":
441
+ logger.error(`[PCMAudioPlayer] Worklet错误:`, e.data.detail);
442
+ break;
443
+ case "destroyConfirmed":
444
+ logger.log(`[PCMAudioPlayer] Worklet已确认销毁`);
445
+ this.destroyResolve?.(); // 触发销毁等待的Promise
446
+ break;
447
+ case "syncVolume":
448
+ // 同步音量到AudioWorkletNode的parameters
449
+ if (
450
+ this.audioWorkletNode?.parameters?.get("volume")?.value !==
451
+ undefined
452
+ ) {
453
+ this.audioWorkletNode.parameters.get("volume")!.value =
454
+ e.data.volume;
455
+ }
456
+ break;
457
+ case "syncSpeed":
458
+ // 同步播放速率到AudioWorkletNode的parameters
459
+ if (
460
+ this.audioWorkletNode?.parameters?.get("speed")?.value !==
461
+ undefined
462
+ ) {
463
+ this.audioWorkletNode.parameters.get("speed")!.value =
464
+ e.data.speed;
465
+ }
466
+ break;
467
+ default:
468
+ logger.log(`[PCMAudioPlayer] Worklet消息:`, e.data);
469
+ }
470
+ };
471
+
472
+ // 7. 标记初始化完成
473
+ this.isInitialized = true;
474
+ this.unsubscribePlaybackSessionRestored =
475
+ subscribeWebAudioPlaybackSessionRestored(() => {
476
+ this.refreshPlaybackRoute();
477
+ });
478
+ logger.log('[PCMAudioPlayer] 播放器初始化完成(Worklet就绪)');
479
+ } catch (err) {
480
+ logger.error("[PCMAudioPlayer] 初始化失败:", err);
481
+ throw err; // 抛出错误,让业务层处理
482
+ }
483
+ }
484
+
485
+ private refreshPlaybackRoute(): boolean {
486
+ if (!this.isInitialized || !this.isPlaying || !this.audioCtx || !this.audioWorkletNode) {
487
+ return false;
488
+ }
489
+
490
+ const logger = (window as any).avatarSDKLogger || console;
491
+ try {
492
+ this.audioWorkletNode.disconnect();
493
+ this.audioWorkletNode.connect(this.audioCtx.destination);
494
+ // 重连 capture worklet(如果已挂载)
495
+ if (this.captureWorklet) {
496
+ this.audioWorkletNode.connect(this.captureWorklet);
497
+ }
498
+ if (this.audioCtx.state !== "running") {
499
+ void this.audioCtx.resume().catch((error) => {
500
+ logger.warn("[PCMAudioPlayer] 刷新路由后恢复 AudioContext 失败:", error);
501
+ });
502
+ }
503
+ logger.log("[PCMAudioPlayer] 已刷新 playback 音频路由");
504
+ return true;
505
+ } catch (error) {
506
+ logger.warn("[PCMAudioPlayer] 刷新 playback 音频路由失败:", error);
507
+ return false;
508
+ }
509
+ }
510
+
511
+ /**
512
+ * 接收PCM音频流(流式下发到Worklet,无本地缓存)
513
+ * @param {Uint8Array} pcmData - 24kHz S16LE单声道PCM数据
514
+ */
515
+ receivePCMStream(pcmData: Uint8Array): void {
516
+ const logger = (window as any).avatarSDKLogger || console;
517
+
518
+ // 1. 前置校验
519
+ if (!this.isInitialized || !this.audioWorkletNode) {
520
+ logger.error("[PCMAudioPlayer] 播放器未初始化,无法接收PCM数据");
521
+ return;
522
+ }
523
+ if (!(pcmData instanceof Uint8Array)) {
524
+ logger.error("[PCMAudioPlayer] 输入PCM数据必须是非空的Uint8Array");
525
+ return;
526
+ }
527
+ logger.log(`[PCMAudioPlayer] 已下发PCM数据: ${pcmData.length}字节`);
528
+
529
+ // 2. 下发PCM数据到Worklet(通过postMessage,数据会被转移/复制到音频线程)
530
+ try {
531
+ this.audioWorkletNode.port.postMessage(
532
+ {
533
+ type: "pcm",
534
+ pcmData: pcmData, // Worklet中会合并到缓存
535
+ },
536
+ [pcmData.buffer] // 可选:Transferable对象,避免数据复制(提升性能)
537
+ );
538
+ } catch (err) {
539
+ logger.error("[PCMAudioPlayer] 下发PCM数据到Worklet失败:", err);
540
+ }
541
+ }
542
+
543
+ /**
544
+ * 开始播放(激活Worklet的音频生成)
545
+ * @returns {Promise<void>} 启动结果
546
+ */
547
+ async start(): Promise<void> {
548
+ if (!this.isInitialized || !this.audioCtx || !this.audioWorkletNode) {
549
+ throw new Error("[PCMAudioPlayer] 请先调用init()初始化播放器");
550
+ }
551
+ const logger = (window as any).avatarSDKLogger || console;
552
+
553
+ // 1. 恢复AudioContext(需用户交互,否则抛异常)
554
+ if (this.audioCtx.state !== "running") {
555
+ await this.audioCtx.resume();
556
+ logger.log('[PCMAudioPlayer] AudioContext已恢复运行');
557
+ }
558
+
559
+ // 2. 发送start消息给Worklet(激活音频生成)
560
+ this.audioWorkletNode.port.postMessage({ type: "start" });
561
+ this.isPlaying = true;
562
+ logger.log('[PCMAudioPlayer] 已启动播放(Worklet激活)');
563
+ }
564
+
565
+ /**
566
+ * 暂停播放(停止Worklet音频生成,保留缓存)
567
+ */
568
+ pause(): void {
569
+ if (!this.isInitialized || !this.audioWorkletNode) return;
570
+ const logger = (window as any).avatarSDKLogger || console;
571
+
572
+ // 发送pause消息给Worklet
573
+ this.audioWorkletNode.port.postMessage({ type: "pause" });
574
+ this.isPlaying = false;
575
+ logger.log('[PCMAudioPlayer] 已暂停播放(Worklet暂停)');
576
+ }
577
+
578
+ /**
579
+ * 停止播放(清空Worklet缓存,重置状态)
580
+ */
581
+ stop(): void {
582
+ if (!this.isInitialized || !this.audioWorkletNode) return;
583
+ const logger = (window as any).avatarSDKLogger || console;
584
+
585
+ // 发送stop消息给Worklet
586
+ this.audioWorkletNode.port.postMessage({ type: "stop" });
587
+ this.isPlaying = false;
588
+ logger.log('[PCMAudioPlayer] 已停止播放(Worklet缓存清空)');
589
+ }
590
+
591
+ /**
592
+ * 销毁播放器(释放所有音频资源,不可恢复)
593
+ * @returns {Promise<void>} 销毁结果
594
+ */
595
+ async destroy(): Promise<void> {
596
+ const logger = (window as any).avatarSDKLogger || console;
597
+ this.unsubscribePlaybackSessionRestored?.();
598
+ this.unsubscribePlaybackSessionRestored = null;
599
+
600
+ // 先卸载捕获节点
601
+ this.detachCapture();
602
+
603
+ // 1. 若未初始化/已销毁,直接返回
604
+ if (!this.isInitialized || !this.audioCtx || !this.audioWorkletNode) {
605
+ logger.log('[PCMAudioPlayer] 播放器未初始化或已销毁');
606
+ return;
607
+ }
608
+
609
+ // 2. 发送销毁指令给Worklet,并等待确认
610
+ await new Promise<void>((resolve) => {
611
+ this.destroyResolve = resolve; // 绑定resolve函数
612
+ // 发送销毁消息(触发Worklet内的#isDestroyed=true)
613
+ this.audioWorkletNode?.port.postMessage({ type: "destroy" });
614
+ // 超时容错:300ms未确认则强制继续(避免Worklet无响应)
615
+ const timeout = setTimeout(() => {
616
+ logger.warn("[PCMAudioPlayer] Worklet销毁超时,强制释放资源");
617
+ this.destroyResolve?.();
618
+ }, 300);
619
+ // 确认后清除超时
620
+ if (this.audioWorkletNode) {
621
+ this.audioWorkletNode.port.onmessage = (e) => {
622
+ if (e.data.type === "destroyConfirmed") {
623
+ clearTimeout(timeout);
624
+ this.destroyResolve?.();
625
+ }
626
+ };
627
+ }
628
+ });
629
+
630
+ // 3. 断开Worklet节点连接(此时Worklet已终止执行)
631
+ this.audioWorkletNode.disconnect();
632
+ this.audioWorkletNode.port.close();
633
+ this.audioWorkletNode = null;
634
+
635
+ // 4. 关闭AudioContext(确保音频线程资源释放)
636
+ if (this.audioCtx.state !== "closed") {
637
+ await this.audioCtx.close();
638
+ }
639
+ this.audioCtx = null;
640
+ logger.log('[PCMAudioPlayer] AudioContext已关闭');
641
+
642
+ // 5. 重置所有状态
643
+ this.isInitialized = false;
644
+ this.isPlaying = false;
645
+ this.destroyResolve = null;
646
+ logger.log("[PCMAudioPlayer] 播放器已完全销毁,所有资源释放完成");
647
+ }
648
+
649
+ /**
650
+ * 挂载音频捕获节点(用于 PCM 模式下 ASR 回声消除)
651
+ * 将 PCM 播放器输出连接到 capture worklet,实时捕获重采样后的 16kHz S16LE 数据
652
+ */
653
+ async attachCapture(callback: (data: { pcmData: ArrayBuffer; hasAudio: boolean }) => void): Promise<void> {
654
+ if (!this.isInitialized || !this.audioCtx || !this.audioWorkletNode) {
655
+ throw new Error('[PCMAudioPlayer] 播放器未初始化,无法挂载捕获');
656
+ }
657
+ if (this.captureWorklet) return;
658
+
659
+ this.captureCallback = callback;
660
+
661
+ try {
662
+ await this.audioCtx.audioWorklet.addModule(captureWorkletUrl);
663
+
664
+ this.captureWorklet = new AudioWorkletNode(
665
+ this.audioCtx,
666
+ 'pcm-capture-processor'
667
+ );
668
+
669
+ // 连接:PCM AudioWorkletNode → capture Worklet
670
+ this.audioWorkletNode.connect(this.captureWorklet);
671
+
672
+ this.captureWorklet.port.onmessage = (e) => {
673
+ if (this.captureCallback && e.data.pcmData) {
674
+ this.captureCallback({
675
+ pcmData: e.data.pcmData,
676
+ hasAudio: e.data.hasAudio,
677
+ });
678
+ }
679
+ };
680
+
681
+ (window as any).avatarSDKLogger?.log(
682
+ '[PCMAudioPlayer] 捕获节点已挂载'
683
+ );
684
+ } catch (err) {
685
+ (window as any).avatarSDKLogger?.error(
686
+ '[PCMAudioPlayer] 挂载捕获节点失败:',
687
+ err
688
+ );
689
+ throw err;
690
+ }
691
+ }
692
+
693
+ /**
694
+ * 卸载音频捕获节点
695
+ */
696
+ detachCapture(): void {
697
+ if (this.captureWorklet) {
698
+ try {
699
+ this.captureWorklet.port.onmessage = null;
700
+ this.audioWorkletNode?.disconnect(this.captureWorklet);
701
+ this.captureWorklet.disconnect();
702
+ } catch {}
703
+ this.captureWorklet = null;
704
+ }
705
+ this.captureCallback = null;
706
+ }
707
+
708
+ /**
709
+ * 清空 capture worklet 内部缓冲区
710
+ * 用于打断/会话切换时,避免旧音频数据混入新 session
711
+ */
712
+ resetCaptureBuffer(): void {
713
+ if (this.captureWorklet) {
714
+ this.captureWorklet.port.postMessage({ type: 'reset' });
715
+ }
716
+ }
717
+
718
+ /**
719
+ * 获取当前播放器状态
720
+ * @returns {Object} 状态对象
721
+ */
722
+ getStatus(): {
723
+ isInitialized: boolean;
724
+ isPlaying: boolean;
725
+ isWorkletReady: boolean;
726
+ } {
727
+ return {
728
+ isInitialized: this.isInitialized,
729
+ isPlaying: this.isPlaying,
730
+ isWorkletReady: !!this.audioWorkletNode, // Worklet节点是否就绪
731
+ };
732
+ }
733
+
734
+ setVolume(valume: number) {
735
+ if (this.isInitialized && this.audioWorkletNode) {
736
+ this.audioWorkletNode.port.postMessage({
737
+ type: "volume",
738
+ volume: valume,
739
+ });
740
+ }
741
+ }
742
+
743
+ setSpeed(speed: number) {
744
+ if (this.isInitialized && this.audioWorkletNode) {
745
+ this.audioWorkletNode.port.postMessage({
746
+ type: "speed",
747
+ speed: speed,
748
+ });
749
+ }
750
+ }
751
+
752
+ /**
753
+ * @returns {Promise<void>} 恢复结果
754
+ */
755
+ async resume(): Promise<void> {
756
+ const logger = (window as any).avatarSDKLogger || console;
757
+ try {
758
+ if (this.audioCtx?.state === "suspended") {
759
+ this.audioCtx
760
+ .resume()
761
+ .then(() => {
762
+ (window as any).avatarSDKLogger.info("worklet音频上下文恢复成功");
763
+ })
764
+ .catch((error) => {
765
+ (window as any).avatarSDKLogger.error(
766
+ "worklet音频上下文恢复失败:",
767
+ error
768
+ );
769
+ });
770
+ }
771
+ } catch (err) {
772
+ logger.error("[PCMAudioPlayer] worklet恢复播放失败:", err);
773
+ throw err;
774
+ }
775
+ }
776
+ }