@xmov/avatar 2.0.0-alpha.38

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (160) hide show
  1. package/README.md +139 -0
  2. package/dist/agent/__tests__/agent.test.d.ts +1 -0
  3. package/dist/agent/audio-debug.d.ts +3 -0
  4. package/dist/agent/audio-uplink.d.ts +62 -0
  5. package/dist/agent/avatar.cjs +2 -0
  6. package/dist/agent/avatar.cjs.map +1 -0
  7. package/dist/agent/avatar.modern.js +2 -0
  8. package/dist/agent/avatar.modern.js.map +1 -0
  9. package/dist/agent/avatar.module.js +2 -0
  10. package/dist/agent/avatar.module.js.map +1 -0
  11. package/dist/agent/avatar.umd.js +2 -0
  12. package/dist/agent/avatar.umd.js.map +1 -0
  13. package/dist/agent/e2e-client.d.ts +32 -0
  14. package/dist/agent/fixed-audio-track.d.ts +48 -0
  15. package/dist/agent/index.cjs +5 -0
  16. package/dist/agent/index.d.ts +104 -0
  17. package/dist/agent/index.umd.js +10 -0
  18. package/dist/agent/microphone.d.ts +49 -0
  19. package/dist/agent/types.d.ts +153 -0
  20. package/dist/baseRender/AudioRenderer.d.ts +46 -0
  21. package/dist/baseRender/AudioWorklet.d.ts +78 -0
  22. package/dist/baseRender/AvatarRenderer.d.ts +226 -0
  23. package/dist/baseRender/MSEAudioPlayer.d.ts +51 -0
  24. package/dist/baseRender/UIRenderer.d.ts +39 -0
  25. package/dist/baseRender/pcm-audio-processor.d.ts +63 -0
  26. package/dist/control/APIForwarder.d.ts +23 -0
  27. package/dist/control/DataCacheQueue.d.ts +69 -0
  28. package/dist/control/EventDispatcher.d.ts +35 -0
  29. package/dist/control/RenderScheduler.d.ts +162 -0
  30. package/dist/control/SaveAndDownload.d.ts +14 -0
  31. package/dist/control/StreamingClient.d.ts +33 -0
  32. package/dist/control/VideoPlayer.d.ts +47 -0
  33. package/dist/control/ttsa.d.ts +111 -0
  34. package/dist/encoding/media-recorder-encoder.d.ts +33 -0
  35. package/dist/encoding/pcm-webm-encoder.d.ts +13 -0
  36. package/dist/encoding/webcodec-opus-encoder.d.ts +41 -0
  37. package/dist/encoding/webm-muxer.d.ts +90 -0
  38. package/dist/index.cjs +2 -0
  39. package/dist/index.cjs.map +1 -0
  40. package/dist/index.d.ts +231 -0
  41. package/dist/index.modern.js +2 -0
  42. package/dist/index.modern.js.map +1 -0
  43. package/dist/index.module.js +2 -0
  44. package/dist/index.module.js.map +1 -0
  45. package/dist/index.umd.js +2 -0
  46. package/dist/index.umd.js.map +1 -0
  47. package/dist/modules/Composition.d.ts +28 -0
  48. package/dist/modules/ResourceManager.d.ts +295 -0
  49. package/dist/modules/TrackRenderer/base-track.d.ts +4 -0
  50. package/dist/modules/TrackRenderer/index.d.ts +12 -0
  51. package/dist/modules/TrackRenderer/render-implements.d.ts +19 -0
  52. package/dist/modules/TrackRenderer/track-pic.d.ts +8 -0
  53. package/dist/modules/TrackRenderer/track-subtitle.d.ts +8 -0
  54. package/dist/modules/cache-manager.d.ts +5 -0
  55. package/dist/modules/decoder.d.ts +84 -0
  56. package/dist/modules/error-handle.d.ts +17 -0
  57. package/dist/modules/network.d.ts +21 -0
  58. package/dist/proto/face_data_pb.d.ts +0 -0
  59. package/dist/types/capability.d.ts +15 -0
  60. package/dist/types/error.d.ts +70 -0
  61. package/dist/types/event.d.ts +25 -0
  62. package/dist/types/frame-data.d.ts +107 -0
  63. package/dist/types/index.d.ts +187 -0
  64. package/dist/types/render.d.ts +18 -0
  65. package/dist/utils/DataInterface.d.ts +113 -0
  66. package/dist/utils/GLDevice.d.ts +46 -0
  67. package/dist/utils/GLPipeline.d.ts +84 -0
  68. package/dist/utils/GLPipelineDebugTools.d.ts +5 -0
  69. package/dist/utils/Math.d.ts +21 -0
  70. package/dist/utils/__tests__/capability-checker.test.d.ts +1 -0
  71. package/dist/utils/audio-session.d.ts +11 -0
  72. package/dist/utils/audio.d.ts +1 -0
  73. package/dist/utils/capability-checker.d.ts +23 -0
  74. package/dist/utils/encodeToken.d.ts +7 -0
  75. package/dist/utils/face.d.ts +1 -0
  76. package/dist/utils/float32-decoder.d.ts +26 -0
  77. package/dist/utils/fpsTracker.d.ts +27 -0
  78. package/dist/utils/index.d.ts +22 -0
  79. package/dist/utils/logger.d.ts +9 -0
  80. package/dist/utils/media-recorder-timestamp.d.ts +15 -0
  81. package/dist/utils/pcmAnalyzer.d.ts +40 -0
  82. package/dist/utils/perfermance.d.ts +15 -0
  83. package/dist/utils/request.d.ts +19 -0
  84. package/dist/utils/requestAnimateFrames.d.ts +31 -0
  85. package/dist/utils/time.d.ts +9 -0
  86. package/dist/view/DebugOverlay.d.ts +54 -0
  87. package/dist/worker/streaming-video.d.ts +1 -0
  88. package/package.json +82 -0
  89. package/src/agent/__tests__/agent.test.ts +3787 -0
  90. package/src/agent/audio-debug.ts +36 -0
  91. package/src/agent/audio-uplink.ts +392 -0
  92. package/src/agent/e2e-client.ts +215 -0
  93. package/src/agent/fixed-audio-track.ts +547 -0
  94. package/src/agent/index.ts +1393 -0
  95. package/src/agent/microphone.ts +534 -0
  96. package/src/agent/types.ts +197 -0
  97. package/src/baseRender/AudioRenderer.ts +317 -0
  98. package/src/baseRender/AudioWorklet.ts +776 -0
  99. package/src/baseRender/AvatarRenderer.ts +1559 -0
  100. package/src/baseRender/MSEAudioPlayer.ts +243 -0
  101. package/src/baseRender/UIRenderer.ts +195 -0
  102. package/src/baseRender/pcm-audio-processor.js +267 -0
  103. package/src/control/APIForwarder.ts +67 -0
  104. package/src/control/DataCacheQueue.ts +372 -0
  105. package/src/control/EventDispatcher.ts +106 -0
  106. package/src/control/RenderScheduler.ts +945 -0
  107. package/src/control/SaveAndDownload.js +77 -0
  108. package/src/control/StreamingClient.ts +138 -0
  109. package/src/control/VideoPlayer.ts +306 -0
  110. package/src/control/ttsa.ts +676 -0
  111. package/src/encoding/media-recorder-encoder.ts +175 -0
  112. package/src/encoding/pcm-webm-encoder.ts +72 -0
  113. package/src/encoding/webcodec-opus-encoder.ts +258 -0
  114. package/src/encoding/webm-muxer.ts +337 -0
  115. package/src/global.d.ts +105 -0
  116. package/src/index.ts +1920 -0
  117. package/src/modules/Composition.ts +79 -0
  118. package/src/modules/ResourceManager.ts +899 -0
  119. package/src/modules/TrackRenderer/base-track.ts +7 -0
  120. package/src/modules/TrackRenderer/index.ts +40 -0
  121. package/src/modules/TrackRenderer/render-implements.ts +269 -0
  122. package/src/modules/TrackRenderer/track-pic.ts +19 -0
  123. package/src/modules/TrackRenderer/track-subtitle.ts +16 -0
  124. package/src/modules/cache-manager.ts +10 -0
  125. package/src/modules/decoder.ts +515 -0
  126. package/src/modules/error-handle.ts +20 -0
  127. package/src/modules/network.ts +71 -0
  128. package/src/proto/face_data.proto +45 -0
  129. package/src/proto/face_data_pb.js +1297 -0
  130. package/src/proto/protobuf.min.js +8 -0
  131. package/src/types/capability.ts +23 -0
  132. package/src/types/error.ts +211 -0
  133. package/src/types/event.ts +38 -0
  134. package/src/types/frame-data.ts +121 -0
  135. package/src/types/index.ts +207 -0
  136. package/src/types/render.ts +18 -0
  137. package/src/utils/DataInterface.ts +746 -0
  138. package/src/utils/GLDevice.ts +286 -0
  139. package/src/utils/GLPipeline.ts +1293 -0
  140. package/src/utils/GLPipelineDebugTools.ts +174 -0
  141. package/src/utils/Math.ts +266 -0
  142. package/src/utils/__tests__/capability-checker.test.ts +159 -0
  143. package/src/utils/audio-session.ts +144 -0
  144. package/src/utils/audio.ts +44 -0
  145. package/src/utils/blueimp-md5.d.ts +3 -0
  146. package/src/utils/capability-checker.ts +379 -0
  147. package/src/utils/encodeToken.ts +100 -0
  148. package/src/utils/face.ts +29 -0
  149. package/src/utils/float32-decoder.js +130 -0
  150. package/src/utils/fpsTracker.ts +100 -0
  151. package/src/utils/index.ts +182 -0
  152. package/src/utils/logger.js +36 -0
  153. package/src/utils/media-recorder-timestamp.ts +79 -0
  154. package/src/utils/pcmAnalyzer.ts +154 -0
  155. package/src/utils/perfermance.ts +141 -0
  156. package/src/utils/request.ts +95 -0
  157. package/src/utils/requestAnimateFrames.ts +195 -0
  158. package/src/utils/time.ts +32 -0
  159. package/src/view/DebugOverlay.ts +404 -0
  160. package/src/worker/streaming-video.ts +4647 -0
@@ -0,0 +1,243 @@
1
+ export default class MediaSourceAudioPlayer {
2
+ private mediaSource: MediaSource | null = null;
3
+ private audioElement: HTMLAudioElement | null = null;
4
+ private sourceBuffer: SourceBuffer | null = null;
5
+ public isInitialized = false;
6
+ public isPlaying = false;
7
+ private volume = 1.0;
8
+ private readonly MIME_TYPE = 'audio/webm;codecs=opus';
9
+ private queue: Array<{ data: ArrayBuffer }> = [];
10
+ private isUpdating = false;
11
+ private currentObjectURL: string | null = null;
12
+ private isDestroyed = false;
13
+
14
+ // 速度同步相关(保留不动)
15
+ private playbackRate = 1.0;
16
+ public firstFrameIndex = -1;
17
+
18
+ // 单例
19
+ private static instance: MediaSourceAudioPlayer;
20
+ public static getInstance(sdk): MediaSourceAudioPlayer {
21
+ if (!MediaSourceAudioPlayer.instance) {
22
+ MediaSourceAudioPlayer.instance = new MediaSourceAudioPlayer(sdk);
23
+ }
24
+ return MediaSourceAudioPlayer.instance;
25
+ }
26
+
27
+ private constructor(sdk) {
28
+ this.createAudioElement(sdk);
29
+ this.createNewMediaSource();
30
+ }
31
+
32
+ /**
33
+ * 只复用 audio 标签!!!
34
+ */
35
+ private createAudioElement(sdk) {
36
+ if (this.audioElement) return;
37
+ const el = document.getElementById("xmov-avatar-audio");
38
+ if (el) {
39
+ this.audioElement = el as HTMLAudioElement;
40
+ return;
41
+ }
42
+ const audio = document.createElement("audio");
43
+ audio.id = "xmov-avatar-audio";
44
+ audio.preload = "auto";
45
+ audio.style.position = "absolute";
46
+ audio.style.width = "1px";
47
+ audio.style.height = "1px";
48
+ audio.style.opacity = "0";
49
+ audio.style.pointerEvents = "none";
50
+ audio.style.zIndex = "-9999";
51
+ sdk.el.appendChild(audio);
52
+ this.audioElement = audio;
53
+ }
54
+
55
+ /**
56
+ * 每次都创建 NEW MediaSource
57
+ * 不复用!不复用!不复用!
58
+ */
59
+ private createNewMediaSource(): Promise<void> {
60
+ return new Promise((resolve, reject) => {
61
+ try {
62
+ if(!this.audioElement) return;
63
+ this.mediaSource = new MediaSource();
64
+ this.currentObjectURL = URL.createObjectURL(this.mediaSource);
65
+ this.audioElement!.src = this.currentObjectURL;
66
+
67
+ this.mediaSource.addEventListener("sourceopen", () => {
68
+ try {
69
+ if (!this.mediaSource) return;
70
+ this.sourceBuffer = this.mediaSource.addSourceBuffer(this.MIME_TYPE);
71
+ this.sourceBuffer.mode = "sequence";
72
+
73
+ this.sourceBuffer.addEventListener("updateend", () => {
74
+ this.isUpdating = false;
75
+ this.processQueue();
76
+ });
77
+
78
+ this.isInitialized = true;
79
+ resolve();
80
+ } catch (e) {
81
+ reject(e);
82
+ }
83
+ });
84
+
85
+ this.mediaSource.addEventListener("sourceclose", () => {
86
+ this.isInitialized = false;
87
+ });
88
+ } catch (e) {
89
+ reject(e);
90
+ }
91
+ });
92
+ }
93
+
94
+ /**
95
+ * 销毁当前 MediaSource 所有资源
96
+ * stop 时调用
97
+ */
98
+ private async destroyCurrentMediaSource(callback: () => void) {
99
+ try {
100
+ if (this.audioElement) {
101
+ this.audioElement.pause();
102
+ this.audioElement.removeAttribute("src");
103
+ this.audioElement.load();
104
+ }
105
+ } catch {}
106
+
107
+ try {
108
+ if (this.currentObjectURL) {
109
+ URL.revokeObjectURL(this.currentObjectURL);
110
+ this.currentObjectURL = null;
111
+ }
112
+ } catch {}
113
+
114
+ this.mediaSource = null;
115
+ this.sourceBuffer = null;
116
+ this.isInitialized = false;
117
+ this.isPlaying = false;
118
+ this.queue = [];
119
+ this.isUpdating = false;
120
+ await this.createNewMediaSource();
121
+ callback();
122
+ }
123
+
124
+ // ==========================
125
+ // 核心:stop 清空一切,只留 audio
126
+ // ==========================
127
+ public stop(callback: () => void) {
128
+ this.destroyCurrentMediaSource(callback);
129
+ }
130
+
131
+ // ==========================
132
+ // 添加数据(自动初始化)
133
+ // ==========================
134
+ async addAudioSegment(audioData: ArrayBuffer): Promise<void> {
135
+ if (this.isDestroyed) return;
136
+ this.queue.push({ data: audioData });
137
+ this.processQueue();
138
+ }
139
+
140
+ private processQueue() {
141
+ if (this.isDestroyed) return;
142
+ if (!this.sourceBuffer || !this.mediaSource) return;
143
+ if (this.isUpdating || this.queue.length === 0) return;
144
+ if (this.sourceBuffer.updating || this.mediaSource.readyState !== "open") return;
145
+
146
+ this.isUpdating = true;
147
+ const item = this.queue.shift()!;
148
+ try {
149
+ this.sourceBuffer.appendBuffer(item.data);
150
+ } catch (e) {
151
+ this.isUpdating = false;
152
+ }
153
+ }
154
+
155
+ // ==========================
156
+ // 播放
157
+ // ==========================
158
+ async start(): Promise<void> {
159
+ if (this.isDestroyed || !this.audioElement || this.isPlaying) return;
160
+
161
+ try {
162
+ this.audioElement.currentTime = 0;
163
+ this.isPlaying = true;
164
+ await this.audioElement.play();
165
+ (window as any).avatarSDKLogger.log("mseAudioPlayer开始播放", this.getStats(), Date.now());
166
+ } catch {}
167
+ }
168
+
169
+ getStats() {
170
+ return {
171
+ isPlaying: this.isPlaying,
172
+ isInitialized: this.isInitialized,
173
+ queueLength: this.queue.length,
174
+ bufferedRanges: this.getBufferedRanges(),
175
+ mediaSourceReadyState: this.mediaSource?.readyState || 'null',
176
+ };
177
+ }
178
+
179
+ setSpeed(speed: number) {
180
+ this.setPlaybackRate(speed);
181
+ }
182
+
183
+ /**
184
+ * 获取缓冲范围
185
+ */
186
+ private getBufferedRanges(): string {
187
+ if (this.sourceBuffer && this.sourceBuffer.buffered.length > 0) {
188
+ const ranges: string[] = [];
189
+ for (let i = 0; i < this.sourceBuffer.buffered.length; i++) {
190
+ const start = this.sourceBuffer.buffered.start(i).toFixed(2);
191
+ const end = this.sourceBuffer.buffered.end(i).toFixed(2);
192
+ ranges.push(`[${start}-${end}]`);
193
+ }
194
+ return ranges.join(", ");
195
+ }
196
+ return "-";
197
+ }
198
+
199
+ setVolume(volume: number): void {
200
+ this.volume = Math.max(0, Math.min(1, volume));
201
+ if (this.audioElement) this.audioElement.volume = this.volume;
202
+ }
203
+
204
+ setPlaybackRate(rate: number): void {
205
+ this.playbackRate = Math.max(0.5, Math.min(4.0, rate));
206
+ if (this.audioElement) this.audioElement.playbackRate = this.playbackRate;
207
+ }
208
+
209
+ destroy() {
210
+ // 0. 标记已销毁,阻止异步操作继续执行
211
+ this.isDestroyed = true;
212
+
213
+ // 1. 停止播放并清除 audio 元素的 src(解除对 blob URL 的引用)
214
+ if (this.audioElement) {
215
+ this.audioElement.pause();
216
+ this.audioElement.removeAttribute("src");
217
+ this.audioElement.load();
218
+ }
219
+
220
+ // 2. 释放 blob URL(必须在 audio 元素清除 src 之后)
221
+ if (this.currentObjectURL) {
222
+ try { URL.revokeObjectURL(this.currentObjectURL); } catch {}
223
+ this.currentObjectURL = null;
224
+ }
225
+
226
+ // 3. 移除 audio 元素
227
+ if (this.audioElement) {
228
+ this.audioElement.remove();
229
+ this.audioElement = null;
230
+ }
231
+
232
+ // 4. 重置所有状态
233
+ this.mediaSource = null;
234
+ this.sourceBuffer = null;
235
+ this.isInitialized = false;
236
+ this.isPlaying = false;
237
+ this.queue = [];
238
+ this.isUpdating = false;
239
+
240
+ // 5. 重置单例,下次 getInstance() 会重新创建
241
+ MediaSourceAudioPlayer.instance = null as any;
242
+ }
243
+ }
@@ -0,0 +1,195 @@
1
+ /**
2
+ * UI 控件渲染,所有轨道的元素
3
+ */
4
+ import { DataCacheQueue } from "../control/DataCacheQueue";
5
+ import DefaultWidgetRenderer from "../modules/TrackRenderer/render-implements";
6
+ import TrackerRenderer from "../modules/TrackRenderer/index";
7
+ import ResourceManager from "../modules/ResourceManager";
8
+ import { performanceConstant } from "../utils/perfermance";
9
+ import XmovAvatar from "../index";
10
+ import { EErrorCode } from "../types/error";
11
+
12
+ type Option = {
13
+ sdk: XmovAvatar;
14
+ dataCacheQueue: DataCacheQueue;
15
+ resourceManager: ResourceManager;
16
+ enableClientInterrupt: boolean;
17
+ onVoiceEnd: (client_speak_id: number | string, speech_id: number) => void;
18
+ onVoiceStart: (duration: number, client_speak_id: number | string) => void;
19
+ onWalkStateChange: (state: string) => void;
20
+ sendSdkPoint: (type: string, data: any, extra?: any) => void;
21
+ onSpeakStateChange: (state: string, client_speak_id: number | string, speech_id: number) => void;
22
+ };
23
+ export default class UIRenderer {
24
+ private TAG = "[UIRenderer]";
25
+ options: Option;
26
+ trackerRenderer: TrackerRenderer[] = [];
27
+ root = document.createElement("div");
28
+ lastFrameIndex: number = -1;
29
+ onVoiceEnd: (client_speak_id: number | string, speech_id: number) => void;
30
+ onVoiceStart: (duration: number, client_speak_id: number | string) => void;
31
+ onWalkStateChange: (state: string) => void;
32
+ initEventsRendered: boolean = false;
33
+ lastSpeechId: number = -1;
34
+ interruptSpeechId: number = -1;
35
+ enableClientInterrupt: boolean = false;
36
+ onSpeakStateChange: (state: string, client_speak_id: number | string, speech_id: number) => void;
37
+ constructor(options: Option) {
38
+ this.options = options;
39
+ this.root.setAttribute("style", "position:absolute;");
40
+ this.onVoiceStart = options.onVoiceStart;
41
+ this.onVoiceEnd = options.onVoiceEnd;
42
+ this.initEventsRendered = false;
43
+ this.onWalkStateChange = options.onWalkStateChange;
44
+ this.enableClientInterrupt = options.enableClientInterrupt;
45
+ this.onSpeakStateChange = options.onSpeakStateChange;
46
+ }
47
+
48
+ render(frame: number) {
49
+ // if(this.options.sdk.getStatus() === AvatarStatus.offline) return;
50
+ let initEvents: any[] = [];
51
+ if (this.lastFrameIndex === -1) {
52
+ const config = this.options.resourceManager.getConfig();
53
+ if(config.init_events && config.init_events.length > 0){
54
+ initEvents = config.init_events.map(item => ({...item, speech_id:0}));
55
+ }
56
+ }
57
+ let event: any = null;
58
+ if (frame - this.lastFrameIndex > 1) {
59
+ event = this.options.dataCacheQueue._getEventInterval(
60
+ this.lastFrameIndex,
61
+ frame
62
+ );
63
+ } else {
64
+ event = this.options.dataCacheQueue._getEvent(frame);
65
+ }
66
+ this.lastFrameIndex = frame;
67
+ // 仅执行一次
68
+ if(!this.initEventsRendered){
69
+ this.initEventsRendered = true;
70
+ if(event){
71
+ event.e = event.e.concat(initEvents);
72
+ } else {
73
+ event = {
74
+ e: initEvents,
75
+ };
76
+ }
77
+ }
78
+ if (!event?.e) {
79
+ return;
80
+ }
81
+ (window as any).avatarSDKLogger.log("当前帧", frame, "获取到的事件", JSON.stringify(event));
82
+ // 过滤出不包含speechId的事件list
83
+ const noSpeechIdEvents = event.e?.filter(item => item?.speech_id === undefined || item?.speech_id === null);
84
+ if(noSpeechIdEvents?.length) {
85
+ const typesStr = noSpeechIdEvents.map(item => item.type).join(',');
86
+ this.options.sdk.onMessage({
87
+ code: EErrorCode.EVENT_WITHOUT_SPEECH_ID,
88
+ message: `Error: 事件中存在不包含speech_id的事件${typesStr}`,
89
+ e: noSpeechIdEvents
90
+ })
91
+ }
92
+ if(this.enableClientInterrupt) {
93
+ // 根据interruptSpeechId过滤旧的数据,加入前端interrupt事件后,可能在前端打断后,服务仍下发数据 如果type是以_end结尾的,不过滤
94
+ event.e = event.e?.filter(item => (item.speech_id > this.interruptSpeechId || item.speech_id === undefined || item.speech_id === null) || item.type.endsWith('_end') || item.type === 'subtitle_off');
95
+ }
96
+ // 过滤出当前语音的事件
97
+ if(this.lastSpeechId !== -1 && this.lastSpeechId) {
98
+ event.e = event.e?.filter(item => item?.speech_id >= this.lastSpeechId);
99
+ }
100
+
101
+ // 1. 处理 speak_start
102
+ if (event.e?.some(item => item.type === "speak_start")) {
103
+ const speakStart = event.e.find(item => item.type === "speak_start");
104
+ this.lastSpeechId = speakStart?.speech_id;
105
+ this.onSpeakStateChange("speak_start", speakStart.client_speak_id, speakStart?.speech_id || 0);
106
+ }
107
+ // 2. 处理 walk_start / voice_start / speak_walk_start
108
+ if (event.e?.some(item => item.type === "walk_start") || event.e?.some(item => item.type === "speak_walk_start")) {
109
+ this.onWalkStateChange("walk_start");
110
+ }
111
+ if(event.e?.some(item => item.type === "voice_start")) {
112
+ const voiceStart = event.e.find(item => item.type === "voice_start");
113
+ const duration = (window as any).performanceTracker.markEnd(performanceConstant.voice_response_play, 'speak');
114
+ this.options.sendSdkPoint('rendering_display', {multi_turn_conversation_id: voiceStart?.multi_turn_conversation_id});
115
+ this.onVoiceStart(duration, voiceStart?.client_speak_id);
116
+ }
117
+ // 3. 处理 walk_end / speak_walk_end / voice_end
118
+ if (event.e?.some(item => item.type === "walk_end") || event.e?.some(item => item.type === "speak_walk_end")) {
119
+ this.onWalkStateChange("walk_end");
120
+ }
121
+ if (event.e?.some(item => item.type === "voice_end")) {
122
+ const voiceEnd = event.e.find(item => item.type === "voice_end");
123
+ this.onVoiceEnd(voiceEnd?.client_speak_id || '', voiceEnd?.speech_id || 0);
124
+ }
125
+ // 4. 处理 speak_end / speak_error
126
+ const speakEndEventTypes = ["speak_end", "speak_error"];
127
+ speakEndEventTypes.forEach(eventType => {
128
+ const speakEvent = event.e.find(item => item.type === eventType);
129
+ if (speakEvent) {
130
+ this.onSpeakStateChange(eventType, speakEvent.client_speak_id, speakEvent?.speech_id || 0);
131
+ }
132
+ });
133
+
134
+
135
+ const remainingWidgets = [...event.e];
136
+
137
+ // 循环遍历 event.e 数组中的所有元素
138
+ for (let i = 0; i < event.e.length; i++) {
139
+ const widgetData = event.e[i];
140
+ if (Object.prototype.hasOwnProperty.call(widgetData, "event_type")) {
141
+ // 先不处理 ka/ka_intent
142
+ continue;
143
+ }
144
+ // 如果有自定义的渲染器,则所有事件都交给自定义的渲染器处理
145
+ if (DefaultWidgetRenderer.CUSTOM_WIDGET) {
146
+ DefaultWidgetRenderer.CUSTOM_WIDGET(widgetData);
147
+ // 从剩余元素中删除已处理的元素
148
+ const index = remainingWidgets.indexOf(widgetData);
149
+ if (index > -1) {
150
+ remainingWidgets.splice(index, 1);
151
+ }
152
+ continue;
153
+ }
154
+ // 如果有代理的渲染器,则根据类型执行代理的渲染器
155
+ if (DefaultWidgetRenderer.PROXY_WIDGET && DefaultWidgetRenderer.PROXY_WIDGET[widgetData.type]) {
156
+ DefaultWidgetRenderer.PROXY_WIDGET[widgetData.type](widgetData);
157
+
158
+ // 从剩余元素中删除已处理的元素
159
+ const index = remainingWidgets.indexOf(widgetData);
160
+ if (index > -1) {
161
+ remainingWidgets.splice(index, 1);
162
+ }
163
+ continue;
164
+ }
165
+ }
166
+ // 对剩余未自定义的元素执行 TrackerRenderer
167
+ if (remainingWidgets.length > 0) {
168
+ // 遍历剩余的元素,每个单独执行 TrackerRenderer
169
+ // 传递 SDK 实例引用,用于区分不同数字人实例的 widget
170
+ for (const widgetData of remainingWidgets) {
171
+ const tracker = new TrackerRenderer(widgetData, this.options.sdk);
172
+ this.trackerRenderer.push(tracker);
173
+ }
174
+ }
175
+ return this.trackerRenderer;
176
+ }
177
+
178
+ clearTrackerRenderer() {
179
+ this.trackerRenderer = [];
180
+ }
181
+
182
+ setInterrupt(speech_id: number) {
183
+ this.interruptSpeechId = speech_id;
184
+ }
185
+
186
+ destroy() {
187
+ this.lastSpeechId = -1;
188
+ this.interruptSpeechId = -1;
189
+ this.lastFrameIndex = -1;
190
+ this.initEventsRendered = false;
191
+ // 传递 SDK 实例引用,只清除当前实例的 widget
192
+ DefaultWidgetRenderer.destroy(this.options.sdk);
193
+ this.trackerRenderer.forEach((tracker) => tracker.destroy());
194
+ }
195
+ }
@@ -0,0 +1,267 @@
1
+ // pcm-audio-processor.js - 运行在音频线程的Worklet处理器
2
+ class PCMAudioProcessor extends AudioWorkletProcessor {
3
+ // 音量配置(与主线程保持一致)
4
+ static get parameterDescriptors() {
5
+ return [
6
+ {
7
+ name: 'volume',
8
+ defaultValue: 1.0,
9
+ minValue: 0.0,
10
+ maxValue: 1.0,
11
+ automationRate: 'a-rate' // 支持音频速率自动化
12
+ }
13
+ ];
14
+ }
15
+
16
+ constructor() {
17
+ super();
18
+ this.config = {
19
+ sampleRate: 24000, // 固定24kHz采样率
20
+ bytesPerSample: 2, // S16LE=2字节/采样点
21
+ };
22
+ this.pcmCache = new Uint8Array(0); // Worklet内的PCM缓存
23
+ this.isActive = false; // 播放音频状态(控制是否生成声音)
24
+ this.isDestroyed = false;
25
+ this.fadeState = {
26
+ // 淡入淡出状态(避免片段切换爆音)
27
+ phase: 'idle', // idle/fadeIn/fadeOut/steady
28
+ currentGain: 0,
29
+ fadeTime: 0.01, // 淡入淡出时间(秒,建议最小0.01s避免爆音)
30
+ fadeSamples: Math.floor(0.01 * 24000), // 淡入淡出样本数(24000Hz下为240样本)
31
+ };
32
+ this.currentSegment = null; // 当前处理的音频段(float32数组)
33
+ this.segmentOffset = 0; // 当前音频段的播放偏移量
34
+
35
+ // 监听主线程消息(接收PCM数据、状态控制等)
36
+ this.port.onmessage = (e) => this.handleMainThreadMessage(e.data);
37
+
38
+ // 初始化淡入淡出样本数
39
+ this.fadeState.fadeSamples = Math.floor(this.fadeState.fadeTime * this.config.sampleRate);
40
+ }
41
+
42
+ /**
43
+ * 处理主线程发来的消息(PCM数据/状态控制等)
44
+ * @param {Object} data - 消息数据
45
+ * @param {string} data.type - 消息类型:'pcm'/'start'/'pause'/'stop'/'config'/'destroy'/'volume'
46
+ * @param {Uint8Array} [data.pcmData] - PCM数据(仅type='pcm'时存在)
47
+ * @param {Object} [data.config] - 配置更新(仅type='config'时存在)
48
+ * @param {number} [data.volume] - 音量值(仅type='volume'时存在)
49
+ */
50
+ handleMainThreadMessage(data) {
51
+ switch (data.type) {
52
+ // 接收主线程下发的PCM流数据
53
+ case 'pcm':
54
+ if (data.pcmData instanceof Uint8Array && data.pcmData.length > 0) {
55
+ // 确保PCM数据字节数为2的整数倍(S16LE要求)
56
+ const validLength = data.pcmData.length - (data.pcmData.length % this.config.bytesPerSample);
57
+ if (validLength <= 0) break;
58
+
59
+ // 合并到Worklet缓存
60
+ const newCache = new Uint8Array(this.pcmCache.length + validLength);
61
+ newCache.set(this.pcmCache, 0);
62
+ newCache.set(data.pcmData.subarray(0, validLength), this.pcmCache.length);
63
+ this.pcmCache = newCache;
64
+
65
+ // 向主线程发送缓存状态
66
+ this.port.postMessage({
67
+ type: 'cacheStatus',
68
+ size: this.pcmCache.length,
69
+ sampleCount: this.pcmCache.length / this.config.bytesPerSample
70
+ });
71
+ }
72
+ break;
73
+
74
+ // 启动播放(继续音频生成)
75
+ case 'start':
76
+ this.isActive = true;
77
+ break;
78
+
79
+ // 暂停播放(停止音频生成,保留缓存)
80
+ case 'pause':
81
+ this.isActive = false;
82
+ this.resetSegmentState();
83
+ break;
84
+
85
+ // 停止播放(清空缓存+重置状态)
86
+ case 'stop':
87
+ this.isActive = false;
88
+ this.pcmCache = new Uint8Array(0);
89
+ this.resetSegmentState();
90
+ break;
91
+
92
+ // 更新配置(如采样率/淡入淡出时间)
93
+ case 'config':
94
+ if (data.config.sampleRate) {
95
+ this.config.sampleRate = data.config.sampleRate;
96
+ // 重新计算淡入淡出样本数
97
+ this.fadeState.fadeSamples = Math.floor(this.fadeState.fadeTime * this.config.sampleRate);
98
+ }
99
+ if (data.config.fadeTime) {
100
+ this.fadeState.fadeTime = data.config.fadeTime;
101
+ this.fadeState.fadeSamples = Math.floor(this.fadeState.fadeTime * this.config.sampleRate);
102
+ }
103
+ break;
104
+
105
+ // 单独更新音量(可选,也可通过参数自动化控制)
106
+ case 'volume':
107
+ this.volume = Number(data.volume);
108
+ // 通知主线程同步AudioWorkletNode的parameters
109
+ this.port.postMessage({
110
+ type: 'syncVolume',
111
+ volume: this.volume
112
+ });
113
+ break;
114
+
115
+ // 销毁处理器
116
+ case 'destroy':
117
+ this.isDestroyed = true;
118
+ this.isActive = false;
119
+ this.pcmCache = new Uint8Array(0);
120
+ this.resetSegmentState();
121
+ this.port.postMessage({ type: 'destroyConfirmed' });
122
+ break;
123
+
124
+ default:
125
+ }
126
+ }
127
+
128
+ /**
129
+ * 重置当前音频段状态(暂停/停止时调用)
130
+ */
131
+ resetSegmentState() {
132
+ this.currentSegment = null;
133
+ this.segmentOffset = 0;
134
+ this.fadeState.phase = 'idle';
135
+ this.fadeState.currentGain = 0;
136
+ }
137
+
138
+ /**
139
+ * S16LE Uint8Array 转 Float32Array(-1~1范围)
140
+ * @param {Uint8Array} pcmData - PCM数据
141
+ * @returns {Float32Array} 转换后的音频样本
142
+ */
143
+ pcmToFloat32(pcmData) {
144
+ const sampleCount = pcmData.length / this.config.bytesPerSample;
145
+ const floatData = new Float32Array(sampleCount);
146
+ const dataView = new DataView(pcmData.buffer);
147
+
148
+ for (let i = 0; i < sampleCount; i++) {
149
+ // S16LE格式:小端序解析,范围-32768~32767 归一化到-1~1
150
+ floatData[i] = dataView.getInt16(i * this.config.bytesPerSample, true) / 32768;
151
+ }
152
+ return floatData;
153
+ }
154
+
155
+ /**
156
+ * 计算当前样本的增益(淡入淡出)
157
+ * @param {number} segmentLength - 当前音频段的总样本数
158
+ * @returns {number} 增益值(0~1)
159
+ */
160
+ calculateGain(segmentLength) {
161
+ const { phase, fadeSamples } = this.fadeState;
162
+ const { segmentOffset } = this;
163
+
164
+ // 边界保护:防止无效值导致的NaN
165
+ if (typeof segmentOffset !== 'number' || typeof segmentLength !== 'number' || segmentLength <= 0) {
166
+ return 0;
167
+ }
168
+
169
+ switch (phase) {
170
+ // 淡入阶段:从0线性增加到1
171
+ case 'fadeIn':
172
+ if (segmentOffset >= fadeSamples) {
173
+ this.fadeState.phase = 'steady';
174
+ return 1;
175
+ }
176
+ return segmentOffset / fadeSamples;
177
+
178
+ // 稳定阶段:增益保持1
179
+ case 'steady':
180
+ // 检测是否进入淡出阶段(接近段尾)
181
+ if (segmentOffset >= segmentLength - fadeSamples) {
182
+ this.fadeState.phase = 'fadeOut';
183
+ }
184
+ return 1;
185
+
186
+ // 淡出阶段:从1线性降低到0.0001(避免分音)
187
+ case 'fadeOut':
188
+ const fadeStart = Math.max(segmentLength - fadeSamples, 0);
189
+ const fadeProgress = (segmentOffset - fadeStart) / fadeSamples;
190
+ return Math.max(1 - fadeProgress, 0.0001);
191
+
192
+ // 空闲阶段:增益0
193
+ case 'idle':
194
+ default:
195
+ return 0;
196
+ }
197
+ }
198
+
199
+ /**
200
+ * AudioWorklet核心方法:实时生成音频样本(音频线程调用)
201
+ * @param {Float32Array[][]} inputs - 输入样本
202
+ * @param {Float32Array[][]} outputs - 输出样本(单声道)
203
+ * @param {Object} parameters - 音频参数
204
+ * @returns {boolean} 是否继续运行
205
+ */
206
+ process(inputs, outputs, parameters) {
207
+ // 销毁状态:终止处理
208
+ if (this.isDestroyed) {
209
+ if (outputs.length > 0 && outputs[0].length > 0) {
210
+ outputs[0][0].fill(0);
211
+ }
212
+ return false;
213
+ }
214
+
215
+ // 未激活或无输出:返回静音
216
+ if (!this.isActive || outputs.length === 0 || outputs[0].length === 0) {
217
+ outputs[0][0].fill(0);
218
+ return true;
219
+ }
220
+
221
+ const outputBuffer = outputs[0][0];
222
+ const bufferLength = outputBuffer.length;
223
+ // 获取音量参数(支持自动化)
224
+ const volume = parameters.volume.length > 1
225
+ ? parameters.volume // 自动化模式:每个样本点可能有不同值
226
+ : parameters.volume[0]; // 控制模式:整个缓冲区使用相同值
227
+
228
+ // 生成输出样本
229
+ for (let i = 0; i < bufferLength; i++) {
230
+ // 如当前无音频段且缓存有数据,创建新段
231
+ if (!this.currentSegment && this.pcmCache.length > 0) {
232
+ // 使用缓存中的所有数据创建新段
233
+ this.currentSegment = this.pcmToFloat32(this.pcmCache);
234
+ // 清空缓存(已处理)
235
+ this.pcmCache = new Uint8Array(0);
236
+ this.segmentOffset = 0;
237
+ // 根据前一段状态决定是否淡入
238
+ this.fadeState.phase = this.fadeState.phase === 'fadeOut' ? 'steady' : 'fadeIn';
239
+ }
240
+
241
+ // 生成当前样本
242
+ let sample = 0;
243
+ if (this.currentSegment) {
244
+ const segmentLength = this.currentSegment.length;
245
+ // 确保偏移量有效
246
+ if (this.segmentOffset < segmentLength) {
247
+ const gain = this.calculateGain(segmentLength);
248
+ // 应用淡入淡出增益和音量控制
249
+ sample = this.currentSegment[this.segmentOffset] * gain * volume;
250
+ this.segmentOffset++;
251
+ } else {
252
+ // 当前段处理完毕,重置
253
+ this.currentSegment = null;
254
+ this.segmentOffset = 0;
255
+ this.fadeState.phase = 'idle';
256
+ }
257
+ }
258
+
259
+ outputBuffer[i] = sample;
260
+ }
261
+
262
+ return true;
263
+ }
264
+ }
265
+
266
+ // 注册Worklet处理器
267
+ registerProcessor('pcm-audio-processor', PCMAudioProcessor);