@xmov/avatar 2.0.0-alpha.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +139 -0
- package/dist/agent/__tests__/agent.test.d.ts +1 -0
- package/dist/agent/audio-debug.d.ts +3 -0
- package/dist/agent/audio-uplink.d.ts +62 -0
- package/dist/agent/avatar.cjs +2 -0
- package/dist/agent/avatar.cjs.map +1 -0
- package/dist/agent/avatar.modern.js +2 -0
- package/dist/agent/avatar.modern.js.map +1 -0
- package/dist/agent/avatar.module.js +2 -0
- package/dist/agent/avatar.module.js.map +1 -0
- package/dist/agent/avatar.umd.js +2 -0
- package/dist/agent/avatar.umd.js.map +1 -0
- package/dist/agent/e2e-client.d.ts +32 -0
- package/dist/agent/fixed-audio-track.d.ts +48 -0
- package/dist/agent/index.cjs +5 -0
- package/dist/agent/index.d.ts +104 -0
- package/dist/agent/index.umd.js +10 -0
- package/dist/agent/microphone.d.ts +49 -0
- package/dist/agent/types.d.ts +153 -0
- package/dist/baseRender/AudioRenderer.d.ts +46 -0
- package/dist/baseRender/AudioWorklet.d.ts +78 -0
- package/dist/baseRender/AvatarRenderer.d.ts +226 -0
- package/dist/baseRender/MSEAudioPlayer.d.ts +51 -0
- package/dist/baseRender/UIRenderer.d.ts +39 -0
- package/dist/baseRender/pcm-audio-processor.d.ts +63 -0
- package/dist/control/APIForwarder.d.ts +23 -0
- package/dist/control/DataCacheQueue.d.ts +69 -0
- package/dist/control/EventDispatcher.d.ts +35 -0
- package/dist/control/RenderScheduler.d.ts +162 -0
- package/dist/control/SaveAndDownload.d.ts +14 -0
- package/dist/control/StreamingClient.d.ts +33 -0
- package/dist/control/VideoPlayer.d.ts +47 -0
- package/dist/control/ttsa.d.ts +111 -0
- package/dist/encoding/media-recorder-encoder.d.ts +33 -0
- package/dist/encoding/pcm-webm-encoder.d.ts +13 -0
- package/dist/encoding/webcodec-opus-encoder.d.ts +41 -0
- package/dist/encoding/webm-muxer.d.ts +90 -0
- package/dist/index.cjs +2 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.ts +231 -0
- package/dist/index.modern.js +2 -0
- package/dist/index.modern.js.map +1 -0
- package/dist/index.module.js +2 -0
- package/dist/index.module.js.map +1 -0
- package/dist/index.umd.js +2 -0
- package/dist/index.umd.js.map +1 -0
- package/dist/modules/Composition.d.ts +28 -0
- package/dist/modules/ResourceManager.d.ts +295 -0
- package/dist/modules/TrackRenderer/base-track.d.ts +4 -0
- package/dist/modules/TrackRenderer/index.d.ts +12 -0
- package/dist/modules/TrackRenderer/render-implements.d.ts +19 -0
- package/dist/modules/TrackRenderer/track-pic.d.ts +8 -0
- package/dist/modules/TrackRenderer/track-subtitle.d.ts +8 -0
- package/dist/modules/cache-manager.d.ts +5 -0
- package/dist/modules/decoder.d.ts +84 -0
- package/dist/modules/error-handle.d.ts +17 -0
- package/dist/modules/network.d.ts +21 -0
- package/dist/proto/face_data_pb.d.ts +0 -0
- package/dist/types/capability.d.ts +15 -0
- package/dist/types/error.d.ts +70 -0
- package/dist/types/event.d.ts +25 -0
- package/dist/types/frame-data.d.ts +107 -0
- package/dist/types/index.d.ts +187 -0
- package/dist/types/render.d.ts +18 -0
- package/dist/utils/DataInterface.d.ts +113 -0
- package/dist/utils/GLDevice.d.ts +46 -0
- package/dist/utils/GLPipeline.d.ts +84 -0
- package/dist/utils/GLPipelineDebugTools.d.ts +5 -0
- package/dist/utils/Math.d.ts +21 -0
- package/dist/utils/__tests__/capability-checker.test.d.ts +1 -0
- package/dist/utils/audio-session.d.ts +11 -0
- package/dist/utils/audio.d.ts +1 -0
- package/dist/utils/capability-checker.d.ts +23 -0
- package/dist/utils/encodeToken.d.ts +7 -0
- package/dist/utils/face.d.ts +1 -0
- package/dist/utils/float32-decoder.d.ts +26 -0
- package/dist/utils/fpsTracker.d.ts +27 -0
- package/dist/utils/index.d.ts +22 -0
- package/dist/utils/logger.d.ts +9 -0
- package/dist/utils/media-recorder-timestamp.d.ts +15 -0
- package/dist/utils/pcmAnalyzer.d.ts +40 -0
- package/dist/utils/perfermance.d.ts +15 -0
- package/dist/utils/request.d.ts +19 -0
- package/dist/utils/requestAnimateFrames.d.ts +31 -0
- package/dist/utils/time.d.ts +9 -0
- package/dist/view/DebugOverlay.d.ts +54 -0
- package/dist/worker/streaming-video.d.ts +1 -0
- package/package.json +82 -0
- package/src/agent/__tests__/agent.test.ts +3787 -0
- package/src/agent/audio-debug.ts +36 -0
- package/src/agent/audio-uplink.ts +392 -0
- package/src/agent/e2e-client.ts +215 -0
- package/src/agent/fixed-audio-track.ts +547 -0
- package/src/agent/index.ts +1393 -0
- package/src/agent/microphone.ts +534 -0
- package/src/agent/types.ts +197 -0
- package/src/baseRender/AudioRenderer.ts +317 -0
- package/src/baseRender/AudioWorklet.ts +776 -0
- package/src/baseRender/AvatarRenderer.ts +1559 -0
- package/src/baseRender/MSEAudioPlayer.ts +243 -0
- package/src/baseRender/UIRenderer.ts +195 -0
- package/src/baseRender/pcm-audio-processor.js +267 -0
- package/src/control/APIForwarder.ts +67 -0
- package/src/control/DataCacheQueue.ts +372 -0
- package/src/control/EventDispatcher.ts +106 -0
- package/src/control/RenderScheduler.ts +945 -0
- package/src/control/SaveAndDownload.js +77 -0
- package/src/control/StreamingClient.ts +138 -0
- package/src/control/VideoPlayer.ts +306 -0
- package/src/control/ttsa.ts +676 -0
- package/src/encoding/media-recorder-encoder.ts +175 -0
- package/src/encoding/pcm-webm-encoder.ts +72 -0
- package/src/encoding/webcodec-opus-encoder.ts +258 -0
- package/src/encoding/webm-muxer.ts +337 -0
- package/src/global.d.ts +105 -0
- package/src/index.ts +1920 -0
- package/src/modules/Composition.ts +79 -0
- package/src/modules/ResourceManager.ts +899 -0
- package/src/modules/TrackRenderer/base-track.ts +7 -0
- package/src/modules/TrackRenderer/index.ts +40 -0
- package/src/modules/TrackRenderer/render-implements.ts +269 -0
- package/src/modules/TrackRenderer/track-pic.ts +19 -0
- package/src/modules/TrackRenderer/track-subtitle.ts +16 -0
- package/src/modules/cache-manager.ts +10 -0
- package/src/modules/decoder.ts +515 -0
- package/src/modules/error-handle.ts +20 -0
- package/src/modules/network.ts +71 -0
- package/src/proto/face_data.proto +45 -0
- package/src/proto/face_data_pb.js +1297 -0
- package/src/proto/protobuf.min.js +8 -0
- package/src/types/capability.ts +23 -0
- package/src/types/error.ts +211 -0
- package/src/types/event.ts +38 -0
- package/src/types/frame-data.ts +121 -0
- package/src/types/index.ts +207 -0
- package/src/types/render.ts +18 -0
- package/src/utils/DataInterface.ts +746 -0
- package/src/utils/GLDevice.ts +286 -0
- package/src/utils/GLPipeline.ts +1293 -0
- package/src/utils/GLPipelineDebugTools.ts +174 -0
- package/src/utils/Math.ts +266 -0
- package/src/utils/__tests__/capability-checker.test.ts +159 -0
- package/src/utils/audio-session.ts +144 -0
- package/src/utils/audio.ts +44 -0
- package/src/utils/blueimp-md5.d.ts +3 -0
- package/src/utils/capability-checker.ts +379 -0
- package/src/utils/encodeToken.ts +100 -0
- package/src/utils/face.ts +29 -0
- package/src/utils/float32-decoder.js +130 -0
- package/src/utils/fpsTracker.ts +100 -0
- package/src/utils/index.ts +182 -0
- package/src/utils/logger.js +36 -0
- package/src/utils/media-recorder-timestamp.ts +79 -0
- package/src/utils/pcmAnalyzer.ts +154 -0
- package/src/utils/perfermance.ts +141 -0
- package/src/utils/request.ts +95 -0
- package/src/utils/requestAnimateFrames.ts +195 -0
- package/src/utils/time.ts +32 -0
- package/src/view/DebugOverlay.ts +404 -0
- package/src/worker/streaming-video.ts +4647 -0
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 3D 渲染(身体+表情)
|
|
3
|
+
*/
|
|
4
|
+
import ResourceManager from "../modules/ResourceManager";
|
|
5
|
+
import { DataCacheQueue, IBodyFrame } from "../control/DataCacheQueue";
|
|
6
|
+
import { GLDevice } from "../utils/GLDevice";
|
|
7
|
+
import { GLPipeline, GLPipelineCharData } from "../utils/GLPipeline";
|
|
8
|
+
import { EErrorCode } from "../types/error";
|
|
9
|
+
import { Layout, RenderState, WalkConfig } from "../types";
|
|
10
|
+
import SaveAndDownload from "../control/SaveAndDownload";
|
|
11
|
+
import { IRawFaceFrameData } from "types/frame-data";
|
|
12
|
+
type Option = {
|
|
13
|
+
dataCacheQueue: DataCacheQueue;
|
|
14
|
+
resourceManager: ResourceManager;
|
|
15
|
+
saveAndDownload: SaveAndDownload;
|
|
16
|
+
forceSyncDecoder: () => void;
|
|
17
|
+
reportMessage: (message: {
|
|
18
|
+
code: EErrorCode;
|
|
19
|
+
message: string;
|
|
20
|
+
e?: object;
|
|
21
|
+
}) => void;
|
|
22
|
+
onDownloadProgress: (progress: number) => void;
|
|
23
|
+
onStateChange: (state: string) => void;
|
|
24
|
+
onRenderChange: (state: RenderState) => void;
|
|
25
|
+
sendVideoInfo: (info: {
|
|
26
|
+
name: string;
|
|
27
|
+
body_id: number;
|
|
28
|
+
id: number;
|
|
29
|
+
}) => void;
|
|
30
|
+
onError: (error: any) => void;
|
|
31
|
+
sendSdkPoint: (point: string, data: any) => void;
|
|
32
|
+
};
|
|
33
|
+
export default class AvatarRender {
|
|
34
|
+
private TAG;
|
|
35
|
+
options: Option;
|
|
36
|
+
canvas: HTMLCanvasElement;
|
|
37
|
+
device: GLDevice | null;
|
|
38
|
+
pipeline: GLPipeline | null;
|
|
39
|
+
currentBodyFrame: IBodyFrame | null;
|
|
40
|
+
lastFaceFrame: IRawFaceFrameData | null;
|
|
41
|
+
private isInit;
|
|
42
|
+
private isFirstRender;
|
|
43
|
+
private lastFrameState;
|
|
44
|
+
private lastRenderState;
|
|
45
|
+
private onDownloadProgress;
|
|
46
|
+
private onStateChange?;
|
|
47
|
+
private onRenderChange?;
|
|
48
|
+
private sendVideoInfo;
|
|
49
|
+
private lostHandler;
|
|
50
|
+
private restoreHandler;
|
|
51
|
+
private avatarCanvasVisible;
|
|
52
|
+
private lastRealFaceFrameData;
|
|
53
|
+
private lastRealFaceFrame;
|
|
54
|
+
private lastWeight;
|
|
55
|
+
private lastFrameIndex;
|
|
56
|
+
private saveAndDownload;
|
|
57
|
+
private canvasOffsetX;
|
|
58
|
+
private baseWalkOffset;
|
|
59
|
+
private lastBodyOffset;
|
|
60
|
+
private pendingCharData;
|
|
61
|
+
private interrupt;
|
|
62
|
+
private walkConfig;
|
|
63
|
+
layout: Layout | null;
|
|
64
|
+
private allLostFrameCount;
|
|
65
|
+
private allLostFaceFrameCount;
|
|
66
|
+
private maxLostFrameCount;
|
|
67
|
+
private forceSyncDecoder;
|
|
68
|
+
private createPipelineLock;
|
|
69
|
+
private restoreTimeout;
|
|
70
|
+
private isRecovering;
|
|
71
|
+
private recoverStartTime;
|
|
72
|
+
private recoveryStage;
|
|
73
|
+
private readonly recoveryMaxAttempts;
|
|
74
|
+
private readonly recoveryBaseDelays;
|
|
75
|
+
private readonly recoveryMaxDelay;
|
|
76
|
+
private currentStageAttempts;
|
|
77
|
+
private isUsingReducedResolution;
|
|
78
|
+
private resolutionRestoreTimer;
|
|
79
|
+
private readonly RESOLUTION_RESTORE_DELAY;
|
|
80
|
+
private scale;
|
|
81
|
+
private sendSdkPoint;
|
|
82
|
+
private copyCanvas;
|
|
83
|
+
private copyCtx;
|
|
84
|
+
constructor(options: Option);
|
|
85
|
+
/**
|
|
86
|
+
* 重置表情相关状态(用于从隐身模式恢复渲染时)
|
|
87
|
+
*/
|
|
88
|
+
resetFaceFrameState(): void;
|
|
89
|
+
/**
|
|
90
|
+
* 创建 pipeline 并设置字符数据
|
|
91
|
+
*/
|
|
92
|
+
createPipeline(): void;
|
|
93
|
+
_lostHandler(event: Event): void;
|
|
94
|
+
_restoreHandler(event: Event): void;
|
|
95
|
+
/**
|
|
96
|
+
* 根据当前阶段分发到对应的恢复方法
|
|
97
|
+
*/
|
|
98
|
+
private _recoverContext;
|
|
99
|
+
/**
|
|
100
|
+
* 阶段 1 — 轻量恢复
|
|
101
|
+
* 复用现有 canvas,仅重建 GLDevice + GLPipeline
|
|
102
|
+
* 适用于:上下文偶发丢失、浏览器后台恢复
|
|
103
|
+
*/
|
|
104
|
+
private _tryLightRecovery;
|
|
105
|
+
/**
|
|
106
|
+
* 阶段 2 — 中等恢复
|
|
107
|
+
* 创建新 canvas 替换 DOM 节点,重建 GLDevice + GLPipeline
|
|
108
|
+
* 适用于:canvas 节点本身出现异常(如全屏切换导致的 DOM 重建)
|
|
109
|
+
*/
|
|
110
|
+
private _tryMediumRecovery;
|
|
111
|
+
/**
|
|
112
|
+
* 阶段 3 — 完全恢复(降级分辨率)
|
|
113
|
+
* 在全量重建基础上降低渲染分辨率,减少 GPU 压力
|
|
114
|
+
* 适用于:GPU 资源受限设备(笔记本集成显卡)连续上下文丢失
|
|
115
|
+
*/
|
|
116
|
+
private _tryFullRecovery;
|
|
117
|
+
/**
|
|
118
|
+
* 恢复成功后的统一收尾处理
|
|
119
|
+
*/
|
|
120
|
+
private _onRecoverySuccess;
|
|
121
|
+
/**
|
|
122
|
+
* 安排恢复原始分辨率
|
|
123
|
+
* 在降级分辨率稳定运行一段时间后尝试恢复
|
|
124
|
+
*/
|
|
125
|
+
private _scheduleResolutionRestore;
|
|
126
|
+
/**
|
|
127
|
+
* 恢复原始分辨率
|
|
128
|
+
* 尝试将 canvas 恢复到原始分辨率,如果再次丢失上下文则保持降级
|
|
129
|
+
*/
|
|
130
|
+
private _restoreOriginalResolution;
|
|
131
|
+
/**
|
|
132
|
+
* 升级恢复阶段:light → medium → full → 放弃
|
|
133
|
+
*/
|
|
134
|
+
private _advanceStage;
|
|
135
|
+
/**
|
|
136
|
+
* 按指数退避延迟安排下一次恢复尝试
|
|
137
|
+
* 延迟计算:baseDelay * 2^attempt,受 maxDelay 上限约束
|
|
138
|
+
*/
|
|
139
|
+
private _scheduleNextAttempt;
|
|
140
|
+
/**
|
|
141
|
+
* 上报单次恢复失败事件
|
|
142
|
+
*/
|
|
143
|
+
private _reportRecoveryFailure;
|
|
144
|
+
init(data: GLPipelineCharData | null): void;
|
|
145
|
+
private style;
|
|
146
|
+
/**
|
|
147
|
+
* 设置画布样式(修复transform失效 + 优化对齐逻辑 + 优雅拼接样式)
|
|
148
|
+
* @param avatar 布局配置
|
|
149
|
+
*/
|
|
150
|
+
setCanvasStyle(layout: Layout): void;
|
|
151
|
+
setWalkConfig(walkConfig: WalkConfig): void;
|
|
152
|
+
setCharacterCanvasAnchor(layout?: Layout): void;
|
|
153
|
+
computeWeight(frameIndex: any): number;
|
|
154
|
+
render(frameIndex: number): {
|
|
155
|
+
canvas: HTMLCanvasElement;
|
|
156
|
+
rendered: boolean;
|
|
157
|
+
};
|
|
158
|
+
renderBackground(): void;
|
|
159
|
+
_setCurrentBodyFrame(bodyFrame?: IBodyFrame): void;
|
|
160
|
+
_getCurrentBodyFrameInfo(frameIndex: number): {
|
|
161
|
+
client_frame: number;
|
|
162
|
+
current_ani: string;
|
|
163
|
+
current_ani_frame: number;
|
|
164
|
+
next_state: string;
|
|
165
|
+
};
|
|
166
|
+
/**
|
|
167
|
+
* 获取当前帧的 body 和 face 原始数据(用于回调)
|
|
168
|
+
* 不包含渲染数据,只返回帧号、body_id 等元数据
|
|
169
|
+
*/
|
|
170
|
+
_getCurrentFrameData(frameIndex: number): {
|
|
171
|
+
body: {
|
|
172
|
+
body_id: number;
|
|
173
|
+
name: string;
|
|
174
|
+
video_frame: number;
|
|
175
|
+
sf: number;
|
|
176
|
+
state: string;
|
|
177
|
+
} | null;
|
|
178
|
+
face: {
|
|
179
|
+
body_id: number;
|
|
180
|
+
sf: number;
|
|
181
|
+
ef: number;
|
|
182
|
+
state: string;
|
|
183
|
+
face_frame_type: number;
|
|
184
|
+
} | null;
|
|
185
|
+
};
|
|
186
|
+
/**
|
|
187
|
+
* 设置canvas的显隐状态
|
|
188
|
+
* @param visible 是否可见
|
|
189
|
+
*/
|
|
190
|
+
setCanvasVisibility(visible: boolean): void;
|
|
191
|
+
/**
|
|
192
|
+
* 获取canvas的显隐状态
|
|
193
|
+
*/
|
|
194
|
+
getCanvasVisibility(): boolean;
|
|
195
|
+
destroy(): void;
|
|
196
|
+
setInterrupt(interrupt: boolean): void;
|
|
197
|
+
/**
|
|
198
|
+
* [测试用] 模拟 WebGL 上下文丢失
|
|
199
|
+
* 用于测试分阶段恢复策略是否正常工作
|
|
200
|
+
* @param restoreAfterMs 自动恢复上下文的延迟时间(毫秒),设置为 0 则不自动恢复
|
|
201
|
+
* @param forceFail 强制恢复失败(用于测试 medium/full 阶段的升级逻辑)
|
|
202
|
+
*/
|
|
203
|
+
simulateContextLoss(restoreAfterMs?: number, forceFail?: boolean): void;
|
|
204
|
+
/**
|
|
205
|
+
* [测试用] 获取当前恢复状态信息
|
|
206
|
+
*/
|
|
207
|
+
getRecoveryStatus(): {
|
|
208
|
+
isRecovering: boolean;
|
|
209
|
+
stage: string;
|
|
210
|
+
attempts: number;
|
|
211
|
+
maxAttempts: number;
|
|
212
|
+
isUsingReducedResolution: boolean;
|
|
213
|
+
};
|
|
214
|
+
/**
|
|
215
|
+
* [测试用] 批量测试分阶段恢复
|
|
216
|
+
* 依次触发多次上下文丢失,观察各阶段的恢复情况
|
|
217
|
+
*/
|
|
218
|
+
testStagedRecovery(): Promise<void>;
|
|
219
|
+
/**
|
|
220
|
+
* 手动重启 WebGL:创建新 canvas + 新 GLDevice + 新 GLPipeline
|
|
221
|
+
* 用于分阶段恢复全部失败后的手动恢复
|
|
222
|
+
* @returns 是否成功
|
|
223
|
+
*/
|
|
224
|
+
restartWebGL(): boolean;
|
|
225
|
+
}
|
|
226
|
+
export {};
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
export default class MediaSourceAudioPlayer {
|
|
2
|
+
private mediaSource;
|
|
3
|
+
private audioElement;
|
|
4
|
+
private sourceBuffer;
|
|
5
|
+
isInitialized: boolean;
|
|
6
|
+
isPlaying: boolean;
|
|
7
|
+
private volume;
|
|
8
|
+
private readonly MIME_TYPE;
|
|
9
|
+
private queue;
|
|
10
|
+
private isUpdating;
|
|
11
|
+
private currentObjectURL;
|
|
12
|
+
private isDestroyed;
|
|
13
|
+
private playbackRate;
|
|
14
|
+
firstFrameIndex: number;
|
|
15
|
+
private static instance;
|
|
16
|
+
static getInstance(sdk: any): MediaSourceAudioPlayer;
|
|
17
|
+
private constructor();
|
|
18
|
+
/**
|
|
19
|
+
* 只复用 audio 标签!!!
|
|
20
|
+
*/
|
|
21
|
+
private createAudioElement;
|
|
22
|
+
/**
|
|
23
|
+
* 每次都创建 NEW MediaSource
|
|
24
|
+
* 不复用!不复用!不复用!
|
|
25
|
+
*/
|
|
26
|
+
private createNewMediaSource;
|
|
27
|
+
/**
|
|
28
|
+
* 销毁当前 MediaSource 所有资源
|
|
29
|
+
* stop 时调用
|
|
30
|
+
*/
|
|
31
|
+
private destroyCurrentMediaSource;
|
|
32
|
+
stop(callback: () => void): void;
|
|
33
|
+
addAudioSegment(audioData: ArrayBuffer): Promise<void>;
|
|
34
|
+
private processQueue;
|
|
35
|
+
start(): Promise<void>;
|
|
36
|
+
getStats(): {
|
|
37
|
+
isPlaying: boolean;
|
|
38
|
+
isInitialized: boolean;
|
|
39
|
+
queueLength: number;
|
|
40
|
+
bufferedRanges: string;
|
|
41
|
+
mediaSourceReadyState: string;
|
|
42
|
+
};
|
|
43
|
+
setSpeed(speed: number): void;
|
|
44
|
+
/**
|
|
45
|
+
* 获取缓冲范围
|
|
46
|
+
*/
|
|
47
|
+
private getBufferedRanges;
|
|
48
|
+
setVolume(volume: number): void;
|
|
49
|
+
setPlaybackRate(rate: number): void;
|
|
50
|
+
destroy(): void;
|
|
51
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* UI 控件渲染,所有轨道的元素
|
|
3
|
+
*/
|
|
4
|
+
import { DataCacheQueue } from "../control/DataCacheQueue";
|
|
5
|
+
import TrackerRenderer from "../modules/TrackRenderer/index";
|
|
6
|
+
import ResourceManager from "../modules/ResourceManager";
|
|
7
|
+
import XmovAvatar from "../index";
|
|
8
|
+
type Option = {
|
|
9
|
+
sdk: XmovAvatar;
|
|
10
|
+
dataCacheQueue: DataCacheQueue;
|
|
11
|
+
resourceManager: ResourceManager;
|
|
12
|
+
enableClientInterrupt: boolean;
|
|
13
|
+
onVoiceEnd: (client_speak_id: number | string, speech_id: number) => void;
|
|
14
|
+
onVoiceStart: (duration: number, client_speak_id: number | string) => void;
|
|
15
|
+
onWalkStateChange: (state: string) => void;
|
|
16
|
+
sendSdkPoint: (type: string, data: any, extra?: any) => void;
|
|
17
|
+
onSpeakStateChange: (state: string, client_speak_id: number | string, speech_id: number) => void;
|
|
18
|
+
};
|
|
19
|
+
export default class UIRenderer {
|
|
20
|
+
private TAG;
|
|
21
|
+
options: Option;
|
|
22
|
+
trackerRenderer: TrackerRenderer[];
|
|
23
|
+
root: HTMLDivElement;
|
|
24
|
+
lastFrameIndex: number;
|
|
25
|
+
onVoiceEnd: (client_speak_id: number | string, speech_id: number) => void;
|
|
26
|
+
onVoiceStart: (duration: number, client_speak_id: number | string) => void;
|
|
27
|
+
onWalkStateChange: (state: string) => void;
|
|
28
|
+
initEventsRendered: boolean;
|
|
29
|
+
lastSpeechId: number;
|
|
30
|
+
interruptSpeechId: number;
|
|
31
|
+
enableClientInterrupt: boolean;
|
|
32
|
+
onSpeakStateChange: (state: string, client_speak_id: number | string, speech_id: number) => void;
|
|
33
|
+
constructor(options: Option);
|
|
34
|
+
render(frame: number): TrackerRenderer[] | undefined;
|
|
35
|
+
clearTrackerRenderer(): void;
|
|
36
|
+
setInterrupt(speech_id: number): void;
|
|
37
|
+
destroy(): void;
|
|
38
|
+
}
|
|
39
|
+
export {};
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
declare class PCMAudioProcessor {
|
|
2
|
+
static get parameterDescriptors(): {
|
|
3
|
+
name: string;
|
|
4
|
+
defaultValue: number;
|
|
5
|
+
minValue: number;
|
|
6
|
+
maxValue: number;
|
|
7
|
+
automationRate: string;
|
|
8
|
+
}[];
|
|
9
|
+
config: {
|
|
10
|
+
sampleRate: number;
|
|
11
|
+
bytesPerSample: number;
|
|
12
|
+
};
|
|
13
|
+
pcmCache: Uint8Array;
|
|
14
|
+
isActive: boolean;
|
|
15
|
+
isDestroyed: boolean;
|
|
16
|
+
fadeState: {
|
|
17
|
+
phase: string;
|
|
18
|
+
currentGain: number;
|
|
19
|
+
fadeTime: number;
|
|
20
|
+
fadeSamples: number;
|
|
21
|
+
};
|
|
22
|
+
currentSegment: Float32Array | null;
|
|
23
|
+
segmentOffset: number;
|
|
24
|
+
/**
|
|
25
|
+
* 处理主线程发来的消息(PCM数据/状态控制等)
|
|
26
|
+
* @param {Object} data - 消息数据
|
|
27
|
+
* @param {string} data.type - 消息类型:'pcm'/'start'/'pause'/'stop'/'config'/'destroy'/'volume'
|
|
28
|
+
* @param {Uint8Array} [data.pcmData] - PCM数据(仅type='pcm'时存在)
|
|
29
|
+
* @param {Object} [data.config] - 配置更新(仅type='config'时存在)
|
|
30
|
+
* @param {number} [data.volume] - 音量值(仅type='volume'时存在)
|
|
31
|
+
*/
|
|
32
|
+
handleMainThreadMessage(data: {
|
|
33
|
+
type: string;
|
|
34
|
+
pcmData?: Uint8Array | undefined;
|
|
35
|
+
config?: any;
|
|
36
|
+
volume?: number | undefined;
|
|
37
|
+
}): void;
|
|
38
|
+
volume: number | undefined;
|
|
39
|
+
/**
|
|
40
|
+
* 重置当前音频段状态(暂停/停止时调用)
|
|
41
|
+
*/
|
|
42
|
+
resetSegmentState(): void;
|
|
43
|
+
/**
|
|
44
|
+
* S16LE Uint8Array 转 Float32Array(-1~1范围)
|
|
45
|
+
* @param {Uint8Array} pcmData - PCM数据
|
|
46
|
+
* @returns {Float32Array} 转换后的音频样本
|
|
47
|
+
*/
|
|
48
|
+
pcmToFloat32(pcmData: Uint8Array): Float32Array;
|
|
49
|
+
/**
|
|
50
|
+
* 计算当前样本的增益(淡入淡出)
|
|
51
|
+
* @param {number} segmentLength - 当前音频段的总样本数
|
|
52
|
+
* @returns {number} 增益值(0~1)
|
|
53
|
+
*/
|
|
54
|
+
calculateGain(segmentLength: number): number;
|
|
55
|
+
/**
|
|
56
|
+
* AudioWorklet核心方法:实时生成音频样本(音频线程调用)
|
|
57
|
+
* @param {Float32Array[][]} inputs - 输入样本
|
|
58
|
+
* @param {Float32Array[][]} outputs - 输出样本(单声道)
|
|
59
|
+
* @param {Object} parameters - 音频参数
|
|
60
|
+
* @returns {boolean} 是否继续运行
|
|
61
|
+
*/
|
|
62
|
+
process(inputs: Float32Array[][], outputs: Float32Array[][], parameters: any): boolean;
|
|
63
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* APIForwarder - 薄客户端模式下的 API 代理转发器
|
|
3
|
+
* 将客户端的 speak/idle/listen 等 API 调用封装为 api_call 消息,
|
|
4
|
+
* 通过 StreamingClient 发送给 Server 端 SDK 执行。
|
|
5
|
+
*/
|
|
6
|
+
import { StreamingClient } from './StreamingClient';
|
|
7
|
+
export declare class APIForwarder {
|
|
8
|
+
private client;
|
|
9
|
+
constructor(client: StreamingClient);
|
|
10
|
+
speak(ssml: string, is_start?: boolean, is_end?: boolean, extra?: any): void;
|
|
11
|
+
idle(): void;
|
|
12
|
+
listen(): void;
|
|
13
|
+
think(): void;
|
|
14
|
+
interactiveidle(): void;
|
|
15
|
+
interrupt(type: string): void;
|
|
16
|
+
changeLayout(layout: any): void;
|
|
17
|
+
changeWalkConfig(walkConfig: any): void;
|
|
18
|
+
setVolume(volume: number): void;
|
|
19
|
+
changeAvatarVisible(visible: boolean): void;
|
|
20
|
+
switchInvisibleMode(): void;
|
|
21
|
+
destroy(reason?: string): void;
|
|
22
|
+
init(config?: any): void;
|
|
23
|
+
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { IRawAudioFrameData, IRawFaceFrameData, IRawEventFrameData, IRawFrameData, StateChangeInfo } from "../types/frame-data";
|
|
2
|
+
export interface IBodyFrame {
|
|
3
|
+
frame: any;
|
|
4
|
+
frameIndex: number;
|
|
5
|
+
frameState: string;
|
|
6
|
+
id: number;
|
|
7
|
+
name: string;
|
|
8
|
+
body_id: number;
|
|
9
|
+
hfd: boolean;
|
|
10
|
+
sf: number;
|
|
11
|
+
offset: number;
|
|
12
|
+
}
|
|
13
|
+
export declare class DataCacheQueue {
|
|
14
|
+
private TAG;
|
|
15
|
+
private _bodyQueue;
|
|
16
|
+
private _facialQueue;
|
|
17
|
+
private _realFacialQueue;
|
|
18
|
+
private audioQueue;
|
|
19
|
+
private eventQueue;
|
|
20
|
+
private videoIdList;
|
|
21
|
+
private _currentPlayState;
|
|
22
|
+
private _currentTtsaState;
|
|
23
|
+
constructor();
|
|
24
|
+
set currentPlayState(state: string);
|
|
25
|
+
get currentPlayState(): string;
|
|
26
|
+
set currentTtsaState(state: StateChangeInfo | null);
|
|
27
|
+
get currentTtsaState(): StateChangeInfo | null;
|
|
28
|
+
get bodyQueue(): IBodyFrame[];
|
|
29
|
+
_updateBodyImageBitmap(data: IBodyFrame): void;
|
|
30
|
+
clearOldFrames(sf: number): void;
|
|
31
|
+
clearOldFaceData(frameNumber: number): void;
|
|
32
|
+
setVideoIdList(videoId: string): void;
|
|
33
|
+
getVideoIdList(): string[];
|
|
34
|
+
/**
|
|
35
|
+
* 获取指定帧并从Map中删除
|
|
36
|
+
* @param frameIndex 要获取的帧索引
|
|
37
|
+
* @returns 找到的帧数据(未找到则返回undefined)
|
|
38
|
+
*/
|
|
39
|
+
_getBodyImageBitmap(frameIndex: number): IBodyFrame | undefined;
|
|
40
|
+
getBodyVideoNameListLength(): number;
|
|
41
|
+
_getFaceImageBitmap(frameIndex: number, body_id: number): IRawFaceFrameData | null;
|
|
42
|
+
_updateFacial(data: Array<IRawFaceFrameData>): void;
|
|
43
|
+
get facialQueue(): IRawFaceFrameData[];
|
|
44
|
+
_getRealFaceImageBitmap(frameIndex: number, body_id: number): IRawFaceFrameData | null;
|
|
45
|
+
_updateRealFacial(data: Array<IRawFaceFrameData>): void;
|
|
46
|
+
get realFacialQueue(): IRawFaceFrameData[];
|
|
47
|
+
/**
|
|
48
|
+
* 清空所有表情数据(用于切换隐身模式时)
|
|
49
|
+
*/
|
|
50
|
+
clearAllFaceData(): void;
|
|
51
|
+
_updateAudio(data: Array<IRawAudioFrameData>): void;
|
|
52
|
+
_clearAudio(speech_id: number): void;
|
|
53
|
+
_getAudio(frameIndex: number): IRawAudioFrameData | undefined;
|
|
54
|
+
_getAudioInterval(startFrame: number, endFrame: number): IRawAudioFrameData | undefined;
|
|
55
|
+
_updateUiEvent(data: Array<IRawEventFrameData>): void;
|
|
56
|
+
clearOldEvent(frame: number): void;
|
|
57
|
+
clearSubtitleOn(speech_id: number): void;
|
|
58
|
+
_clearEvent(): void;
|
|
59
|
+
_getEvent(frame: number): any;
|
|
60
|
+
_getEventInterval(startFrame: number, endFrame: number): any;
|
|
61
|
+
_mergeEventsBySf(events: any[]): any;
|
|
62
|
+
/**
|
|
63
|
+
* 检查数据是否因seek等原因失效,并清理缓存。
|
|
64
|
+
* @param data 新的数据帧数组
|
|
65
|
+
* @param type 数据类型
|
|
66
|
+
*/
|
|
67
|
+
checkValidData(data: Array<IRawFrameData>, type: string): void;
|
|
68
|
+
destroy(): void;
|
|
69
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* EventDispatcher - 薄客户端模式下的事件分发器
|
|
3
|
+
* 接收来自 Streaming Server 转发的 event_proxy 消息,
|
|
4
|
+
* 分发到用户注册的回调函数。
|
|
5
|
+
*/
|
|
6
|
+
import { AvatarStatus, RenderState, IFrameDataUpdate } from '../types/index';
|
|
7
|
+
import type { IRawWidgetData } from '../types/frame-data';
|
|
8
|
+
import { FPSStats } from '../utils/fpsTracker';
|
|
9
|
+
export interface EventDispatcherCallbacks {
|
|
10
|
+
onStateChange?: (state: string) => void;
|
|
11
|
+
onSpeakStateChange?: (state: string, client_speak_id: string | number) => void;
|
|
12
|
+
onVoiceStateChange?: (state: string, duration?: number, client_speak_id?: string | number) => void;
|
|
13
|
+
onStatusChange?: (status: AvatarStatus) => void;
|
|
14
|
+
onRenderChange?: (state: RenderState) => void;
|
|
15
|
+
onWalkStateChange?: (state: string) => void;
|
|
16
|
+
onWidgetEvent?: (data: IRawWidgetData) => void;
|
|
17
|
+
onMessage?: (error: any) => void;
|
|
18
|
+
onNetworkInfo?: (info: any) => void;
|
|
19
|
+
onFPSUpdate?: (stats: FPSStats) => void;
|
|
20
|
+
onDownloadProgress?: (progress: number) => void;
|
|
21
|
+
onSessionId?: (sessionId: string) => void;
|
|
22
|
+
onFrameDataUpdate?: (data: IFrameDataUpdate) => void;
|
|
23
|
+
}
|
|
24
|
+
export declare class EventDispatcher {
|
|
25
|
+
private callbacks;
|
|
26
|
+
private subtitleText;
|
|
27
|
+
private subtitleVisible;
|
|
28
|
+
constructor(callbacks: EventDispatcherCallbacks);
|
|
29
|
+
/**
|
|
30
|
+
* 处理来自 Streaming Server 的 event_proxy 消息
|
|
31
|
+
*/
|
|
32
|
+
dispatch(eventType: string, data: any, _serverFrame?: number): void;
|
|
33
|
+
getSubtitleText(): string;
|
|
34
|
+
isSubtitleVisible(): boolean;
|
|
35
|
+
}
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
import ResourceManager from "../modules/ResourceManager";
|
|
2
|
+
import { EFrameDataType, IRawFrameData, IRawFaceFrameData, ITtsFaceFrameData, StateChangeInfo, SYNC_FRAME_DATA } from "../types/frame-data";
|
|
3
|
+
import { EErrorCode } from "../types/error";
|
|
4
|
+
import AudioRenderer from "../baseRender/AudioRenderer";
|
|
5
|
+
import type XmovAvatar from "../index";
|
|
6
|
+
import { RenderState, WalkConfig, IFrameDataUpdate } from "../types/index";
|
|
7
|
+
import { Layout } from "../types";
|
|
8
|
+
import { FPSStats } from "../utils/fpsTracker";
|
|
9
|
+
interface CONFIG {
|
|
10
|
+
sdkInstance: XmovAvatar;
|
|
11
|
+
container: Element;
|
|
12
|
+
hardwareAcceleration: string;
|
|
13
|
+
resourceManager: ResourceManager;
|
|
14
|
+
enableDebugger?: boolean;
|
|
15
|
+
enableClientInterrupt?: boolean;
|
|
16
|
+
audioOnlyMode?: boolean;
|
|
17
|
+
onDownloadProgress: (progress: number) => void;
|
|
18
|
+
onStateChange: (state: string) => void;
|
|
19
|
+
onSpeakStateChange: (state: string, client_speak_id: number | string) => void;
|
|
20
|
+
onRenderChange: (state: RenderState, oldState?: RenderState) => void;
|
|
21
|
+
onVoiceStateChange: (state: string, duration?: number, client_speak_id?: string | number) => void;
|
|
22
|
+
onWalkStateChange: (state: string) => void;
|
|
23
|
+
sendVideoInfo: (info: {
|
|
24
|
+
name: string;
|
|
25
|
+
body_id: number;
|
|
26
|
+
id: number;
|
|
27
|
+
}) => void;
|
|
28
|
+
sendMessage: (type: string, payload: any) => void;
|
|
29
|
+
setAudioInfo: (info: {
|
|
30
|
+
sf: number;
|
|
31
|
+
ef: number;
|
|
32
|
+
ad: Uint8Array;
|
|
33
|
+
}) => void;
|
|
34
|
+
setEventData: (info: {
|
|
35
|
+
sf: number;
|
|
36
|
+
ef: number;
|
|
37
|
+
event: Array<any>;
|
|
38
|
+
}) => void;
|
|
39
|
+
renderFrameCallback: (frame: number) => void;
|
|
40
|
+
reportMessage: (message: {
|
|
41
|
+
code: EErrorCode;
|
|
42
|
+
message: string;
|
|
43
|
+
e?: object;
|
|
44
|
+
}) => void;
|
|
45
|
+
sendSdkPoint: (type: string, data: any, extra?: any) => void;
|
|
46
|
+
onFPSUpdate: (stats: FPSStats) => void;
|
|
47
|
+
onFrameDataUpdate?: (data: IFrameDataUpdate) => void;
|
|
48
|
+
}
|
|
49
|
+
export default class RenderScheduler {
|
|
50
|
+
private TAG;
|
|
51
|
+
private dataCacheQueue;
|
|
52
|
+
private sdk;
|
|
53
|
+
private config;
|
|
54
|
+
private avatarRenderer;
|
|
55
|
+
private audioRenderer;
|
|
56
|
+
private uiRenderer;
|
|
57
|
+
private composition;
|
|
58
|
+
private currentSpeechId;
|
|
59
|
+
private resourceManager;
|
|
60
|
+
private frameAnimationController;
|
|
61
|
+
private isStartPlay;
|
|
62
|
+
private renderState;
|
|
63
|
+
private onDownloadProgress;
|
|
64
|
+
private onRenderChange;
|
|
65
|
+
private decoder;
|
|
66
|
+
private saveAndDownload;
|
|
67
|
+
lastSpeechId: number;
|
|
68
|
+
private enableClientInterrupt;
|
|
69
|
+
private audioOnlyMode;
|
|
70
|
+
private _speedResetTimer;
|
|
71
|
+
private offsetFrame;
|
|
72
|
+
private hasSendTimeJson;
|
|
73
|
+
private onFPSUpdate;
|
|
74
|
+
private setAudioInfo;
|
|
75
|
+
private setEventData;
|
|
76
|
+
private reportMessage;
|
|
77
|
+
private sendSdkPoint;
|
|
78
|
+
private sendMessage;
|
|
79
|
+
constructor(config: CONFIG);
|
|
80
|
+
init(): void;
|
|
81
|
+
/**
|
|
82
|
+
* 处理数据:拆解并添加到缓存队列中
|
|
83
|
+
* @param data TTSA 下发的数据
|
|
84
|
+
*/
|
|
85
|
+
handleData(type: EFrameDataType, data: IRawFrameData[] | SYNC_FRAME_DATA): Promise<void>;
|
|
86
|
+
setOffsetFrame(offsetFrame: number): void;
|
|
87
|
+
setVolume(volume: number): void;
|
|
88
|
+
runStartFrameIndex(): void;
|
|
89
|
+
stateChangeHandle(e: any): void;
|
|
90
|
+
stopAudio(speech_id: number): void;
|
|
91
|
+
render(): void;
|
|
92
|
+
stop(): void;
|
|
93
|
+
/**
|
|
94
|
+
* 暂停渲染(停止渲染循环和音频播放)
|
|
95
|
+
* 但继续接收和处理后端推送的数据并放入缓存队列,避免切换到在线时丢失数据导致丢帧
|
|
96
|
+
*/
|
|
97
|
+
pauseRender(): void;
|
|
98
|
+
/**
|
|
99
|
+
* 恢复渲染
|
|
100
|
+
*/
|
|
101
|
+
resumeRender(): void;
|
|
102
|
+
/**
|
|
103
|
+
* 切换隐身模式(暂停/恢复渲染)
|
|
104
|
+
*/
|
|
105
|
+
switchInvisibleMode(): void;
|
|
106
|
+
/**
|
|
107
|
+
* 获取渲染状态
|
|
108
|
+
*/
|
|
109
|
+
getRenderState(): RenderState;
|
|
110
|
+
destroy(): void;
|
|
111
|
+
handleFaceData(data: ITtsFaceFrameData[]): IRawFaceFrameData[];
|
|
112
|
+
forceSyncDecoder(): void;
|
|
113
|
+
_reload(): void;
|
|
114
|
+
_getResumeInfo(): {
|
|
115
|
+
client_frame: number;
|
|
116
|
+
current_ani: string;
|
|
117
|
+
current_ani_frame: number;
|
|
118
|
+
next_state: string;
|
|
119
|
+
};
|
|
120
|
+
sendVoiceEnd(): void;
|
|
121
|
+
_offlineMode(): void;
|
|
122
|
+
_offlineRun(): void;
|
|
123
|
+
ttsaStateChangeHandle(state: StateChangeInfo): void;
|
|
124
|
+
resume(): void;
|
|
125
|
+
/**
|
|
126
|
+
* 设置数字人canvas的显隐状态
|
|
127
|
+
* @param visible 是否可见
|
|
128
|
+
*/
|
|
129
|
+
setAvatarCanvasVisible(visible: boolean): void;
|
|
130
|
+
setCharacterCanvasLayout(layout?: Layout): void;
|
|
131
|
+
setWalkConfig(walkConfig: WalkConfig): void;
|
|
132
|
+
interrupt(type?: string): number;
|
|
133
|
+
seekToFrame(frame: number): void;
|
|
134
|
+
getCurrentFrameIndex(): number;
|
|
135
|
+
getSpeechId(): number;
|
|
136
|
+
getAudioRenderer(): AudioRenderer;
|
|
137
|
+
/**
|
|
138
|
+
* [测试用] 模拟 WebGL 上下文丢失
|
|
139
|
+
*/
|
|
140
|
+
simulateContextLoss(restoreAfterMs?: number, forceFail?: boolean): void;
|
|
141
|
+
/**
|
|
142
|
+
* [测试用] 获取当前恢复状态信息
|
|
143
|
+
*/
|
|
144
|
+
getRecoveryStatus(): {
|
|
145
|
+
isRecovering: boolean;
|
|
146
|
+
stage: string;
|
|
147
|
+
attempts: number;
|
|
148
|
+
maxAttempts: number;
|
|
149
|
+
isUsingReducedResolution: boolean;
|
|
150
|
+
};
|
|
151
|
+
/**
|
|
152
|
+
* [测试用] 批量测试分阶段恢复
|
|
153
|
+
*/
|
|
154
|
+
testStagedRecovery(): Promise<void>;
|
|
155
|
+
/**
|
|
156
|
+
* 手动重启 WebGL:创建新 canvas + 新 GLDevice + 新 GLPipeline
|
|
157
|
+
* 用于分阶段恢复全部失败后的手动恢复
|
|
158
|
+
* @returns 是否成功
|
|
159
|
+
*/
|
|
160
|
+
restartWebGL(): boolean;
|
|
161
|
+
}
|
|
162
|
+
export {};
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export default SaveAndDownload;
|
|
2
|
+
declare class SaveAndDownload {
|
|
3
|
+
constructor(fileName?: string, enabled?: boolean);
|
|
4
|
+
fileName: string;
|
|
5
|
+
enabled: boolean;
|
|
6
|
+
data: {};
|
|
7
|
+
downloadButton: HTMLAnchorElement | null;
|
|
8
|
+
initDownloadButton(): void;
|
|
9
|
+
writeFields(fields: any): void;
|
|
10
|
+
appendMultipleToArray(fieldName: any, items: any): void;
|
|
11
|
+
download(): void;
|
|
12
|
+
generateJSONContent(): string;
|
|
13
|
+
downloadFile(content: any, fileName: any, contentType: any): void;
|
|
14
|
+
}
|