@xmov/avatar 2.0.0-alpha.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +139 -0
- package/dist/agent/__tests__/agent.test.d.ts +1 -0
- package/dist/agent/audio-debug.d.ts +3 -0
- package/dist/agent/audio-uplink.d.ts +62 -0
- package/dist/agent/avatar.cjs +2 -0
- package/dist/agent/avatar.cjs.map +1 -0
- package/dist/agent/avatar.modern.js +2 -0
- package/dist/agent/avatar.modern.js.map +1 -0
- package/dist/agent/avatar.module.js +2 -0
- package/dist/agent/avatar.module.js.map +1 -0
- package/dist/agent/avatar.umd.js +2 -0
- package/dist/agent/avatar.umd.js.map +1 -0
- package/dist/agent/e2e-client.d.ts +32 -0
- package/dist/agent/fixed-audio-track.d.ts +48 -0
- package/dist/agent/index.cjs +5 -0
- package/dist/agent/index.d.ts +104 -0
- package/dist/agent/index.umd.js +10 -0
- package/dist/agent/microphone.d.ts +49 -0
- package/dist/agent/types.d.ts +153 -0
- package/dist/baseRender/AudioRenderer.d.ts +46 -0
- package/dist/baseRender/AudioWorklet.d.ts +78 -0
- package/dist/baseRender/AvatarRenderer.d.ts +226 -0
- package/dist/baseRender/MSEAudioPlayer.d.ts +51 -0
- package/dist/baseRender/UIRenderer.d.ts +39 -0
- package/dist/baseRender/pcm-audio-processor.d.ts +63 -0
- package/dist/control/APIForwarder.d.ts +23 -0
- package/dist/control/DataCacheQueue.d.ts +69 -0
- package/dist/control/EventDispatcher.d.ts +35 -0
- package/dist/control/RenderScheduler.d.ts +162 -0
- package/dist/control/SaveAndDownload.d.ts +14 -0
- package/dist/control/StreamingClient.d.ts +33 -0
- package/dist/control/VideoPlayer.d.ts +47 -0
- package/dist/control/ttsa.d.ts +111 -0
- package/dist/encoding/media-recorder-encoder.d.ts +33 -0
- package/dist/encoding/pcm-webm-encoder.d.ts +13 -0
- package/dist/encoding/webcodec-opus-encoder.d.ts +41 -0
- package/dist/encoding/webm-muxer.d.ts +90 -0
- package/dist/index.cjs +2 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.ts +231 -0
- package/dist/index.modern.js +2 -0
- package/dist/index.modern.js.map +1 -0
- package/dist/index.module.js +2 -0
- package/dist/index.module.js.map +1 -0
- package/dist/index.umd.js +2 -0
- package/dist/index.umd.js.map +1 -0
- package/dist/modules/Composition.d.ts +28 -0
- package/dist/modules/ResourceManager.d.ts +295 -0
- package/dist/modules/TrackRenderer/base-track.d.ts +4 -0
- package/dist/modules/TrackRenderer/index.d.ts +12 -0
- package/dist/modules/TrackRenderer/render-implements.d.ts +19 -0
- package/dist/modules/TrackRenderer/track-pic.d.ts +8 -0
- package/dist/modules/TrackRenderer/track-subtitle.d.ts +8 -0
- package/dist/modules/cache-manager.d.ts +5 -0
- package/dist/modules/decoder.d.ts +84 -0
- package/dist/modules/error-handle.d.ts +17 -0
- package/dist/modules/network.d.ts +21 -0
- package/dist/proto/face_data_pb.d.ts +0 -0
- package/dist/types/capability.d.ts +15 -0
- package/dist/types/error.d.ts +70 -0
- package/dist/types/event.d.ts +25 -0
- package/dist/types/frame-data.d.ts +107 -0
- package/dist/types/index.d.ts +187 -0
- package/dist/types/render.d.ts +18 -0
- package/dist/utils/DataInterface.d.ts +113 -0
- package/dist/utils/GLDevice.d.ts +46 -0
- package/dist/utils/GLPipeline.d.ts +84 -0
- package/dist/utils/GLPipelineDebugTools.d.ts +5 -0
- package/dist/utils/Math.d.ts +21 -0
- package/dist/utils/__tests__/capability-checker.test.d.ts +1 -0
- package/dist/utils/audio-session.d.ts +11 -0
- package/dist/utils/audio.d.ts +1 -0
- package/dist/utils/capability-checker.d.ts +23 -0
- package/dist/utils/encodeToken.d.ts +7 -0
- package/dist/utils/face.d.ts +1 -0
- package/dist/utils/float32-decoder.d.ts +26 -0
- package/dist/utils/fpsTracker.d.ts +27 -0
- package/dist/utils/index.d.ts +22 -0
- package/dist/utils/logger.d.ts +9 -0
- package/dist/utils/media-recorder-timestamp.d.ts +15 -0
- package/dist/utils/pcmAnalyzer.d.ts +40 -0
- package/dist/utils/perfermance.d.ts +15 -0
- package/dist/utils/request.d.ts +19 -0
- package/dist/utils/requestAnimateFrames.d.ts +31 -0
- package/dist/utils/time.d.ts +9 -0
- package/dist/view/DebugOverlay.d.ts +54 -0
- package/dist/worker/streaming-video.d.ts +1 -0
- package/package.json +82 -0
- package/src/agent/__tests__/agent.test.ts +3787 -0
- package/src/agent/audio-debug.ts +36 -0
- package/src/agent/audio-uplink.ts +392 -0
- package/src/agent/e2e-client.ts +215 -0
- package/src/agent/fixed-audio-track.ts +547 -0
- package/src/agent/index.ts +1393 -0
- package/src/agent/microphone.ts +534 -0
- package/src/agent/types.ts +197 -0
- package/src/baseRender/AudioRenderer.ts +317 -0
- package/src/baseRender/AudioWorklet.ts +776 -0
- package/src/baseRender/AvatarRenderer.ts +1559 -0
- package/src/baseRender/MSEAudioPlayer.ts +243 -0
- package/src/baseRender/UIRenderer.ts +195 -0
- package/src/baseRender/pcm-audio-processor.js +267 -0
- package/src/control/APIForwarder.ts +67 -0
- package/src/control/DataCacheQueue.ts +372 -0
- package/src/control/EventDispatcher.ts +106 -0
- package/src/control/RenderScheduler.ts +945 -0
- package/src/control/SaveAndDownload.js +77 -0
- package/src/control/StreamingClient.ts +138 -0
- package/src/control/VideoPlayer.ts +306 -0
- package/src/control/ttsa.ts +676 -0
- package/src/encoding/media-recorder-encoder.ts +175 -0
- package/src/encoding/pcm-webm-encoder.ts +72 -0
- package/src/encoding/webcodec-opus-encoder.ts +258 -0
- package/src/encoding/webm-muxer.ts +337 -0
- package/src/global.d.ts +105 -0
- package/src/index.ts +1920 -0
- package/src/modules/Composition.ts +79 -0
- package/src/modules/ResourceManager.ts +899 -0
- package/src/modules/TrackRenderer/base-track.ts +7 -0
- package/src/modules/TrackRenderer/index.ts +40 -0
- package/src/modules/TrackRenderer/render-implements.ts +269 -0
- package/src/modules/TrackRenderer/track-pic.ts +19 -0
- package/src/modules/TrackRenderer/track-subtitle.ts +16 -0
- package/src/modules/cache-manager.ts +10 -0
- package/src/modules/decoder.ts +515 -0
- package/src/modules/error-handle.ts +20 -0
- package/src/modules/network.ts +71 -0
- package/src/proto/face_data.proto +45 -0
- package/src/proto/face_data_pb.js +1297 -0
- package/src/proto/protobuf.min.js +8 -0
- package/src/types/capability.ts +23 -0
- package/src/types/error.ts +211 -0
- package/src/types/event.ts +38 -0
- package/src/types/frame-data.ts +121 -0
- package/src/types/index.ts +207 -0
- package/src/types/render.ts +18 -0
- package/src/utils/DataInterface.ts +746 -0
- package/src/utils/GLDevice.ts +286 -0
- package/src/utils/GLPipeline.ts +1293 -0
- package/src/utils/GLPipelineDebugTools.ts +174 -0
- package/src/utils/Math.ts +266 -0
- package/src/utils/__tests__/capability-checker.test.ts +159 -0
- package/src/utils/audio-session.ts +144 -0
- package/src/utils/audio.ts +44 -0
- package/src/utils/blueimp-md5.d.ts +3 -0
- package/src/utils/capability-checker.ts +379 -0
- package/src/utils/encodeToken.ts +100 -0
- package/src/utils/face.ts +29 -0
- package/src/utils/float32-decoder.js +130 -0
- package/src/utils/fpsTracker.ts +100 -0
- package/src/utils/index.ts +182 -0
- package/src/utils/logger.js +36 -0
- package/src/utils/media-recorder-timestamp.ts +79 -0
- package/src/utils/pcmAnalyzer.ts +154 -0
- package/src/utils/perfermance.ts +141 -0
- package/src/utils/request.ts +95 -0
- package/src/utils/requestAnimateFrames.ts +195 -0
- package/src/utils/time.ts +32 -0
- package/src/view/DebugOverlay.ts +404 -0
- package/src/worker/streaming-video.ts +4647 -0
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import AvatarRender from "../baseRender/AvatarRenderer";
|
|
2
|
+
import UIRenderer from "../baseRender/UIRenderer";
|
|
3
|
+
import AudioRenderer from "../baseRender/AudioRenderer";
|
|
4
|
+
import { FPSTracker, FPSStats } from "../utils/fpsTracker";
|
|
5
|
+
type TRenderer = UIRenderer;
|
|
6
|
+
export default class Composition {
|
|
7
|
+
TAG = "[Composition]";
|
|
8
|
+
container: Element;
|
|
9
|
+
avatarRenderer: AvatarRender;
|
|
10
|
+
audioRenderer: AudioRenderer;
|
|
11
|
+
restRenderers: TRenderer[];
|
|
12
|
+
private fpsTracker: FPSTracker;
|
|
13
|
+
private onFPSUpdate?: (stats: FPSStats) => void;
|
|
14
|
+
audioOnlyMode: boolean;
|
|
15
|
+
|
|
16
|
+
constructor(options: {
|
|
17
|
+
container: Element;
|
|
18
|
+
avatarRenderer: AvatarRender;
|
|
19
|
+
restRenderers: TRenderer[];
|
|
20
|
+
audioRenderer: AudioRenderer;
|
|
21
|
+
audioOnlyMode?: boolean;
|
|
22
|
+
onFPSUpdate?: (stats: FPSStats) => void;
|
|
23
|
+
}) {
|
|
24
|
+
this.container = options.container;
|
|
25
|
+
this.avatarRenderer = options.avatarRenderer;
|
|
26
|
+
this.restRenderers = options.restRenderers;
|
|
27
|
+
this.audioRenderer = options.audioRenderer;
|
|
28
|
+
this.onFPSUpdate = options.onFPSUpdate;
|
|
29
|
+
this.audioOnlyMode = options.audioOnlyMode || false;
|
|
30
|
+
this.fpsTracker = new FPSTracker();
|
|
31
|
+
if (!this.audioOnlyMode) {
|
|
32
|
+
this.container.appendChild(options.avatarRenderer.canvas);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
compose(frameIndex: number) {
|
|
37
|
+
if (!this.audioOnlyMode) {
|
|
38
|
+
const avatarResult = this.avatarRenderer.render(frameIndex);
|
|
39
|
+
const rendered = avatarResult.rendered;
|
|
40
|
+
|
|
41
|
+
this.fpsTracker.tick(rendered);
|
|
42
|
+
|
|
43
|
+
const stats = this.fpsTracker.getStats();
|
|
44
|
+
if (this.onFPSUpdate && stats.current > 0) {
|
|
45
|
+
this.onFPSUpdate(stats);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// Audio rendering always executes
|
|
50
|
+
this.audioRenderer.render(frameIndex);
|
|
51
|
+
// UI event rendering always executes
|
|
52
|
+
for (const r of this.restRenderers) {
|
|
53
|
+
const layer = r.render(frameIndex);
|
|
54
|
+
r.clearTrackerRenderer();
|
|
55
|
+
if (layer) {
|
|
56
|
+
for (const l of layer) {
|
|
57
|
+
const ele = l?.render();
|
|
58
|
+
if (ele) {
|
|
59
|
+
this.container.appendChild(ele as Node);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
getFPSStats(): FPSStats {
|
|
67
|
+
return this.fpsTracker.getStats();
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
stop() {
|
|
71
|
+
this.audioRenderer.stop(-1);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
destroy() {
|
|
75
|
+
this.avatarRenderer.destroy();
|
|
76
|
+
this.audioRenderer.destroy();
|
|
77
|
+
this.restRenderers.forEach((r) => r.destroy());
|
|
78
|
+
}
|
|
79
|
+
}
|