@xmov/avatar 2.4.0 → 2.5.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. package/README.md +34 -6
  2. package/package-dist/agent/avatar.cjs +1 -0
  3. package/package-dist/agent/avatar.modern.js +1 -0
  4. package/package-dist/agent/avatar.module.js +1 -0
  5. package/package-dist/agent/avatar.umd.js +1 -0
  6. package/package-dist/agent/index.d.ts +42 -0
  7. package/{dist/agent/types.d.ts → package-dist/agent/public-types.d.ts} +115 -116
  8. package/package-dist/index.cjs +3 -0
  9. package/package-dist/index.d.ts +34 -0
  10. package/package-dist/index.modern.js +1 -0
  11. package/package-dist/index.module.js +1 -0
  12. package/package-dist/index.umd.js +14 -0
  13. package/package-dist/public-types.d.ts +205 -0
  14. package/package.json +41 -33
  15. package/dist/agent/avatar.cjs +0 -1
  16. package/dist/agent/avatar.modern.js +0 -1
  17. package/dist/agent/avatar.module.js +0 -1
  18. package/dist/agent/avatar.umd.js +0 -1
  19. package/dist/agent/index.d.ts +0 -137
  20. package/dist/baseRender/AudioRenderer.d.ts +0 -46
  21. package/dist/baseRender/AudioWorklet.d.ts +0 -78
  22. package/dist/baseRender/AvatarRenderer.d.ts +0 -225
  23. package/dist/baseRender/MSEAudioPlayer.d.ts +0 -51
  24. package/dist/baseRender/UIRenderer.d.ts +0 -39
  25. package/dist/baseRender/pcm-audio-processor.d.ts +0 -63
  26. package/dist/control/APIForwarder.d.ts +0 -23
  27. package/dist/control/DataCacheQueue.d.ts +0 -69
  28. package/dist/control/EventDispatcher.d.ts +0 -35
  29. package/dist/control/RenderScheduler.d.ts +0 -171
  30. package/dist/control/SaveAndDownload.d.ts +0 -14
  31. package/dist/control/StreamingClient.d.ts +0 -33
  32. package/dist/control/VideoPlayer.d.ts +0 -47
  33. package/dist/control/ttsa.d.ts +0 -125
  34. package/dist/encoding/media-recorder-encoder.d.ts +0 -33
  35. package/dist/encoding/pcm-webm-encoder.d.ts +0 -13
  36. package/dist/encoding/webcodec-opus-encoder.d.ts +0 -41
  37. package/dist/encoding/webm-muxer.d.ts +0 -90
  38. package/dist/index.cjs +0 -2
  39. package/dist/index.cjs.map +0 -1
  40. package/dist/index.d.ts +0 -251
  41. package/dist/index.modern.js +0 -2
  42. package/dist/index.modern.js.map +0 -1
  43. package/dist/index.module.js +0 -2
  44. package/dist/index.module.js.map +0 -1
  45. package/dist/index.umd.js +0 -2
  46. package/dist/index.umd.js.map +0 -1
  47. package/dist/modules/Composition.d.ts +0 -28
  48. package/dist/modules/ResourceManager.d.ts +0 -299
  49. package/dist/modules/TrackRenderer/base-track.d.ts +0 -4
  50. package/dist/modules/TrackRenderer/index.d.ts +0 -12
  51. package/dist/modules/TrackRenderer/render-implements.d.ts +0 -19
  52. package/dist/modules/TrackRenderer/track-pic.d.ts +0 -8
  53. package/dist/modules/TrackRenderer/track-subtitle.d.ts +0 -8
  54. package/dist/modules/cache-manager.d.ts +0 -5
  55. package/dist/modules/decoder.d.ts +0 -84
  56. package/dist/modules/error-handle.d.ts +0 -17
  57. package/dist/modules/network.d.ts +0 -21
  58. package/dist/modules/sentry-reporter.d.ts +0 -58
  59. package/dist/proto/face_data_pb.d.ts +0 -0
  60. package/dist/types/capability.d.ts +0 -15
  61. package/dist/types/error.d.ts +0 -70
  62. package/dist/types/event.d.ts +0 -25
  63. package/dist/types/frame-data.d.ts +0 -107
  64. package/dist/types/index.d.ts +0 -196
  65. package/dist/types/render.d.ts +0 -18
  66. package/dist/utils/DataInterface.d.ts +0 -113
  67. package/dist/utils/GLDevice.d.ts +0 -46
  68. package/dist/utils/GLPipeline.d.ts +0 -84
  69. package/dist/utils/GLPipelineDebugTools.d.ts +0 -5
  70. package/dist/utils/Math.d.ts +0 -21
  71. package/dist/utils/__tests__/capability-checker.test.d.ts +0 -1
  72. package/dist/utils/audio-session.d.ts +0 -11
  73. package/dist/utils/audio.d.ts +0 -1
  74. package/dist/utils/capability-checker.d.ts +0 -28
  75. package/dist/utils/encodeToken.d.ts +0 -14
  76. package/dist/utils/face.d.ts +0 -1
  77. package/dist/utils/float32-decoder.d.ts +0 -26
  78. package/dist/utils/fpsTracker.d.ts +0 -27
  79. package/dist/utils/index.d.ts +0 -22
  80. package/dist/utils/logger.d.ts +0 -9
  81. package/dist/utils/media-recorder-timestamp.d.ts +0 -15
  82. package/dist/utils/pcmAnalyzer.d.ts +0 -40
  83. package/dist/utils/perfermance.d.ts +0 -15
  84. package/dist/utils/request.d.ts +0 -20
  85. package/dist/utils/requestAnimateFrames.d.ts +0 -31
  86. package/dist/utils/time.d.ts +0 -9
  87. package/dist/utils/video-probe-asset.d.ts +0 -28
  88. package/dist/view/DebugOverlay.d.ts +0 -54
  89. package/dist/worker/streaming-video.d.ts +0 -1
  90. package/src/baseRender/AudioRenderer.ts +0 -328
  91. package/src/baseRender/AudioWorklet.ts +0 -776
  92. package/src/baseRender/AvatarRenderer.ts +0 -1550
  93. package/src/baseRender/MSEAudioPlayer.ts +0 -254
  94. package/src/baseRender/UIRenderer.ts +0 -195
  95. package/src/baseRender/pcm-audio-processor.js +0 -267
  96. package/src/control/APIForwarder.ts +0 -67
  97. package/src/control/DataCacheQueue.ts +0 -372
  98. package/src/control/EventDispatcher.ts +0 -106
  99. package/src/control/RenderScheduler.ts +0 -1000
  100. package/src/control/SaveAndDownload.js +0 -77
  101. package/src/control/StreamingClient.ts +0 -138
  102. package/src/control/VideoPlayer.ts +0 -306
  103. package/src/control/ttsa.ts +0 -754
  104. package/src/encoding/media-recorder-encoder.ts +0 -175
  105. package/src/encoding/pcm-webm-encoder.ts +0 -72
  106. package/src/encoding/webcodec-opus-encoder.ts +0 -258
  107. package/src/encoding/webm-muxer.ts +0 -337
  108. package/src/global.d.ts +0 -112
  109. package/src/index.ts +0 -2053
  110. package/src/modules/Composition.ts +0 -79
  111. package/src/modules/ResourceManager.ts +0 -926
  112. package/src/modules/TrackRenderer/base-track.ts +0 -7
  113. package/src/modules/TrackRenderer/index.ts +0 -40
  114. package/src/modules/TrackRenderer/render-implements.ts +0 -269
  115. package/src/modules/TrackRenderer/track-pic.ts +0 -19
  116. package/src/modules/TrackRenderer/track-subtitle.ts +0 -16
  117. package/src/modules/cache-manager.ts +0 -10
  118. package/src/modules/decoder.ts +0 -548
  119. package/src/modules/error-handle.ts +0 -20
  120. package/src/modules/network.ts +0 -71
  121. package/src/modules/sentry-reporter.ts +0 -171
  122. package/src/proto/face_data.proto +0 -45
  123. package/src/proto/face_data_pb.js +0 -1297
  124. package/src/proto/protobuf.min.js +0 -8
  125. package/src/types/capability.ts +0 -24
  126. package/src/types/error.ts +0 -211
  127. package/src/types/event.ts +0 -38
  128. package/src/types/frame-data.ts +0 -121
  129. package/src/types/index.ts +0 -220
  130. package/src/types/render.ts +0 -18
  131. package/src/utils/DataInterface.ts +0 -746
  132. package/src/utils/GLDevice.ts +0 -286
  133. package/src/utils/GLPipeline.ts +0 -1293
  134. package/src/utils/GLPipelineDebugTools.ts +0 -174
  135. package/src/utils/Math.ts +0 -266
  136. package/src/utils/__tests__/capability-checker.test.ts +0 -159
  137. package/src/utils/audio-session.ts +0 -144
  138. package/src/utils/audio.ts +0 -44
  139. package/src/utils/blueimp-md5.d.ts +0 -3
  140. package/src/utils/capability-checker.ts +0 -762
  141. package/src/utils/encodeToken.ts +0 -115
  142. package/src/utils/face.ts +0 -29
  143. package/src/utils/float32-decoder.js +0 -130
  144. package/src/utils/fpsTracker.ts +0 -100
  145. package/src/utils/index.ts +0 -182
  146. package/src/utils/logger.js +0 -36
  147. package/src/utils/media-recorder-timestamp.ts +0 -79
  148. package/src/utils/pcmAnalyzer.ts +0 -154
  149. package/src/utils/perfermance.ts +0 -141
  150. package/src/utils/request.ts +0 -107
  151. package/src/utils/requestAnimateFrames.ts +0 -195
  152. package/src/utils/time.ts +0 -32
  153. package/src/utils/video-probe-asset.ts +0 -99
  154. package/src/view/DebugOverlay.ts +0 -409
  155. package/src/worker/streaming-video.ts +0 -4647
  156. /package/{dist → package-dist}/agent/index.cjs +0 -0
  157. /package/{dist → package-dist}/agent/index.umd.js +0 -0
@@ -1,107 +0,0 @@
1
- import type { IWidget } from "./event";
2
- import { IBRAnimationFrameData_NN } from "../utils/DataInterface";
3
- export declare enum EFrameDataType {
4
- AUDIO = "tts_audio",
5
- BODY = "body_data",
6
- FACE = "face_data",
7
- EVENT = "event_data",
8
- AA_FRAME = "aa_frame",
9
- SYNC_FRAME = "sync_frame"
10
- }
11
- export declare const FRAME_SOURCE_SOURCE = "xmov_avatar_js_sdk";
12
- /** socket.io 原数据格式 */
13
- export interface IRawBaseFrameData {
14
- sf: number;
15
- ef: number;
16
- }
17
- export interface IRawBodyFrameData extends IRawBaseFrameData {
18
- hfd: boolean;
19
- aef: number;
20
- asf: number;
21
- id: number;
22
- n: string;
23
- s: string;
24
- body_id: number;
25
- x_offset: Uint8Array;
26
- }
27
- /**
28
- * tts下发的face数据
29
- */
30
- export interface ITtsFaceFrameData {
31
- body_id: number;
32
- bsw: number[];
33
- ef: number;
34
- htmi?: number[];
35
- htpw?: number[];
36
- id: number;
37
- mjt?: number[][];
38
- s: string;
39
- sf: number;
40
- cs: string;
41
- ttmi?: number[];
42
- ttpw?: number[];
43
- ms: {
44
- index: number;
45
- weights: number[];
46
- }[];
47
- js: {
48
- translate: number[];
49
- rotate: number[];
50
- }[];
51
- face_frame_type: number;
52
- asf?: number;
53
- aef?: number;
54
- n?: string;
55
- }
56
- export interface IRawFaceFrameData extends IRawBaseFrameData {
57
- FaceFrameData: IBRAnimationFrameData_NN;
58
- frameIndex: number;
59
- state: string;
60
- id: number;
61
- body_id: number;
62
- cs: string;
63
- face_frame_type: number;
64
- }
65
- export interface IRawAudioFrameData extends IRawBaseFrameData {
66
- ad: any;
67
- id: number;
68
- sid: number;
69
- multi_turn_conversation_id: string;
70
- }
71
- export interface IRawWidgetData {
72
- type: string;
73
- data?: IWidget;
74
- text?: string;
75
- speech_id?: number;
76
- client_speak_id?: number;
77
- }
78
- export interface IRawEventFrameData extends IRawBaseFrameData {
79
- id: number;
80
- s: string;
81
- e: IRawWidgetData[];
82
- }
83
- export type IRawFrameData = IRawBodyFrameData | ITtsFaceFrameData | IRawFaceFrameData | IRawAudioFrameData | IRawEventFrameData;
84
- export interface SYNC_FRAME_DATA {
85
- server_time: number;
86
- client_time: number;
87
- server_client_frame: number;
88
- source: string;
89
- }
90
- export type bodyFile = {
91
- name: string;
92
- id: number;
93
- frameState: string;
94
- start: number;
95
- end: number;
96
- data: ArrayBuffer;
97
- hfd: boolean;
98
- startFrameIndex: number;
99
- endFrameIndex: number;
100
- body_id: number;
101
- x_offset: any;
102
- };
103
- export interface StateChangeInfo {
104
- state: string;
105
- params: object;
106
- start_frame: number;
107
- }
@@ -1,196 +0,0 @@
1
- import type XmovAvatar from "../index";
2
- import type { IRawWidgetData } from "./frame-data";
3
- import type { SDKError } from "./error";
4
- import type { TDownloadProgress } from "../modules/ResourceManager";
5
- import { FPSStats } from "../utils/fpsTracker";
6
- export { ERROR_CODE_MAP, ERROR_NAME_MAP } from './error';
7
- export type { CapabilitySeverity, CapabilityName, ICapabilityCheckItem, ICapabilityCheckReport, } from './capability';
8
- export declare enum AvatarStatus {
9
- online = 0,
10
- offline = 1,
11
- network_on = 2,
12
- network_off = 3,
13
- close = 4,
14
- visible = 5,
15
- invisible = 6,
16
- stopped = 7
17
- }
18
- /**
19
- * 渲染状态枚举
20
- * 用于表示渲染器的状态,与数字人状态(AvatarStatus)分离
21
- */
22
- export declare enum RenderState {
23
- init = "init",
24
- rendering = "rendering",
25
- pausing = "pausing",
26
- paused = "paused",
27
- resumed = "resumed",
28
- stopped = "stopped"
29
- }
30
- /**
31
- * 网络情况
32
- * @params downlink 下载速率(Mbps)
33
- * @params rtt 延迟(毫秒)
34
- */
35
- export interface INetworkInfo {
36
- downlink: number;
37
- rtt: number;
38
- networkStatus: boolean;
39
- }
40
- export declare enum InitModel {
41
- normal = "normal",
42
- invisible = "invisible"
43
- }
44
- export interface IInitParams {
45
- scaleDownBy?: any;
46
- useAlphaMask?: any;
47
- initModel?: InitModel;
48
- onDownloadProgress: TDownloadProgress;
49
- }
50
- export interface IAvatarOptions {
51
- sdkInstance?: XmovAvatar;
52
- containerId: string;
53
- container?: HTMLElement;
54
- appId: string;
55
- appSecret: string;
56
- gatewayServer: string;
57
- tag?: string;
58
- gateway_type?: string;
59
- asr_id?: number;
60
- llm_id?: number;
61
- session_speak_req_id?: 0 | 1;
62
- headers?: Record<string, string>;
63
- cacheServer?: string;
64
- env?: string;
65
- config?: any;
66
- hardwareAcceleration?: string;
67
- enableClientInterrupt?: boolean;
68
- enableLogger?: boolean;
69
- enableDebugger?: boolean;
70
- audioOnlyMode?: boolean;
71
- renderMode?: 'client' | 'server';
72
- streamingServer?: string;
73
- useAlphaMask?: boolean;
74
- scaleDownBy?: number;
75
- onMessage: (error: SDKError) => void;
76
- onStartSessionWarning?: (message: Object) => void;
77
- onAAFrameHandle?: (data: any) => void;
78
- onNetworkInfo?(networkInfo: INetworkInfo): void;
79
- onWidgetEvent?(data: IRawWidgetData): void;
80
- proxyWidget?: {
81
- [key: string]: (data: any) => void;
82
- };
83
- onStateChange?(state: string): void;
84
- onStateRenderChange?(state: string, duration: number): void;
85
- onSpeakStateChange?(state: string, client_speak_id: string | number): void;
86
- onRenderChange?(state: RenderState): void;
87
- onVoiceStateChange?(state: string, duration?: number, client_speak_id?: string | number): void;
88
- onStatusChange?(status: AvatarStatus): void;
89
- onWalkStateChange?(state: string): void;
90
- onFPSUpdate?(stats: FPSStats): void;
91
- /**
92
- * 数字人音频实时播放数据回调
93
- * - MSE/WebM 模式(raw_audio=false):通过 AudioWorklet + WebCodecs 捕获和编码
94
- * - PCM 模式(raw_audio=true):通过 capture worklet + WebCodecs/MediaRecorder 编码
95
- * 两种模式输出格式完全一致(WebM/Opus),业务侧无需区分
96
- */
97
- onAudioPlaybackData?(data: IAudioPlaybackData): void;
98
- /**
99
- * Sentry 错误监控配置
100
- * 仅在公有云构建(ENABLE_SENTRY=true)时生效,私有化构建会忽略此配置
101
- * 设为 { enabled: false } 可禁用 Sentry 上报
102
- */
103
- sentry?: {
104
- enabled?: boolean;
105
- /**
106
- * 性能 Traces 采样率(0-1),默认 1.0(100% 全量)
107
- * 容量受限时可下调,如 0.1(10%)
108
- */
109
- tracesSampleRate?: number;
110
- };
111
- onFrameDataUpdate?(data: IFrameDataUpdate): void;
112
- }
113
- /**
114
- * 音频播放数据回调参数(farend 格式)
115
- * 用于 ASR 回声消除(AEC),由 Agent 封装为 AU v1 farend Binary 帧
116
- */
117
- export interface IAudioPlaybackData {
118
- data: ArrayBuffer;
119
- codec: string;
120
- isFirstChunk: boolean;
121
- sampleRate: number;
122
- samples: number;
123
- timestamp: number;
124
- speech_id: number;
125
- }
126
- /**
127
- * 帧数据回调参数
128
- * 用于返回每一帧的 face + body 原始数据(不包含渲染数据)
129
- */
130
- export interface IFrameDataUpdate {
131
- /** 客户端帧号 */
132
- frameIndex: number;
133
- /** body 数据 */
134
- body: {
135
- /** body_id */
136
- body_id: number;
137
- /** 视频名称 */
138
- name: string;
139
- /** 视频内帧号(相对于视频起始的帧偏移) */
140
- video_frame: number;
141
- /** 起始帧号 */
142
- sf: number;
143
- /** 当前状态 */
144
- state: string;
145
- } | null;
146
- /** face 数据(不包含骨骼渲染数据) */
147
- face: {
148
- /** body_id */
149
- body_id: number;
150
- /** 起始帧号 */
151
- sf: number;
152
- /** 结束帧号 */
153
- ef: number;
154
- /** 状态 */
155
- state: string;
156
- /** face 帧类型:0=原始数据,1=实时数据 */
157
- face_frame_type: number;
158
- } | null;
159
- }
160
- export interface DeviceInfo {
161
- device_id: string;
162
- device_brand: string;
163
- device_model: string;
164
- os_version: string;
165
- browser: string;
166
- browser_version: string;
167
- browser_ua: string;
168
- sdk_version: string;
169
- resolution: {
170
- width: number;
171
- height: number;
172
- };
173
- download_speed: number;
174
- decode_speed: number;
175
- render_speed: number;
176
- }
177
- export interface Layout {
178
- container: {
179
- size: number[];
180
- };
181
- avatar: {
182
- v_align: string;
183
- h_align: string;
184
- scale: number | string;
185
- offset_x: number;
186
- offset_y: number;
187
- };
188
- }
189
- export interface WalkConfig {
190
- min_x_offset: number;
191
- max_x_offset: number;
192
- walk_points: {
193
- [key: string]: number;
194
- };
195
- init_point?: number;
196
- }
@@ -1,18 +0,0 @@
1
- export interface IBackgroundRender {
2
- render(): HTMLElement;
3
- }
4
- export interface IImgRender {
5
- render(): HTMLElement;
6
- }
7
- export interface ITextRender {
8
- render(): HTMLElement;
9
- }
10
- export interface IVideoRender {
11
- render(): HTMLElement;
12
- }
13
- export interface I3DRender {
14
- render(): HTMLCanvasElement;
15
- }
16
- export interface ISubtitleRender {
17
- render(): HTMLElement;
18
- }
@@ -1,113 +0,0 @@
1
- import * as GLMath from "./Math";
2
- export interface DataViewWithCustomOffset {
3
- offset: number;
4
- dataview: DataView;
5
- }
6
- declare const TensorDataTypeMap: {
7
- B: Uint8ArrayConstructor;
8
- b: Int8ArrayConstructor;
9
- H: Uint16ArrayConstructor;
10
- h: Int16ArrayConstructor;
11
- I: Uint32ArrayConstructor;
12
- i: Int32ArrayConstructor;
13
- L: BigUint64ArrayConstructor;
14
- l: BigInt64ArrayConstructor;
15
- f: Float32ArrayConstructor;
16
- d: Float64ArrayConstructor;
17
- };
18
- type ValueOf<T> = T[keyof T];
19
- export type TensorDataConstructorType = ValueOf<typeof TensorDataTypeMap>;
20
- export type TensorDataType = InstanceType<TensorDataConstructorType>;
21
- export declare class Tensor {
22
- size: number[];
23
- data: TensorDataType;
24
- constructor(size: number[], data: TensorDataType);
25
- itemSize(): number;
26
- byteSize(): number;
27
- static zeros(size: number[], dataType?: TensorDataConstructorType): Tensor;
28
- part(...index: number[]): Tensor;
29
- static deserialize(buffer: ArrayBuffer | DataViewWithCustomOffset): Tensor;
30
- }
31
- export declare class CameraConfig {
32
- focalLength: number;
33
- filmAperture: [number, number];
34
- translation: GLMath.vec3;
35
- rotation: GLMath.vec3;
36
- getIntrinsicMatrix(resolution: [number, number]): GLMath.mat3;
37
- getProjMatrix(resolution: [number, number], near?: number, far?: number): GLMath.mat4;
38
- getExtrinsicMatrix(): GLMath.mat4;
39
- static deserialize(buffer: ArrayBuffer | DataViewWithCustomOffset): CameraConfig;
40
- }
41
- export declare class RigidTransform {
42
- translation: GLMath.vec3;
43
- rotQuaternion: GLMath.vec4;
44
- constructor(translation?: GLMath.vec3, rotQuaternion?: GLMath.vec4);
45
- apply(x: GLMath.vec3 | RigidTransform): GLMath.vec3 | RigidTransform;
46
- inv(): RigidTransform;
47
- matrix(): GLMath.mat3;
48
- homogeneous_matrix(): GLMath.mat4;
49
- static deserialize(buffer: ArrayBuffer | DataViewWithCustomOffset): RigidTransform;
50
- }
51
- export interface JointInfo {
52
- parent: number | null;
53
- local_transform: RigidTransform;
54
- }
55
- export interface IBRMeshInfoSpec {
56
- version: [number, number, number];
57
- maxJointsPerVertex: number;
58
- blendshapeIndices?: number[];
59
- archivedItems?: unknown[];
60
- }
61
- export declare class IBRMeshInfo {
62
- genMask: boolean;
63
- vertices: Tensor;
64
- opacity?: Tensor;
65
- triangles: Tensor;
66
- UVCoord: Tensor;
67
- blendshapeIndices: number[];
68
- blendshapes: Tensor;
69
- jointIndex: Uint8Array;
70
- jointWeight: Float32Array;
71
- textureModels: {
72
- data: Tensor;
73
- scalingFactor?: Tensor;
74
- }[];
75
- constructor(buffer: ArrayBuffer | DataViewWithCustomOffset, spec: IBRMeshInfoSpec);
76
- }
77
- export declare class IBRMeshFrameInfo {
78
- textureModelIndex: number;
79
- texturePCAWeights: Float32Array;
80
- constructor(textureModelIndex: number, texturePCAWeights: Float32Array);
81
- static deserialize(buffer: ArrayBuffer | DataViewWithCustomOffset, char_data: IBRMeshInfo): IBRMeshFrameInfo;
82
- }
83
- export declare class IBRAnimationGeneratorCharInfo_NN {
84
- cameraConfig: CameraConfig;
85
- blendshapeCount: number;
86
- mesh: IBRMeshInfo[];
87
- skeleton: JointInfo[];
88
- jointEvalOrder: number[];
89
- movableJoints: number[];
90
- maxJointsPerVertex: number;
91
- constructor(buffer: ArrayBuffer | DataViewWithCustomOffset, spec?: {
92
- blendshapeMap?: number[][];
93
- });
94
- evalSkeleton(): RigidTransform[];
95
- evalSkeletonFromMovable(movableJointTransforms: RigidTransform[]): RigidTransform[];
96
- }
97
- export declare class IBRAnimationFrameData_NN {
98
- mesh: IBRMeshFrameInfo[];
99
- blendshapeWeights: Float32Array;
100
- movableJointTransforms: RigidTransform[];
101
- constructor();
102
- static deserialize(buffer: ArrayBuffer | DataViewWithCustomOffset, char_data: IBRAnimationGeneratorCharInfo_NN, version: [number, number, number]): IBRAnimationFrameData_NN;
103
- static interp(frame1: IBRAnimationFrameData_NN, frame2: IBRAnimationFrameData_NN, frameJointRef: IBRAnimationFrameData_NN, t: number, jointEvalOrder: [number, number][]): IBRAnimationFrameData_NN;
104
- }
105
- export declare function unpackIBRAnimation(buffer: ArrayBuffer | DataViewWithCustomOffset, char_data: IBRAnimationGeneratorCharInfo_NN): IBRAnimationFrameData_NN[];
106
- export declare class TextureFloat_3D {
107
- size: [number, number, number, number];
108
- data: Float32Array;
109
- constructor(buffer: ArrayBuffer | DataViewWithCustomOffset);
110
- }
111
- export declare function transformMJT(mjt: number[][]): RigidTransform[];
112
- export declare function formatMJT(a1: number[], a2: number[]): RigidTransform;
113
- export {};
@@ -1,46 +0,0 @@
1
- export declare class GLDevice {
2
- private _ElementTypeMap_JS2GL;
3
- private _FormatTypeMap_Ext2Int;
4
- private _FormatTypeMap_Ext2Str;
5
- private shaderStage;
6
- private running;
7
- private syncMedia;
8
- private syncMediaTime;
9
- private _lost_context_sim;
10
- canvas: HTMLCanvasElement;
11
- gl: WebGL2RenderingContext;
12
- frameID: number | null;
13
- cbRender: (device: GLDevice) => void;
14
- private _isDestroyed;
15
- private _onContextLost;
16
- private _onContextRestored;
17
- private _onContextCreationError;
18
- constructor(canvas: HTMLCanvasElement);
19
- compileShaderProgram(shaders: {
20
- readonly [index: string]: string;
21
- }): WebGLProgram;
22
- getShaderProgramUniformLocation(program: WebGLProgram, attribNames: Array<string>): {
23
- [index: string]: WebGLUniformLocation;
24
- };
25
- getShaderProgramUniformBlockLocation(program: WebGLProgram, attribNames: Array<string>): {
26
- [index: string]: GLint;
27
- };
28
- getShaderProgramAttribLocation(program: WebGLProgram, attribNames: Array<string>): {
29
- [index: string]: GLint;
30
- };
31
- getGLArrayElementType(arr: ArrayBufferView<ArrayBufferLike>): number | null;
32
- getGLTexInternalFormat(externalFormat: number, elementType: number): number | null;
33
- texImage2D(target: number, level: number, width: number, height: number, format: number, srcData: ArrayBufferView<ArrayBufferLike>): void;
34
- texImage3D(target: number, level: number, width: number, height: number, depth: number, format: number, srcData: ArrayBufferView<ArrayBufferLike>): void;
35
- run(fn: (device: GLDevice) => void, syncMedia?: HTMLVideoElement): void;
36
- refresh(): void;
37
- getSyncMediaTime(): number | null;
38
- stop(): void;
39
- simulateContextFault(contextAvailable: boolean): void;
40
- captureFrame(): {
41
- width: number;
42
- height: number;
43
- data: Uint8ClampedArray;
44
- };
45
- destroy(): void;
46
- }
@@ -1,84 +0,0 @@
1
- import { GLDevice } from './GLDevice';
2
- import { IBRAnimationGeneratorCharInfo_NN, IBRAnimationFrameData_NN, TextureFloat_3D } from './DataInterface';
3
- import { vec3, mat3 } from './Math';
4
- export interface ScaleTranslate2D {
5
- offsetX: number;
6
- offsetY: number;
7
- scaleX: number;
8
- scaleY: number;
9
- }
10
- export interface GLPipelineCharData {
11
- char: IBRAnimationGeneratorCharInfo_NN | null;
12
- LUT: TextureFloat_3D | null;
13
- transform: ScaleTranslate2D;
14
- multisample: number | null;
15
- }
16
- export interface GLPipelineFrameData {
17
- backgroundTexture: HTMLImageElement | HTMLVideoElement | null;
18
- charBodyTexture: HTMLImageElement | HTMLVideoElement;
19
- data: IBRAnimationFrameData_NN | null;
20
- fx: {
21
- [index: string]: unknown;
22
- } | null;
23
- onPostRender: (pipeline: GLPipeline) => GLPipelineFrameData | null;
24
- }
25
- export interface GLPipelineFXShadow {
26
- refPos: vec3;
27
- light: vec3;
28
- lightType: "point" | "directional";
29
- intensity: number;
30
- blurRadius: number;
31
- tint: vec3;
32
- }
33
- interface GLPipelineFXShadowGeometryInfo {
34
- Mat_ShadowImage2World: mat3;
35
- Mat_World2ObjImage: mat3;
36
- }
37
- export declare class GLPipeline {
38
- readonly device: GLDevice;
39
- private compat_features;
40
- private charData;
41
- private backgroundPipelineInfo;
42
- private backgroundVAO;
43
- private backgroundBuffers;
44
- private backgroundTextures;
45
- private backgroundTextureSrc;
46
- private meshPipelineInfo;
47
- private maskPipelineInfo;
48
- private meshInfos;
49
- private meshStatistics;
50
- private FrameTexture_meshColor;
51
- private FrameTexture_meshAlpha;
52
- private LUTTexture;
53
- private FrameBuffer_mesh;
54
- private FrameBuffer_meshColor;
55
- private FrameBuffer_meshAlpha;
56
- private initSkeletonStatus;
57
- private frameDataCallback;
58
- private currentGamma;
59
- private currentColorBalance;
60
- private _ub_shadow_fx_data;
61
- private _ub_rig_info_data;
62
- private _ub_pca_info_data;
63
- private first_webgl_render;
64
- private sendSdkPoint;
65
- private assembleBackgroundPipeline;
66
- assembleMeshPipelines(data: GLPipelineCharData): void;
67
- private getProjMat;
68
- private initFrame;
69
- private renderBackground;
70
- private renderMesh;
71
- constructor(device: GLDevice, sendSdkPoint: (point: string, data: any) => void);
72
- reinitialize(): void;
73
- setSyncMedia(syncMedia?: HTMLVideoElement): void;
74
- setCharData(charData?: GLPipelineCharData): void;
75
- setFrameDataCallback(cb: () => GLPipelineFrameData | null): void;
76
- setGamma(gammaR: number, gammaG: number, gammaB: number): void;
77
- setColorBalance(rc: number, gm: number, by: number): void;
78
- getBillboardTransform2D(pos_orig: vec3, pos_new: vec3): ScaleTranslate2D;
79
- getBillboardShadowGeometryInfo(pos_orig: vec3, light: vec3, point_src: boolean): GLPipelineFXShadowGeometryInfo;
80
- private render;
81
- renderFrame(frame: GLPipelineFrameData): void;
82
- destroy(): void;
83
- }
84
- export {};
@@ -1,5 +0,0 @@
1
- import { GLPipelineCharData } from './GLPipeline';
2
- import { IBRAnimationFrameData_NN } from './DataInterface';
3
- export declare function getVertices(data: GLPipelineCharData, frame_data: IBRAnimationFrameData_NN): number[][][];
4
- export declare function getPCATextures(data: GLPipelineCharData, frame_data: IBRAnimationFrameData_NN): ImageData[];
5
- export declare function getWavefrontObjFromVertices(data: GLPipelineCharData, vertices: number[][][]): string;
@@ -1,21 +0,0 @@
1
- export type vec3 = [number, number, number];
2
- export type vec4 = [number, number, number, number];
3
- export type mat3 = [vec3, vec3, vec3];
4
- export type mat4 = [vec4, vec4, vec4, vec4];
5
- export declare function length(vec: number[]): number;
6
- export declare function normalize(vec: number[]): number[];
7
- export declare function mvmul(lhs: number[][], rhs: number[]): number[];
8
- export declare function mmul(lhs: number[][], rhs: number[][]): number[][];
9
- export declare function transpose(mat: number[][]): number[][];
10
- export declare function identity(size: number): number[][];
11
- export declare function det3(mat: mat3): number;
12
- export declare function inv3(mat: mat3): mat3;
13
- export declare function inv4(mat: mat4): mat4;
14
- export declare function quaternion_conj(q: vec4): vec4;
15
- export declare function quaternion_mul(lhs: vec4, rhs: vec4): vec4;
16
- export declare function quaternion_to_mat(q: vec4): mat3;
17
- export declare function mat_to_quaternion(mat: mat3 | mat4): vec4;
18
- export declare function GivensTransform(mat: number[][], row0: number, row1: number, angle: number): void;
19
- export declare function Euler_to_mat(angles: [number, number, number], order?: string): mat3;
20
- export declare function make_homogeneous(mat: mat3, translation: vec3): mat4;
21
- export declare function flatten(mat: number[][]): number[];
@@ -1 +0,0 @@
1
- export {};
@@ -1,11 +0,0 @@
1
- export type WebAudioSessionType = "playback" | "play-and-record";
2
- type WebAudioPlaybackSessionRestoredListener = () => void;
3
- /**
4
- * Keep Web Audio audible through the iOS Ring/Silent switch. Do not override
5
- * a play-and-record session that may belong to an active microphone track.
6
- */
7
- export declare function configureWebAudioPlaybackSession(target?: Navigator): boolean;
8
- export declare function beginWebAudioCaptureSession(target?: Navigator): boolean;
9
- export declare function subscribeWebAudioPlaybackSessionRestored(listener: WebAudioPlaybackSessionRestoredListener, target?: Navigator): () => void;
10
- export declare function endWebAudioCaptureSession(target?: Navigator): boolean;
11
- export {};
@@ -1 +0,0 @@
1
- export declare function pcmToWav(pcmData: Uint8Array): ArrayBufferLike;
@@ -1,28 +0,0 @@
1
- import type { ICapabilityCheckReport } from '../types/capability';
2
- /**
3
- * 同步检查所有浏览器能力(用于 constructor 快速校验)
4
- *
5
- * 检查项:
6
- * - secureContext:安全上下文(HTTPS / localhost)
7
- * - webgl2:WebGL2 渲染能力
8
- * - audio:音频播放能力(PCM 或 MSE 任一可用即可)
9
- * - webcodecs:WebCodecs API 存在性
10
- * - videoDecode:VideoDecoder API 存在性(真实解码探测见异步 checkVideoDecodeDetailed)
11
- * - worker:Web Worker 支持
12
- *
13
- * 音频检查逻辑:SDK 有两种音频播放器(PCM 和 MSE),只需其中一种可用即可
14
- * - PCM 播放器依赖:AudioContext + AudioWorkletNode
15
- * - MSE 播放器依赖:MediaSource
16
- * - 两者都不可用时,音频能力检查失败
17
- */
18
- export declare function checkCapabilitySync(audioOnlyMode?: boolean): ICapabilityCheckReport;
19
- /**
20
- * 异步检查所有浏览器能力(详细版,用于 init 阶段详细探测)
21
- *
22
- * 包含真实视频解码探测(videoDecode)和 Worker 执行测试。
23
- * audioOnlyMode 下跳过画面相关检查(webgl2 / webcodecs / videoDecode)。
24
- *
25
- * @param audioOnlyMode 纯音频模式,仅检查安全上下文、音频与 Worker
26
- * @param hardwareAcceleration 客户传入的解码加速配置,需与生产解码保持一致
27
- */
28
- export declare function checkCapability(audioOnlyMode?: boolean, hardwareAcceleration?: string): Promise<ICapabilityCheckReport>;
@@ -1,14 +0,0 @@
1
- /**
2
- * 生成 request_id:用于 startSession / stopSession 的幂等与兜底销毁。
3
- * 组成:md5(机器指纹 + 毫秒时间戳 + appId)
4
- * 同一次 init → destroy 生命周期内始终不变,即使 session_id 尚未返回
5
- * 也能用 request_id 让服务端销毁已占用的房间资源。
6
- */
7
- export declare function generateRequestId(appId: string): string;
8
- export interface Headers {
9
- [key: string]: string;
10
- }
11
- export declare function headersNeedSign(ak: string, sk: string, method: string, url: string, data: Record<string, any>): {
12
- headers: Headers;
13
- data: Record<string, any>;
14
- };
@@ -1 +0,0 @@
1
- export declare const translateBufferToFloatArray: (buf: any) => number[];
@@ -1,26 +0,0 @@
1
- /**
2
- * 服务端 floats_to_scaled_int16_bytes 反向函数:Base64 → float32 数组
3
- * 核心逻辑:Base64→int16字节流→int16数值(兼容round+clip)→÷0x7FFF→float32
4
- * @param base64Str 服务端返回的 Base64 编码字符串
5
- * @returns float32 精度的浮点数数组(与服务端原始 floats 一致)
6
- */
7
- export function scaledInt16BytesToFloat32(base64Str: any): number[];
8
- /**
9
- * 可选:带平滑滤波的版本(解决“脸色闪”问题,优化视觉效果)
10
- * @param base64Str 服务端 Base64 字符串
11
- * @param lastData 上一帧数据(用于平滑)
12
- * @param alpha 平滑系数(0~1,默认0.2,越大响应越快,越小越平滑)
13
- * @returns 平滑后的 float32 数组
14
- */
15
- export function scaledInt16BytesToSmoothFloat32(base64Str: any, lastData?: any[], alpha?: number): number[];
16
- /**
17
- * 解析后端传输的 float32 字节流为普通数组
18
- * @param {string|ArrayBuffer|Uint8Array} bytes - 后端传来的二进制数据(注意格式兼容)
19
- * @returns {number[]} 解析后的浮点数组(空数据返回空数组)
20
- */
21
- /**
22
- * 从 Uint8Array 解析出 float32 数组(核心修复版)
23
- * @param {Uint8Array} uint8Arr - 后端传来的 Uint8Array 原始字节
24
- * @returns {number[]} 解析后的浮点数组
25
- */
26
- export function parseUint8ToFloat32(uint8Arr: Uint8Array): number[];