@xmov/avatar 2.0.0-alpha.38

package/dist/modules/Composition.d.ts

@@ -0,0 +1,28 @@
+import AvatarRender from "../baseRender/AvatarRenderer";
+import UIRenderer from "../baseRender/UIRenderer";
+import AudioRenderer from "../baseRender/AudioRenderer";
+import { FPSStats } from "../utils/fpsTracker";
+type TRenderer = UIRenderer;
+export default class Composition {
+    TAG: string;
+    container: Element;
+    avatarRenderer: AvatarRender;
+    audioRenderer: AudioRenderer;
+    restRenderers: TRenderer[];
+    private fpsTracker;
+    private onFPSUpdate?;
+    audioOnlyMode: boolean;
+    constructor(options: {
+        container: Element;
+        avatarRenderer: AvatarRender;
+        restRenderers: TRenderer[];
+        audioRenderer: AudioRenderer;
+        audioOnlyMode?: boolean;
+        onFPSUpdate?: (stats: FPSStats) => void;
+    });
+    compose(frameIndex: number): void;
+    getFPSStats(): FPSStats;
+    stop(): void;
+    destroy(): void;
+}
+export {};

package/dist/modules/ResourceManager.d.ts

@@ -0,0 +1,295 @@
+import { IAvatarOptions, INetworkInfo, Layout, WalkConfig } from "../types/index";
+import XmovAvatar from "../index";
+/**
+ * 资源处理模块，加载并存储资源
+ */
+type TOptions = Pick<IAvatarOptions, "appId" | "appSecret" | "gatewayServer" | 'cacheServer' | "sdkInstance" | "config" | "headers" | "gateway_type" | "tag" | "asr_id" | "llm_id" | "session_speak_req_id"> & {
+    sessionRequestData?: Record<string, unknown>;
+    onNetworkInfo(quality: INetworkInfo): void;
+    onStartSessionWarning?: (message: Object) => void;
+};
+export type TDownloadProgress = (progress: number) => void;
+export interface IOfflineIdle {
+    asf: number;
+    aef: number;
+    n: string;
+}
+export interface IResumeParams {
+    client_frame: number;
+    current_ani: string;
+    current_ani_frame: number;
+    next_state: string;
+}
+export interface ISessionResponse {
+    session_id: string;
+    room: string;
+    token: string;
+    socket_io_url: string;
+    e2e_resp?: {
+        ws_url?: string;
+        e2e_token?: string;
+        monitor_ws_url?: string;
+        features?: {
+            speech_frontend?: {
+                enable_aec?: boolean;
+                [key: string]: unknown;
+            };
+            [key: string]: unknown;
+        };
+    };
+    features?: {
+        speech_frontend?: {
+            enable_aec?: boolean;
+            [key: string]: unknown;
+        };
+        [key: string]: unknown;
+    };
+    session_start_time: string;
+    resource_pack: {
+        enable_robot_expression: boolean;
+        body_data_dir: string;
+        face_ani_char_data: string;
+        face_ani_preload_data: string;
+        offline_idle: IOfflineIdle[];
+        interpolate_joints: [number, number][];
+    };
+    reconnect_timeout: number;
+    reconnect_client_timeout: number;
+    config: {
+        background_img: string;
+        frame_rate: number;
+        look_name: string;
+        tts_vcn_id: string;
+        sta_face_id: string;
+        mp_service_id: string;
+        raw_audio: boolean;
+        resolution: {
+            width: number;
+            height: number;
+        };
+        init_events: [
+            {
+                type: string;
+                data: {
+                    image: string;
+                    axis_id: number;
+                };
+            }
+        ];
+        fx?: {
+            transform?: {
+                offsetX: number;
+                offsetY: number;
+                scaleX: number;
+                scaleY: number;
+            };
+            shadow?: {
+                refPos: number[];
+                light: number[];
+                lightType: string;
+                blurRadius: number;
+                tint: number[];
+                intensity: number;
+            };
+        };
+    };
+}
+export default class ResourceManager {
+    private TAG;
+    options: TOptions;
+    sdk: XmovAvatar;
+    session_id?: string;
+    mouthShapeLib: any;
+    offlineIdle: IOfflineIdle[];
+    offlineCache: Map<string, {
+        data: ArrayBuffer;
+    }>;
+    networkInfos: INetworkInfo[];
+    private bg;
+    private destroyed;
+    resource_pack: {
+        enable_robot_expression: boolean;
+        body_data_dir: string;
+        face_ani_char_data: string;
+        face_ani_preload_data: string;
+        blendshape_map?: number[][];
+        interpolate_joints: [number, number][];
+    };
+    progress: number;
+    config: {
+        background_img: string;
+        frame_rate: number;
+        look_name: string;
+        tts_vcn_id: string;
+        sta_face_id: string;
+        mp_service_id: string;
+        raw_audio: boolean;
+        resolution: {
+            width: number;
+            height: number;
+        };
+        init_events: [
+            {
+                type: string;
+                "x_location"?: number;
+                "y_location"?: number;
+                width?: number;
+                height: number;
+                speech_id: number;
+                data?: {
+                    image: string;
+                    axis_id: number;
+                };
+            }
+        ];
+        framedata_proto_version: number;
+        layout: Layout;
+        walk_config: WalkConfig;
+        fx?: {
+            transform?: {
+                offsetX: number;
+                offsetY: number;
+                scaleX: number;
+                scaleY: number;
+            };
+            shadow?: {
+                refPos: number[];
+                light: number[];
+                lightType: string;
+                blurRadius: number;
+                tint: number[];
+                intensity: number;
+            };
+        };
+    };
+    private videoCache;
+    private maxCacheSize;
+    private maxCacheEntries;
+    private cacheCleanupInterval;
+    private isProcessingVideo;
+    private cacheServer?;
+    private startSessionTimer;
+    private downloadingVideos;
+    first_load: boolean;
+    constructor(options: TOptions);
+    getAppInfo(): {
+        appId: string;
+    };
+    /**
+     * 加载背景图片
+     * @returns Promise<void>
+     */
+    getBackgroundImage(): Promise<void>;
+    getIsEnableRobotExpression(): boolean;
+    /**
+     * 获取背景图片
+     * @returns HTMLImageElement | null
+     */
+    getBackgroundImageElement(): HTMLImageElement | null;
+    getConfig(): {
+        background_img: string;
+        frame_rate: number;
+        look_name: string;
+        tts_vcn_id: string;
+        sta_face_id: string;
+        mp_service_id: string;
+        raw_audio: boolean;
+        resolution: {
+            width: number;
+            height: number;
+        };
+        init_events: [{
+            type: string;
+            x_location?: number | undefined;
+            y_location?: number | undefined;
+            width?: number | undefined;
+            height: number;
+            speech_id: number;
+            data?: {
+                image: string;
+                axis_id: number;
+            } | undefined;
+        }];
+        framedata_proto_version: number;
+        layout: Layout;
+        walk_config: WalkConfig;
+        fx?: {
+            transform?: {
+                offsetX: number;
+                offsetY: number;
+                scaleX: number;
+                scaleY: number;
+            } | undefined;
+            shadow?: {
+                refPos: number[];
+                light: number[];
+                lightType: string;
+                blurRadius: number;
+                tint: number[];
+                intensity: number;
+            } | undefined;
+        } | undefined;
+    };
+    load(onDownloadProgress?: TDownloadProgress): Promise<ISessionResponse | null>;
+    startSession(): Promise<any>;
+    _startSession(): Promise<any> | null;
+    stopSession(stop_reason: string): Promise<any>;
+    /**
+     * 加载表情数据
+     */
+    loadMouthShapeLib(onDownloadProgress?: TDownloadProgress): Promise<void>;
+    getMouthShapeLib(): any;
+    getVideoUrl(name: string): string;
+    /**
+     * 预加载视频
+     * @param name 视频名称
+     * @returns Promise<ArrayBuffer>
+     */
+    preloadVideo(name: string): Promise<ArrayBuffer | undefined>;
+    /**
+     * 加载视频
+     * @param name 视频名称
+     * @returns Promise<ArrayBuffer>
+     */
+    loadVideo(name: string): Promise<ArrayBuffer | undefined>;
+    /**
+     * 实际下载视频的方法
+     * @param name 视频名称
+     * @returns Promise<ArrayBuffer>
+     */
+    private downloadVideo;
+    private _fetchVideo;
+    /**
+     * 添加数据到缓存
+     */
+    private addToCache;
+    /**
+     * 启动缓存清理定时器
+     */
+    private startCacheCleanup;
+    /**
+     * 清理缓存
+     */
+    private cleanupCache;
+    /**
+     * 更新缓存访问信息
+     */
+    private updateCacheAccess;
+    /**
+     * 清空所有缓存
+     */
+    clearAllCache(): void;
+    /** 获取离线 idle */
+    _getOfflineIdle(): IOfflineIdle[];
+    /** 获取上一次 session_id */
+    _getSessionId(): string;
+    /** 离线重连 */
+    _reload(): Promise<ISessionResponse | null>;
+    _setOfflineCache(): Promise<void>;
+    private _runOfflineCache;
+    /**
+     * 销毁资源管理器
+     */
+    destroy(): void;
+}
+export {};

package/dist/modules/TrackRenderer/base-track.d.ts

@@ -0,0 +1,4 @@
+export default class BaseTrack {
+    render(): void;
+    stop(): void;
+}

package/dist/modules/TrackRenderer/index.d.ts

@@ -0,0 +1,12 @@
+import { IRawWidgetData } from '../../types/frame-data';
+import TrackPic from './track-pic';
+import TrackSubtitle from './track-subtitle';
+export default class TrackRenderer {
+    private TAG;
+    tracker: TrackPic | TrackSubtitle;
+    private instance?;
+    constructor(event: IRawWidgetData, instance?: any);
+    render(): HTMLDivElement | null;
+    stop(): void;
+    destroy(): void;
+}

package/dist/modules/TrackRenderer/render-implements.d.ts

@@ -0,0 +1,19 @@
+/**
+ * widget_xx 的默认实现
+ */
+import { IWidgetPic, IWidgetSubtitle, IWidget } from "../../types/event";
+import { IRawWidgetData } from "../../types/frame-data";
+declare const WidgetDefaultRenderer: {
+    _el(data: IWidget): HTMLDivElement;
+    WIDGET_PIC(data: IWidgetPic, instance?: any): HTMLDivElement;
+    WIDGET_SUBTITLE(data: IWidgetSubtitle, instance?: any): HTMLDivElement | null;
+    CUSTOM_WIDGET?(data: IRawWidgetData): void;
+    PROXY_WIDGET?: {
+        [key: string]: (data: any) => void;
+    };
+    destroy(instance?: any): void;
+    _getWidgetKey(type: string, axisId?: number): string;
+    _replaceWidget(type: string, axisId: number, newElement: HTMLDivElement, instance?: any): void;
+    _currentInstance?: any;
+};
+export default WidgetDefaultRenderer;

package/dist/modules/TrackRenderer/track-pic.d.ts

@@ -0,0 +1,8 @@
+import { IWidgetPic } from "../../types/event";
+import BaseTrack from "./base-track";
+export default class ImageTrack extends BaseTrack {
+    data: IWidgetPic;
+    private instance?;
+    constructor(data: IWidgetPic, instance?: any);
+    render(): HTMLDivElement | null;
+}

package/dist/modules/TrackRenderer/track-subtitle.d.ts

@@ -0,0 +1,8 @@
+import { IWidgetSubtitle } from "../../types/event";
+import BaseTrack from "./base-track";
+export default class SubtitleTrack extends BaseTrack {
+    data: IWidgetSubtitle;
+    private instance?;
+    constructor(data: IWidgetSubtitle, instance?: any);
+    render(): HTMLDivElement | null;
+}

package/dist/modules/cache-manager.d.ts

@@ -0,0 +1,5 @@
+export default class CacheManager {
+    cacheServer: string;
+    constructor(cacheServer: string);
+    getVideo(url: string): string;
+}

package/dist/modules/decoder.d.ts

@@ -0,0 +1,84 @@
+import { bodyFile, IRawBodyFrameData } from "../types/frame-data";
+import ResourceManager, { IOfflineIdle } from "./ResourceManager";
+import SaveAndDownload from "../control/SaveAndDownload";
+import { DataCacheQueue } from "../control/DataCacheQueue";
+import { EErrorCode } from "../types/error";
+export default class ParallelDecoder {
+    private tasks;
+    private queue;
+    private maxParallel;
+    private currentParallel;
+    private _locked;
+    private currentTaskId;
+    private resourceManager;
+    private onFrame;
+    private onDone;
+    private isFirstDecode;
+    private pendingAbort;
+    private abortAfterEf;
+    private pendingNewQueue;
+    private currentDecodedFrameIndex;
+    private abortAfterFrame;
+    private cacheVideoCount;
+    private saveAndDownload;
+    private dataCacheQueue;
+    private bodyFrameCountMap;
+    private maxVideoCount;
+    private hardwareAcceleration;
+    private MAX_LOAD_VIDEO_TIMEOUT_MS;
+    _offlineLastFrame: number;
+    _offlineIdle: IOfflineIdle[];
+    _offlineIdleIndex: number;
+    firstDecodeFrame: boolean;
+    reportMessage: (message: {
+        code: EErrorCode;
+        message: string;
+        e?: object;
+    }) => void;
+    constructor(options: {
+        hardwareAcceleration: string;
+        resourceManager: ResourceManager;
+        saveAndDownload: SaveAndDownload;
+        dataCacheQueue: DataCacheQueue;
+        reportMessage: (message: {
+            code: EErrorCode;
+            message: string;
+            e?: object;
+        }) => void;
+    });
+    getRandomTaskId(): string;
+    decode(files: Array<IRawBodyFrameData>, onFrame: (file: bodyFile, frame: any, index: number) => void): void;
+    clearOldQueue(frame: number): void;
+    updateQueue(files: Array<IRawBodyFrameData>): void;
+    _isIOS(): boolean;
+    getQueue(): {
+        x_offset: never[];
+        hfd: boolean;
+        aef: number;
+        asf: number;
+        id: number;
+        n: string;
+        s: string;
+        body_id: number;
+        sf: number;
+        ef: number;
+    }[];
+    timeoutPromise(ms: number, reason?: string): Promise<unknown>;
+    loadVideoWithTimeout(videoKey: string): Promise<ArrayBuffer | null>;
+    _tryStartNext(): Promise<void>;
+    _startWorker(file: bodyFile): void;
+    cacheVideo(): void;
+    abort(): void;
+    abortOne(id: string): void;
+    destroy(): void;
+    _handlePendingAbort(file: bodyFile, index: number): void;
+    private _cleanupOldWorkers;
+    syncDecode(currentFrameIndex: number): void;
+    /** 进入正常模式 */
+    _reload(): void;
+    /** 进入离线，清空剩余身体数据 */
+    _offLineMode(offlineIdle: IOfflineIdle[], currentFrame: number): void;
+    _offlineRun(): void;
+    /** 设置 idle 视频 */
+    _putOfflineBodyData(idle: IOfflineIdle): void;
+}

package/dist/modules/error-handle.d.ts

@@ -0,0 +1,17 @@
+import { EErrorCode } from "../types/error";
+export default function Errors(params: {
+    code: EErrorCode;
+    message: string;
+    e?: any;
+    session_id?: string;
+    client_speak_id?: string;
+}): {
+    code: EErrorCode;
+    error_code: number;
+    error_name: string;
+    message: string;
+    session_id: string;
+    client_speak_id: string | undefined;
+    timestamp: number;
+    originalError: string | null;
+};

package/dist/modules/network.d.ts

@@ -0,0 +1,21 @@
+interface NetworkOptions {
+    offlineCallback(message?: string): void;
+    onlineCallback(message?: string): void;
+}
+export default class NetworkMonitor {
+    private TAG;
+    static ONLINE: boolean;
+    private readonly options;
+    private retryCount;
+    private retryTimer;
+    private bindHandleOnline;
+    private bindHandleOffline;
+    constructor(options: NetworkOptions);
+    private _initEventListeners;
+    private _handleOnline;
+    private _handleOffline;
+    private _attempt;
+    setState(isOnline: boolean): void;
+    destroy(): void;
+}
+export {};

package/dist/types/capability.d.ts

@@ -0,0 +1,15 @@
+export type CapabilitySeverity = 'fatal' | 'warn';
+export type CapabilityName = 'secureContext' | 'webgl2' | 'audio' | 'webcodecs' | 'worker';
+export interface ICapabilityCheckItem {
+    name: CapabilityName;
+    supported: boolean;
+    severity: CapabilitySeverity;
+    reason?: string;
+    detail?: unknown;
+}
+export interface ICapabilityCheckReport {
+    supported: boolean;
+    fatalCount: number;
+    warnCount: number;
+    items: ICapabilityCheckItem[];
+}

package/dist/types/error.d.ts

@@ -0,0 +1,70 @@
+/**
+ * @file 错误码和定义
+ */
+export declare enum EErrorCode {
+    CONTAINER_NOT_FOUND = 10001,
+    CONNECT_SOCKET_ERROR = 10002,
+    START_SESSION_ERROR = 10003,
+    STOP_SESSION_ERROR = 10004,
+    INIT_ERROR = 10005,
+    SOCKET_QUIT = 10006,
+    SOCKET_DISCONNECT = 10007,
+    VIDEO_FRAME_EXTRACT_ERROR = 20001,
+    INIT_WORKER_ERROR = 20002,
+    PROCESS_VIDEO_STREAM_ERROR = 20003,
+    FACE_PROCESSING_ERROR = 20004,
+    RENDER_BODY_ERROR = 20005,
+    RENDER_FACE_ERROR = 20006,
+    WebGL_RENDER_ERROR = 20007,
+    RENDER_DATA_MISSING = 20008,
+    RENDER_FACE_DATA_MISSING = 20009,
+    EVENT_WITHOUT_SPEECH_ID = 20010,
+    EVENT_TRIGGERED = 20011,
+    BACKGROUND_IMAGE_LOAD_ERROR = 30001,
+    FACE_BIN_LOAD_ERROR = 30002,
+    INVALID_BODY_NAME = 30003,
+    VIDEO_DOWNLOAD_ERROR = 30004,
+    BODY_DATA_EXPIRED = 30005,
+    EVENT_SPEAK_START_EXPIRED = 30006,
+    EVENT_VOICE_START_EXPIRED = 30007,
+    AUDIO_DECODE_ERROR = 40001,
+    FACE_DECODE_ERROR = 40002,
+    VIDEO_DECODE_ERROR = 40003,
+    EVENT_DECODE_ERROR = 40004,
+    INVALID_DATA_STRUCTURE = 40005,
+    TTSA_ERROR = 40006,
+    AUDIO_DATA_EXPIRED = 40007,
+    NETWORK_DOWN = 50001,
+    NETWORK_UP = 50002,
+    NETWORK_RETRY = 50003,
+    NETWORK_BREAK = 50004,
+    WEBGL_CONTEXT_LOST = 60001,
+    WEBGL_CONTEXT_RESTORED = 60002,
+    /** @deprecated 未在任何场景实际触发，仅保留枚举值以兼容旧文档解析 */
+    WEBGL_CONTEXT_RESTORE_TIMEOUT = 60003,
+    WEBGL_CONTEXT_RECOVERED = 60004,
+    WEBGL_CONTEXT_RECOVERY_FAILED = 60005,
+    WEBGL_CONTEXT_RECOVERY_MAX_ATTEMPTS = 60006,
+    WEBGL_RESOLUTION_RESTORED = 60007,
+    WEBGL_CONTEXT_MANUAL_RESTARTED = 60008
+}
+/**
+ * 错误码新旧映射表
+ * key: 旧错误码（1xxxx-6xxxx）
+ * value: 新错误码（31xx-36xx）
+ */
+export declare const ERROR_CODE_MAP: Record<number, number>;
+/**
+ * 错误码名称映射
+ */
+export declare const ERROR_NAME_MAP: Record<number, string>;
+export interface SDKError {
+    code: EErrorCode;
+    error_code: number;
+    error_name: string;
+    message: string;
+    session_id: string;
+    client_speak_id?: string;
+    timestamp: number;
+    originalError?: any;
+}

package/dist/types/event.d.ts

@@ -0,0 +1,25 @@
+export type TWidgetType = 'widget_pic' | 'subtitle_on' | 'subtitle_off' | any;
+/** UI 控件位置 */
+export interface IWidgetCommon {
+    axis_id: number;
+    'x_location'?: number;
+    'y_location'?: number;
+    width?: number;
+    height?: number;
+}
+export interface IWidgetPic extends IWidgetCommon {
+    image: string;
+}
+export interface IWidgetSubtitle extends IWidgetCommon {
+    text: string;
+    axis_id: number;
+    type: TWidgetType;
+}
+export type IWidget = IWidgetPic | IWidgetSubtitle | any;
+export interface ISetCharacterCanvasAnchor {
+    type: string;
+    'x_location': number;
+    'y_location': number;
+    width: number;
+    height: number;
+}