@spatialwalk/avatarkit 1.0.0-beta.1 → 1.0.0-beta.100

package/dist/core/Avatar.d.ts

@@ -1,23 +1,13 @@
 import { CharacterMeta } from '../types';
-import { PreloadResult } from './AvatarDownloader';
 export declare class Avatar {
     readonly id: string;
     private characterMeta;
     private resources;
+    /** Local-only field: tracks whether cached model is "standard" or "compressed". */
+    _cachedModelType: string;
     /**
-     * 构造函数（内部使用）
-     * @param id 数字人 ID
-     * @param characterMeta 角色元数据
-     * @param resources 资源数据
-     */
-    constructor(id: string, characterMeta: CharacterMeta, resources: PreloadResult);
-    /**
-     * 获取角色元数据（内部使用）
+     * Get character metadata
+     * @returns Character metadata, including all configuration information (version, resource URLs, camera config, etc.)
      */
     getCharacterMeta(): CharacterMeta;
-    /**
-     * 获取资源数据（内部使用）
-     */
-    getResources(): PreloadResult;
 }
-//# sourceMappingURL=Avatar.d.ts.map

package/dist/core/AvatarController.d.ts

@@ -1,125 +1,140 @@
-import { Flame } from '../generated/driveningress/v1/driveningress';
 import { Avatar } from './Avatar';
-import { AnimationPlayer } from '../audio/AnimationPlayer';
-import { AvatarState, ConnectionState } from '../types';
+import { ConnectionState, AvatarError, DrivingServiceMode, ConversationState, PostProcessingConfig } from '../types';
+import { FrameRateInfo } from '../performance/FrameRateMonitor';
 export declare class AvatarController {
-    protected avatar: Avatar;
-    protected isConnected: boolean;
-    protected isMuted: boolean;
-    protected currentState: AvatarState;
-    protected animationPlayer: AnimationPlayer | null;
-    protected currentKeyframes: Flame[];
-    protected transitionKeyframes: Flame[];
-    protected currentReqId: string | null;
-    protected isPlaying: boolean;
-    private hasSentEnd;
+    private networkLayer?;
+    private readonly playbackMode;
+    private isStartingPlayback;
+    private currentConversationId;
+    private reqEnd;
     onConnectionState: ((state: ConnectionState) => void) | null;
-    onAvatarState: ((state: AvatarState) => void) | null;
-    onError: ((error: Error) => void) | null;
+    onConversationState: ((state: ConversationState) => void) | null;
+    onError: ((error: AvatarError) => void) | null;
     private eventListeners;
-    private wsClient;
-    private pendingAudioChunks;
-    constructor(avatar: Avatar);
-    /**
-     * 启动服务
+    private readonly frameRateMonitor;
+    /** Frame rate monitoring callback. Fires with aggregated metrics from a 2-second sliding window. */
+    get onFrameRateInfo(): ((info: FrameRateInfo) => void) | null;
+    set onFrameRateInfo(value: ((info: FrameRateInfo) => void) | null);
+    /** Whether frame rate monitoring is enabled. Default is false (zero overhead when disabled). */
+    get frameRateMonitorEnabled(): boolean;
+    set frameRateMonitorEnabled(value: boolean);
+    private renderCallback?;
+    private characterHandle;
+    private characterId;
+    private postProcessingConfig;
+    private playbackLoopId;
+    private playbackLoopGeneration;
+    private lastRenderedFrameIndex;
+    private keyframesOffset;
+    private readonly MAX_KEYFRAMES;
+    private readonly KEYFRAMES_CLEANUP_THRESHOLD;
+    private lastSyncLogTime;
+    private lastOutOfBoundsState;
+    private isFallbackMode;
+    private frameStarvationEvents;
+    private isFrameStarved;
+    private playbackStuckCheckState;
+    private readonly MAX_AUDIO_TIME_ZERO_COUNT;
+    private readonly MAX_AUDIO_TIME_STUCK_COUNT;
+    private readonly AUDIO_TIME_STUCK_THRESHOLD;
+    private hostModeMetrics;
+    private readonly audioBytesPerSecond;
+    constructor(avatar: Avatar, options?: {
+        playbackMode?: DrivingServiceMode;
+    });
+    private handleVisibilityChange;
+    private shouldReportPlaybackStats;
+    private _getDeviceScoreProps;
+    /**
+     * Get current conversation ID
+     * Returns the current conversation ID for the active audio session
+     * @returns Current conversation ID, or null if no active session
+     */
+    getCurrentConversationId(): string | null;
+    /**
+     * Initialize audio context (must be called in user gesture context)
+     *
+     * This method must be called before any audio operations (send, yieldAudioData, etc.)
+     * to ensure AudioContext is created and initialized in a user gesture context.
+     *
+     * @example
+     * // In user click handler
+     * button.addEventListener('click', async () => {
+     *   await avatarView.controller.initializeAudioContext()
+     *   // Now you can safely use send() or yieldAudioData()
+     * })
+     */
+    initializeAudioContext(): Promise<void>;
+    /**
+     * Start service (SDK mode only)
      */
     start(): Promise<void>;
     /**
-     * 发送音频数据
-     */
-    send(audioData: ArrayBuffer, end?: boolean): void;
-    /**
-     * 发送文字消息（Animation Only 模式不支持文字输入）
+     * Send audio to server (SDK mode only)
+     * Also cache to data layer for playback
+     * @returns conversationId - Conversation ID for this audio session
      */
-    sendText(_text: string): void;
-    /**
-     * 打断当前对话
-     */
-    interrupt(): void;
+    send(audioData: ArrayBuffer, end?: boolean): string | null;
     /**
-     * 关闭服务（断开连接）
+     * Close service (SDK mode only)
      */
     close(): void;
     /**
-     * 静音切换
-     */
-    get mute(): boolean;
-    set mute(value: boolean);
-    /**
-     * 获取当前状态
-     */
-    get state(): AvatarState;
-    /**
-     * 获取动画播放器实例
-     * @internal
-     */
-    getAnimationPlayer(): AnimationPlayer | null;
-    /**
-     * 提供给视图层读取的过渡关键帧（Idle->Speaking），独立于主序列
+     * Send audio data (host mode)
+     * Stream additional audio data after playback()
+     * @returns conversationId - Conversation ID for this audio session
      */
-    protected getTransitionKeyframes(): Flame[];
+    yieldAudioData(data: Uint8Array, isLast?: boolean): string | null;
     /**
-     * 获取连接状态
+     * Send animation keyframes (host mode or SDK mode)
+     * Stream additional animation data after playback()
+     *
+     * Public API: accepts binary data array (protobuf encoded Message array)
+     * @param keyframesDataArray - Animation keyframes binary data array (each element is a protobuf encoded Message) or empty array to trigger audio-only mode
+     * @param conversationId - Conversation ID (required). If conversationId doesn't match current conversationId, keyframes will be discarded.
+     *                Use getCurrentConversationId() to get the current conversationId.
+     * @returns `true` if the server has sent all animation data for this conversation (end signal received), `false` otherwise.
      */
-    get connected(): boolean;
+    yieldFramesData(keyframesDataArray: (Uint8Array | ArrayBuffer)[], conversationId: string): boolean;
     /**
-     * 为 AvatarView 提供注册内部事件的接口
-     * @internal
+     * Pause playback (can be resumed later)
+     * Pause audio playback and stop render loop, but preserve all state (keyframes, audio buffers, etc.)
      */
-    setupInternalEventListeners(callbacks: {
-        onKeyframesUpdate?: (keyframes: Flame[]) => void;
-        onStartRendering?: () => void;
-        onStopRendering?: () => void;
-    }): void;
+    pause(): void;
     /**
-     * 初始化播放器和客户端
+     * Resume playback (from paused state)
+     * Resume audio playback and restart render loop
+     * Animation will continue from paused frame (because animation time base comes from audio, will auto-sync)
      */
-    protected initializePlayers(): Promise<void>;
+    resume(): Promise<void>;
     /**
-     * 清理播放器和客户端
+     * Interrupt current playback
      */
-    protected cleanupPlayers(): void;
-    /**
-     * 停止播放
-     */
-    protected stopPlayback(): void;
-    /**
-     * 开始播放音频和动画
-     */
-    protected startPlayback(): Promise<void>;
-    /**
-     * 注册内部事件监听器（仅供内部使用）
-     */
-    private registerEventListener;
-    /**
-     * 触发内部事件（仅供内部使用）
-     */
-    private emit;
-    /**
-     * 处理接收到的消息
-     */
-    private handleMessage;
+    interrupt(): void;
     /**
-     * 处理动画消息（从服务器接收）
+     * Clear all data and resources
      */
-    private handleAnimationMessage;
+    clear(): void;
     /**
-     * 开始流式播放（准备播放器、添加音频数据、开始播放动画）
+     * Get point cloud count of the current avatar
+     * @returns Point cloud count, or null if avatar is not loaded
      */
-    private startStreamingPlayback;
+    getPointCount(): number | null;
     /**
-     * 开始同步播放（用户音频 + 动画）
-     * 注意：流式模式下，这个方法由 startStreamingPlayback() 调用
+     * Set post-processing configuration
+     * These parameters will be applied in real-time to animation parameters returned by the server
+     * @param config Post-processing configuration, or null to clear
      */
-    private startSynchronizedPlayback;
+    setPostProcessingConfig(config: PostProcessingConfig | null): void;
     /**
-     * 将 PCM 数据转换为 WAV Blob URL
-     * 注意：流式模式下不再需要此方法，保留用于兼容性
+     * Set audio playback volume
+     * Note: This only controls the avatar audio player volume, not the system volume
+     * @param volume Volume value, range from 0.0 to 1.0 (0.0 = mute, 1.0 = max volume)
      */
-    private createAudioBlobURL;
+    setVolume(volume: number): void;
     /**
-     * 设置 WebSocket 事件监听器
+     * Get current audio playback volume
+     * @returns Current volume value (0.0 - 1.0)
      */
-    private setupWebSocketListeners;
+    getVolume(): number;
 }
-//# sourceMappingURL=AvatarController.d.ts.map

package/dist/core/AvatarManager.d.ts

@@ -3,24 +3,44 @@ import { Avatar } from './Avatar';
 export declare class AvatarManager {
     private static _instance;
     private avatarDownloader;
-    private _templateInitialized;
+    private avatarCache;
+    /** 下载队列：FIFO 顺序 */
+    private downloadQueue;
+    /** 当前正在执行的任务 */
+    private currentTask;
+    /** 任务索引：快速查找 id 对应的任务 */
+    private taskIndex;
     /**
-     * 通过全局单例来访问
+     * Access via global singleton
      */
     static get shared(): AvatarManager;
     /**
-     * 加载数字人
-     * @param id 数字人 ID
-     * @param onProgress 进度回调
-     * @param options 加载选项
+     * Load avatar
+     * @param id Avatar ID
+     * @param onProgress Progress callback
+     * @param useCompressedModel Use compressed model resource (~30% of original size, with minor quality degradation). Defaults to false.
      * @returns Promise<Avatar>
      */
-    load(id: string, onProgress?: (progress: LoadProgressInfo) => void, options?: {
-        enableMonologue?: boolean;
-    }): Promise<Avatar>;
+    load(id: string, onProgress?: (progress: LoadProgressInfo) => void, useCompressedModel?: boolean): Promise<Avatar>;
     /**
-     * 清理资源加载器缓存
+     * Cancel a pending or running download task
+     * @param id Avatar ID to cancel
+     * @returns true if task was found and cancelled
      */
-    clearCache(): void;
+    cancelLoad(id: string): boolean;
+    /**
+     * Get cached avatar
+     * @param id Avatar ID
+     * @returns Avatar instance, or undefined if not in cache
+     */
+    retrieve(id: string): Avatar | undefined;
+    /**
+     * Clear cached avatar for specified ID
+     * @param id Avatar ID
+     */
+    clear(id: string): void;
+    /**
+     * Clear all avatar cache and cancel all tasks
+     */
+    clearAll(): void;
 }
-//# sourceMappingURL=AvatarManager.d.ts.map

package/dist/core/AvatarSDK.d.ts

@@ -0,0 +1,58 @@
+import { Configuration } from '../types';
+export declare class AvatarSDK {
+    private static _initializationState;
+    private static _initializingPromise;
+    private static _configuration;
+    private static readonly _version;
+    private static _avatarCore;
+    private static _dynamicSdkConfig;
+    private static _cachedDeviceScore;
+    private static _rendererBackend;
+    /**
+     * Initialize SDK
+     * @param appId Application ID to be included in both HTTP Headers and WebSocket Headers
+     * @param configuration Configuration parameters
+     */
+    static initialize(appId: string, configuration: Configuration): Promise<void>;
+    private static _initializeInternal;
+    /**
+     * Set sessionToken
+     * Developer Client -> Developer Server -> AvatarKit Server -> return sessionToken (max 1 hour validity)
+     * Include in WebSocket Headers for avatar WebSocket service authentication
+     */
+    static setSessionToken(token: string): void;
+    /**
+     * Set userId
+     * Optional interface for developers, SDK includes this in telemetry logs
+     */
+    static setUserId(userId: string): void;
+    static get isInitialized(): boolean;
+    static get appId(): string | null;
+    static get configuration(): Configuration | null;
+    static get sessionToken(): string | null;
+    static get userId(): string | null;
+    static get version(): string;
+    /**
+     * Cleanup resources
+     */
+    static cleanup(): void;
+    /**
+     * Device performance score (CPU + GPU).
+     * CPU: matrix multiply throughput in Web Worker (warmup 1s + measure 1s).
+     * GPU: WebGPU compute shader throughput (warmup 1s + measure 1s).
+     * Returns 0 for GPU if WebGPU is not available.
+     */
+    static deviceScore(): Promise<{
+        cpuScore: number;
+        gpuScore: number;
+    }>;
+    private static readonly CPU_SCORE_THRESHOLD;
+    private static readonly GPU_SCORE_THRESHOLD;
+    /**
+     * Check if the current device can run the avatar.
+     * Runs a ~2s benchmark, reports device info and scores to PostHog.
+     * Currently always returns true — thresholds will be calibrated from production data.
+     */
+    static isDeviceSupported(): Promise<boolean>;
+    private static _getGPURenderer;
+}

package/dist/core/AvatarView.d.ts

@@ -1,133 +1,52 @@
-import { CameraConfig, RenderPerformanceStats } from '../types';
+import { CameraConfig, KeyframeData } from '../types';
 import { Avatar } from './Avatar';
 import { AvatarController } from './AvatarController';
 export declare class AvatarView {
-    readonly avatarController: AvatarController;
-    backgroundImage: string | null;
-    backgroundOpaque: boolean;
+    private readonly avatarController;
+    private readonly avatar;
+    onFirstRendering?: () => void;
     private canvas;
     private renderSystem;
     private isInitialized;
     private cameraConfig;
-    private isRealtimePlaying;
+    private renderingState;
     private currentKeyframes;
     private lastRenderedFrameIndex;
-    private realtimeStartTime;
     private lastRealtimeProtoFrame;
-    private idleResumeIndexPending;
     private idleAnimationLoopId;
     private realtimeAnimationLoopId;
     private resizeObserver;
     private onWindowResize;
+    private frameCount;
+    private lastFpsUpdate;
+    private currentFPS;
     private transitionKeyframes;
-    private isTransitioning;
     private transitionStartTime;
-    private readonly transitionDurationMs;
-    private endToIdleAfterTransition;
+    private readonly startTransitionDurationMs;
+    private readonly endTransitionDurationMs;
     private cachedIdleFirstFrame;
     private idleCurrentFrameIndex;
+    private currentPlayingFrame;
+    private characterHandle;
+    private characterId;
+    private isPureRenderingMode;
+    private _renderingEnabled;
+    private avatarActiveTimer;
+    private readonly AVATAR_ACTIVE_INTERVAL;
     /**
-     * 对齐两端 Flame 维度：标量统一长度，expression 取最大长度并零填充
+     * Constructor
+     * Creates a unified AvatarController, internally composes network layer based on configuration
+     * @param avatar - Avatar instance
+     * @param container - Canvas container element (required)
      */
-    private alignFlamePair;
+    constructor(avatar: Avatar, container: HTMLElement);
     /**
-     * 判断是否为空 Flame（用于跳过 isLast=true 的哨兵帧）
+     * Get controller (public interface)
      */
-    private isEmptyFlame;
+    get controller(): AvatarController;
     /**
-     * 生成并对齐过渡帧，确保首尾帧与起止帧完全一致
-     */
-    private generateAndAlignTransitionFrames;
-    /**
-     * 获取缓存的 Idle 首帧，如果未缓存则获取并缓存
-     */
-    private getCachedIdleFirstFrame;
-    /**
-     * 构造函数
-     * 使用 AvatarController（纯动画驱动模式）
-     */
-    constructor(avatar: Avatar, canvasContainer?: HTMLElement);
-    /**
-     * 创建canvas元素
-     */
-    private createCanvas;
-    /**
-     * 获取canvas元素（供外部访问）
-     */
-    getCanvas(): HTMLCanvasElement;
-    /**
-     * 初始化视图系统
-     */
-    private initializeView;
-    /**
-     * 初始化渲染系统
-     */
-    private initializeRenderSystem;
-    /**
-     * 获取默认相机配置
-     */
-    private getDefaultCameraConfig;
-    /**
-     * 根据 PreloadCameraSettings 转换为 CameraConfig
-     */
-    private convertPreloadCameraToConfig;
-    /**
-     * 渲染第一帧
-     */
-    private renderFirstFrame;
-    /**
-     * 开始idle动画循环
-     */
-    private startIdleAnimationLoop;
-    /**
-     * 开始实时对话动画循环
-     */
-    private startRealtimeAnimationLoop;
-    /**
-     * 停止idle动画循环
-     */
-    private stopIdleAnimationLoop;
-    /**
-     * 停止实时对话动画循环
-     */
-    private stopRealtimeAnimationLoop;
-    /**
-     * 停止所有动画循环
-     */
-    private stopAllAnimationLoops;
-    /**
-     * 渲染实时对话帧
-     */
-    private renderRealtimeFrame;
-    /**
-     * 设置 AvatarController 事件监听器
-     */
-    private setupControllerEventListeners;
-    /**
-     * 设置当前关键帧数据
-     */
-    private setCurrentKeyframes;
-    /**
-     * 开始实时对话渲染
-     */
-    private startRealtimeRendering;
-    /**
-     * 停止实时对话渲染
-     */
-    private stopRealtimeRendering;
-    /**
-     * 设置背景图
-     * 支持开发者来设置渲染数字人背景图
-     */
-    setBackgroundImage(imageUrl: string | null): void;
-    /**
-     * 设置背景是否不透明
-     * 支持开发者来设置是否背景不透明
-     */
-    setBackgroundOpaque(opaque: boolean): void;
-    /**
-     * 清理视图资源
-     * 关闭 avatarController 并清理所有相关资源
+     * Cleanup view resources
+     * Closes avatarController and cleans up all related resources
      */
     dispose(): void;
     /**
@@ -139,20 +58,110 @@ export declare class AvatarView {
      */
     updateCameraConfig(cameraConfig: CameraConfig): void;
     /**
-     * 处理尺寸变化：通知渲染系统更新视口与投影
-     */
-    private handleResize;
-    /**
-     * 获取渲染性能统计
-     * @returns 渲染性能统计数据，如果渲染系统未初始化则返回 null
-     */
-    getPerformanceStats(): RenderPerformanceStats | null;
-    /**
-     * 将 CameraInfo 转换为 CameraConfig
-     */
-    /**
-     * 设置 canvas 的 CSS 背景（与 app 对齐）
-     */
-    private setCanvasBackground;
+     * Render specified keyframe (pure rendering mode, no audio-animation synchronization)
+     * Suitable for scenarios where external application controls audio playback and animation playback
+     * @param keyframeData - Keyframe data
+     * @param enableIdleRendering - Whether to enable idle loop rendering. true: Enable idle rendering and return immediately (skip this keyframe); false: Disable idle rendering and process keyframe (default)
+     */
+    renderFlame(keyframeData: KeyframeData, enableIdleRendering?: boolean): Promise<void>;
+    generateTransition(options: {
+        from?: KeyframeData;
+        to?: KeyframeData;
+        frameCount: number;
+        useLinear?: boolean;
+    }): Promise<KeyframeData[]>;
+    /**
+     * Render a single animation frame from raw protobuf data.
+     *
+     * Decodes the protobuf Message internally and renders the first keyframe.
+     * This is the preferred method for RTC consumers that receive raw animation bytes.
+     *
+     * @param data - Raw protobuf bytes (a single Message containing ServerResponseAnimation)
+     */
+    renderFromProtobuf(data: ArrayBuffer | Uint8Array): Promise<void>;
+    /**
+     * Play a transition from idle to the target frame in the protobuf data,
+     * then resolve when the transition is complete.
+     *
+     * The transition frames are generated and played internally at 25fps.
+     * The caller should wait for the returned Promise before pushing streaming frames.
+     *
+     * @param data - Raw protobuf bytes containing the target frame
+     * @param frameCount - Number of transition frames to generate
+     * @returns Promise that resolves when the transition playback finishes
+     */
+    playTransitionFromProtobuf(data: ArrayBuffer | Uint8Array, frameCount: number): Promise<void>;
+    /**
+     * Play a transition from current animation back to idle,
+     * then start the idle animation loop.
+     *
+     * Generates reverse transition frames from idle→lastFrame, reverses them,
+     * plays at 25fps, then starts idle.
+     *
+     * @param data - Raw protobuf bytes containing the last animation frame
+     * @param frameCount - Number of transition frames to generate
+     * @returns Promise that resolves when idle animation starts
+     */
+    playTransitionToIdleFromProtobuf(data: ArrayBuffer | Uint8Array, frameCount: number): Promise<void>;
+    /**
+     * Start idle animation (stop pure rendering mode, resume idle loop).
+     */
+    startIdle(): void;
+    /**
+     * Cancel any in-progress frame sequence playback.
+     * Called by renderFromProtobuf when streaming frames arrive during transition.
+     */
+    cancelFrameSequence(): void;
+    /**
+     * Pause rendering loop
+     *
+     * When called:
+     * - Rendering loop stops (no GPU/canvas updates)
+     * - Audio playback continues normally
+     * - Animation state machine continues running
+     *
+     * Use `resumeRendering()` to resume rendering.
+     *
+     * @example
+     * // Stop rendering to save GPU resources (audio continues)
+     * avatarView.pauseRendering()
+     */
+    pauseRendering(): void;
+    /**
+     * Resume rendering loop
+     *
+     * When called:
+     * - Rendering loop resumes from current state
+     * - If in Idle state, immediately renders current frame to restore display
+     *
+     * @example
+     * // Resume rendering
+     * avatarView.resumeRendering()
+     */
+    resumeRendering(): void;
+    /**
+     * Check if rendering is currently enabled
+     * @returns true if rendering is enabled, false if paused
+     */
+    isRenderingEnabled(): boolean;
+    /**
+     * Get or set avatar transform in canvas
+     *
+     * @example
+     * // Get current transform
+     * const current = avatarView.avatarTransform
+     *
+     * // Set transform
+     * avatarView.avatarTransform = { x: 0.5, y: 0, scale: 2.0 }
+     */
+    get avatarTransform(): {
+        x: number;
+        y: number;
+        scale: number;
+    };
+    set avatarTransform(value: {
+        x: number;
+        y: number;
+        scale: number;
+    });
 }
-//# sourceMappingURL=AvatarView.d.ts.map