@spatialwalk/avatarkit 1.0.0-beta.9 → 1.0.0-beta.91

package/dist/core/Avatar.d.ts

@@ -1,23 +1,13 @@
 import { CharacterMeta } from '../types';
-import { PreloadResult } from './AvatarDownloader';
 export declare class Avatar {
     readonly id: string;
     private characterMeta;
     private resources;
+    /** Local-only field: tracks whether cached model is "standard" or "compressed". */
+    _cachedModelType: string;
     /**
-     * 构造函数（内部使用）
-     * @param id 数字人 ID
-     * @param characterMeta 角色元数据
-     * @param resources 资源数据
-     */
-    constructor(id: string, characterMeta: CharacterMeta, resources: PreloadResult);
-    /**
-     * 获取角色元数据（内部使用）
+     * Get character metadata
+     * @returns Character metadata, including all configuration information (version, resource URLs, camera config, etc.)
      */
     getCharacterMeta(): CharacterMeta;
-    /**
-     * 获取资源数据（内部使用）
-     */
-    getResources(): PreloadResult;
 }
-//# sourceMappingURL=Avatar.d.ts.map

package/dist/core/AvatarController.d.ts

@@ -1,157 +1,150 @@
-import { Flame } from '../generated/driveningress/v1/driveningress';
 import { Avatar } from './Avatar';
-import { AnimationPlayer } from '../audio/AnimationPlayer';
-import { AvatarState, ConnectionState, AvatarPlaybackMode } from '../types';
+import { ConnectionState, DrivingServiceMode, ConversationState, PostProcessingConfig, KeyframeData } from '../types';
+import { FrameRateInfo } from '../performance/FrameRateMonitor';
 export declare class AvatarController {
     private networkLayer?;
-    readonly playbackMode: AvatarPlaybackMode;
-    protected avatar: Avatar;
-    protected animationPlayer: AnimationPlayer | null;
-    protected currentKeyframes: Flame[];
-    protected pendingAudioChunks: Array<{
-        data: Uint8Array;
-        isLast: boolean;
-    }>;
-    protected isPlaying: boolean;
-    protected isConnected: boolean;
-    protected currentState: AvatarState;
+    private readonly playbackMode;
+    private isStartingPlayback;
+    private currentConversationId;
+    private reqEnd;
     onConnectionState: ((state: ConnectionState) => void) | null;
-    onAvatarState: ((state: AvatarState) => void) | null;
+    onConversationState: ((state: ConversationState) => void) | null;
     onError: ((error: Error) => void) | null;
     private eventListeners;
+    private readonly frameRateMonitor;
+    /** Frame rate monitoring callback. Fires with aggregated metrics from a 2-second sliding window. */
+    get onFrameRateInfo(): ((info: FrameRateInfo) => void) | null;
+    set onFrameRateInfo(value: ((info: FrameRateInfo) => void) | null);
+    /** Whether frame rate monitoring is enabled. Default is false (zero overhead when disabled). */
+    get frameRateMonitorEnabled(): boolean;
+    set frameRateMonitorEnabled(value: boolean);
     private renderCallback?;
-    private transitionCompleteCallback?;
+    private characterHandle;
+    private characterId;
+    private postProcessingConfig;
     private playbackLoopId;
+    private playbackLoopGeneration;
     private lastRenderedFrameIndex;
+    private keyframesOffset;
+    private readonly MAX_KEYFRAMES;
+    private readonly KEYFRAMES_CLEANUP_THRESHOLD;
+    private lastSyncLogTime;
+    private lastOutOfBoundsState;
+    private isFallbackMode;
+    private playbackStuckCheckState;
+    private readonly MAX_AUDIO_TIME_ZERO_COUNT;
+    private readonly MAX_AUDIO_TIME_STUCK_COUNT;
+    private readonly AUDIO_TIME_STUCK_THRESHOLD;
+    private hostModeMetrics;
+    private readonly audioBytesPerSecond;
     constructor(avatar: Avatar, options?: {
-        playbackMode?: AvatarPlaybackMode;
+        playbackMode?: DrivingServiceMode;
     });
+    private handleVisibilityChange;
+    private shouldReportPlaybackStats;
+    private _getDeviceScoreProps;
     /**
-     * 获取 Avatar ID（供 NetworkLayer 使用）
+     * Get current conversation ID
+     * Returns the current conversation ID for the active audio session
+     * @returns Current conversation ID, or null if no active session
      */
-    getAvatarId(): string;
+    getCurrentConversationId(): string | null;
     /**
-     * 获取播放状态（供 NetworkLayer 使用）
+     * Initialize audio context (must be called in user gesture context)
+     *
+     * This method must be called before any audio operations (send, yieldAudioData, etc.)
+     * to ensure AudioContext is created and initialized in a user gesture context.
+     *
+     * @example
+     * // In user click handler
+     * button.addEventListener('click', async () => {
+     *   await avatarView.controller.initializeAudioContext()
+     *   // Now you can safely use send() or yieldAudioData()
+     * })
      */
-    getIsPlaying(): boolean;
+    initializeAudioContext(): Promise<void>;
     /**
-     * 设置连接状态（供 NetworkLayer 使用）
-     */
-    setConnected(connected: boolean): void;
-    /**
-     * 获取连接状态
-     */
-    get connected(): boolean;
-    /**
-     * 获取当前状态
-     */
-    get state(): AvatarState;
-    /**
-     * 获取动画播放器实例
-     * @internal
-     */
-    getAnimationPlayer(): AnimationPlayer | null;
-    /**
-     * 启动服务（仅网络模式）
+     * Start service (SDK mode only)
      */
     start(): Promise<void>;
     /**
-     * 发送音频到服务器（仅网络模式）
-     * 同时缓存到数据层用于播放
+     * Send audio to server (SDK mode only)
+     * Also cache to data layer for playback
+     * @returns conversationId - Conversation ID for this audio session
      */
-    send(audioData: ArrayBuffer, end?: boolean): void;
+    send(audioData: ArrayBuffer, end?: boolean): string | null;
     /**
-     * 关闭服务（仅网络模式）
+     * Close service (SDK mode only)
      */
     close(): void;
     /**
-     * 开始播放（外部数据模式）
-     * 必须先调用此方法，传入初始数据开始播放
-     */
-    play(initialAudioChunks?: Array<{
-        data: Uint8Array;
-        isLast: boolean;
-    }>, initialKeyframes?: Flame[]): Promise<void>;
-    /**
-     * 发送音频数据（外部数据模式）
-     * 在 play() 之后流式添加剩余音频数据
-     */
-    sendAudioChunk(data: Uint8Array, isLast?: boolean): void;
-    /**
-     * 发送动画关键帧（外部数据模式或网络模式）
-     * 在 play() 之后流式添加剩余动画数据
-     */
-    sendKeyframes(keyframes: Flame[]): void;
-    /**
-     * 打断当前播放
+     * Send audio data (host mode)
+     * Stream additional audio data after playback()
+     * @returns conversationId - Conversation ID for this audio session
      */
-    interrupt(): void;
-    /**
-     * 清理所有数据和资源
-     */
-    clear(): void;
-    /**
-     * 销毁控制器，清理所有回调避免内存泄漏
-     * 应该在 AvatarView.dispose() 时调用
-     */
-    dispose(): void;
-    /**
-     * 开始流式播放（内部方法，由 NetworkLayer 或 play() 调用）
-     */
-    startStreamingPlayback(): Promise<void>;
-    /**
-     * 设置渲染回调（由 AvatarView 调用）
-     */
-    setRenderCallback(callback: (splatData: Float32Array, frameIndex: number) => void): void;
+    yieldAudioData(data: Uint8Array, isLast?: boolean): string | null;
     /**
-     * 设置过渡完成回调（由 AvatarView 调用）
+     * Send animation keyframes (host mode or SDK mode)
+     * Stream additional animation data after playback()
+     *
+     * Public API: accepts binary data array (protobuf encoded Message array)
+     * @param keyframesDataArray - Animation keyframes binary data array (each element is a protobuf encoded Message) or empty array to trigger audio-only mode
+     * @param conversationId - Conversation ID (required). If conversationId doesn't match current conversationId, keyframes will be discarded.
+     *                Use getCurrentConversationId() to get the current conversationId.
      */
-    setTransitionCompleteCallback(callback: () => void): void;
+    yieldFramesData(keyframesDataArray: (Uint8Array | ArrayBuffer)[], conversationId: string): void;
     /**
-     * 过渡完成通知（由 AvatarView 调用）
+     * Send animation keyframes (host mode)
+     * Stream animation keyframes data after yieldAudioData()
+     *
+     * ⚠️ **Not recommended**: Prefer using `yieldFramesData()` method, which accepts binary data (protobuf encoded Message) with better performance.
+     * This method is only for scenarios where you already have decoded KeyframeData arrays.
+     *
+     * Public API: accepts decoded keyframes
+     * @param keyframes - Animation keyframes array (KeyframeData[]) or empty array to trigger audio-only mode
+     * @param conversationId - Conversation ID (required). If conversationId doesn't match current conversationId, keyframes will be discarded.
+     *                Use getCurrentConversationId() to get the current conversationId.
      */
-    onTransitionComplete(): void;
+    yieldKeyframes(keyframes: KeyframeData[], conversationId: string): void;
     /**
-     * 为 AvatarView 提供注册内部事件的接口
-     * @internal
+     * Pause playback (can be resumed later)
+     * Pause audio playback and stop render loop, but preserve all state (keyframes, audio buffers, etc.)
      */
-    setupInternalEventListeners(callbacks: {
-        onKeyframesUpdate?: (keyframes: Flame[]) => void;
-        onStartRendering?: () => void;
-        onStopRendering?: () => void;
-        onInterrupt?: () => void;
-    }): void;
+    pause(): void;
     /**
-     * 开始流式播放（内部实现）
+     * Resume playback (from paused state)
+     * Resume audio playback and restart render loop
+     * Animation will continue from paused frame (because animation time base comes from audio, will auto-sync)
      */
-    private startStreamingPlaybackInternal;
+    resume(): Promise<void>;
     /**
-     * 播放循环：根据音频时间计算动画帧，通知渲染层渲染
+     * Interrupt current playback
      */
-    private startPlaybackLoop;
-    /**
-     * 停止播放循环
-     */
-    private stopPlaybackLoop;
+    interrupt(): void;
     /**
-     * 停止播放
+     * Clear all data and resources
      */
-    protected stopPlayback(): void;
+    clear(): void;
     /**
-     * 清理播放器
+     * Get point cloud count of the current avatar
+     * @returns Point cloud count, or null if avatar is not loaded
      */
-    protected cleanupPlayers(): void;
+    getPointCount(): number | null;
     /**
-     * 添加音频块到缓冲区
+     * Set post-processing configuration
+     * These parameters will be applied in real-time to animation parameters returned by the server
+     * @param config Post-processing configuration, or null to clear
      */
-    private addAudioChunkToBuffer;
+    setPostProcessingConfig(config: PostProcessingConfig | null): void;
     /**
-     * 事件系统
+     * Set audio playback volume
+     * Note: This only controls the avatar audio player volume, not the system volume
+     * @param volume Volume value, range from 0.0 to 1.0 (0.0 = mute, 1.0 = max volume)
      */
-    private registerEventListener;
+    setVolume(volume: number): void;
     /**
-     * 触发事件
+     * Get current audio playback volume
+     * @returns Current volume value (0.0 - 1.0)
      */
-    protected emit(event: string, data?: any): void;
+    getVolume(): number;
 }
-//# sourceMappingURL=AvatarController.d.ts.map

package/dist/core/AvatarManager.d.ts

@@ -3,24 +3,44 @@ import { Avatar } from './Avatar';
 export declare class AvatarManager {
     private static _instance;
     private avatarDownloader;
-    private _templateInitialized;
+    private avatarCache;
+    /** 下载队列：FIFO 顺序 */
+    private downloadQueue;
+    /** 当前正在执行的任务 */
+    private currentTask;
+    /** 任务索引：快速查找 id 对应的任务 */
+    private taskIndex;
     /**
-     * 通过全局单例来访问
+     * Access via global singleton
      */
     static get shared(): AvatarManager;
     /**
-     * 加载数字人
-     * @param id 数字人 ID
-     * @param onProgress 进度回调
-     * @param options 加载选项
+     * Load avatar
+     * @param id Avatar ID
+     * @param onProgress Progress callback
+     * @param useCompressedModel Use compressed model resource (~30% of original size, with minor quality degradation). Defaults to false.
      * @returns Promise<Avatar>
      */
-    load(id: string, onProgress?: (progress: LoadProgressInfo) => void, options?: {
-        enableMonologue?: boolean;
-    }): Promise<Avatar>;
+    load(id: string, onProgress?: (progress: LoadProgressInfo) => void, useCompressedModel?: boolean): Promise<Avatar>;
     /**
-     * 清理资源加载器缓存
+     * Cancel a pending or running download task
+     * @param id Avatar ID to cancel
+     * @returns true if task was found and cancelled
      */
-    clearCache(): void;
+    cancelLoad(id: string): boolean;
+    /**
+     * Get cached avatar
+     * @param id Avatar ID
+     * @returns Avatar instance, or undefined if not in cache
+     */
+    retrieve(id: string): Avatar | undefined;
+    /**
+     * Clear cached avatar for specified ID
+     * @param id Avatar ID
+     */
+    clear(id: string): void;
+    /**
+     * Clear all avatar cache and cancel all tasks
+     */
+    clearAll(): void;
 }
-//# sourceMappingURL=AvatarManager.d.ts.map

package/dist/core/AvatarSDK.d.ts

@@ -0,0 +1,58 @@
+import { Configuration } from '../types';
+export declare class AvatarSDK {
+    private static _initializationState;
+    private static _initializingPromise;
+    private static _configuration;
+    private static readonly _version;
+    private static _avatarCore;
+    private static _dynamicSdkConfig;
+    private static _cachedDeviceScore;
+    private static _rendererBackend;
+    /**
+     * Initialize SDK
+     * @param appId Application ID to be included in both HTTP Headers and WebSocket Headers
+     * @param configuration Configuration parameters
+     */
+    static initialize(appId: string, configuration: Configuration): Promise<void>;
+    private static _initializeInternal;
+    /**
+     * Set sessionToken
+     * Developer Client -> Developer Server -> AvatarKit Server -> return sessionToken (max 1 hour validity)
+     * Include in WebSocket Headers for avatar WebSocket service authentication
+     */
+    static setSessionToken(token: string): void;
+    /**
+     * Set userId
+     * Optional interface for developers, SDK includes this in telemetry logs
+     */
+    static setUserId(userId: string): void;
+    static get isInitialized(): boolean;
+    static get appId(): string | null;
+    static get configuration(): Configuration | null;
+    static get sessionToken(): string | null;
+    static get userId(): string | null;
+    static get version(): string;
+    /**
+     * Cleanup resources
+     */
+    static cleanup(): void;
+    /**
+     * Device performance score (CPU + GPU).
+     * CPU: matrix multiply throughput in Web Worker (warmup 1s + measure 1s).
+     * GPU: WebGPU compute shader throughput (warmup 1s + measure 1s).
+     * Returns 0 for GPU if WebGPU is not available.
+     */
+    static deviceScore(): Promise<{
+        cpuScore: number;
+        gpuScore: number;
+    }>;
+    private static readonly CPU_SCORE_THRESHOLD;
+    private static readonly GPU_SCORE_THRESHOLD;
+    /**
+     * Check if the current device can run the avatar.
+     * Runs a ~2s benchmark, reports device info and scores to PostHog.
+     * Currently always returns true — thresholds will be calibrated from production data.
+     */
+    static isDeviceSupported(): Promise<boolean>;
+    private static _getGPURenderer;
+}

package/dist/core/AvatarView.d.ts

@@ -1,10 +1,10 @@
-import { CameraConfig, RenderPerformanceStats, AvatarViewOptions, AvatarPlaybackMode } from '../types';
+import { CameraConfig, KeyframeData } from '../types';
 import { Avatar } from './Avatar';
 import { AvatarController } from './AvatarController';
 export declare class AvatarView {
     private readonly avatarController;
-    readonly playbackMode: AvatarPlaybackMode;
     private readonly avatar;
+    onFirstRendering?: () => void;
     private canvas;
     private renderSystem;
     private isInitialized;
@@ -17,137 +17,36 @@ export declare class AvatarView {
     private realtimeAnimationLoopId;
     private resizeObserver;
     private onWindowResize;
+    private frameCount;
+    private lastFpsUpdate;
+    private currentFPS;
     private transitionKeyframes;
     private transitionStartTime;
-    private readonly transitionDurationMs;
+    private readonly startTransitionDurationMs;
+    private readonly endTransitionDurationMs;
     private cachedIdleFirstFrame;
     private idleCurrentFrameIndex;
+    private currentPlayingFrame;
+    private characterHandle;
+    private characterId;
+    private isPureRenderingMode;
+    private _renderingEnabled;
+    private avatarActiveTimer;
+    private readonly AVATAR_ACTIVE_INTERVAL;
     /**
-     * 对齐两端 Flame 维度：标量统一长度，expression 取最大长度并零填充
+     * Constructor
+     * Creates a unified AvatarController, internally composes network layer based on configuration
+     * @param avatar - Avatar instance
+     * @param container - Canvas container element (required)
      */
-    private alignFlamePair;
+    constructor(avatar: Avatar, container: HTMLElement);
     /**
-     * 判断是否为空 Flame（用于跳过 isLast=true 的哨兵帧）
-     */
-    private isEmptyFlame;
-    /**
-     * 生成并对齐过渡帧，确保首尾帧与起止帧完全一致
-     */
-    private generateAndAlignTransitionFrames;
-    /**
-     * 获取缓存的 Idle 首帧，如果未缓存则获取并缓存
-     */
-    private getCachedIdleFirstFrame;
-    /**
-     * 构造函数
-     * 创建统一的 AvatarController，内部根据配置自动组合网络层
-     */
-    constructor(avatar: Avatar, options?: AvatarViewOptions);
-    /**
-     * 获取控制器（公共接口）
+     * Get controller (public interface)
      */
     get controller(): AvatarController;
     /**
-     * 创建canvas元素
-     */
-    private createCanvas;
-    /**
-     * 获取canvas元素（供外部访问）
-     */
-    getCanvas(): HTMLCanvasElement;
-    /**
-     * 初始化视图系统
-     */
-    private initializeView;
-    /**
-     * 初始化渲染系统
-     */
-    private initializeRenderSystem;
-    /**
-     * 获取默认相机配置
-     */
-    private getDefaultCameraConfig;
-    /**
-     * 根据资源解析最终的相机配置，优先使用角色设置，其次 camera.json
-     */
-    private resolveCameraConfig;
-    /**
-     * 从角色设置中推导相机配置
-     */
-    private deriveCameraConfigFromSettings;
-    /**
-     * 渲染第一帧
-     */
-    private renderFirstFrame;
-    /**
-     * 开始idle动画循环
-     */
-    private startIdleAnimationLoop;
-    /**
-     * 开始实时对话动画循环
-     */
-    private startRealtimeAnimationLoop;
-    /**
-     * 停止idle动画循环
-     */
-    private stopIdleAnimationLoop;
-    /**
-     * 停止实时对话动画循环
-     */
-    private stopRealtimeAnimationLoop;
-    /**
-     * 停止所有动画循环
-     */
-    private stopAllAnimationLoops;
-    /**
-     * 渲染实时帧（由播放层回调调用）
-     */
-    private renderRealtimeFrame;
-    /**
-     * 过渡完成回调（通知播放层启动音频）
-     */
-    private onTransitionComplete;
-    /**
-     * 状态转换方法
-     * 统一管理状态转换，确保状态一致性
-     */
-    private setState;
-    /**
-     * 检查是否在实时播放状态（Speaking 或过渡到 Speaking）
-     */
-    private get isRealtimePlaying();
-    /**
-     * 检查是否在过渡中
-     */
-    private get isTransitioning();
-    /**
-     * 检查过渡结束后是否回到 Idle
-     */
-    private get endToIdleAfterTransition();
-    /**
-     * 处理打断
-     * 打断时应该生成过渡动画，而不是直接跳回 Idle
-     */
-    private handleInterrupt;
-    /**
-     * 设置 AvatarController 事件监听器
-     */
-    private setupControllerEventListeners;
-    /**
-     * 准备实时渲染（生成 Idle -> Speaking 过渡）
-     */
-    private prepareRealtimeRendering;
-    /**
-     * 开始实时渲染循环
-     */
-    private startRealtimeRendering;
-    /**
-     * 停止实时对话渲染
-     */
-    private stopRealtimeRendering;
-    /**
-     * 清理视图资源
-     * 关闭 avatarController 并清理所有相关资源
+     * Cleanup view resources
+     * Closes avatarController and cleans up all related resources
      */
     dispose(): void;
     /**
@@ -159,13 +58,68 @@ export declare class AvatarView {
      */
     updateCameraConfig(cameraConfig: CameraConfig): void;
     /**
-     * 处理尺寸变化：通知渲染系统更新视口与投影
-     */
-    private handleResize;
-    /**
-     * 获取渲染性能统计
-     * @returns 渲染性能统计数据，如果渲染系统未初始化则返回 null
-     */
-    getPerformanceStats(): RenderPerformanceStats | null;
+     * Render specified keyframe (pure rendering mode, no audio-animation synchronization)
+     * Suitable for scenarios where external application controls audio playback and animation playback
+     * @param keyframeData - Keyframe data
+     * @param enableIdleRendering - Whether to enable idle loop rendering. true: Enable idle rendering and return immediately (skip this keyframe); false: Disable idle rendering and process keyframe (default)
+     */
+    renderFlame(keyframeData: KeyframeData, enableIdleRendering?: boolean): Promise<void>;
+    generateTransition(options: {
+        from?: KeyframeData;
+        to?: KeyframeData;
+        frameCount: number;
+        useLinear?: boolean;
+    }): Promise<KeyframeData[]>;
+    /**
+     * Pause rendering loop
+     *
+     * When called:
+     * - Rendering loop stops (no GPU/canvas updates)
+     * - Audio playback continues normally
+     * - Animation state machine continues running
+     *
+     * Use `resumeRendering()` to resume rendering.
+     *
+     * @example
+     * // Stop rendering to save GPU resources (audio continues)
+     * avatarView.pauseRendering()
+     */
+    pauseRendering(): void;
+    /**
+     * Resume rendering loop
+     *
+     * When called:
+     * - Rendering loop resumes from current state
+     * - If in Idle state, immediately renders current frame to restore display
+     *
+     * @example
+     * // Resume rendering
+     * avatarView.resumeRendering()
+     */
+    resumeRendering(): void;
+    /**
+     * Check if rendering is currently enabled
+     * @returns true if rendering is enabled, false if paused
+     */
+    isRenderingEnabled(): boolean;
+    /**
+     * Get or set avatar transform in canvas
+     *
+     * @example
+     * // Get current transform
+     * const current = avatarView.transform
+     *
+     * // Set transform
+     * avatarView.transform = { x: 0.5, y: 0, scale: 2.0 }
+     */
+    get transform(): {
+        x: number;
+        y: number;
+        scale: number;
+    };
+    set transform(value: {
+        x: number;
+        y: number;
+        scale: number;
+    });
 }
-//# sourceMappingURL=AvatarView.d.ts.map

package/dist/demo/src/main.d.ts

@@ -0,0 +1 @@
+export {};