@xmov/avatar 2.0.0-alpha.38

package/src/control/APIForwarder.ts

@@ -0,0 +1,67 @@
+/**
+ * APIForwarder - 薄客户端模式下的 API 代理转发器
+ * 将客户端的 speak/idle/listen 等 API 调用封装为 api_call 消息，
+ * 通过 StreamingClient 发送给 Server 端 SDK 执行。
+ */
+
+import { StreamingClient } from './StreamingClient';
+
+export class APIForwarder {
+  private client: StreamingClient;
+
+  constructor(client: StreamingClient) {
+    this.client = client;
+  }
+
+  speak(ssml: string, is_start: boolean = true, is_end: boolean = true, extra: any = {}) {
+    this.client.sendApiCall('speak', [ssml, is_start, is_end, extra]);
+  }
+
+  idle() {
+    this.client.sendApiCall('idle', []);
+  }
+
+  listen() {
+    this.client.sendApiCall('listen', []);
+  }
+
+  think() {
+    this.client.sendApiCall('think', []);
+  }
+
+  interactiveidle() {
+    this.client.sendApiCall('interactiveidle', []);
+  }
+
+  interrupt(type: string) {
+    this.client.sendApiCall('interrupt', [type]);
+  }
+
+  changeLayout(layout: any) {
+    this.client.sendApiCall('changeLayout', [layout]);
+  }
+
+  changeWalkConfig(walkConfig: any) {
+    this.client.sendApiCall('changeWalkConfig', [walkConfig]);
+  }
+
+  setVolume(volume: number) {
+    this.client.sendApiCall('setVolume', [volume]);
+  }
+
+  changeAvatarVisible(visible: boolean) {
+    this.client.sendApiCall('changeAvatarVisible', [visible]);
+  }
+
+  switchInvisibleMode() {
+    this.client.sendApiCall('switchInvisibleMode', []);
+  }
+
+  destroy(reason: string = 'user') {
+    this.client.sendApiCall('destroy', [reason]);
+  }
+
+  init(config?: any) {
+    this.client.send({ type: 'init', config });
+  }
+}

package/src/control/DataCacheQueue.ts

@@ -0,0 +1,372 @@
+import {
+  EFrameDataType,
+  IRawAudioFrameData,
+  IRawFaceFrameData,
+  IRawEventFrameData,
+  IRawFrameData,
+  StateChangeInfo,
+} from "../types/frame-data";
+
+export interface IBodyFrame {
+  frame: any;
+  frameIndex: number;
+  frameState: string;
+  id: number;
+  name: string;
+  body_id: number;
+  hfd: boolean;
+  sf: number;
+  offset: number;
+}
+
+export class DataCacheQueue {
+  private TAG = "[DataCacheQueue]";
+  // body视频抽帧副本，使用 Map 优化查找和删除性能
+  private _bodyQueue: Map<number, IBodyFrame> = new Map();
+  // 表情match信息列表
+  private _facialQueue: Array<IRawFaceFrameData> = [];
+  // 原始表情数据
+  private _realFacialQueue: Array<IRawFaceFrameData> = [];
+  // 音频队列
+  private audioQueue: Array<IRawAudioFrameData> = [];
+  // UI事件队列
+  private eventQueue: Array<IRawEventFrameData> = [];
+  // 当前处理好的视频id组
+  private videoIdList: string[] = [];
+  // 当前状态
+  private _currentPlayState: string = "idle";
+  // 当前ttsa状态
+  private _currentTtsaState: StateChangeInfo | null = null;
+  constructor() {
+    // @ts-ignore
+    window["__dev_event_queue__"] = this.eventQueue;
+  }
+
+  set currentPlayState(state: string) {
+    this._currentPlayState = state;
+  }
+
+  get currentPlayState() {
+    return this._currentPlayState;
+  }
+
+  set currentTtsaState(state: StateChangeInfo | null) {
+    this._currentTtsaState = state;
+  }
+  get currentTtsaState() {
+    return this._currentTtsaState;
+  }
+
+  get bodyQueue() {
+    return Array.from(this._bodyQueue.values());
+  }
+
+  // 更新body视频抽帧副本
+  _updateBodyImageBitmap(data: IBodyFrame) {
+    const old = this._bodyQueue.get(data.frameIndex);
+    if (old && old.frame && typeof old.frame.close === "function") {
+      old.frame.close();
+    }
+    // 清理过期帧，根据data.body_id,删除所有小于的帧
+    for (const [curIndex, curFrame] of this._bodyQueue.entries()) {
+      if (curFrame.body_id < data.body_id && curFrame.frameIndex >= data.frameIndex) {
+        curFrame.frame.close();
+        this._bodyQueue.delete(curIndex);
+      }
+    }
+    this._bodyQueue.set(data.frameIndex, data);
+  }
+
+  clearOldFrames(sf: number) {
+    for (const [curIndex, curFrame] of this._bodyQueue.entries()) {
+      if (curIndex >= sf) {
+        curFrame.frame.close();
+        this._bodyQueue.delete(curIndex);
+      }
+    }
+  }
+
+  clearOldFaceData(frameNumber: number) {
+    this._facialQueue = this._facialQueue.filter(
+      (item) => item.frameIndex >= frameNumber
+    );
+    this._realFacialQueue = this._realFacialQueue.filter(
+      (item) => item.frameIndex >= frameNumber
+    );
+  }
+
+  setVideoIdList(videoId: string) {
+    if (!this.videoIdList.includes(videoId)) {
+      this.videoIdList.push(videoId);
+    }
+  }
+
+  getVideoIdList() {
+    return this.videoIdList;
+  }
+
+  /**
+   * 获取指定帧并从Map中删除
+   * @param frameIndex 要获取的帧索引
+   * @returns 找到的帧数据（未找到则返回undefined）
+   */
+  _getBodyImageBitmap(frameIndex: number) {
+    const frame = this._bodyQueue.get(frameIndex);
+    
+    // 清理过期帧
+    for (const [curIndex, curFrame] of this._bodyQueue.entries()) {
+      if (curIndex < frameIndex) {
+        curFrame.frame.close();
+        this._bodyQueue.delete(curIndex);
+      }
+    }
+
+    if (frame) {
+      return frame;
+    }
+    return undefined;
+  }
+
+  // 获取body内的视频名称list
+  getBodyVideoNameListLength() {
+    const list: string[] = [];
+    this.bodyQueue.forEach((item) => {
+      if (!list.includes(item.name)) {
+        list.push(item.name);
+      }
+    });
+    return list.length;
+  }
+
+  _getFaceImageBitmap(frameIndex: number, body_id: number) {
+    // 找出face中frameIndex和body_id相同的帧
+    let targetItem: IRawFaceFrameData | null = null;
+    for (let i = this._facialQueue.length - 1; i >= 0; i--) {
+      const item = this._facialQueue[i];
+      if (item.frameIndex === frameIndex && item.body_id === body_id) {
+        targetItem = item;
+        break; // 找到后立即退出循环
+      }
+    }
+    // 删除所有小于frameIndex的帧
+    this._facialQueue = this._facialQueue.filter(
+      (item) => item.frameIndex >= frameIndex
+    );
+    return targetItem;
+  }
+  // 更新表情队列
+  _updateFacial(data: Array<IRawFaceFrameData>) {
+    this._facialQueue.push(...data);
+  }
+  get facialQueue() {
+    return this._facialQueue;
+  }
+  _getRealFaceImageBitmap(frameIndex: number, body_id: number) {
+    // 找出face中frameIndex和body_id相同的帧
+    let targetItem: IRawFaceFrameData | null = null;
+    for (let i = this._realFacialQueue.length - 1; i >= 0; i--) {
+      const item = this._realFacialQueue[i];
+      if (item?.frameIndex === frameIndex && item?.body_id === body_id) {
+        targetItem = item;
+        break; // 找到后立即退出循环
+      }
+    }
+    // 删除所有小于frameIndex的帧
+    this._realFacialQueue = this._realFacialQueue.filter(
+      (item) => item?.frameIndex >= frameIndex
+    );
+    return targetItem;
+  }
+  _updateRealFacial(data: Array<IRawFaceFrameData>) {
+    this._realFacialQueue.push(...data);
+  }
+  get realFacialQueue() {
+    return this._realFacialQueue;
+  }
+
+  /**
+   * 清空所有表情数据（用于切换隐身模式时）
+   */
+  clearAllFaceData() {
+    this._facialQueue = [];
+    this._realFacialQueue = [];
+    (window as any).avatarSDKLogger?.log(
+      this.TAG,
+      "已清空所有表情数据"
+    );
+  }
+  _updateAudio(data: Array<IRawAudioFrameData>) {
+    this.audioQueue.push(...data);
+  }
+
+  _clearAudio(speech_id: number) {
+    if (speech_id !== -1) {
+      this.audioQueue = this.audioQueue.filter(item => item.sid !== speech_id);
+    } else {
+      this.audioQueue = [];
+    }
+  }
+
+  _getAudio(frameIndex: number) {
+    const targetIndex = this.audioQueue.findIndex(
+      (item) => item.sf === frameIndex
+    );
+    if (targetIndex === -1) {
+      return;
+    }
+    const [targetItem] = this.audioQueue.splice(targetIndex, 1);
+    return targetItem;
+  }
+
+  _getAudioInterval(startFrame: number, endFrame: number) {
+    const audioList = this.audioQueue.filter(
+      (item) => item.sf >= startFrame && item.sf <= endFrame
+    );
+    if (audioList.length === 0) {
+      return;
+    }
+    const audio = audioList[audioList.length - 1];
+    const index = this.audioQueue.findIndex((item) => item.id === audio.id);
+    this.audioQueue.splice(index, 1);
+    return audio;
+  }
+
+  _updateUiEvent(data: Array<IRawEventFrameData>) {
+    this.eventQueue.push(...data);
+  }
+
+  clearOldEvent(frame: number) {
+    this.eventQueue = this.eventQueue.filter(item => item.sf >= frame);
+  }
+
+  clearSubtitleOn(speech_id: number) {
+    this.eventQueue = this.eventQueue.map(item => ({
+      ...item,
+      e: item.e?.filter((subItem: any) => subItem.speech_id > speech_id || subItem.type !== "subtitle_on")
+    }));
+  }
+
+  _clearEvent() {
+    this.eventQueue = [];
+  }
+
+  // 获取指定帧的事件，并合并相同 sf 的 e 数组
+  _getEvent(frame: number) {
+    // 1. 找出所有匹配 sf 的事件
+    const matchedItems = this.eventQueue.filter(item => item.sf === frame);
+    if (matchedItems.length === 0) return;
+
+    // 2. 从原队列删除这些事件
+    this.eventQueue = this.eventQueue.filter(item => item.sf !== frame);
+
+    // 3. 合并所有 e 数组（核心逻辑）
+    return this._mergeEventsBySf(matchedItems);
+  }
+
+  // 获取区间内事件，并合并相同 sf 的 e 数组
+  _getEventInterval(startFrame: number, endFrame: number) {
+    // 1. 找出所有区间内的事件
+    const matchedItems = this.eventQueue.filter(
+      item => item.sf >= startFrame && item.sf <= endFrame
+    );
+    if (matchedItems.length === 0) return;
+
+    // 2. 从原队列删除这些事件
+    this.eventQueue = this.eventQueue.filter(
+      item => !(item.sf >= startFrame && item.sf <= endFrame)
+    );
+
+    // 3. 合并所有 e 数组（核心逻辑）
+    return this._mergeEventsBySf(matchedItems);
+  }
+
+  // 【工具方法】合并相同 sf 的事件，把 e 数组拼接在一起
+  _mergeEventsBySf(events: any[]) {
+    // 以 sf 为 key 分组合并
+    const grouped = {};
+    for (const event of events) {
+      const key = event.sf;
+      if (!grouped[key]) {
+        // 第一次遇到该 sf，创建基础结构
+        grouped[key] = { ...event, e: [] };
+      }
+      // 把当前事件的 e 数组 push 进去（合并）
+      grouped[key].e.push(...event.e);
+    }
+
+    // 取出合并后的结果（你这里 sf 都相同，只会有一个）
+    const mergedList:any = Object.values(grouped);
+    if(mergedList.length === 0) return null
+    if (mergedList.length > 1) {
+      // 合并所有帧的e到一个数组
+      const allE: any[] = [];
+      mergedList.forEach((item:any) => allE.push(...item.e));
+      // 构造单个event对象，和你原来代码格式完全一致
+      return {
+        ...mergedList[0],
+        mergeFrameFlag: 1,
+        e: allE
+      };
+    } else{
+      return mergedList[0];
+    }
+  }
+
+  /**
+   * 检查数据是否因seek等原因失效，并清理缓存。
+   * @param data 新的数据帧数组
+   * @param type 数据类型
+   */
+  checkValidData(data: Array<IRawFrameData>, type: string) {
+    switch (type) {
+      case EFrameDataType.BODY:
+        break;
+      case EFrameDataType.FACE: {
+        // 根据bodyId获取face，暂时去掉过滤
+        // if (this._facialQueue.length === 0 || data.length === 0) {
+        //   return;
+        // }
+        // const isDiscontinuous =
+        //   this._facialQueue[this._facialQueue.length - 1]?.ef >= data[0].sf ||
+        //   data[0].sf <= this._facialQueue[0]?.sf;
+        // if (isDiscontinuous) {
+        //   const facialIndex = this._facialQueue.findIndex(
+        //     (item) => item.sf >= data[0].sf
+        //   );
+        //   if (facialIndex !== -1) {
+        //     this._facialQueue.length = facialIndex;
+        //   }
+        // }
+        break;
+      }
+      case EFrameDataType.AUDIO: {
+        if (this.audioQueue.length === 0 || data.length === 0) {
+          return;
+        }
+        const isDiscontinuous =
+          this.audioQueue[this.audioQueue.length - 1]?.ef < data[0].sf ||
+          data[0].sf < this.audioQueue[0]?.sf;
+        if (isDiscontinuous) {
+          const audioIndex = this.audioQueue.findIndex(
+            (item) => item.sf >= data[0].sf
+          );
+          if (audioIndex !== -1) {
+            this.audioQueue.length = audioIndex;
+          }
+        }
+        break;
+      }
+    }
+  }
+  destroy() {
+    this._bodyQueue.forEach((item) => {
+      if (item.frame && typeof item.frame.close === "function") {
+        item.frame.close();
+      }
+    });
+    this._bodyQueue.clear();
+    this._facialQueue = [];
+    this.audioQueue = [];
+    this.eventQueue = [];
+  }
+}

package/src/control/EventDispatcher.ts

@@ -0,0 +1,106 @@
+/**
+ * EventDispatcher - 薄客户端模式下的事件分发器
+ * 接收来自 Streaming Server 转发的 event_proxy 消息，
+ * 分发到用户注册的回调函数。
+ */
+
+import { AvatarStatus, RenderState, IFrameDataUpdate } from '../types/index';
+import type { IRawWidgetData } from '../types/frame-data';
+import { FPSStats } from '../utils/fpsTracker';
+
+export interface EventDispatcherCallbacks {
+  onStateChange?: (state: string) => void;
+  onSpeakStateChange?: (state: string, client_speak_id: string | number) => void;
+  onVoiceStateChange?: (state: string, duration?: number, client_speak_id?: string | number) => void;
+  onStatusChange?: (status: AvatarStatus) => void;
+  onRenderChange?: (state: RenderState) => void;
+  onWalkStateChange?: (state: string) => void;
+  onWidgetEvent?: (data: IRawWidgetData) => void;
+  onMessage?: (error: any) => void;
+  onNetworkInfo?: (info: any) => void;
+  onFPSUpdate?: (stats: FPSStats) => void;
+  onDownloadProgress?: (progress: number) => void;
+  onSessionId?: (sessionId: string) => void;
+  onFrameDataUpdate?: (data: IFrameDataUpdate) => void;
+}
+
+export class EventDispatcher {
+  private callbacks: EventDispatcherCallbacks;
+  private subtitleText = '';
+  private subtitleVisible = false;
+
+  constructor(callbacks: EventDispatcherCallbacks) {
+    this.callbacks = callbacks;
+  }
+
+  /**
+   * 处理来自 Streaming Server 的 event_proxy 消息
+   */
+  dispatch(eventType: string, data: any, _serverFrame?: number) {
+    switch (eventType) {
+      case 'state_change':
+        this.callbacks.onStateChange?.(data.state || data);
+        break;
+
+      case 'speak_state_change':
+        this.callbacks.onSpeakStateChange?.(data.state, data.client_speak_id);
+        break;
+
+      case 'voice_state_change':
+        this.callbacks.onVoiceStateChange?.(data.state, data.duration, data.client_speak_id || '');
+        break;
+
+      case 'status_change':
+        this.callbacks.onStatusChange?.(data.status);
+        break;
+
+      case 'render_change':
+        this.callbacks.onRenderChange?.(data.state);
+        break;
+
+      case 'walk_state_change':
+        this.callbacks.onWalkStateChange?.(data.state || data);
+        break;
+
+      case 'widget_event':
+        this.callbacks.onWidgetEvent?.(data);
+        break;
+
+      case 'message':
+      case 'error':
+        this.callbacks.onMessage?.(data);
+        break;
+
+      case 'network_info':
+        this.callbacks.onNetworkInfo?.(data);
+        break;
+
+      case 'fps_update':
+        this.callbacks.onFPSUpdate?.(data);
+        break;
+
+      case 'download_progress':
+        this.callbacks.onDownloadProgress?.(data.progress);
+        break;
+
+      case 'session_id':
+        this.callbacks.onSessionId?.(data.session_id);
+        break;
+
+      case 'frame_data_update':
+        this.callbacks.onFrameDataUpdate?.(data);
+        break;
+
+      default:
+        console.log('[EventDispatcher] 未处理的事件类型:', eventType, data);
+    }
+  }
+
+  getSubtitleText(): string {
+    return this.subtitleText;
+  }
+
+  isSubtitleVisible(): boolean {
+    return this.subtitleVisible;
+  }
+}