@xmov/avatar 2.4.0 → 2.5.0-alpha.1

package/src/baseRender/pcm-audio-processor.js

@@ -1,267 +0,0 @@
-// pcm-audio-processor.js - 运行在音频线程的Worklet处理器
-class PCMAudioProcessor extends AudioWorkletProcessor {
-  // 音量配置（与主线程保持一致）
-  static get parameterDescriptors() {
-    return [
-      {
-        name: 'volume',
-        defaultValue: 1.0,
-        minValue: 0.0,
-        maxValue: 1.0,
-        automationRate: 'a-rate' // 支持音频速率自动化
-      }
-    ];
-  }
-
-  constructor() {
-    super();
-    this.config = {
-      sampleRate: 24000, // 固定24kHz采样率
-      bytesPerSample: 2, // S16LE=2字节/采样点
-    };
-    this.pcmCache = new Uint8Array(0); // Worklet内的PCM缓存
-    this.isActive = false; // 播放音频状态（控制是否生成声音）
-    this.isDestroyed = false;
-    this.fadeState = {
-      // 淡入淡出状态（避免片段切换爆音）
-      phase: 'idle', // idle/fadeIn/fadeOut/steady
-      currentGain: 0,
-      fadeTime: 0.01, // 淡入淡出时间（秒，建议最小0.01s避免爆音）
-      fadeSamples: Math.floor(0.01 * 24000), // 淡入淡出样本数（24000Hz下为240样本）
-    };
-    this.currentSegment = null; // 当前处理的音频段（float32数组）
-    this.segmentOffset = 0; // 当前音频段的播放偏移量
-    
-    // 监听主线程消息（接收PCM数据、状态控制等）
-    this.port.onmessage = (e) => this.handleMainThreadMessage(e.data);
-    
-    // 初始化淡入淡出样本数
-    this.fadeState.fadeSamples = Math.floor(this.fadeState.fadeTime * this.config.sampleRate);
-  }
-
-  /**
-   * 处理主线程发来的消息（PCM数据/状态控制等）
-   * @param {Object} data - 消息数据
-   * @param {string} data.type - 消息类型：'pcm'/'start'/'pause'/'stop'/'config'/'destroy'/'volume'
-   * @param {Uint8Array} [data.pcmData] - PCM数据（仅type='pcm'时存在）
-   * @param {Object} [data.config] - 配置更新（仅type='config'时存在）
-   * @param {number} [data.volume] - 音量值（仅type='volume'时存在）
-   */
-  handleMainThreadMessage(data) {
-    switch (data.type) {
-      // 接收主线程下发的PCM流数据
-      case 'pcm':
-        if (data.pcmData instanceof Uint8Array && data.pcmData.length > 0) {
-          // 确保PCM数据字节数为2的整数倍（S16LE要求）
-          const validLength = data.pcmData.length - (data.pcmData.length % this.config.bytesPerSample);
-          if (validLength <= 0) break;
-          
-          // 合并到Worklet缓存
-          const newCache = new Uint8Array(this.pcmCache.length + validLength);
-          newCache.set(this.pcmCache, 0);
-          newCache.set(data.pcmData.subarray(0, validLength), this.pcmCache.length);
-          this.pcmCache = newCache;
-          
-          // 向主线程发送缓存状态
-          this.port.postMessage({
-            type: 'cacheStatus',
-            size: this.pcmCache.length,
-            sampleCount: this.pcmCache.length / this.config.bytesPerSample
-          });
-        }
-        break;
-
-      // 启动播放（继续音频生成）
-      case 'start':
-        this.isActive = true;
-        break;
-
-      // 暂停播放（停止音频生成，保留缓存）
-      case 'pause':
-        this.isActive = false;
-        this.resetSegmentState();
-        break;
-
-      // 停止播放（清空缓存+重置状态）
-      case 'stop':
-        this.isActive = false;
-        this.pcmCache = new Uint8Array(0);
-        this.resetSegmentState();
-        break;
-
-      // 更新配置（如采样率/淡入淡出时间）
-      case 'config':
-        if (data.config.sampleRate) {
-          this.config.sampleRate = data.config.sampleRate;
-          // 重新计算淡入淡出样本数
-          this.fadeState.fadeSamples = Math.floor(this.fadeState.fadeTime * this.config.sampleRate);
-        }
-        if (data.config.fadeTime) {
-          this.fadeState.fadeTime = data.config.fadeTime;
-          this.fadeState.fadeSamples = Math.floor(this.fadeState.fadeTime * this.config.sampleRate);
-        }
-        break;
-
-      // 单独更新音量（可选，也可通过参数自动化控制）
-      case 'volume':
-        this.volume = Number(data.volume);
-        // 通知主线程同步AudioWorkletNode的parameters
-        this.port.postMessage({
-          type: 'syncVolume',
-          volume: this.volume
-        });
-        break;
-
-      // 销毁处理器
-      case 'destroy':
-        this.isDestroyed = true;
-        this.isActive = false;
-        this.pcmCache = new Uint8Array(0);
-        this.resetSegmentState();
-        this.port.postMessage({ type: 'destroyConfirmed' });
-        break;
-  
-      default:
-    }
-  }
-
-  /**
-   * 重置当前音频段状态（暂停/停止时调用）
-   */
-  resetSegmentState() {
-    this.currentSegment = null;
-    this.segmentOffset = 0;
-    this.fadeState.phase = 'idle';
-    this.fadeState.currentGain = 0;
-  }
-
-  /**
-   * S16LE Uint8Array 转 Float32Array（-1~1范围）
-   * @param {Uint8Array} pcmData - PCM数据
-   * @returns {Float32Array} 转换后的音频样本
-   */
-  pcmToFloat32(pcmData) {
-    const sampleCount = pcmData.length / this.config.bytesPerSample;
-    const floatData = new Float32Array(sampleCount);
-    const dataView = new DataView(pcmData.buffer);
-
-    for (let i = 0; i < sampleCount; i++) {
-      // S16LE格式：小端序解析，范围-32768~32767 归一化到-1~1
-      floatData[i] = dataView.getInt16(i * this.config.bytesPerSample, true) / 32768;
-    }
-    return floatData;
-  }
-
-  /**
-   * 计算当前样本的增益（淡入淡出）
-   * @param {number} segmentLength - 当前音频段的总样本数
-   * @returns {number} 增益值（0~1）
-   */
-  calculateGain(segmentLength) {
-    const { phase, fadeSamples } = this.fadeState;
-    const { segmentOffset } = this;
-
-    // 边界保护：防止无效值导致的NaN
-    if (typeof segmentOffset !== 'number' || typeof segmentLength !== 'number' || segmentLength <= 0) {
-      return 0;
-    }
-
-    switch (phase) {
-      // 淡入阶段：从0线性增加到1
-      case 'fadeIn':
-        if (segmentOffset >= fadeSamples) {
-          this.fadeState.phase = 'steady';
-          return 1;
-        }
-        return segmentOffset / fadeSamples;
-
-      // 稳定阶段：增益保持1
-      case 'steady':
-        // 检测是否进入淡出阶段（接近段尾）
-        if (segmentOffset >= segmentLength - fadeSamples) {
-          this.fadeState.phase = 'fadeOut';
-        }
-        return 1;
-
-      // 淡出阶段：从1线性降低到0.0001（避免分音）
-      case 'fadeOut':
-        const fadeStart = Math.max(segmentLength - fadeSamples, 0);
-        const fadeProgress = (segmentOffset - fadeStart) / fadeSamples;
-        return Math.max(1 - fadeProgress, 0.0001);
-
-      // 空闲阶段：增益0
-      case 'idle':
-      default:
-        return 0;
-    }
-  }
-
-  /**
-   * AudioWorklet核心方法：实时生成音频样本（音频线程调用）
-   * @param {Float32Array[][]} inputs - 输入样本
-   * @param {Float32Array[][]} outputs - 输出样本（单声道）
-   * @param {Object} parameters - 音频参数
-   * @returns {boolean} 是否继续运行
-   */
-  process(inputs, outputs, parameters) {
-    // 销毁状态：终止处理
-    if (this.isDestroyed) {
-      if (outputs.length > 0 && outputs[0].length > 0) {
-        outputs[0][0].fill(0);
-      }
-      return false;
-    }
-
-    // 未激活或无输出：返回静音
-    if (!this.isActive || outputs.length === 0 || outputs[0].length === 0) {
-      outputs[0][0].fill(0);
-      return true;
-    }
-
-    const outputBuffer = outputs[0][0];
-    const bufferLength = outputBuffer.length;
-    // 获取音量参数（支持自动化）
-    const volume = parameters.volume.length > 1 
-      ? parameters.volume // 自动化模式：每个样本点可能有不同值
-      : parameters.volume[0]; // 控制模式：整个缓冲区使用相同值
-
-    // 生成输出样本
-    for (let i = 0; i < bufferLength; i++) {
-      // 如当前无音频段且缓存有数据，创建新段
-      if (!this.currentSegment && this.pcmCache.length > 0) {
-        // 使用缓存中的所有数据创建新段
-        this.currentSegment = this.pcmToFloat32(this.pcmCache);
-        // 清空缓存（已处理）
-        this.pcmCache = new Uint8Array(0);
-        this.segmentOffset = 0;
-        // 根据前一段状态决定是否淡入
-        this.fadeState.phase = this.fadeState.phase === 'fadeOut' ? 'steady' : 'fadeIn';
-      }
-
-      // 生成当前样本
-      let sample = 0;
-      if (this.currentSegment) {
-        const segmentLength = this.currentSegment.length;
-        // 确保偏移量有效
-        if (this.segmentOffset < segmentLength) {
-          const gain = this.calculateGain(segmentLength);
-          // 应用淡入淡出增益和音量控制
-          sample = this.currentSegment[this.segmentOffset] * gain * volume;
-          this.segmentOffset++;
-        } else {
-          // 当前段处理完毕，重置
-          this.currentSegment = null;
-          this.segmentOffset = 0;
-          this.fadeState.phase = 'idle';
-        }
-      }
-
-      outputBuffer[i] = sample;
-    }
-
-    return true;
-  }
-}
-
-// 注册Worklet处理器
-registerProcessor('pcm-audio-processor', PCMAudioProcessor);

package/src/control/APIForwarder.ts

@@ -1,67 +0,0 @@
-/**
- * APIForwarder - 薄客户端模式下的 API 代理转发器
- * 将客户端的 speak/idle/listen 等 API 调用封装为 api_call 消息，
- * 通过 StreamingClient 发送给 Server 端 SDK 执行。
- */
-
-import { StreamingClient } from './StreamingClient';
-
-export class APIForwarder {
-  private client: StreamingClient;
-
-  constructor(client: StreamingClient) {
-    this.client = client;
-  }
-
-  speak(ssml: string, is_start: boolean = true, is_end: boolean = true, extra: any = {}) {
-    this.client.sendApiCall('speak', [ssml, is_start, is_end, extra]);
-  }
-
-  idle() {
-    this.client.sendApiCall('idle', []);
-  }
-
-  listen() {
-    this.client.sendApiCall('listen', []);
-  }
-
-  think() {
-    this.client.sendApiCall('think', []);
-  }
-
-  interactiveidle() {
-    this.client.sendApiCall('interactiveidle', []);
-  }
-
-  interrupt(type: string) {
-    this.client.sendApiCall('interrupt', [type]);
-  }
-
-  changeLayout(layout: any) {
-    this.client.sendApiCall('changeLayout', [layout]);
-  }
-
-  changeWalkConfig(walkConfig: any) {
-    this.client.sendApiCall('changeWalkConfig', [walkConfig]);
-  }
-
-  setVolume(volume: number) {
-    this.client.sendApiCall('setVolume', [volume]);
-  }
-
-  changeAvatarVisible(visible: boolean) {
-    this.client.sendApiCall('changeAvatarVisible', [visible]);
-  }
-
-  switchInvisibleMode() {
-    this.client.sendApiCall('switchInvisibleMode', []);
-  }
-
-  destroy(reason: string = 'user') {
-    this.client.sendApiCall('destroy', [reason]);
-  }
-
-  init(config?: any) {
-    this.client.send({ type: 'init', config });
-  }
-}

package/src/control/DataCacheQueue.ts

@@ -1,372 +0,0 @@
-import {
-  EFrameDataType,
-  IRawAudioFrameData,
-  IRawFaceFrameData,
-  IRawEventFrameData,
-  IRawFrameData,
-  StateChangeInfo,
-} from "../types/frame-data";
-
-export interface IBodyFrame {
-  frame: any;
-  frameIndex: number;
-  frameState: string;
-  id: number;
-  name: string;
-  body_id: number;
-  hfd: boolean;
-  sf: number;
-  offset: number;
-}
-
-export class DataCacheQueue {
-  private TAG = "[DataCacheQueue]";
-  // body视频抽帧副本，使用 Map 优化查找和删除性能
-  private _bodyQueue: Map<number, IBodyFrame> = new Map();
-  // 表情match信息列表
-  private _facialQueue: Array<IRawFaceFrameData> = [];
-  // 原始表情数据
-  private _realFacialQueue: Array<IRawFaceFrameData> = [];
-  // 音频队列
-  private audioQueue: Array<IRawAudioFrameData> = [];
-  // UI事件队列
-  private eventQueue: Array<IRawEventFrameData> = [];
-  // 当前处理好的视频id组
-  private videoIdList: string[] = [];
-  // 当前状态
-  private _currentPlayState: string = "idle";
-  // 当前ttsa状态
-  private _currentTtsaState: StateChangeInfo | null = null;
-  constructor() {
-    // @ts-ignore
-    window["__dev_event_queue__"] = this.eventQueue;
-  }
-
-  set currentPlayState(state: string) {
-    this._currentPlayState = state;
-  }
-
-  get currentPlayState() {
-    return this._currentPlayState;
-  }
-
-  set currentTtsaState(state: StateChangeInfo | null) {
-    this._currentTtsaState = state;
-  }
-  get currentTtsaState() {
-    return this._currentTtsaState;
-  }
-
-  get bodyQueue() {
-    return Array.from(this._bodyQueue.values());
-  }
-
-  // 更新body视频抽帧副本
-  _updateBodyImageBitmap(data: IBodyFrame) {
-    const old = this._bodyQueue.get(data.frameIndex);
-    if (old && old.frame && typeof old.frame.close === "function") {
-      old.frame.close();
-    }
-    // 清理过期帧，根据data.body_id,删除所有小于的帧
-    for (const [curIndex, curFrame] of this._bodyQueue.entries()) {
-      if (curFrame.body_id < data.body_id && curFrame.frameIndex >= data.frameIndex) {
-        curFrame.frame.close();
-        this._bodyQueue.delete(curIndex);
-      }
-    }
-    this._bodyQueue.set(data.frameIndex, data);
-  }
-
-  clearOldFrames(sf: number) {
-    for (const [curIndex, curFrame] of this._bodyQueue.entries()) {
-      if (curIndex >= sf) {
-        curFrame.frame.close();
-        this._bodyQueue.delete(curIndex);
-      }
-    }
-  }
-
-  clearOldFaceData(frameNumber: number) {
-    this._facialQueue = this._facialQueue.filter(
-      (item) => item.frameIndex >= frameNumber
-    );
-    this._realFacialQueue = this._realFacialQueue.filter(
-      (item) => item.frameIndex >= frameNumber
-    );
-  }
-
-  setVideoIdList(videoId: string) {
-    if (!this.videoIdList.includes(videoId)) {
-      this.videoIdList.push(videoId);
-    }
-  }
-
-  getVideoIdList() {
-    return this.videoIdList;
-  }
-
-  /**
-   * 获取指定帧并从Map中删除
-   * @param frameIndex 要获取的帧索引
-   * @returns 找到的帧数据（未找到则返回undefined）
-   */
-  _getBodyImageBitmap(frameIndex: number) {
-    const frame = this._bodyQueue.get(frameIndex);
-    
-    // 清理过期帧
-    for (const [curIndex, curFrame] of this._bodyQueue.entries()) {
-      if (curIndex < frameIndex) {
-        curFrame.frame.close();
-        this._bodyQueue.delete(curIndex);
-      }
-    }
-
-    if (frame) {
-      return frame;
-    }
-    return undefined;
-  }
-
-  // 获取body内的视频名称list
-  getBodyVideoNameListLength() {
-    const list: string[] = [];
-    this.bodyQueue.forEach((item) => {
-      if (!list.includes(item.name)) {
-        list.push(item.name);
-      }
-    });
-    return list.length;
-  }
-
-  _getFaceImageBitmap(frameIndex: number, body_id: number) {
-    // 找出face中frameIndex和body_id相同的帧
-    let targetItem: IRawFaceFrameData | null = null;
-    for (let i = this._facialQueue.length - 1; i >= 0; i--) {
-      const item = this._facialQueue[i];
-      if (item.frameIndex === frameIndex && item.body_id === body_id) {
-        targetItem = item;
-        break; // 找到后立即退出循环
-      }
-    }
-    // 删除所有小于frameIndex的帧
-    this._facialQueue = this._facialQueue.filter(
-      (item) => item.frameIndex >= frameIndex
-    );
-    return targetItem;
-  }
-  // 更新表情队列
-  _updateFacial(data: Array<IRawFaceFrameData>) {
-    this._facialQueue.push(...data);
-  }
-  get facialQueue() {
-    return this._facialQueue;
-  }
-  _getRealFaceImageBitmap(frameIndex: number, body_id: number) {
-    // 找出face中frameIndex和body_id相同的帧
-    let targetItem: IRawFaceFrameData | null = null;
-    for (let i = this._realFacialQueue.length - 1; i >= 0; i--) {
-      const item = this._realFacialQueue[i];
-      if (item?.frameIndex === frameIndex && item?.body_id === body_id) {
-        targetItem = item;
-        break; // 找到后立即退出循环
-      }
-    }
-    // 删除所有小于frameIndex的帧
-    this._realFacialQueue = this._realFacialQueue.filter(
-      (item) => item?.frameIndex >= frameIndex
-    );
-    return targetItem;
-  }
-  _updateRealFacial(data: Array<IRawFaceFrameData>) {
-    this._realFacialQueue.push(...data);
-  }
-  get realFacialQueue() {
-    return this._realFacialQueue;
-  }
-
-  /**
-   * 清空所有表情数据（用于切换隐身模式时）
-   */
-  clearAllFaceData() {
-    this._facialQueue = [];
-    this._realFacialQueue = [];
-    (window as any).avatarSDKLogger?.log(
-      this.TAG,
-      "已清空所有表情数据"
-    );
-  }
-  _updateAudio(data: Array<IRawAudioFrameData>) {
-    this.audioQueue.push(...data);
-  }
-
-  _clearAudio(speech_id: number) {
-    if (speech_id !== -1) {
-      this.audioQueue = this.audioQueue.filter(item => item.sid !== speech_id);
-    } else {
-      this.audioQueue = [];
-    }
-  }
-
-  _getAudio(frameIndex: number) {
-    const targetIndex = this.audioQueue.findIndex(
-      (item) => item.sf === frameIndex
-    );
-    if (targetIndex === -1) {
-      return;
-    }
-    const [targetItem] = this.audioQueue.splice(targetIndex, 1);
-    return targetItem;
-  }
-
-  _getAudioInterval(startFrame: number, endFrame: number) {
-    const audioList = this.audioQueue.filter(
-      (item) => item.sf >= startFrame && item.sf <= endFrame
-    );
-    if (audioList.length === 0) {
-      return;
-    }
-    const audio = audioList[audioList.length - 1];
-    const index = this.audioQueue.findIndex((item) => item.id === audio.id);
-    this.audioQueue.splice(index, 1);
-    return audio;
-  }
-
-  _updateUiEvent(data: Array<IRawEventFrameData>) {
-    this.eventQueue.push(...data);
-  }
-
-  clearOldEvent(frame: number) {
-    this.eventQueue = this.eventQueue.filter(item => item.sf >= frame);
-  }
-
-  clearSubtitleOn(speech_id: number) {
-    this.eventQueue = this.eventQueue.map(item => ({
-      ...item,
-      e: item.e?.filter((subItem: any) => subItem.speech_id > speech_id || subItem.type !== "subtitle_on")
-    }));
-  }
-
-  _clearEvent() {
-    this.eventQueue = [];
-  }
-
-  // 获取指定帧的事件，并合并相同 sf 的 e 数组
-  _getEvent(frame: number) {
-    // 1. 找出所有匹配 sf 的事件
-    const matchedItems = this.eventQueue.filter(item => item.sf === frame);
-    if (matchedItems.length === 0) return;
-
-    // 2. 从原队列删除这些事件
-    this.eventQueue = this.eventQueue.filter(item => item.sf !== frame);
-
-    // 3. 合并所有 e 数组（核心逻辑）
-    return this._mergeEventsBySf(matchedItems);
-  }
-
-  // 获取区间内事件，并合并相同 sf 的 e 数组
-  _getEventInterval(startFrame: number, endFrame: number) {
-    // 1. 找出所有区间内的事件
-    const matchedItems = this.eventQueue.filter(
-      item => item.sf >= startFrame && item.sf <= endFrame
-    );
-    if (matchedItems.length === 0) return;
-
-    // 2. 从原队列删除这些事件
-    this.eventQueue = this.eventQueue.filter(
-      item => !(item.sf >= startFrame && item.sf <= endFrame)
-    );
-
-    // 3. 合并所有 e 数组（核心逻辑）
-    return this._mergeEventsBySf(matchedItems);
-  }
-
-  // 【工具方法】合并相同 sf 的事件，把 e 数组拼接在一起
-  _mergeEventsBySf(events: any[]) {
-    // 以 sf 为 key 分组合并
-    const grouped = {};
-    for (const event of events) {
-      const key = event.sf;
-      if (!grouped[key]) {
-        // 第一次遇到该 sf，创建基础结构
-        grouped[key] = { ...event, e: [] };
-      }
-      // 把当前事件的 e 数组 push 进去（合并）
-      grouped[key].e.push(...event.e);
-    }
-
-    // 取出合并后的结果（你这里 sf 都相同，只会有一个）
-    const mergedList:any = Object.values(grouped);
-    if(mergedList.length === 0) return null
-    if (mergedList.length > 1) {
-      // 合并所有帧的e到一个数组
-      const allE: any[] = [];
-      mergedList.forEach((item:any) => allE.push(...item.e));
-      // 构造单个event对象，和你原来代码格式完全一致
-      return {
-        ...mergedList[0],
-        mergeFrameFlag: 1,
-        e: allE
-      };
-    } else{
-      return mergedList[0];
-    }
-  }
-
-  /**
-   * 检查数据是否因seek等原因失效，并清理缓存。
-   * @param data 新的数据帧数组
-   * @param type 数据类型
-   */
-  checkValidData(data: Array<IRawFrameData>, type: string) {
-    switch (type) {
-      case EFrameDataType.BODY:
-        break;
-      case EFrameDataType.FACE: {
-        // 根据bodyId获取face，暂时去掉过滤
-        // if (this._facialQueue.length === 0 || data.length === 0) {
-        //   return;
-        // }
-        // const isDiscontinuous =
-        //   this._facialQueue[this._facialQueue.length - 1]?.ef >= data[0].sf ||
-        //   data[0].sf <= this._facialQueue[0]?.sf;
-        // if (isDiscontinuous) {
-        //   const facialIndex = this._facialQueue.findIndex(
-        //     (item) => item.sf >= data[0].sf
-        //   );
-        //   if (facialIndex !== -1) {
-        //     this._facialQueue.length = facialIndex;
-        //   }
-        // }
-        break;
-      }
-      case EFrameDataType.AUDIO: {
-        if (this.audioQueue.length === 0 || data.length === 0) {
-          return;
-        }
-        const isDiscontinuous =
-          this.audioQueue[this.audioQueue.length - 1]?.ef < data[0].sf ||
-          data[0].sf < this.audioQueue[0]?.sf;
-        if (isDiscontinuous) {
-          const audioIndex = this.audioQueue.findIndex(
-            (item) => item.sf >= data[0].sf
-          );
-          if (audioIndex !== -1) {
-            this.audioQueue.length = audioIndex;
-          }
-        }
-        break;
-      }
-    }
-  }
-  destroy() {
-    this._bodyQueue.forEach((item) => {
-      if (item.frame && typeof item.frame.close === "function") {
-        item.frame.close();
-      }
-    });
-    this._bodyQueue.clear();
-    this._facialQueue = [];
-    this.audioQueue = [];
-    this.eventQueue = [];
-  }
-}