@xmov/avatar 2.4.0 → 2.5.0-alpha.1

package/src/baseRender/MSEAudioPlayer.ts

@@ -1,254 +0,0 @@
-export default class MediaSourceAudioPlayer {
-  private mediaSource: MediaSource | null = null;
-  private audioElement: HTMLAudioElement | null = null;
-  private sourceBuffer: SourceBuffer | null = null;
-  public isInitialized = false;
-  public isPlaying = false;
-  private volume = 1.0;
-  private readonly MIME_TYPE = 'audio/webm;codecs=opus';
-  private queue: Array<{ data: ArrayBuffer }> = [];
-  private isUpdating = false;
-  private currentObjectURL: string | null = null;
-  private isDestroyed = false;
-
-  // 速度同步相关（保留不动）
-  private playbackRate = 1.0;
-  public firstFrameIndex = -1;
-
-  // 单例
-  private static instance: MediaSourceAudioPlayer;
-  public static getInstance(sdk): MediaSourceAudioPlayer {
-    if (!MediaSourceAudioPlayer.instance) {
-      MediaSourceAudioPlayer.instance = new MediaSourceAudioPlayer(sdk);
-    }
-    return MediaSourceAudioPlayer.instance;
-  }
-
-  private constructor(sdk) {
-    this.createAudioElement(sdk);
-    this.createNewMediaSource();
-  }
-
-  /**
-   * 只复用 audio 标签！！！
-   */
-  private createAudioElement(sdk) {
-    if (this.audioElement) return;
-    const el = document.getElementById("xmov-avatar-audio");
-    if (el) {
-      this.audioElement = el as HTMLAudioElement;
-      return;
-    }
-    const audio = document.createElement("audio");
-    audio.id = "xmov-avatar-audio";
-    audio.preload = "auto";
-    audio.style.position = "absolute";
-    audio.style.width = "1px";
-    audio.style.height = "1px";
-    audio.style.opacity = "0";
-    audio.style.pointerEvents = "none";
-    audio.style.zIndex = "-9999";
-    sdk.el.appendChild(audio);
-    this.audioElement = audio;
-  }
-
-  /**
-   * 每次都创建 NEW MediaSource
-   * 不复用！不复用！不复用！
-   */
-  private createNewMediaSource(): Promise<void> {
-    return new Promise((resolve, reject) => {
-      try {
-        if(!this.audioElement) return;
-        this.mediaSource = new MediaSource();
-        this.currentObjectURL = URL.createObjectURL(this.mediaSource);
-        this.audioElement!.src = this.currentObjectURL;
-
-        this.mediaSource.addEventListener("sourceopen", () => {
-          try {
-            if (!this.mediaSource) return;
-            this.sourceBuffer = this.mediaSource.addSourceBuffer(this.MIME_TYPE);
-            this.sourceBuffer.mode = "sequence";
-
-            this.sourceBuffer.addEventListener("updateend", () => {
-              this.isUpdating = false;
-              this.processQueue();
-            });
-
-            this.isInitialized = true;
-            resolve();
-          } catch (e) {
-            reject(e);
-          }
-        });
-
-        this.mediaSource.addEventListener("sourceclose", () => {
-          this.isInitialized = false;
-        });
-      } catch (e) {
-        reject(e);
-      }
-    });
-  }
-
-  /**
-   * 销毁当前 MediaSource 所有资源
-   * stop 时调用
-   */
-  private async destroyCurrentMediaSource(callback: () => void) {
-    try {
-      if (this.audioElement) {
-        this.audioElement.pause();
-        this.audioElement.removeAttribute("src");
-        this.audioElement.load();
-      }
-    } catch {}
-
-    try {
-      if (this.currentObjectURL) {
-        URL.revokeObjectURL(this.currentObjectURL);
-        this.currentObjectURL = null;
-      }
-    } catch {}
-
-    this.mediaSource = null;
-    this.sourceBuffer = null;
-    this.isInitialized = false;
-    this.isPlaying = false;
-    this.queue = [];
-    this.isUpdating = false;
-    await this.createNewMediaSource();
-    callback();
-  }
-
-  // ==========================
-  // 核心：stop 清空一切，只留 audio
-  // ==========================
-  public stop(callback: () => void) {
-    this.destroyCurrentMediaSource(callback);
-  }
-
-  // ==========================
-  // 添加数据（自动初始化）
-  // ==========================
-  async addAudioSegment(audioData: ArrayBuffer): Promise<void> {
-    if (this.isDestroyed) return;
-    this.queue.push({ data: audioData });
-    this.processQueue();
-  }
-
-  private processQueue() {
-    if (this.isDestroyed) return;
-    if (!this.sourceBuffer || !this.mediaSource) return;
-    if (this.isUpdating || this.queue.length === 0) return;
-    if (this.sourceBuffer.updating || this.mediaSource.readyState !== "open") return;
-
-    this.isUpdating = true;
-    const item = this.queue.shift()!;
-    try {
-      this.sourceBuffer.appendBuffer(item.data);
-    } catch (e) {
-      this.isUpdating = false;
-    }
-  }
-
-  // ==========================
-  // 播放
-  // ==========================
-  async start(): Promise<void> {
-    if (this.isDestroyed || !this.audioElement || this.isPlaying) return;
-
-    try {
-      this.audioElement.currentTime = 0;
-      this.isPlaying = true;
-      await this.audioElement.play();
-      (window as any).avatarSDKLogger.log("mseAudioPlayer开始播放", this.getStats(), Date.now());
-    } catch {}
-  }
-
-  getStats() {
-    return {
-      isPlaying: this.isPlaying,
-      isInitialized: this.isInitialized,
-      queueLength: this.queue.length,
-      bufferedRanges: this.getBufferedRanges(),
-      mediaSourceReadyState: this.mediaSource?.readyState || 'null',
-    };
-  }
-
-  setSpeed(speed: number) {
-    this.setPlaybackRate(speed);
-  }
-  
-  /**
-   * 获取缓冲范围
-   */
-  private getBufferedRanges(): string {
-    try {
-      // stop/destroy 流程中浏览器会先移除 SourceBuffer 再置空引用，
-      // 此时访问 .buffered 会抛 InvalidStateError，需要 readyState 校验 + 兜底
-      if (
-        this.sourceBuffer &&
-        this.mediaSource?.readyState === "open" &&
-        this.sourceBuffer.buffered.length > 0
-      ) {
-        const ranges: string[] = [];
-        for (let i = 0; i < this.sourceBuffer.buffered.length; i++) {
-          const start = this.sourceBuffer.buffered.start(i).toFixed(2);
-          const end = this.sourceBuffer.buffered.end(i).toFixed(2);
-          ranges.push(`[${start}-${end}]`);
-        }
-        return ranges.join(", ");
-      }
-    } catch (e) {
-      // SourceBuffer 已被移除（stop/destroy 竞态），视为无缓冲数据
-      return "-";
-    }
-    return "-";
-  }
-
-  setVolume(volume: number): void {
-    this.volume = Math.max(0, Math.min(1, volume));
-    if (this.audioElement) this.audioElement.volume = this.volume;
-  }
-
-  setPlaybackRate(rate: number): void {
-    this.playbackRate = Math.max(0.5, Math.min(4.0, rate));
-    if (this.audioElement) this.audioElement.playbackRate = this.playbackRate;
-  }
-
-  destroy() {
-    // 0. 标记已销毁，阻止异步操作继续执行
-    this.isDestroyed = true;
-
-    // 1. 停止播放并清除 audio 元素的 src（解除对 blob URL 的引用）
-    if (this.audioElement) {
-      this.audioElement.pause();
-      this.audioElement.removeAttribute("src");
-      this.audioElement.load();
-    }
-
-    // 2. 释放 blob URL（必须在 audio 元素清除 src 之后）
-    if (this.currentObjectURL) {
-      try { URL.revokeObjectURL(this.currentObjectURL); } catch {}
-      this.currentObjectURL = null;
-    }
-
-    // 3. 移除 audio 元素
-    if (this.audioElement) {
-      this.audioElement.remove();
-      this.audioElement = null;
-    }
-
-    // 4. 重置所有状态
-    this.mediaSource = null;
-    this.sourceBuffer = null;
-    this.isInitialized = false;
-    this.isPlaying = false;
-    this.queue = [];
-    this.isUpdating = false;
-
-    // 5. 重置单例，下次 getInstance() 会重新创建
-    MediaSourceAudioPlayer.instance = null as any;
-  }
-}

package/src/baseRender/UIRenderer.ts

@@ -1,195 +0,0 @@
-/**
- * UI 控件渲染，所有轨道的元素
- */
-import { DataCacheQueue } from "../control/DataCacheQueue";
-import DefaultWidgetRenderer from "../modules/TrackRenderer/render-implements";
-import TrackerRenderer from "../modules/TrackRenderer/index";
-import ResourceManager from "../modules/ResourceManager";
-import { performanceConstant } from "../utils/perfermance";
-import XmovAvatar from "../index";
-import { EErrorCode } from "../types/error";
-
-type Option = {
-  sdk: XmovAvatar;
-  dataCacheQueue: DataCacheQueue;
-  resourceManager: ResourceManager;
-  enableClientInterrupt: boolean;
-  onVoiceEnd: (client_speak_id: number | string, speech_id: number) => void;
-  onVoiceStart: (duration: number, client_speak_id: number | string) => void;
-  onWalkStateChange: (state: string) => void;
-  sendSdkPoint: (type: string, data: any, extra?: any) => void;
-  onSpeakStateChange: (state: string, client_speak_id: number | string, speech_id: number) => void;
-};
-export default class UIRenderer {
-  private TAG = "[UIRenderer]";
-  options: Option;
-  trackerRenderer: TrackerRenderer[] = [];
-  root = document.createElement("div");
-  lastFrameIndex: number = -1;
-  onVoiceEnd: (client_speak_id: number | string, speech_id: number) => void;
-  onVoiceStart: (duration: number, client_speak_id: number | string) => void;
-  onWalkStateChange: (state: string) => void;
-  initEventsRendered: boolean = false;  
-  lastSpeechId: number = -1;  
-  interruptSpeechId: number = -1;
-  enableClientInterrupt: boolean = false;
-  onSpeakStateChange: (state: string, client_speak_id: number | string, speech_id: number) => void;
-  constructor(options: Option) {
-    this.options = options;
-    this.root.setAttribute("style", "position:absolute;");
-    this.onVoiceStart = options.onVoiceStart;
-    this.onVoiceEnd = options.onVoiceEnd;
-    this.initEventsRendered = false;  
-    this.onWalkStateChange = options.onWalkStateChange;
-    this.enableClientInterrupt = options.enableClientInterrupt;
-    this.onSpeakStateChange = options.onSpeakStateChange;
-  }
-
-  render(frame: number) {
-    // if(this.options.sdk.getStatus() === AvatarStatus.offline) return;
-    let initEvents: any[] = [];
-    if (this.lastFrameIndex === -1) {
-      const config = this.options.resourceManager.getConfig();
-      if(config.init_events && config.init_events.length > 0){
-        initEvents = config.init_events.map(item => ({...item, speech_id:0}));
-      }
-    }
-    let event: any = null;
-    if (frame - this.lastFrameIndex > 1) {
-      event = this.options.dataCacheQueue._getEventInterval(
-        this.lastFrameIndex,
-        frame
-      );
-    } else {
-      event = this.options.dataCacheQueue._getEvent(frame);
-    }
-    this.lastFrameIndex = frame;
-    // 仅执行一次
-    if(!this.initEventsRendered){
-      this.initEventsRendered = true;
-      if(event){
-        event.e = event.e.concat(initEvents);
-      } else {
-        event = {
-          e: initEvents,
-        };
-      }
-    }
-    if (!event?.e) {
-      return;
-    }
-    (window as any).avatarSDKLogger.log("当前帧", frame, "获取到的事件", JSON.stringify(event));
-    // 过滤出不包含speechId的事件list
-    const noSpeechIdEvents = event.e?.filter(item => item?.speech_id === undefined || item?.speech_id === null);
-    if(noSpeechIdEvents?.length) {
-      const typesStr = noSpeechIdEvents.map(item => item.type).join(',');
-      this.options.sdk.onMessage({
-        code: EErrorCode.EVENT_WITHOUT_SPEECH_ID,
-        message: `Error: 事件中存在不包含speech_id的事件${typesStr}`,
-        e: noSpeechIdEvents
-      })
-    }
-    if(this.enableClientInterrupt) {
-      // 根据interruptSpeechId过滤旧的数据,加入前端interrupt事件后，可能在前端打断后，服务仍下发数据 如果type是以_end结尾的，不过滤
-      event.e = event.e?.filter(item => (item.speech_id > this.interruptSpeechId || item.speech_id === undefined || item.speech_id === null) || item.type.endsWith('_end') || item.type === 'subtitle_off');
-    }
-    // 过滤出当前语音的事件
-    if(this.lastSpeechId !== -1 && this.lastSpeechId) {
-      event.e = event.e?.filter(item => item?.speech_id >= this.lastSpeechId);
-    }
-    
-    // 1. 处理 speak_start
-    if (event.e?.some(item => item.type === "speak_start")) {
-      const speakStart = event.e.find(item => item.type === "speak_start");
-      this.lastSpeechId = speakStart?.speech_id;
-      this.onSpeakStateChange("speak_start", speakStart.client_speak_id, speakStart?.speech_id || 0);
-    }
-    // 2. 处理 walk_start / voice_start / speak_walk_start
-    if (event.e?.some(item => item.type === "walk_start") || event.e?.some(item => item.type === "speak_walk_start")) {
-      this.onWalkStateChange("walk_start");
-    }
-    if(event.e?.some(item => item.type === "voice_start")) {
-      const voiceStart = event.e.find(item => item.type === "voice_start");
-      const duration = (window as any).performanceTracker.markEnd(performanceConstant.voice_response_play, 'speak');
-      this.options.sendSdkPoint('rendering_display', {multi_turn_conversation_id: voiceStart?.multi_turn_conversation_id});
-      this.onVoiceStart(duration, voiceStart?.client_speak_id);
-    }
-    // 3. 处理 walk_end / speak_walk_end / voice_end
-    if (event.e?.some(item => item.type === "walk_end") || event.e?.some(item => item.type === "speak_walk_end")) {
-      this.onWalkStateChange("walk_end");
-    }
-    if (event.e?.some(item => item.type === "voice_end")) {
-      const voiceEnd = event.e.find(item => item.type === "voice_end");
-      this.onVoiceEnd(voiceEnd?.client_speak_id || '', voiceEnd?.speech_id || 0);
-    }
-    // 4. 处理 speak_end / speak_error
-    const speakEndEventTypes = ["speak_end", "speak_error"];
-    speakEndEventTypes.forEach(eventType => {
-      const speakEvent = event.e.find(item => item.type === eventType);
-      if (speakEvent) {
-        this.onSpeakStateChange(eventType, speakEvent.client_speak_id, speakEvent?.speech_id || 0);
-      }
-    });
-
-
-    const remainingWidgets = [...event.e];
-
-    // 循环遍历 event.e 数组中的所有元素
-    for (let i = 0; i < event.e.length; i++) {
-      const widgetData = event.e[i];
-      if (Object.prototype.hasOwnProperty.call(widgetData, "event_type")) {
-        // 先不处理 ka/ka_intent
-        continue;
-      }
-      // 如果有自定义的渲染器，则所有事件都交给自定义的渲染器处理
-      if (DefaultWidgetRenderer.CUSTOM_WIDGET) {
-        DefaultWidgetRenderer.CUSTOM_WIDGET(widgetData);
-        // 从剩余元素中删除已处理的元素
-        const index = remainingWidgets.indexOf(widgetData);
-        if (index > -1) {
-          remainingWidgets.splice(index, 1);
-        }
-        continue;
-      }
-      // 如果有代理的渲染器，则根据类型执行代理的渲染器
-      if (DefaultWidgetRenderer.PROXY_WIDGET && DefaultWidgetRenderer.PROXY_WIDGET[widgetData.type]) {
-        DefaultWidgetRenderer.PROXY_WIDGET[widgetData.type](widgetData);
-        
-        // 从剩余元素中删除已处理的元素
-        const index = remainingWidgets.indexOf(widgetData);
-        if (index > -1) {
-          remainingWidgets.splice(index, 1);
-        }
-        continue;
-      }
-    }
-    // 对剩余未自定义的元素执行 TrackerRenderer
-    if (remainingWidgets.length > 0) {
-      // 遍历剩余的元素，每个单独执行 TrackerRenderer
-      // 传递 SDK 实例引用，用于区分不同数字人实例的 widget
-      for (const widgetData of remainingWidgets) {
-        const tracker = new TrackerRenderer(widgetData, this.options.sdk);
-        this.trackerRenderer.push(tracker);
-      }
-    }
-    return this.trackerRenderer;
-  }
-
-  clearTrackerRenderer() {
-    this.trackerRenderer = [];
-  }
-
-  setInterrupt(speech_id: number) {
-    this.interruptSpeechId = speech_id;
-  }
-
-  destroy() {
-    this.lastSpeechId = -1;
-    this.interruptSpeechId = -1;
-    this.lastFrameIndex = -1;
-    this.initEventsRendered = false;
-    // 传递 SDK 实例引用，只清除当前实例的 widget
-    DefaultWidgetRenderer.destroy(this.options.sdk);
-    this.trackerRenderer.forEach((tracker) => tracker.destroy());
-  }
-}