@xmov/avatar 2.0.0-alpha.38

package/src/agent/types.ts

@@ -0,0 +1,197 @@
+import type { IAvatarOptions, IInitParams, INetworkInfo, RenderState } from "../types";
+
+export type AgentState =
+  | "idle"
+  | "initializing"
+  | "ready"
+  | "running"
+  | "reconnecting"
+  | "stopped"
+  | "destroyed"
+  | "failed";
+export type AgentSocketState = "idle" | "connecting" | "open" | "closed" | "error";
+export type AgentASRState = "idle" | "requesting-permission" | "starting" | "listening" | "stopping" | "failed";
+export type AgentConversationState =
+  | "idle"
+  | "asking"
+  | "speaking-directly"
+  | "speaking"
+  | "completed"
+  | "interrupted"
+  | "failed";
+
+export type AgentErrorDomain = "permission" | "network" | "asr" | "brain" | "ttsa" | "quota" | "sdk";
+
+export interface AgentAudioOptions {
+  chunkMs?: number;
+  inputEnabled?: boolean;
+  /**
+   * @deprecated Session 返回 features.speech_frontend.enable_aec 后，以服务端返回值为准。
+   */
+  echoCancellationEnabled?: boolean;
+}
+
+export interface AgentReconnectOptions {
+  enabled?: boolean;
+  maxAttempts?: number;
+  initialDelayMs?: number;
+  maxDelayMs?: number;
+}
+
+export interface AgentError {
+  code: string;
+  message: string;
+  domain: AgentErrorDomain;
+  retryable?: boolean;
+  cause?: unknown;
+}
+
+export interface AgentASRResult {
+  text: string;
+  isFinal: boolean;
+  raw?: unknown;
+}
+
+export interface AgentASRStartOptions {
+  /**
+   * Optional caller-owned audio stream used instead of requesting the physical microphone.
+   * The Agent clones one live audio track and never stops the caller-owned stream.
+   */
+  inputStream?: MediaStream;
+}
+
+export interface AgentLLMUsage {
+  promptTokens: number;
+  completionTokens: number;
+  totalTokens: number;
+  cachedTokens: number;
+}
+
+export interface AgentLLMResponse {
+  event: "chunk" | "done";
+  text?: string;
+  isFirst?: boolean;
+  usage?: AgentLLMUsage;
+  raw?: unknown;
+}
+
+export interface AgentSemanticJudgeResult {
+  query: string;
+  meaningful: boolean;
+  action: string;
+  raw?: unknown;
+}
+
+export interface AgentSpeakStateEvent {
+  state: string;
+  clientSpeakId?: string | number;
+  raw?: unknown;
+}
+
+export interface AgentConversationEvent {
+  state: AgentConversationState;
+  text?: string;
+  raw?: unknown;
+}
+
+export interface XingyunAvatarAgentCallbacks {
+  onAgentStateChange?: (state: AgentState) => void;
+  onSocketStateChange?: (state: AgentSocketState) => void;
+  onASRStateChange?: (state: AgentASRState) => void;
+  onASRResult?: (result: AgentASRResult) => void;
+  onLLMResponse?: (response: AgentLLMResponse) => void;
+  onSpeakStateChange?: (event: AgentSpeakStateEvent) => void;
+  onConversationChange?: (event: AgentConversationEvent) => void;
+  onSemanticJudgeResult?: (result: AgentSemanticJudgeResult) => void;
+  onRenderChange?: (state: RenderState) => void;
+  onNetworkInfo?: (info: INetworkInfo) => void;
+  onError?: (error: AgentError) => void;
+}
+
+export type AgentServiceConfig = Record<string, unknown>;
+
+export interface AgentBrainConfig extends AgentServiceConfig {
+  extra_body?: Record<string, unknown>;
+}
+
+export interface AgentFeatures extends AgentServiceConfig {
+  anti_interference?: {
+    semantic_judge_enabled?: boolean;
+  };
+  speech_frontend?: {
+    enabled?: boolean;
+    enable_aec?: boolean;
+    enable_speech_separation?: boolean;
+  };
+  vad_merge_mode?: boolean;
+  volume_and_repetition_text_detection?: boolean;
+}
+
+export interface XingyunAvatarAgentOptions extends IAvatarOptions {
+  asr_id?: number;
+  asr_config?: AgentServiceConfig;
+  tts_config?: AgentServiceConfig;
+  features?: AgentFeatures;
+  extras?: AgentServiceConfig;
+  llm_id?: number;
+  brain_config?: AgentBrainConfig;
+  session_speak_req_id?: 0 | 1;
+  e2eServer?: string;
+  authToken?: string;
+  audio?: AgentAudioOptions;
+  reconnect?: AgentReconnectOptions;
+  agentCallbacks?: XingyunAvatarAgentCallbacks;
+  webSocketCtor?: typeof WebSocket;
+}
+
+export type XingyunAvatarAgentInitParams = Omit<IInitParams, "onDownloadProgress"> & {
+  onDownloadProgress?: IInitParams["onDownloadProgress"];
+};
+
+export interface AgentSessionInfo {
+  ws_url?: string;
+  e2e_token?: string;
+  monitor_ws_url?: string;
+}
+
+export interface AgentClientMessage {
+  type: string;
+  message?: unknown;
+  [key: string]: unknown;
+}
+
+export interface AgentServerEvent {
+  type: string;
+  domain?: string;
+  code?: string | number;
+  event?: string;
+  message?: unknown;
+  text?: string;
+  is_first?: boolean;
+  is_final?: boolean;
+  usage?: {
+    prompt_tokens?: number;
+    completion_tokens?: number;
+    total_tokens?: number;
+    cached_tokens?: number;
+  };
+  [key: string]: unknown;
+}
+
+export interface AgentAudioMetadata {
+  format: "webm_opus";
+  mimeType: "audio/webm;codecs=opus";
+  /** Fixed output track sample rate passed to the MediaRecorder pipeline. */
+  sampleRate: number;
+  /** Fixed output track channel count passed to the MediaRecorder pipeline. */
+  channels: number;
+  chunkMs: number;
+}
+
+export const DEFAULT_AGENT_AUDIO: AgentAudioMetadata = {
+  format: "webm_opus",
+  mimeType: "audio/webm;codecs=opus",
+  sampleRate: 16000,
+  channels: 1,
+  chunkMs: 100,
+};

package/src/baseRender/AudioRenderer.ts

@@ -0,0 +1,317 @@
+/**
+ * 音频渲染
+ * 使用 AudioContext 控制
+ */
+
+import ResourceManager from "../modules/ResourceManager";
+import { DataCacheQueue } from "../control/DataCacheQueue";
+// import PCMAudioPlayer from "./Audio";
+import AudioWorklet from "./AudioWorklet";
+import { performanceConstant } from "../utils/perfermance";
+import XmovAvatar from "../index";
+import { AvatarStatus } from "../types";
+import MediaSourceAudioPlayer from "./MSEAudioPlayer";
+import { isSupportMES } from "../utils";
+
+type Option = {
+  sdk: XmovAvatar;
+  dataCacheQueue: DataCacheQueue;
+  resourceManager: ResourceManager;
+};
+
+export default class AudioRender {
+  private TAG = "[AudioRenderer]";
+  private options: Option;
+  isPlaying = false;
+  lastFrameIndex: number = -1;
+  audio: AudioWorklet | null = null;
+  firstFrameIndex: number = -1;
+  resourceManager: ResourceManager;
+  valume: number = 1;
+  speech_id: number = -1; // 当前播放的音频id
+  cacheFirstFrameIndex: number = -1; // 缓存下来的第一个音频的索引
+  cacheAudioData: Uint8Array[] = []; // 缓存下来的音频数据
+  oldSpeechId: number = -1; // 上一个播放的音频id
+  mseCacheAudioData: Array<{ data: ArrayBuffer; frameIndex?: number; speechId?: number }> = [];
+  private mseAudioPlayer: MediaSourceAudioPlayer | null = null;
+  
+  constructor(options: Option) {
+    this.options = options;
+    this.resourceManager = options.resourceManager;
+    if(this.resourceManager.config.raw_audio) {
+      this.audio = new AudioWorklet();
+      this.audio.init();
+    } else {
+      this.mseAudioPlayer = MediaSourceAudioPlayer.getInstance(options.sdk);
+    }
+  }
+  updateAudioData(audioList: any) {
+    if (
+      !this.resourceManager.config.raw_audio ||
+      audioList[0].sid === this.oldSpeechId
+    )
+      return;
+    if (this.firstFrameIndex === -1) {
+      this.firstFrameIndex = audioList[0].sf;
+    }
+    let totalLength = 0;
+    for (const audio of audioList) {
+      totalLength += audio.ad?.length || 0;
+    }
+
+    // 创建合并后的Uint8Array
+    const mergedData = new Uint8Array(totalLength);
+    let offset = 0;
+
+    // 填充合并数据
+    for (const audio of audioList) {
+      if (audio.ad) {
+        mergedData.set(new Uint8Array(audio.ad), offset);
+        offset += audio.ad.length;
+      }
+    }
+    if (this.speech_id === -1) {
+      this.speech_id = audioList[0].sid;
+      this.audio?.receivePCMStream(mergedData);
+    } else {
+      if (this.speech_id !== audioList[0].sid) {
+        // 先缓存下来
+        if (this.cacheFirstFrameIndex === -1) {
+          this.cacheFirstFrameIndex = audioList[0].sf;
+        }
+        this.cacheAudioData.push(mergedData);
+      } else {
+        // 直接播放
+        this.audio?.receivePCMStream(mergedData);
+      }
+    }
+  }
+
+  _updateAudio(audioList: any) {
+    if (!this.mseAudioPlayer || audioList.length === 0) return;
+
+    const firstAudio = audioList[0];
+    
+    // 检查是否是旧的 speech_id
+    if (firstAudio.sid === this.oldSpeechId) {
+      return;
+    }
+
+    // 如果 speech_id 变化，停止当前播放并重置
+    if (this.speech_id !== -1 && firstAudio.sid !== this.speech_id) {
+      (window as any).avatarSDKLogger?.log(
+        this.TAG,
+        `检测到新的 speech_id: ${firstAudio.sid}, 当前 WebM 播放 (旧: ${this.speech_id}), 新: ${firstAudio.sid}, 缓存数据长度: ${this.mseCacheAudioData.length}`
+      );
+      if(this.cacheFirstFrameIndex === -1) {
+        this.cacheFirstFrameIndex = firstAudio.sf;
+      }
+      for(const item of audioList) {
+        this.mseCacheAudioData.push({
+          data: item.ad,
+          frameIndex: item.sf,
+          speechId: item.sid,
+        })
+      }
+    } else {
+      // 记录第一个音频段的帧索引
+      if (this.firstFrameIndex === -1) {
+        this.firstFrameIndex = firstAudio.sf;
+      }
+
+      // 更新 speech_id
+      if (this.speech_id === -1) {
+        this.speech_id = firstAudio.sid;
+      } else if (this.speech_id !== firstAudio.sid) {
+        this.speech_id = firstAudio.sid;
+      }
+
+      // 处理每个音频段
+      for (const audio of audioList) {
+        if (audio.ad) {
+          // 将 Uint8Array 转换为 ArrayBuffer
+          const audioData = audio.ad instanceof ArrayBuffer 
+            ? audio.ad 
+            : audio.ad.buffer.slice(audio.ad.byteOffset, audio.ad.byteOffset + audio.ad.byteLength);
+          
+          this.mseAudioPlayer.addAudioSegment(
+            audioData,
+          );
+        }
+      }
+    }
+  }
+
+  async render(frameIndex: number) {
+    (window as any).avatarSDKLogger.log("当前帧索引:", frameIndex, "音频开始播放帧：", this.firstFrameIndex);
+    if (this.lastFrameIndex === -1) {
+      this.lastFrameIndex = frameIndex;
+    }
+    if (this.options.sdk.getStatus() === AvatarStatus.offline) return;
+    // PCM 流式播放（raw_audio = true）
+    if (
+      frameIndex >= this.firstFrameIndex &&
+      this.firstFrameIndex != -1 &&
+      !this.isPlaying &&
+      this.resourceManager.config.raw_audio
+    ) {
+      (window as any).performanceTracker.markEnd(
+        performanceConstant.start_action_render,
+        "speak"
+      );
+      this.isPlaying = true;
+      this.audio?.start();
+      return;
+    }
+
+    // WebM 流式播放（raw_audio = false）
+    if (!this.resourceManager.config.raw_audio && this.mseAudioPlayer) {
+      // 检查是否应该开始播放（帧对齐）
+      if (
+        this.firstFrameIndex !== -1 &&
+        frameIndex >= this.firstFrameIndex &&
+        !this.mseAudioPlayer.isPlaying &&
+        !this.isPlaying
+      ) {
+        // 优化：确保有足够的数据缓冲再开始播放
+        const stats = this.mseAudioPlayer.getStats();
+        const hasEnoughData = stats.queueLength > 0 || stats.bufferedRanges !== '-';
+        if (hasEnoughData) {
+          try {
+            (window as any).avatarSDKLogger.log("开始播放", frameIndex);
+            this.isPlaying = true;
+            await this.mseAudioPlayer.start();
+            (window as any).performanceTracker.markEnd(
+              performanceConstant.start_action_render,
+              "speak"
+            );
+          } catch (error) {
+            (window as any).avatarSDKLogger.error("[AudioRenderer] WebM 音频播放失败:", {
+              error,
+              frameIndex,
+              firstFrameIndex: this.firstFrameIndex,
+              queueLength: stats.queueLength,
+            });
+          }
+        } else {
+          // 数据不足，等待更多数据
+          (window as any).avatarSDKLogger.debug?.(
+            "[AudioRenderer]",
+            `等待更多 WebM 音频数据 (frameIndex: ${frameIndex}, queueLength: ${stats.queueLength})`
+          );
+        }
+      }
+      return; // WebM 模式下不需要继续处理 PCM 播放逻辑
+    }
+  }
+
+  resume() {
+    if (this.audio && this.resourceManager.config.raw_audio) {
+      this.audio?.resume();
+    } else {
+      // if (this.audioCtx.state === "suspended") {
+      //   this.audioCtx
+      //     .resume()
+      //     .then(() => {
+      //       (window as any).avatarSDKLogger.info("音频上下文恢复成功");
+      //     })
+      //     .catch((error) => {
+      //       (window as any).avatarSDKLogger.error("音频上下文恢复失败:", error);
+      //     });
+      // }
+    }
+  }
+
+  pause() {
+    this.isPlaying = false;
+    this.options.dataCacheQueue._clearAudio(this.speech_id);
+    this.firstFrameIndex = -1;
+    // 停止 PCM 播放
+    this.audio?.stop();
+    
+    // 停止 WebM 播放
+    this.mseAudioPlayer?.stop(this.addNewAudioSegment.bind(this));
+    this.speech_id = -1;
+  }
+
+  stop(speech_id: number) {
+    // 如果当前需要结束的是旧的语音ID，直接返回
+    if(this.oldSpeechId >= speech_id) return;
+    
+    this.isPlaying = false;
+    this.options.dataCacheQueue._clearAudio(speech_id);
+    this.firstFrameIndex = -1;
+    // 处理缓存的音频数据
+    (window as any).avatarSDKLogger?.log(
+      this.TAG,
+      `清除缓存数据 (speech_id: ${this.speech_id}, PCM缓存数据长度: ${this.cacheAudioData.length}, WebM缓存数据长度: ${this.mseCacheAudioData.length})`
+    );
+    if (this.resourceManager.config.raw_audio) {
+      // 停止 PCM 播放
+      this.audio?.stop();
+    } else {
+      // 停止 WebM 播放
+      this.mseAudioPlayer?.stop(this.addNewAudioSegment.bind(this));
+    }
+    this.oldSpeechId = speech_id;
+    this.speech_id = -1;
+    if (this.cacheAudioData.length > 0) {
+      this.firstFrameIndex = this.cacheFirstFrameIndex;
+      this.cacheFirstFrameIndex = -1;
+      // 新的缓存下来的数据，推入音频流
+      for (const data of this.cacheAudioData) {
+        this.audio?.receivePCMStream(data);
+      }
+      this.cacheAudioData = [];
+    }
+  }
+
+  addNewAudioSegment() {
+    if(this.mseCacheAudioData.length > 0) {
+      this.firstFrameIndex = this.cacheFirstFrameIndex;
+      this.cacheFirstFrameIndex = -1;
+      // 新的缓存下来的数据，推入音频流
+      for (const audio of this.mseCacheAudioData) {
+        if (audio.data) {
+          this.mseAudioPlayer?.addAudioSegment(
+            audio.data,
+          );
+        }
+      }
+      this.mseCacheAudioData = [];
+    }
+  }
+
+  setVolume(valume: number) {
+    this.valume = valume;
+    if (this.resourceManager.config.raw_audio) {
+      this.audio?.setVolume(valume);
+    } else {
+      // WebM 模式：使用 MSEAudioPlayer 设置音量
+      this.mseAudioPlayer?.setVolume(valume);
+    }
+  }
+
+  setSpeed(speed: number) {
+    if (this.resourceManager.config.raw_audio) {
+      this.audio?.setSpeed(speed);
+    } else {
+      // WebM 模式：使用 MSEAudioPlayer 设置播放速度
+      this.mseAudioPlayer?.setSpeed(speed);
+    }
+  }
+
+  getAudioPlayer(): AudioWorklet | null {
+    return this.audio;
+  }
+
+  destroy() {
+    this.stop(-1);
+    this.cacheAudioData = [];
+    this.mseCacheAudioData = [];
+    this.audio?.destroy();
+    // 直接调用 destroy()，不需要先 stop()（stop 会重建 MediaSource，浪费资源）
+    this.mseAudioPlayer?.destroy();
+    this.mseAudioPlayer = null;
+  }
+}