@xmov/avatar 2.0.0-alpha.38

package/dist/agent/e2e-client.d.ts

@@ -0,0 +1,32 @@
+import type { AgentClientMessage, AgentServerEvent, AgentSocketState } from "./types";
+export interface AgentE2EClientOptions {
+    wsUrl: string;
+    token: string;
+    audioInputEnabled?: boolean;
+    WebSocketCtor?: typeof WebSocket;
+    onStateChange?: (state: AgentSocketState) => void;
+    onEvent?: (event: AgentServerEvent) => void;
+    onError?: (error: Error) => void;
+    onClose?: (event: CloseEvent) => void;
+}
+export declare class AgentE2EClient {
+    private ws;
+    private connectPromise;
+    private rejectConnect;
+    private audioInputEnabled;
+    private readonly WebSocketCtor;
+    private readonly options;
+    constructor(options: AgentE2EClientOptions);
+    get isOpen(): boolean;
+    connect(): Promise<void>;
+    disconnect(code?: number, reason?: string): void;
+    send(message: AgentClientMessage): void;
+    trySend(message: AgentClientMessage): boolean;
+    sendBinary(frame: ArrayBuffer | ArrayBufferView | Blob): void;
+    ping(): void;
+    setAudioInputEnabled(enabled: boolean): void;
+    private assertOpen;
+    private parseMessage;
+    private buildWsUrl;
+    private updateAudioInputQuery;
+}

package/dist/agent/fixed-audio-track.d.ts

@@ -0,0 +1,48 @@
+export interface FixedAudioTrackOptions {
+    onError?: (error: Error) => void;
+}
+/**
+ * Converts a microphone MediaStreamTrack to a verifiable 16 kHz mono track
+ * before it is handed to MediaRecorder. WebM/Opus remains the container/codec
+ * used on the wire; this class only owns the decoded audio track conversion.
+ */
+export declare class FixedAudioTrack {
+    private readonly options;
+    private inputTrack;
+    private reader;
+    private writer;
+    private generator;
+    private audioContext;
+    private audioSource;
+    private audioDestination;
+    private pumpPromise;
+    private pumpFinished;
+    private outputClosed;
+    private draining;
+    private inputSampleRate;
+    private resampleBuffer;
+    private resamplePosition;
+    private outputBuffer;
+    private outputTimestamp;
+    private startGeneration;
+    constructor(options?: FixedAudioTrackOptions);
+    start(inputTrack: MediaStreamTrack): Promise<MediaStream>;
+    private startWebAudio;
+    stop(): Promise<void>;
+    /** Stop input and flush decoded samples while keeping the output track live. */
+    drain(): Promise<void>;
+    private closeOutput;
+    private closeWebAudioOutput;
+    private pump;
+    private toMono;
+    private resample;
+    private writeChunk;
+    private validateAudioData;
+    private delay;
+    private stopGenerator;
+    private unsupported;
+    private contextSuspended;
+    private assertStartActive;
+    private normalizeWebAudioError;
+    private validateOutputTrack;
+}

package/dist/agent/index.cjs

@@ -0,0 +1,5 @@
+const entry = require("./avatar.cjs");
+module.exports = entry.default;
+module.exports.AgentE2EClient = entry.AgentE2EClient;
+module.exports.DEFAULT_AGENT_AUDIO = entry.DEFAULT_AGENT_AUDIO;
+module.exports.MicrophoneController = entry.MicrophoneController;

package/dist/agent/index.d.ts

@@ -0,0 +1,104 @@
+import XmovAvatar from "../index";
+import type { ISessionResponse } from "../modules/ResourceManager";
+import type { AgentASRStartOptions, AgentASRState, AgentState, XingyunAvatarAgentInitParams, XingyunAvatarAgentOptions } from "./types";
+export default class XingyunAvatarAgent extends XmovAvatar {
+    private e2eClient;
+    private microphone;
+    private stopASRPromise;
+    private initializationPromise;
+    private rejectInitialization;
+    private initialSessionAttempt;
+    private initialSessionGeneration;
+    private readonly initialTtsaReadySessionIds;
+    private initialTtsaReadyBeforeSession;
+    private asrStartGeneration;
+    private agentState;
+    private asrState;
+    private agentDestroyed;
+    private agentDestroying;
+    private suppressNextVoiceEnd;
+    private audioInputEnabled;
+    private farendAudioEnabled;
+    private readonly audioUplink;
+    private originalOnAudioPlaybackData?;
+    private playbackDataHandler?;
+    private readonly agentOptions;
+    private readonly reconnectOptions;
+    private reconnectTimer;
+    private reconnectAttempt;
+    private reconnectInFlight;
+    private sessionReloadFallback;
+    private waitingForTtsaReload;
+    private stateBeforeReconnect;
+    private reportingInitializationFailure;
+    private readonly handleOnlineForReconnect;
+    constructor(options: XingyunAvatarAgentOptions);
+    getAgentState(): AgentState;
+    getState(): AgentState;
+    getASRState(): AgentASRState;
+    init(params?: XingyunAvatarAgentInitParams): Promise<ISessionResponse | null | undefined>;
+    private initialize;
+    private beginInitialSession;
+    start(): void;
+    protected onTtsaReady(sessionInfo?: ISessionResponse): void;
+    private abortInitialization;
+    stop(): Promise<void>;
+    speak(...args: Parameters<XmovAvatar["speak"]>): string | undefined;
+    destroy(reason?: string): Promise<void>;
+    ask(text: string): Promise<void>;
+    speakByE2E(text: string): Promise<void>;
+    startASR(options?: AgentASRStartOptions): Promise<void>;
+    private cloneASRInputStream;
+    private assertASRStartActive;
+    stopASR(): Promise<void>;
+    private finishStopASR;
+    interruptConversation(reason?: string): Promise<void>;
+    protected onSessionReloaded(sessionInfo: ISessionResponse): Promise<void>;
+    protected onSessionReloadExhausted(error?: unknown): void;
+    reloadSuccess(): void;
+    private replaceE2EConnection;
+    private resolveE2EConnection;
+    private sendControl;
+    private assertAudioChunkDuration;
+    private sendAudioFrame;
+    private handleAudioUplinkError;
+    private applySessionFeatures;
+    private forwardFarendAudio;
+    private handleMicrophoneError;
+    private finishFailedASR;
+    private enableAudioInput;
+    private disableAudioInput;
+    private handleServerEvent;
+    private handleASRResult;
+    private handleLLMResponse;
+    private handleSemanticJudgeResult;
+    private forwardVoiceEnd;
+    private readFiniteNumber;
+    private handleBackendError;
+    private handleSocketClose;
+    private shouldReconnect;
+    private beginReconnect;
+    private scheduleE2EReconnect;
+    private attemptE2EReconnect;
+    private fallbackToSessionReload;
+    private finishReconnect;
+    private failReconnect;
+    private cancelReconnect;
+    private clearReconnectTimer;
+    private getReconnectDelay;
+    private readRestorableState;
+    private normalizeReconnectOptions;
+    private readNonNegativeNumber;
+    private stopASRAfterSocketClose;
+    private setAgentState;
+    private setASRState;
+    private emitConversation;
+    private emitError;
+    private safeEmit;
+    private assertRunning;
+    private assertE2EOpen;
+    private assertNotDestroyed;
+}
+export * from "./types";
+export { AgentE2EClient } from "./e2e-client";
+export { MicrophoneController } from "./microphone";

package/dist/agent/index.umd.js

@@ -0,0 +1,10 @@
+(function (root) {
+  var entry = root.XingyunAvatarAgentModule;
+  if (!entry || !entry.default) {
+    throw new Error("XingyunAvatarAgentModule must be loaded before XingyunAvatarAgent");
+  }
+  root.XingyunAvatarAgent = entry.default;
+  root.XingyunAvatarAgent.AgentE2EClient = entry.AgentE2EClient;
+  root.XingyunAvatarAgent.DEFAULT_AGENT_AUDIO = entry.DEFAULT_AGENT_AUDIO;
+  root.XingyunAvatarAgent.MicrophoneController = entry.MicrophoneController;
+}(typeof self !== "undefined" ? self : this));

package/dist/agent/microphone.d.ts

@@ -0,0 +1,49 @@
+import type { AgentAudioMetadata, AgentAudioOptions } from "./types";
+export interface MicrophoneControllerOptions {
+    audio?: AgentAudioOptions;
+    debugAudioChunks?: boolean;
+    /** Agent-owned input stream. When omitted, the controller requests the microphone. */
+    inputStream?: MediaStream;
+    /** Use deterministic 100ms PCM buckets with WebCodecs Opus encoding. */
+    usePcmWebCodecs?: boolean;
+    onMetadata?: (metadata: AgentAudioMetadata) => void;
+    /** WebM/Opus Blob callback; timestamp is the media-block start time. */
+    onFrame: (frame: Blob, timestamp?: number) => void;
+    onError?: (error: Error) => void;
+}
+export declare class MicrophoneController {
+    private stream;
+    private pendingInputStream;
+    private recordingStream;
+    private audioTransformer;
+    private mediaRecorder;
+    private pcmAudioContext;
+    private pcmSource;
+    private pcmWorklet;
+    private pcmEncoder;
+    private pcmEncodeQueue;
+    private pcmStartedAt;
+    private pcmGeneration;
+    private resolvePcmFlush;
+    private startGeneration;
+    private starting;
+    private hasAudioSessionCapture;
+    private metadata;
+    private readonly options;
+    private readonly usesProvidedInput;
+    constructor(options: MicrophoneControllerOptions);
+    get isRecording(): boolean;
+    getAudioMetadata(): AgentAudioMetadata;
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    private startWebMRecorder;
+    private assertStartActive;
+    private cancelled;
+    private webMOpusUnsupported;
+    private webCodecsOpusUnsupported;
+    private startPcmWebMRecorder;
+    private stopPcmWebMRecorder;
+    private monotonicEpochNow;
+    private audioContextSuspended;
+    private stopMediaRecorder;
+}

package/dist/agent/types.d.ts

@@ -0,0 +1,153 @@
+import type { IAvatarOptions, IInitParams, INetworkInfo, RenderState } from "../types";
+export type AgentState = "idle" | "initializing" | "ready" | "running" | "reconnecting" | "stopped" | "destroyed" | "failed";
+export type AgentSocketState = "idle" | "connecting" | "open" | "closed" | "error";
+export type AgentASRState = "idle" | "requesting-permission" | "starting" | "listening" | "stopping" | "failed";
+export type AgentConversationState = "idle" | "asking" | "speaking-directly" | "speaking" | "completed" | "interrupted" | "failed";
+export type AgentErrorDomain = "permission" | "network" | "asr" | "brain" | "ttsa" | "quota" | "sdk";
+export interface AgentAudioOptions {
+    chunkMs?: number;
+    inputEnabled?: boolean;
+    /**
+     * @deprecated Session 返回 features.speech_frontend.enable_aec 后，以服务端返回值为准。
+     */
+    echoCancellationEnabled?: boolean;
+}
+export interface AgentReconnectOptions {
+    enabled?: boolean;
+    maxAttempts?: number;
+    initialDelayMs?: number;
+    maxDelayMs?: number;
+}
+export interface AgentError {
+    code: string;
+    message: string;
+    domain: AgentErrorDomain;
+    retryable?: boolean;
+    cause?: unknown;
+}
+export interface AgentASRResult {
+    text: string;
+    isFinal: boolean;
+    raw?: unknown;
+}
+export interface AgentASRStartOptions {
+    /**
+     * Optional caller-owned audio stream used instead of requesting the physical microphone.
+     * The Agent clones one live audio track and never stops the caller-owned stream.
+     */
+    inputStream?: MediaStream;
+}
+export interface AgentLLMUsage {
+    promptTokens: number;
+    completionTokens: number;
+    totalTokens: number;
+    cachedTokens: number;
+}
+export interface AgentLLMResponse {
+    event: "chunk" | "done";
+    text?: string;
+    isFirst?: boolean;
+    usage?: AgentLLMUsage;
+    raw?: unknown;
+}
+export interface AgentSemanticJudgeResult {
+    query: string;
+    meaningful: boolean;
+    action: string;
+    raw?: unknown;
+}
+export interface AgentSpeakStateEvent {
+    state: string;
+    clientSpeakId?: string | number;
+    raw?: unknown;
+}
+export interface AgentConversationEvent {
+    state: AgentConversationState;
+    text?: string;
+    raw?: unknown;
+}
+export interface XingyunAvatarAgentCallbacks {
+    onAgentStateChange?: (state: AgentState) => void;
+    onSocketStateChange?: (state: AgentSocketState) => void;
+    onASRStateChange?: (state: AgentASRState) => void;
+    onASRResult?: (result: AgentASRResult) => void;
+    onLLMResponse?: (response: AgentLLMResponse) => void;
+    onSpeakStateChange?: (event: AgentSpeakStateEvent) => void;
+    onConversationChange?: (event: AgentConversationEvent) => void;
+    onSemanticJudgeResult?: (result: AgentSemanticJudgeResult) => void;
+    onRenderChange?: (state: RenderState) => void;
+    onNetworkInfo?: (info: INetworkInfo) => void;
+    onError?: (error: AgentError) => void;
+}
+export type AgentServiceConfig = Record<string, unknown>;
+export interface AgentBrainConfig extends AgentServiceConfig {
+    extra_body?: Record<string, unknown>;
+}
+export interface AgentFeatures extends AgentServiceConfig {
+    anti_interference?: {
+        semantic_judge_enabled?: boolean;
+    };
+    speech_frontend?: {
+        enabled?: boolean;
+        enable_aec?: boolean;
+        enable_speech_separation?: boolean;
+    };
+    vad_merge_mode?: boolean;
+    volume_and_repetition_text_detection?: boolean;
+}
+export interface XingyunAvatarAgentOptions extends IAvatarOptions {
+    asr_id?: number;
+    asr_config?: AgentServiceConfig;
+    tts_config?: AgentServiceConfig;
+    features?: AgentFeatures;
+    extras?: AgentServiceConfig;
+    llm_id?: number;
+    brain_config?: AgentBrainConfig;
+    session_speak_req_id?: 0 | 1;
+    e2eServer?: string;
+    authToken?: string;
+    audio?: AgentAudioOptions;
+    reconnect?: AgentReconnectOptions;
+    agentCallbacks?: XingyunAvatarAgentCallbacks;
+    webSocketCtor?: typeof WebSocket;
+}
+export type XingyunAvatarAgentInitParams = Omit<IInitParams, "onDownloadProgress"> & {
+    onDownloadProgress?: IInitParams["onDownloadProgress"];
+};
+export interface AgentSessionInfo {
+    ws_url?: string;
+    e2e_token?: string;
+    monitor_ws_url?: string;
+}
+export interface AgentClientMessage {
+    type: string;
+    message?: unknown;
+    [key: string]: unknown;
+}
+export interface AgentServerEvent {
+    type: string;
+    domain?: string;
+    code?: string | number;
+    event?: string;
+    message?: unknown;
+    text?: string;
+    is_first?: boolean;
+    is_final?: boolean;
+    usage?: {
+        prompt_tokens?: number;
+        completion_tokens?: number;
+        total_tokens?: number;
+        cached_tokens?: number;
+    };
+    [key: string]: unknown;
+}
+export interface AgentAudioMetadata {
+    format: "webm_opus";
+    mimeType: "audio/webm;codecs=opus";
+    /** Fixed output track sample rate passed to the MediaRecorder pipeline. */
+    sampleRate: number;
+    /** Fixed output track channel count passed to the MediaRecorder pipeline. */
+    channels: number;
+    chunkMs: number;
+}
+export declare const DEFAULT_AGENT_AUDIO: AgentAudioMetadata;

package/dist/baseRender/AudioRenderer.d.ts

@@ -0,0 +1,46 @@
+/**
+ * 音频渲染
+ * 使用 AudioContext 控制
+ */
+import ResourceManager from "../modules/ResourceManager";
+import { DataCacheQueue } from "../control/DataCacheQueue";
+import AudioWorklet from "./AudioWorklet";
+import XmovAvatar from "../index";
+type Option = {
+    sdk: XmovAvatar;
+    dataCacheQueue: DataCacheQueue;
+    resourceManager: ResourceManager;
+};
+export default class AudioRender {
+    private TAG;
+    private options;
+    isPlaying: boolean;
+    lastFrameIndex: number;
+    audio: AudioWorklet | null;
+    firstFrameIndex: number;
+    resourceManager: ResourceManager;
+    valume: number;
+    speech_id: number;
+    cacheFirstFrameIndex: number;
+    cacheAudioData: Uint8Array[];
+    oldSpeechId: number;
+    mseCacheAudioData: Array<{
+        data: ArrayBuffer;
+        frameIndex?: number;
+        speechId?: number;
+    }>;
+    private mseAudioPlayer;
+    constructor(options: Option);
+    updateAudioData(audioList: any): void;
+    _updateAudio(audioList: any): void;
+    render(frameIndex: number): Promise<void>;
+    resume(): void;
+    pause(): void;
+    stop(speech_id: number): void;
+    addNewAudioSegment(): void;
+    setVolume(valume: number): void;
+    setSpeed(speed: number): void;
+    getAudioPlayer(): AudioWorklet | null;
+    destroy(): void;
+}
+export {};

package/dist/baseRender/AudioWorklet.d.ts

@@ -0,0 +1,78 @@
+/**
+ * Web Audio Worklet版PCM流式播放器（24kHz采样率、S16LE单声道）
+ * 核心：无定时器，依赖AudioWorklet的process方法实时处理流式数据
+ */
+export default class PCMAudioPlayer {
+    readonly SAMPLE_RATE: number;
+    readonly FRAME_ALIGNMENT: number;
+    readonly WORKLET_SCRIPT_URL: string;
+    audioCtx: AudioContext | null;
+    audioWorkletNode: AudioWorkletNode | null;
+    isInitialized: boolean;
+    isPlaying: boolean;
+    private destroyResolve;
+    private unsubscribePlaybackSessionRestored;
+    private captureWorklet;
+    private captureCallback;
+    /**
+     * 初始化（加载Worklet + 创建AudioContext + 连接节点）
+     * @returns {Promise<void>} 初始化结果（需用户交互中调用）
+     */
+    init(): Promise<void>;
+    private refreshPlaybackRoute;
+    /**
+     * 接收PCM音频流（流式下发到Worklet，无本地缓存）
+     * @param {Uint8Array} pcmData - 24kHz S16LE单声道PCM数据
+     */
+    receivePCMStream(pcmData: Uint8Array): void;
+    /**
+     * 开始播放（激活Worklet的音频生成）
+     * @returns {Promise<void>} 启动结果
+     */
+    start(): Promise<void>;
+    /**
+     * 暂停播放（停止Worklet音频生成，保留缓存）
+     */
+    pause(): void;
+    /**
+     * 停止播放（清空Worklet缓存，重置状态）
+     */
+    stop(): void;
+    /**
+     * 销毁播放器（释放所有音频资源，不可恢复）
+     * @returns {Promise<void>} 销毁结果
+     */
+    destroy(): Promise<void>;
+    /**
+     * 挂载音频捕获节点（用于 PCM 模式下 ASR 回声消除）
+     * 将 PCM 播放器输出连接到 capture worklet，实时捕获重采样后的 16kHz S16LE 数据
+     */
+    attachCapture(callback: (data: {
+        pcmData: ArrayBuffer;
+        hasAudio: boolean;
+    }) => void): Promise<void>;
+    /**
+     * 卸载音频捕获节点
+     */
+    detachCapture(): void;
+    /**
+     * 清空 capture worklet 内部缓冲区
+     * 用于打断/会话切换时，避免旧音频数据混入新 session
+     */
+    resetCaptureBuffer(): void;
+    /**
+     * 获取当前播放器状态
+     * @returns {Object} 状态对象
+     */
+    getStatus(): {
+        isInitialized: boolean;
+        isPlaying: boolean;
+        isWorkletReady: boolean;
+    };
+    setVolume(valume: number): void;
+    setSpeed(speed: number): void;
+    /**
+     * @returns {Promise<void>} 恢复结果
+     */
+    resume(): Promise<void>;
+}