@xmov/avatar 2.0.1 → 2.1.1

package/src/agent/microphone.ts

@@ -1,534 +0,0 @@
-import type { AgentAudioMetadata, AgentAudioOptions } from "./types";
-import { FixedAudioTrack } from "./fixed-audio-track";
-import { DEFAULT_AGENT_AUDIO } from "./types";
-import { logAudioChunkContent } from "./audio-debug";
-import {
-  beginWebAudioCaptureSession,
-  endWebAudioCaptureSession,
-} from "../utils/audio-session";
-import { MediaRecorderTimestamp } from "../utils/media-recorder-timestamp";
-import { PcmWebMEncoder } from "../encoding/pcm-webm-encoder";
-
-const WEBM_OPUS_MIME_TYPE = "audio/webm;codecs=opus";
-const TARGET_SAMPLE_RATE = 16_000;
-const TARGET_CHANNELS = 1;
-const PCM_BLOCK_MS = 100;
-const PCM_CAPTURE_PROCESSOR = "agent-pcm-capture-processor";
-const PCM_CAPTURE_WORKLET = `
-class AgentPcmCaptureProcessor extends AudioWorkletProcessor {
-  constructor() {
-    super();
-    this.buffer = new Float32Array(1600);
-    this.bufferIndex = 0;
-    this.sequence = 0;
-    this.port.onmessage = (event) => {
-      if (event.data?.type === 'flush') {
-        this.emitBlock(true);
-        this.port.postMessage({ type: 'flushed' });
-      }
-    };
-  }
-
-  emitBlock(padPartial = false) {
-    if (this.bufferIndex === 0 || (!padPartial && this.bufferIndex < 1600)) return;
-    const pcm = new Int16Array(1600);
-    for (let index = 0; index < this.bufferIndex; index += 1) {
-      const sample = Math.max(-1, Math.min(1, this.buffer[index]));
-      pcm[index] = sample < 0 ? sample * 0x8000 : sample * 0x7fff;
-    }
-    this.port.postMessage(
-      { type: 'frame', pcm: pcm.buffer, sequence: this.sequence++ },
-      [pcm.buffer],
-    );
-    this.buffer.fill(0);
-    this.bufferIndex = 0;
-  }
-
-  process(inputs, outputs) {
-    for (const output of outputs) {
-      for (const channel of output) channel.fill(0);
-    }
-    const channels = inputs[0];
-    if (!channels?.length || !channels[0]?.length) return true;
-    for (let frame = 0; frame < channels[0].length; frame += 1) {
-      let sample = 0;
-      for (let channel = 0; channel < channels.length; channel += 1) {
-        sample += channels[channel][frame] || 0;
-      }
-      this.buffer[this.bufferIndex++] = sample / channels.length;
-      if (this.bufferIndex === 1600) this.emitBlock();
-    }
-    return true;
-  }
-}
-registerProcessor('${PCM_CAPTURE_PROCESSOR}', AgentPcmCaptureProcessor);
-`;
-type AgentCodedError = Error & {
-  agentCode?: string;
-  retryable?: boolean;
-};
-
-export interface MicrophoneControllerOptions {
-  audio?: AgentAudioOptions;
-  debugAudioChunks?: boolean;
-  /** Agent-owned input stream. When omitted, the controller requests the microphone. */
-  inputStream?: MediaStream;
-  /** Use deterministic 100ms PCM buckets with WebCodecs Opus encoding. */
-  usePcmWebCodecs?: boolean;
-  onMetadata?: (metadata: AgentAudioMetadata) => void;
-  /** WebM/Opus Blob callback; timestamp is the media-block start time. */
-  onFrame: (frame: Blob, timestamp?: number) => void;
-  onError?: (error: Error) => void;
-}
-
-export class MicrophoneController {
-  private stream: MediaStream | null = null;
-  private pendingInputStream: MediaStream | null;
-  private recordingStream: MediaStream | null = null;
-  private audioTransformer: FixedAudioTrack | null = null;
-  private mediaRecorder: MediaRecorder | null = null;
-  private pcmAudioContext: AudioContext | null = null;
-  private pcmSource: MediaStreamAudioSourceNode | null = null;
-  private pcmWorklet: AudioWorkletNode | null = null;
-  private pcmEncoder: PcmWebMEncoder | null = null;
-  private pcmEncodeQueue: Promise<void> = Promise.resolve();
-  private pcmStartedAt = 0;
-  private pcmGeneration = 0;
-  private resolvePcmFlush: (() => void) | null = null;
-  private startGeneration = 0;
-  private starting = false;
-  private hasAudioSessionCapture = false;
-  private metadata: AgentAudioMetadata;
-  private readonly options: MicrophoneControllerOptions;
-  private readonly usesProvidedInput: boolean;
-
-  constructor(options: MicrophoneControllerOptions) {
-    this.options = options;
-    this.pendingInputStream = options.inputStream ?? null;
-    this.usesProvidedInput = Boolean(options.inputStream);
-    this.metadata = {
-      ...DEFAULT_AGENT_AUDIO,
-      chunkMs: options.audio?.chunkMs || DEFAULT_AGENT_AUDIO.chunkMs,
-    };
-  }
-
-  get isRecording() {
-    return this.starting || Boolean(this.stream) || Boolean(this.pendingInputStream);
-  }
-
-  getAudioMetadata() {
-    return this.metadata;
-  }
-
-  async start() {
-    if (this.stream || this.starting) {
-      return;
-    }
-
-    const startGeneration = ++this.startGeneration;
-    this.starting = true;
-    try {
-      const inputStream = this.pendingInputStream;
-      this.stream = inputStream ?? null;
-      this.pendingInputStream = null;
-      const mediaDevices = navigator.mediaDevices;
-      if (!this.stream && !mediaDevices?.getUserMedia) {
-        throw new Error("当前浏览器不支持麦克风采集");
-      }
-
-      const MediaRecorderCtor = globalThis.MediaRecorder;
-      if (this.options.usePcmWebCodecs) {
-        if (!await PcmWebMEncoder.isSupported()) {
-          throw this.webCodecsOpusUnsupported();
-        }
-      } else if (!MediaRecorderCtor?.isTypeSupported?.(WEBM_OPUS_MIME_TYPE)) {
-        throw this.webMOpusUnsupported();
-      }
-      this.assertStartActive(startGeneration);
-
-      // A playback-only session can prevent iOS WebKit from starting a
-      // microphone track. Switch categories before requesting capture.
-      if (!this.stream) {
-        if (this.usesProvidedInput) {
-          throw inputStream ? this.cancelled() : new Error("ASR 输入流已释放");
-        }
-        this.hasAudioSessionCapture = beginWebAudioCaptureSession();
-        this.stream = await mediaDevices.getUserMedia({
-          audio: {
-            sampleRate: { ideal: TARGET_SAMPLE_RATE },
-            channelCount: { ideal: TARGET_CHANNELS },
-            echoCancellation: true,
-            noiseSuppression: true,
-            autoGainControl: true,
-          },
-        });
-      }
-      this.assertStartActive(startGeneration);
-      const inputTrack = this.stream.getAudioTracks()[0];
-      if (!inputTrack) {
-        throw new Error("麦克风未返回音频轨道");
-      }
-      if (this.options.usePcmWebCodecs) {
-        await this.startPcmWebMRecorder(this.stream);
-        this.assertStartActive(startGeneration);
-        this.metadata = {
-          ...this.metadata,
-          sampleRate: TARGET_SAMPLE_RATE,
-          channels: TARGET_CHANNELS,
-          chunkMs: PCM_BLOCK_MS,
-        };
-        this.options.onMetadata?.(this.metadata);
-        return;
-      }
-      const audioTransformer = new FixedAudioTrack({
-        onError: (error) => {
-          const normalized = error instanceof Error ? error : new Error("16 kHz mono 音频处理失败");
-          normalized.name = "MediaRecorderError";
-          this.options.onError?.(normalized);
-        },
-      });
-      this.audioTransformer = audioTransformer;
-      this.recordingStream = await audioTransformer.start(inputTrack);
-      this.assertStartActive(startGeneration);
-      const outputTrack = this.recordingStream.getAudioTracks()[0];
-      const outputSettings = outputTrack?.getSettings() as
-        | (MediaTrackSettings & { sampleRate?: number; channelCount?: number })
-        | undefined;
-      this.metadata = {
-        ...this.metadata,
-        sampleRate: outputSettings?.sampleRate || TARGET_SAMPLE_RATE,
-        channels: outputSettings?.channelCount || TARGET_CHANNELS,
-      };
-      this.startWebMRecorder(MediaRecorderCtor);
-      this.options.onMetadata?.(this.metadata);
-    } catch (error) {
-      const startCancelled = startGeneration !== this.startGeneration;
-      await this.stop();
-      const normalized: AgentCodedError = startCancelled
-        ? this.cancelled()
-        : error instanceof Error
-          ? error
-          : new Error("麦克风启动失败");
-      if (normalized.agentCode !== "AUDIO_FIXED_TRACK_CANCELLED") {
-        this.options.onError?.(normalized);
-      }
-      throw normalized;
-    } finally {
-      if (this.startGeneration === startGeneration) {
-        this.starting = false;
-      }
-    }
-  }
-
-  async stop() {
-    this.startGeneration += 1;
-    this.starting = false;
-    try {
-      const audioTransformer = this.audioTransformer;
-      try {
-        // Drain the processor first, but keep the generator track live so the
-        // recorder can capture the flushed tail before it is explicitly stopped.
-        await audioTransformer?.drain();
-      } finally {
-        await this.stopPcmWebMRecorder();
-        await this.stopMediaRecorder();
-        await audioTransformer?.stop();
-        this.audioTransformer = null;
-      }
-      this.recordingStream = null;
-
-      if (this.stream && !audioTransformer) {
-        this.stream.getTracks().forEach((track) => track.stop());
-      }
-      this.stream = null;
-      this.pendingInputStream?.getTracks().forEach((track) => track.stop());
-      this.pendingInputStream = null;
-    } finally {
-      if (this.hasAudioSessionCapture) {
-        this.hasAudioSessionCapture = false;
-        endWebAudioCaptureSession();
-      }
-    }
-  }
-
-  private startWebMRecorder(MediaRecorderCtor: typeof MediaRecorder) {
-    if (!this.recordingStream) {
-      throw new Error("16 kHz mono 音频流未就绪");
-    }
-
-    let recorder: MediaRecorder;
-    try {
-      recorder = new MediaRecorderCtor(this.recordingStream, {
-        mimeType: WEBM_OPUS_MIME_TYPE,
-        audioBitsPerSecond: 24000,
-      });
-    } catch (error) {
-      if (error instanceof Error && error.name === "NotSupportedError") {
-        throw this.webMOpusUnsupported(error);
-      }
-      throw error;
-    }
-    const recorderTimestamp = new MediaRecorderTimestamp(this.metadata.chunkMs);
-    recorder.ondataavailable = (event) => {
-      if (event.data.size > 0) {
-        const timestamp = recorderTimestamp.resolve(event.timecode);
-        (globalThis as typeof globalThis & { avatarSDKLogger?: { log?: (...args: unknown[]) => void } })
-          .avatarSDKLogger?.log?.("[Agent][Microphone]", "audio chunk", {
-            size: event.data.size,
-            type: event.data.type,
-            chunkMs: this.metadata.chunkMs,
-            sampleRate: this.metadata.sampleRate,
-            channels: this.metadata.channels,
-            recorderState: recorder.state,
-            timecode: event.timecode,
-          });
-        if (this.options.debugAudioChunks) {
-          void logAudioChunkContent("[Agent][Microphone] nearend WebM/Opus chunk", event.data, {
-            chunkMs: this.metadata.chunkMs,
-            sampleRate: this.metadata.sampleRate,
-            channels: this.metadata.channels,
-            recorderState: recorder.state,
-            timecode: event.timecode,
-          });
-        }
-        this.options.onFrame(event.data, timestamp);
-      }
-    };
-    recorder.onerror = (event) => {
-      const error = (event as ErrorEvent).error;
-      const recorderError = new Error(error instanceof Error ? error.message : "WebM/Opus 麦克风录制失败");
-      recorderError.name = "MediaRecorderError";
-      this.options.onError?.(recorderError);
-    };
-    this.mediaRecorder = recorder;
-    try {
-      recorder.start(this.metadata.chunkMs);
-    } catch (error) {
-      if (error instanceof Error && error.name === "NotSupportedError") {
-        throw this.webMOpusUnsupported(error);
-      }
-      throw error;
-    }
-  }
-
-  private assertStartActive(startGeneration: number) {
-    if (startGeneration !== this.startGeneration) {
-      throw this.cancelled();
-    }
-  }
-
-  private cancelled() {
-    const error = new Error("麦克风启动已取消") as AgentCodedError;
-    error.name = "AbortError";
-    error.agentCode = "AUDIO_FIXED_TRACK_CANCELLED";
-    error.retryable = true;
-    return error;
-  }
-
-  private webMOpusUnsupported(cause?: unknown) {
-    const error = new Error("当前浏览器不支持 audio/webm;codecs=opus") as AgentCodedError & {
-      cause?: unknown;
-    };
-    error.name = "NotSupportedError";
-    error.agentCode = "AUDIO_WEBM_OPUS_UNSUPPORTED";
-    error.retryable = false;
-    error.cause = cause;
-    return error;
-  }
-
-  private webCodecsOpusUnsupported(cause?: unknown) {
-    const error = new Error("当前浏览器不支持 WebCodecs AudioEncoder Opus") as AgentCodedError & {
-      cause?: unknown;
-    };
-    error.name = "NotSupportedError";
-    error.agentCode = "AUDIO_WEBCODECS_OPUS_UNSUPPORTED";
-    error.retryable = false;
-    error.cause = cause;
-    return error;
-  }
-
-  private async startPcmWebMRecorder(stream: MediaStream) {
-    const globals = globalThis as typeof globalThis & {
-      webkitAudioContext?: typeof AudioContext;
-    };
-    const AudioContextCtor = globalThis.AudioContext || globals.webkitAudioContext;
-    if (!AudioContextCtor || typeof globalThis.AudioWorkletNode === "undefined") {
-      throw this.webCodecsOpusUnsupported(new Error("AudioWorklet 不可用"));
-    }
-
-    const encoder = new PcmWebMEncoder();
-    await encoder.init();
-    const context = new AudioContextCtor({ sampleRate: TARGET_SAMPLE_RATE });
-    if (context.sampleRate !== TARGET_SAMPLE_RATE) {
-      await encoder.destroy();
-      await context.close().catch(() => undefined);
-      throw this.webCodecsOpusUnsupported(
-        new Error(`浏览器无法创建 ${TARGET_SAMPLE_RATE} Hz AudioContext`),
-      );
-    }
-    this.pcmAudioContext = context;
-    this.pcmEncoder = encoder;
-
-    const workletUrl = URL.createObjectURL(
-      new Blob([PCM_CAPTURE_WORKLET], { type: "application/javascript" }),
-    );
-    try {
-      await context.audioWorklet.addModule(workletUrl);
-    } finally {
-      URL.revokeObjectURL(workletUrl);
-    }
-    if (context.state === "suspended") {
-      await context.resume();
-    }
-    if (context.state !== "running") {
-      throw this.audioContextSuspended();
-    }
-
-    const source = context.createMediaStreamSource(stream);
-    const worklet = new AudioWorkletNode(context, PCM_CAPTURE_PROCESSOR, {
-      numberOfInputs: 1,
-      numberOfOutputs: 1,
-      outputChannelCount: [1],
-    });
-    const generation = ++this.pcmGeneration;
-    this.pcmSource = source;
-    this.pcmWorklet = worklet;
-    this.pcmEncodeQueue = Promise.resolve();
-    this.pcmStartedAt = this.monotonicEpochNow();
-
-    worklet.port.onmessage = (event) => {
-      if (event.data?.type === "flushed") {
-        this.resolvePcmFlush?.();
-        this.resolvePcmFlush = null;
-        return;
-      }
-      if (event.data?.type !== "frame" || !(event.data.pcm instanceof ArrayBuffer)) {
-        return;
-      }
-      const timestamp = this.pcmStartedAt + Number(event.data.sequence || 0) * PCM_BLOCK_MS;
-      const pcm = new Int16Array(event.data.pcm);
-      this.pcmEncodeQueue = this.pcmEncodeQueue
-        .then(async () => {
-          const webm = await encoder.encode(pcm);
-          if (generation !== this.pcmGeneration || webm.byteLength === 0) {
-            return;
-          }
-          const frame = new Blob([webm], { type: WEBM_OPUS_MIME_TYPE });
-          if (this.options.debugAudioChunks) {
-            void logAudioChunkContent("[Agent][Microphone] nearend PCM/WebCodecs chunk", frame, {
-              chunkMs: PCM_BLOCK_MS,
-              sampleRate: TARGET_SAMPLE_RATE,
-              channels: TARGET_CHANNELS,
-              sequence: event.data.sequence,
-            });
-          }
-          this.options.onFrame(frame, timestamp);
-        })
-        .catch((cause) => {
-          const error = cause instanceof Error ? cause : new Error("PCM/Opus 麦克风编码失败");
-          error.name = "AudioEncoderError";
-          this.options.onError?.(error);
-        });
-    };
-    source.connect(worklet);
-    worklet.connect(context.destination);
-  }
-
-  private async stopPcmWebMRecorder() {
-    const worklet = this.pcmWorklet;
-    const source = this.pcmSource;
-    const context = this.pcmAudioContext;
-    const encoder = this.pcmEncoder;
-    if (!worklet && !context && !encoder) {
-      return;
-    }
-
-    source?.disconnect();
-    let flushCompleted = true;
-    if (worklet) {
-      flushCompleted = await Promise.race([
-        new Promise<boolean>((resolve) => {
-          this.resolvePcmFlush = () => resolve(true);
-          worklet.port.postMessage({ type: "flush" });
-        }),
-        new Promise<boolean>((resolve) => globalThis.setTimeout(() => resolve(false), 100)),
-      ]);
-      if (!flushCompleted) {
-        this.resolvePcmFlush = null;
-        worklet.port.onmessage = null;
-      }
-    }
-    await this.pcmEncodeQueue;
-    this.pcmGeneration += 1;
-    this.resolvePcmFlush = null;
-    if (worklet) {
-      worklet.port.onmessage = null;
-      worklet.disconnect();
-      worklet.port.close();
-    }
-    if (context && context.state !== "closed") {
-      await context.close().catch(() => undefined);
-    }
-    await encoder?.destroy();
-    this.pcmWorklet = null;
-    this.pcmSource = null;
-    this.pcmAudioContext = null;
-    this.pcmEncoder = null;
-    this.pcmEncodeQueue = Promise.resolve();
-  }
-
-  private monotonicEpochNow() {
-    const clock = globalThis.performance;
-    if (clock && Number.isFinite(clock.timeOrigin)) {
-      return Math.round(clock.timeOrigin + clock.now());
-    }
-    return Date.now();
-  }
-
-  private audioContextSuspended() {
-    const error = new Error("AudioContext 未运行，请在用户操作后重试") as AgentCodedError;
-    error.name = "NotAllowedError";
-    error.agentCode = "AUDIO_CONTEXT_SUSPENDED";
-    error.retryable = true;
-    return error;
-  }
-
-  private async stopMediaRecorder() {
-    const recorder = this.mediaRecorder;
-    this.mediaRecorder = null;
-    if (!recorder) {
-      return;
-    }
-
-    const shouldStop = recorder.state !== "inactive";
-    await new Promise<void>((resolve) => {
-      let settled = false;
-      const finish = () => {
-        if (settled) {
-          return;
-        }
-        settled = true;
-        globalThis.clearTimeout(timeoutId);
-        resolve();
-      };
-      const onError = recorder.onerror;
-      const timeoutId = globalThis.setTimeout(finish, 1000);
-      recorder.onstop = finish;
-      recorder.onerror = (event) => {
-        onError?.call(recorder, event);
-        finish();
-      };
-      if (shouldStop) {
-        try {
-          recorder.stop();
-        } catch {
-          finish();
-        }
-      }
-    });
-    recorder.ondataavailable = null;
-    recorder.onerror = null;
-    recorder.onstop = null;
-  }
-
-}