@xmov/avatar 2.0.1 → 2.1.1

package/src/agent/fixed-audio-track.ts

@@ -1,547 +0,0 @@
-const TARGET_SAMPLE_RATE = 16_000;
-const TARGET_CHANNELS = 1;
-const OUTPUT_FRAME_SIZE = 320;
-const MIN_INPUT_SAMPLE_RATE = 8_000;
-const MAX_INPUT_SAMPLE_RATE = 192_000;
-const MAX_INPUT_FRAMES_PER_READ = 96_000;
-const MAX_RESAMPLE_BUFFER_LENGTH = MAX_INPUT_FRAMES_PER_READ * 2;
-
-interface AudioDataLike {
-  sampleRate: number;
-  numberOfChannels: number;
-  numberOfFrames: number;
-  timestamp: number;
-  copyTo(destination: Float32Array, options?: { format?: string; planeIndex?: number }): void;
-  close(): void;
-}
-
-interface AudioDataConstructorLike {
-  new (init: {
-    format: "f32-planar";
-    sampleRate: number;
-    numberOfFrames: number;
-    numberOfChannels: number;
-    timestamp: number;
-    data: Float32Array;
-  }): AudioDataLike;
-}
-
-interface AudioTrackProcessorLike {
-  readable: ReadableStream<AudioDataLike>;
-}
-
-interface AudioTrackProcessorConstructorLike {
-  new (init: { track: MediaStreamTrack }): AudioTrackProcessorLike;
-}
-
-interface AudioTrackGeneratorLike extends MediaStreamTrack {
-  writable: WritableStream<AudioDataLike>;
-}
-
-interface AudioTrackGeneratorConstructorLike {
-  new (init: { kind: "audio" }): AudioTrackGeneratorLike;
-}
-
-interface AudioRuntimeGlobals {
-  AudioData?: AudioDataConstructorLike;
-  MediaStreamTrackProcessor?: AudioTrackProcessorConstructorLike;
-  MediaStreamTrackGenerator?: AudioTrackGeneratorConstructorLike;
-  webkitAudioContext?: typeof AudioContext;
-}
-
-type AgentCodedError = Error & {
-  agentCode?: string;
-  retryable?: boolean;
-  cause?: unknown;
-};
-
-export interface FixedAudioTrackOptions {
-  onError?: (error: Error) => void;
-}
-
-/**
- * Converts a microphone MediaStreamTrack to a verifiable 16 kHz mono track
- * before it is handed to MediaRecorder. WebM/Opus remains the container/codec
- * used on the wire; this class only owns the decoded audio track conversion.
- */
-export class FixedAudioTrack {
-  private readonly options: FixedAudioTrackOptions;
-  private inputTrack: MediaStreamTrack | null = null;
-  private reader: ReadableStreamDefaultReader<AudioDataLike> | null = null;
-  private writer: WritableStreamDefaultWriter<AudioDataLike> | null = null;
-  private generator: AudioTrackGeneratorLike | null = null;
-  private audioContext: AudioContext | null = null;
-  private audioSource: MediaStreamAudioSourceNode | null = null;
-  private audioDestination: MediaStreamAudioDestinationNode | null = null;
-  private pumpPromise: Promise<void> | null = null;
-  private pumpFinished = false;
-  private outputClosed = false;
-  private draining = false;
-  private inputSampleRate: number | null = null;
-  private resampleBuffer: number[] = [];
-  private resamplePosition = 0;
-  private outputBuffer: number[] = [];
-  private outputTimestamp: number | null = null;
-  private startGeneration = 0;
-
-  constructor(options: FixedAudioTrackOptions = {}) {
-    this.options = options;
-  }
-
-  async start(inputTrack: MediaStreamTrack): Promise<MediaStream> {
-    const startGeneration = ++this.startGeneration;
-    this.inputTrack = inputTrack;
-    this.inputSampleRate = null;
-    this.resampleBuffer = [];
-    this.resamplePosition = 0;
-    this.outputBuffer = [];
-    this.outputTimestamp = null;
-    this.pumpFinished = false;
-    this.outputClosed = false;
-    this.draining = false;
-    const globals = globalThis as typeof globalThis & AudioRuntimeGlobals;
-    const ProcessorCtor = globals.MediaStreamTrackProcessor;
-    const GeneratorCtor = globals.MediaStreamTrackGenerator;
-    const AudioDataCtor = globals.AudioData;
-    if (!ProcessorCtor || !GeneratorCtor || !AudioDataCtor) {
-      return this.startWebAudio(inputTrack, globals, startGeneration);
-    }
-
-    const processor = new ProcessorCtor({ track: inputTrack });
-    const generator = new GeneratorCtor({ kind: "audio" });
-    const stream = new MediaStream([generator]);
-    const outputTrack = stream.getAudioTracks()[0];
-    const settings = outputTrack?.getSettings() as (MediaTrackSettings & {
-      sampleRate?: number;
-      channelCount?: number;
-    }) | undefined;
-    // Chrome does not expose generator settings until its first AudioData is
-    // written. The format is nevertheless fixed by every AudioData created in
-    // writeChunk; when settings are available, reject an implementation that
-    // reports a different output track.
-    if (
-      (settings?.sampleRate !== undefined && settings.sampleRate !== TARGET_SAMPLE_RATE) ||
-      (settings?.channelCount !== undefined && settings.channelCount !== TARGET_CHANNELS)
-    ) {
-      generator.stop();
-      throw this.unsupported(
-        `浏览器音频输出格式不符合要求: ${settings?.sampleRate || "unknown"} Hz / ${settings?.channelCount || "unknown"} channel`,
-      );
-    }
-
-    try {
-      this.reader = processor.readable.getReader();
-      this.writer = generator.writable.getWriter();
-    } catch (error) {
-      generator.stop();
-      if (inputTrack.readyState !== "ended") {
-        inputTrack.stop();
-      }
-      this.inputTrack = null;
-      throw error;
-    }
-    this.generator = generator;
-    this.pumpPromise = this.pump(AudioDataCtor);
-    return stream;
-  }
-
-  private async startWebAudio(
-    inputTrack: MediaStreamTrack,
-    globals: AudioRuntimeGlobals,
-    startGeneration: number,
-  ) {
-    const AudioContextCtor = globalThis.AudioContext || globals.webkitAudioContext;
-    if (!AudioContextCtor) {
-      throw this.unsupported("当前浏览器不支持 16 kHz mono 音频处理");
-    }
-
-    let context: AudioContext | null = null;
-    let source: MediaStreamAudioSourceNode | null = null;
-    let destination: MediaStreamAudioDestinationNode | null = null;
-    try {
-      context = new AudioContextCtor({ sampleRate: TARGET_SAMPLE_RATE });
-      this.audioContext = context;
-      if (context.sampleRate !== TARGET_SAMPLE_RATE) {
-        throw this.unsupported(`浏览器无法创建 16 kHz AudioContext: ${context.sampleRate} Hz`);
-      }
-      if (context.state === "suspended") {
-        try {
-          await context.resume();
-        } catch (cause) {
-          throw this.contextSuspended(cause);
-        }
-      }
-      this.assertStartActive(startGeneration);
-      if (context.state !== "running") {
-        throw this.contextSuspended();
-      }
-
-      source = context.createMediaStreamSource(new MediaStream([inputTrack]));
-      this.audioSource = source;
-      destination = context.createMediaStreamDestination();
-      this.audioDestination = destination;
-      destination.channelCount = TARGET_CHANNELS;
-      destination.channelCountMode = "explicit";
-      destination.channelInterpretation = "speakers";
-      source.connect(destination);
-
-      const outputTrack = destination.stream.getAudioTracks()[0];
-      if (!outputTrack) {
-        throw new Error("Web Audio 未返回音频输出轨道");
-      }
-      this.validateOutputTrack(outputTrack);
-      this.assertStartActive(startGeneration);
-      return destination.stream;
-    } catch (error) {
-      source?.disconnect();
-      destination?.stream.getTracks().forEach((track) => track.stop());
-      if (context && context.state !== "closed") {
-        await context.close().catch(() => undefined);
-      }
-      if (inputTrack.readyState !== "ended") {
-        inputTrack.stop();
-      }
-      this.inputTrack = null;
-      if (this.audioContext === context) {
-        this.audioContext = null;
-        this.audioSource = null;
-        this.audioDestination = null;
-      }
-      throw this.normalizeWebAudioError(error);
-    }
-  }
-
-  async stop() {
-    await this.drain();
-    if (this.audioContext) {
-      await this.closeWebAudioOutput();
-      return;
-    }
-    await this.closeOutput();
-  }
-
-  /** Stop input and flush decoded samples while keeping the output track live. */
-  async drain() {
-    this.startGeneration += 1;
-    this.draining = true;
-    this.inputTrack?.stop();
-    this.inputTrack = null;
-
-    if (this.audioContext) {
-      return;
-    }
-
-    const pump = this.pumpPromise;
-    if (pump) {
-      const settled = pump.then(
-        () => true,
-        () => true,
-      );
-      const finished = await Promise.race([settled, this.delay(1000)]);
-      if (!finished) {
-        await Promise.race([
-          this.reader?.cancel().catch(() => undefined),
-          this.delay(100),
-        ]);
-        await Promise.race([
-          this.writer?.abort(new Error("16 kHz mono 音频处理停止超时")).catch(() => undefined),
-          this.delay(100),
-        ]);
-        this.stopGenerator();
-        // Do not await an implementation that is stuck in writer.write(). The
-        // rejection handler above keeps the eventual pump settlement observed.
-        void settled;
-      }
-    }
-  }
-
-  private async closeOutput() {
-    const writer = this.writer;
-    this.outputClosed = true;
-    if (writer && this.pumpFinished) {
-      await Promise.race([
-        writer.close().catch(() => undefined),
-        this.delay(100),
-      ]);
-    } else {
-      await Promise.race([
-        writer?.abort(new Error("16 kHz mono 音频处理停止超时")).catch(() => undefined),
-        this.delay(100),
-      ]);
-    }
-    this.stopGenerator();
-    if (writer && this.pumpFinished) {
-      writer.releaseLock();
-    }
-    this.reader = null;
-    this.writer = null;
-    this.generator = null;
-    this.pumpPromise = null;
-  }
-
-  private async closeWebAudioOutput() {
-    const context = this.audioContext;
-    const source = this.audioSource;
-    const destination = this.audioDestination;
-    this.audioContext = null;
-    this.audioSource = null;
-    this.audioDestination = null;
-
-    source?.disconnect();
-    destination?.disconnect();
-    destination?.stream.getTracks().forEach((track) => track.stop());
-    if (context && context.state !== "closed") {
-      await context.close().catch(() => undefined);
-    }
-  }
-
-  private async pump(AudioDataCtor: AudioDataConstructorLike) {
-    const reader = this.reader;
-    const writer = this.writer;
-    if (!reader || !writer) {
-      return;
-    }
-
-    try {
-      let inputEnded = false;
-      while (true) {
-        const result = await reader.read();
-        if (result.done) {
-          inputEnded = true;
-          break;
-        }
-        if (!result.value) {
-          continue;
-        }
-
-        const audioData = result.value;
-        try {
-          this.validateAudioData(audioData);
-          const mono = this.toMono(audioData);
-          const chunks = this.resample(mono, audioData.sampleRate, false);
-          if (this.outputTimestamp === null) {
-            this.outputTimestamp = Number.isFinite(audioData.timestamp) ? audioData.timestamp : 0;
-          }
-          for (const chunk of chunks) {
-            await this.writeChunk(writer, AudioDataCtor, chunk);
-          }
-        } finally {
-          audioData.close();
-        }
-      }
-
-      for (const chunk of this.resample(new Float32Array(), this.inputSampleRate || TARGET_SAMPLE_RATE, true)) {
-        await this.writeChunk(writer, AudioDataCtor, chunk);
-      }
-      if (inputEnded && !this.draining && this.inputTrack?.readyState === "ended") {
-        const error = new Error("麦克风音频轨道已结束");
-        error.name = "AudioTransformError";
-        this.options.onError?.(error);
-        this.stopGenerator();
-      }
-    } catch (error) {
-      const normalized = error instanceof Error ? error : new Error("16 kHz mono 音频处理失败");
-      normalized.name = "AudioTransformError";
-      this.options.onError?.(normalized);
-    } finally {
-      reader.releaseLock();
-      this.pumpFinished = true;
-      if (this.outputClosed) {
-        writer.releaseLock();
-      }
-    }
-  }
-
-  private toMono(audioData: AudioDataLike) {
-    const channels = Math.max(1, audioData.numberOfChannels);
-    const mono = new Float32Array(audioData.numberOfFrames);
-    for (let channel = 0; channel < channels; channel += 1) {
-      const plane = new Float32Array(audioData.numberOfFrames);
-      audioData.copyTo(plane, { format: "f32-planar", planeIndex: channel });
-      for (let index = 0; index < plane.length; index += 1) {
-        mono[index] += plane[index] / channels;
-      }
-    }
-    return mono;
-  }
-
-  private resample(input: Float32Array, inputSampleRate: number, flush: boolean) {
-    if (this.inputSampleRate === null) {
-      this.inputSampleRate = inputSampleRate;
-    } else if (this.inputSampleRate !== inputSampleRate) {
-      throw new Error(`输入音频采样率发生变化: ${this.inputSampleRate} -> ${inputSampleRate}`);
-    }
-
-    for (const sample of input) {
-      this.resampleBuffer.push(sample);
-    }
-    if (this.resampleBuffer.length > MAX_RESAMPLE_BUFFER_LENGTH) {
-      throw new Error("输入音频重采样缓冲区超限");
-    }
-    if (flush && this.resampleBuffer.length > 0) {
-      this.resampleBuffer.push(this.resampleBuffer[this.resampleBuffer.length - 1]);
-    }
-
-    const step = inputSampleRate / TARGET_SAMPLE_RATE;
-    const chunks: Float32Array[] = [];
-    while (this.resamplePosition + 1 < this.resampleBuffer.length) {
-      const before = Math.floor(this.resamplePosition);
-      const after = Math.min(before + 1, this.resampleBuffer.length - 1);
-      const weight = this.resamplePosition - before;
-      this.outputBuffer.push(
-        this.resampleBuffer[before] * (1 - weight) + this.resampleBuffer[after] * weight,
-      );
-      this.resamplePosition += step;
-
-      const consumed = Math.floor(this.resamplePosition);
-      if (consumed > 0) {
-        this.resampleBuffer.splice(0, consumed);
-        this.resamplePosition -= consumed;
-      }
-
-      if (this.outputBuffer.length >= OUTPUT_FRAME_SIZE) {
-        chunks.push(new Float32Array(this.outputBuffer.splice(0, OUTPUT_FRAME_SIZE)));
-      }
-    }
-
-    if (flush && this.outputBuffer.length > 0) {
-      chunks.push(new Float32Array(this.outputBuffer.splice(0)));
-    }
-    return chunks;
-  }
-
-  private async writeChunk(
-    writer: WritableStreamDefaultWriter<AudioDataLike>,
-    AudioDataCtor: AudioDataConstructorLike,
-    samples: Float32Array,
-  ) {
-    if (!samples.length) {
-      return;
-    }
-    const timestamp = this.outputTimestamp || 0;
-    const audioData = new AudioDataCtor({
-      format: "f32-planar",
-      sampleRate: TARGET_SAMPLE_RATE,
-      numberOfFrames: samples.length,
-      numberOfChannels: TARGET_CHANNELS,
-      timestamp,
-      data: samples,
-    });
-    this.outputTimestamp = timestamp + Math.round((samples.length * 1_000_000) / TARGET_SAMPLE_RATE);
-    await writer.write(audioData);
-    audioData.close();
-    const settings = this.generator?.getSettings() as
-      | (MediaTrackSettings & { sampleRate?: number; channelCount?: number })
-      | undefined;
-    if (
-      settings &&
-      ((settings.sampleRate !== undefined && settings.sampleRate !== TARGET_SAMPLE_RATE) ||
-        (settings.channelCount !== undefined && settings.channelCount !== TARGET_CHANNELS))
-    ) {
-      throw this.unsupported(
-        `浏览器音频输出格式不符合要求: ${settings.sampleRate || "unknown"} Hz / ${settings.channelCount || "unknown"} channel`,
-      );
-    }
-  }
-
-  private validateAudioData(audioData: AudioDataLike) {
-    if (
-      !Number.isFinite(audioData.sampleRate) ||
-      !Number.isInteger(audioData.sampleRate) ||
-      audioData.sampleRate < MIN_INPUT_SAMPLE_RATE ||
-      audioData.sampleRate > MAX_INPUT_SAMPLE_RATE
-    ) {
-      throw new Error(`输入音频采样率无效: ${audioData.sampleRate}`);
-    }
-    if (
-      !Number.isInteger(audioData.numberOfChannels) ||
-      audioData.numberOfChannels < 1 ||
-      audioData.numberOfChannels > 8
-    ) {
-      throw new Error(`输入音频声道数无效: ${audioData.numberOfChannels}`);
-    }
-    if (
-      !Number.isInteger(audioData.numberOfFrames) ||
-      audioData.numberOfFrames < 1 ||
-      audioData.numberOfFrames > MAX_INPUT_FRAMES_PER_READ
-    ) {
-      throw new Error(`输入音频帧数无效: ${audioData.numberOfFrames}`);
-    }
-    if (!Number.isFinite(audioData.timestamp) || Math.abs(audioData.timestamp) > 1e15) {
-      throw new Error(`输入音频时间戳无效: ${audioData.timestamp}`);
-    }
-  }
-
-  private delay(timeoutMs: number) {
-    return new Promise<boolean>((resolve) => {
-      globalThis.setTimeout(() => resolve(false), timeoutMs);
-    });
-  }
-
-  private stopGenerator() {
-    if (this.generator && this.generator.readyState !== "ended") {
-      this.generator.stop();
-    }
-  }
-
-  private unsupported(message: string, cause?: unknown) {
-    const error = new Error(message) as AgentCodedError;
-    error.name = "NotSupportedError";
-    error.agentCode = "AUDIO_FIXED_TRACK_UNSUPPORTED";
-    error.retryable = false;
-    error.cause = cause;
-    return error;
-  }
-
-  private contextSuspended(cause?: unknown) {
-    const error = new Error("AudioContext 未运行，请在用户操作后重试") as AgentCodedError;
-    error.name = "NotAllowedError";
-    error.agentCode = "AUDIO_CONTEXT_SUSPENDED";
-    error.retryable = true;
-    error.cause = cause;
-    return error;
-  }
-
-  private assertStartActive(startGeneration: number) {
-    if (startGeneration !== this.startGeneration) {
-      const error = new Error("16 kHz mono 音频处理启动已取消") as AgentCodedError;
-      error.name = "AbortError";
-      error.agentCode = "AUDIO_FIXED_TRACK_CANCELLED";
-      error.retryable = true;
-      throw error;
-    }
-  }
-
-  private normalizeWebAudioError(error: unknown) {
-    const normalized: AgentCodedError = error instanceof Error
-      ? error
-      : new Error(String(error));
-    if (normalized.agentCode) {
-      return normalized;
-    }
-    if (normalized.name === "NotSupportedError") {
-      return this.unsupported(`浏览器无法初始化 16 kHz mono Web Audio: ${normalized.message}`, normalized);
-    }
-    const coded = new Error(`16 kHz mono Web Audio 初始化失败: ${normalized.message}`) as AgentCodedError;
-    coded.name = normalized.name;
-    coded.agentCode = "AUDIO_FIXED_TRACK_FAILED";
-    coded.retryable = true;
-    coded.cause = normalized;
-    return coded;
-  }
-
-  private validateOutputTrack(outputTrack: MediaStreamTrack) {
-    const settings = outputTrack.getSettings() as MediaTrackSettings & {
-      sampleRate?: number;
-      channelCount?: number;
-    };
-    if (
-      (settings.sampleRate !== undefined && settings.sampleRate !== TARGET_SAMPLE_RATE) ||
-      (settings.channelCount !== undefined && settings.channelCount !== TARGET_CHANNELS)
-    ) {
-      throw this.unsupported(
-        `浏览器音频输出格式不符合要求: ${settings.sampleRate || "unknown"} Hz / ${settings.channelCount || "unknown"} channel`,
-      );
-    }
-  }
-}