@xmov/avatar 2.0.0-alpha.38

package/src/encoding/media-recorder-encoder.ts

@@ -0,0 +1,175 @@
+/**
+ * MediaRecorder 降级编码器
+ * 对于不支持 WebCodecs Opus 但支持 MediaRecorder webm/opus 的平台（Safari 18.4+）
+ * 通过 16kHz AudioContext + PCMInjectorProcessor + MediaRecorder 实现 Opus 编码
+ */
+
+export class MediaRecorderEncoder {
+  private audioCtx: AudioContext | null = null;
+  private workletNode: AudioWorkletNode | null = null;
+  private destination: MediaStreamAudioDestinationNode | null = null;
+  private recorder: MediaRecorder | null = null;
+  private isRecording = false;
+  private nextResolve: ((data: ArrayBuffer | null) => void) | null = null;
+  private dataQueue: ArrayBuffer[] = []; // 缓冲 ondataavailable 产生的数据
+
+  /**
+   * 检测当前浏览器是否支持 MediaRecorder webm/opus
+   */
+  static isSupported(): boolean {
+    if (typeof MediaRecorder === 'undefined') return false;
+    return MediaRecorder.isTypeSupported('audio/webm;codecs=opus');
+  }
+
+  /**
+   * 初始化：创建 16kHz AudioContext + 中转 Worklet + MediaRecorder
+   */
+  async init(): Promise<void> {
+    const AudioContextCtor = window.AudioContext || (window as any).webkitAudioContext;
+    this.audioCtx = new AudioContextCtor({ sampleRate: 16000 });
+
+    // PCM 注入处理器：接收主线程传入的 S16LE PCM 数据，输出到音频流
+    const workletCode = `
+      class PCMInjectorProcessor extends AudioWorkletProcessor {
+        constructor() {
+          super();
+          this.buffer = new Float32Array(0);
+          this.port.onmessage = (e) => {
+            if (e.data.type === 'pcm') {
+              const int16 = new Int16Array(e.data.pcm);
+              const float32 = new Float32Array(int16.length);
+              for (let i = 0; i < int16.length; i++) {
+                float32[i] = int16[i] / 32768;
+              }
+              const newBuffer = new Float32Array(this.buffer.length + float32.length);
+              newBuffer.set(this.buffer);
+              newBuffer.set(float32, this.buffer.length);
+              this.buffer = newBuffer;
+            }
+          };
+        }
+        process(inputs, outputs) {
+          const output = outputs[0][0];
+          if (!output) return true;
+          const len = Math.min(output.length, this.buffer.length);
+          for (let i = 0; i < len; i++) {
+            output[i] = this.buffer[i];
+          }
+          this.buffer = this.buffer.slice(len);
+          for (let i = len; i < output.length; i++) {
+            output[i] = 0;
+          }
+          return true;
+        }
+      }
+      registerProcessor('pcm-injector-processor', PCMInjectorProcessor);
+    `;
+    const blob = new Blob([workletCode], { type: 'application/javascript' });
+    const url = URL.createObjectURL(blob);
+    await this.audioCtx.audioWorklet.addModule(url);
+    URL.revokeObjectURL(url);
+
+    this.workletNode = new AudioWorkletNode(this.audioCtx, 'pcm-injector-processor');
+    this.destination = this.audioCtx.createMediaStreamDestination();
+    this.workletNode.connect(this.destination);
+
+    // 创建 MediaRecorder
+    const mimeType = MediaRecorder.isTypeSupported('audio/webm;codecs=opus')
+      ? 'audio/webm;codecs=opus'
+      : 'audio/webm';
+    this.recorder = new MediaRecorder(this.destination.stream, {
+      mimeType,
+      audioBitsPerSecond: 24000,
+    });
+
+    this.recorder.ondataavailable = (ev: BlobEvent) => {
+      if (ev.data && ev.data.size > 0) {
+        ev.data.arrayBuffer().then((buffer) => {
+          // 如果有等待中的 resolve，立即触发
+          if (this.nextResolve) {
+            const resolve = this.nextResolve;
+            this.nextResolve = null;
+            resolve(buffer);
+          } else {
+            // 否则加入队列缓冲
+            this.dataQueue.push(buffer);
+          }
+        });
+      }
+    };
+
+    // 恢复 AudioContext
+    if (this.audioCtx.state === 'suspended') {
+      await this.audioCtx.resume();
+    }
+  }
+
+  /**
+   * 注入 PCM 数据并等待编码输出
+   * @param pcmData S16LE PCM ArrayBuffer (16kHz)
+   * @returns 编码后的 WebM 数据，或 null
+   */
+  async feedPCM(pcmData: ArrayBuffer): Promise<ArrayBuffer | null> {
+    if (!this.workletNode || !this.recorder) return null;
+
+    // 启动录制（首次）
+    if (!this.isRecording) {
+      this.recorder.start(100);
+      this.isRecording = true;
+    }
+
+    // 发送 PCM 到 worklet
+    this.workletNode.port.postMessage({ type: 'pcm', pcm: pcmData }, [pcmData]);
+
+    // 如果队列有缓冲数据，立即返回
+    if (this.dataQueue.length > 0) {
+      return this.dataQueue.shift()!;
+    }
+
+    // 否则等待下一个 ondataavailable
+    return new Promise<ArrayBuffer | null>((resolve) => {
+      const timeout = setTimeout(() => {
+        this.nextResolve = null;
+        resolve(null);
+      }, 200); // 超时 200ms
+
+      this.nextResolve = (data) => {
+        clearTimeout(timeout);
+        resolve(data);
+      };
+    });
+  }
+
+  /**
+   * 重置（新 speak 段开始时调用）
+   */
+  reset(): void {
+    if (this.recorder && this.recorder.state !== 'inactive') {
+      this.recorder.stop();
+    }
+    this.isRecording = false;
+    this.nextResolve = null;
+    this.dataQueue = []; // 清空队列缓冲
+  }
+
+  destroy(): void {
+    this.nextResolve = null;
+    this.dataQueue = []; // 清空队列缓冲
+    if (this.recorder) {
+      try {
+        if (this.recorder.state !== 'inactive') {
+          this.recorder.stop();
+        }
+      } catch {}
+    }
+    this.workletNode?.disconnect();
+    this.destination?.disconnect();
+    try {
+      this.audioCtx?.close();
+    } catch {}
+    this.recorder = null;
+    this.workletNode = null;
+    this.destination = null;
+    this.audioCtx = null;
+  }
+}

package/src/encoding/pcm-webm-encoder.ts

@@ -0,0 +1,72 @@
+import { WebCodecOpusEncoder } from './webcodec-opus-encoder';
+import { WebMMuxer } from './webm-muxer';
+
+export class PcmWebMEncoder {
+  private readonly opusEncoder = new WebCodecOpusEncoder({
+    sampleRate: 16000,
+    channels: 1,
+    bitrate: 24000,
+    frameSize: 1600,
+  });
+  private readonly muxer = new WebMMuxer(16000);
+  private queue: Promise<void> = Promise.resolve();
+  private generation = 0;
+  private destroyed = false;
+  private pendingBlocks = 0;
+
+  static isSupported() {
+    return WebCodecOpusEncoder.isSupported();
+  }
+
+  async init() {
+    await this.opusEncoder.init();
+  }
+
+  async encode(pcm: Int16Array) {
+    if (pcm.length !== 1600) {
+      throw new Error(`PCM 音频块必须为 1600 samples，实际为 ${pcm.length}`);
+    }
+    if (this.pendingBlocks >= 20) {
+      throw new Error("PCM/Opus 编码队列超过 2 秒");
+    }
+    this.pendingBlocks += 1;
+    const generation = this.generation;
+    const pcmCopy = new Int16Array(pcm);
+    const operation = this.queue.then(async () => {
+      if (this.destroyed || generation !== this.generation) {
+        return new ArrayBuffer(0);
+      }
+      const frames = await this.opusEncoder.encode(pcmCopy);
+      if (this.destroyed || generation !== this.generation) {
+        return new ArrayBuffer(0);
+      }
+      for (const frame of frames) {
+        this.muxer.feedOpusFrame(frame.data, frame.durationMs);
+      }
+      return this.muxer.flush();
+    });
+    const trackedOperation = operation.finally(() => {
+      this.pendingBlocks -= 1;
+    });
+    this.queue = trackedOperation.then(() => undefined, () => undefined);
+    return trackedOperation;
+  }
+
+  reset() {
+    const generation = ++this.generation;
+    this.queue = this.queue.then(() => {
+      if (!this.destroyed && generation === this.generation) {
+        this.opusEncoder.reset();
+        this.muxer.reset();
+      }
+    });
+  }
+
+  async destroy() {
+    this.destroyed = true;
+    this.generation += 1;
+    await this.opusEncoder.destroy();
+    await this.queue;
+    this.muxer.reset();
+  }
+}

package/src/encoding/webcodec-opus-encoder.ts

@@ -0,0 +1,258 @@
+/**
+ * WebCodecs AudioEncoder Opus 编码封装
+ * 使用浏览器原生 WebCodecs API 将 PCM 音频编码为 Opus 帧
+ */
+
+export interface OpusEncoderConfig {
+  sampleRate: number;      // 16000
+  channels: number;        // 1 (mono)
+  bitrate: number;         // 24000
+  frameSize: number;       // 1600 (100ms @ 16kHz)
+}
+
+export interface EncodedOpusFrame {
+  data: Uint8Array;
+  durationMs: number;
+}
+
+const OPUS_FRAME_DURATION_US = 20_000;
+const ENCODE_TIMEOUT_MS = 2_000;
+
+interface PendingEncode {
+  startTimestampUs: number;
+  endTimestampUs: number;
+  expectedFrameCount: number;
+  chunks: EncodedAudioChunk[];
+  resolve(frames: EncodedOpusFrame[]): void;
+  reject(error: Error): void;
+  timeoutId: number;
+}
+
+function buildAudioEncoderConfig(config: OpusEncoderConfig) {
+  return {
+    codec: 'opus',
+    sampleRate: config.sampleRate,
+    numberOfChannels: config.channels,
+    bitrate: config.bitrate,
+    // Safari may otherwise choose a 120 ms Opus packet for a 100 ms AudioData
+    // input. The AU protocol clock is fixed at 100 ms, so pin packetization to
+    // five 20 ms packets per input block.
+    opus: {
+      format: 'opus' as const,
+      frameDuration: OPUS_FRAME_DURATION_US,
+    },
+  };
+}
+
+export class WebCodecOpusEncoder {
+  private encoder: AudioEncoder | null = null;
+  private config: OpusEncoderConfig;
+  private isInitialized = false;
+  private pendingEncode: PendingEncode | null = null;
+  private timestamp: number = 0; // 单位：微秒
+
+  constructor(config: Partial<OpusEncoderConfig> = {}) {
+    this.config = {
+      sampleRate: 16000,
+      channels: 1,
+      bitrate: 24000,
+      frameSize: 1600,
+      ...config,
+    };
+  }
+
+  /**
+   * 检测当前浏览器是否支持 WebCodecs Opus 编码
+   */
+  static async isSupported(): Promise<boolean> {
+    if (typeof AudioEncoder === 'undefined') return false;
+    try {
+      const support = await AudioEncoder.isConfigSupported(buildAudioEncoderConfig({
+        sampleRate: 16000,
+        channels: 1,
+        bitrate: 24000,
+        frameSize: 1600,
+      }));
+      return support.supported === true
+        && support.config?.opus?.frameDuration === OPUS_FRAME_DURATION_US;
+    } catch {
+      return false;
+    }
+  }
+
+  async init(): Promise<void> {
+    if (this.isInitialized) return;
+
+    this.encoder = this.createEncoder();
+
+    this.isInitialized = true;
+  }
+
+  /**
+   * 异步编码 PCM 为 Opus 帧
+   * @param pcm S16LE PCM 数据 (Int16Array, 1600 samples = 100ms)
+   * @returns 保留 packet 边界和时长的 Opus 帧数组
+   */
+  async encode(pcm: Int16Array): Promise<EncodedOpusFrame[]> {
+    if (!this.isInitialized || !this.encoder) {
+      return Promise.reject(new Error('WebCodecOpusEncoder 未初始化'));
+    }
+    if (this.pendingEncode) {
+      return Promise.reject(new Error('WebCodecs Opus 编码器仍在处理上一音频块'));
+    }
+
+    // 构造 AudioData（S16 格式）
+    // 复制为独立 ArrayBuffer，避免 SharedArrayBuffer 类型不兼容
+    // 同时确保数据边界精确（Int16Array 可能是更大 buffer 的 view）
+    const pcmCopy = new Int16Array(pcm);
+
+    const startTimestampUs = this.timestamp;
+    const blockDurationUs = Math.round((pcm.length * 1_000_000) / this.config.sampleRate);
+    const endTimestampUs = startTimestampUs + blockDurationUs;
+    const expectedFrameCount = blockDurationUs / OPUS_FRAME_DURATION_US;
+    if (!Number.isInteger(expectedFrameCount)) {
+      return Promise.reject(new Error('PCM 音频块无法按 20ms Opus packet 完整分包'));
+    }
+
+    const audioData = new AudioData({
+      format: 's16',
+      sampleRate: this.config.sampleRate,
+      numberOfFrames: pcmCopy.length,
+      numberOfChannels: this.config.channels,
+      timestamp: startTimestampUs,
+      data: pcmCopy,
+    });
+
+    const result = new Promise<EncodedOpusFrame[]>((resolve, reject) => {
+      const timeoutId = globalThis.setTimeout(() => {
+        this.failPending(new Error('WebCodecs Opus 编码输出超过 2 秒'));
+      }, ENCODE_TIMEOUT_MS);
+      this.pendingEncode = {
+        startTimestampUs,
+        endTimestampUs,
+        expectedFrameCount,
+        chunks: [],
+        resolve,
+        reject,
+        timeoutId,
+      };
+    });
+
+    try {
+      this.encoder.encode(audioData);
+      this.timestamp = endTimestampUs;
+    } catch (cause) {
+      this.failPending(cause instanceof Error ? cause : new Error('WebCodecs Opus 编码失败'));
+    } finally {
+      audioData.close();
+    }
+
+    return result;
+  }
+
+  /**
+   * 重置时间戳（新 speak 段开始时调用）
+   */
+  reset(): void {
+    this.timestamp = 0;
+    this.failPending(new Error('WebCodecs Opus 编码已重置'));
+    if (this.encoder) {
+      try {
+        this.encoder.close();
+      } catch {}
+      this.encoder = this.createEncoder();
+    }
+  }
+
+  async destroy(): Promise<void> {
+    const encoder = this.encoder;
+    if (encoder) {
+      try {
+        await encoder.flush();
+      } catch {}
+      this.failPending(new Error('WebCodecs Opus 编码器已销毁'));
+      try {
+        encoder.close();
+      } catch {}
+    }
+    this.encoder = null;
+    this.isInitialized = false;
+  }
+
+  private createEncoder() {
+    let encoder: AudioEncoder;
+    encoder = new AudioEncoder({
+      output: (chunk) => {
+        if (this.encoder === encoder) {
+          this.handleOutput(chunk);
+        }
+      },
+      error: (cause) => {
+        if (this.encoder !== encoder) return;
+        const error = cause instanceof Error ? cause : new Error('WebCodecs Opus 编码失败');
+        const logger = (globalThis as typeof globalThis & {
+          avatarSDKLogger?: Pick<Console, 'error'>;
+        }).avatarSDKLogger || console;
+        logger.error('[WebCodecOpusEncoder] 编码错误:', error);
+        this.failPending(error);
+      },
+    });
+    encoder.configure(buildAudioEncoderConfig(this.config));
+    return encoder;
+  }
+
+  private handleOutput(chunk: EncodedAudioChunk) {
+    const pending = this.pendingEncode;
+    if (!pending) {
+      // flush() may emit one codec-delay/padding packet after all real PCM input.
+      return;
+    }
+    if (
+      chunk.timestamp < pending.startTimestampUs
+      || chunk.timestamp >= pending.endTimestampUs
+      || (chunk.duration !== null && chunk.duration !== OPUS_FRAME_DURATION_US)
+    ) {
+      this.failPending(new Error('WebCodecs Opus 未按 20ms 分包，无法生成固定 100ms AU 音频'));
+      return;
+    }
+
+    pending.chunks.push(chunk);
+    if (pending.chunks.length > pending.expectedFrameCount) {
+      this.failPending(new Error('WebCodecs Opus 未按 20ms 分包，无法生成固定 100ms AU 音频'));
+      return;
+    }
+    if (pending.chunks.length < pending.expectedFrameCount) {
+      return;
+    }
+
+    const frames = pending.chunks.map((encodedChunk, index) => {
+      const data = new Uint8Array(encodedChunk.byteLength);
+      encodedChunk.copyTo(data);
+      const nextTimestamp = pending.chunks[index + 1]?.timestamp ?? pending.endTimestampUs;
+      const durationUs = encodedChunk.duration && encodedChunk.duration > 0
+        ? encodedChunk.duration
+        : Math.max(1, nextTimestamp - encodedChunk.timestamp);
+      return { data, durationMs: durationUs / 1000 };
+    });
+    const totalDurationMs = frames.reduce((total, frame) => total + frame.durationMs, 0);
+    if (
+      frames.some((frame) => frame.durationMs !== OPUS_FRAME_DURATION_US / 1000)
+      || totalDurationMs !== (pending.endTimestampUs - pending.startTimestampUs) / 1000
+    ) {
+      this.failPending(new Error('WebCodecs Opus 未按 20ms 分包，无法生成固定 100ms AU 音频'));
+      return;
+    }
+
+    globalThis.clearTimeout(pending.timeoutId);
+    this.pendingEncode = null;
+    pending.resolve(frames);
+  }
+
+  private failPending(error: Error) {
+    const pending = this.pendingEncode;
+    if (!pending) return;
+    globalThis.clearTimeout(pending.timeoutId);
+    this.pendingEncode = null;
+    pending.reject(error);
+  }
+}