@xmov/avatar 2.0.0-alpha.38

package/src/encoding/webm-muxer.ts

@@ -0,0 +1,337 @@
+/**
+ * WebM EBML 封装器
+ * 将 Opus 帧封装为 WebM 容器格式，产出与 MSE 模式一致的 WebM/Opus 数据
+ *
+ * WebM 结构：
+ * EBML Header → Segment → SegmentInfo → Tracks → Cluster(s) → SimpleBlock(s)
+ */
+
+// EBML Element IDs
+const EBML_ID = {
+  EBML: 0x1A45DFA3,
+  EBMLVersion: 0x4286,
+  EBMLReadVersion: 0x42F7,
+  EBMLMaxIDLength: 0x42F2,
+  EBMLMaxSizeLength: 0x42F3,
+  DocType: 0x4282,
+  DocTypeVersion: 0x4287,
+  DocTypeReadVersion: 0x4285,
+
+  Segment: 0x18538067,
+  SegmentInfo: 0x1549A966,
+  TimecodeScale: 0x2AD7B1,
+  MuxingApp: 0x4D80,
+  WritingApp: 0x5741,
+
+  Tracks: 0x1654AE6B,
+  TrackEntry: 0xAE,
+  TrackNumber: 0xD7,
+  TrackUID: 0x73C5,
+  TrackType: 0x83,
+  FlagLacing: 0x9C,
+  Language: 0x22B59C,
+  CodecID: 0x86,
+  CodecPrivate: 0x63A2,
+  CodecDelay: 0x56AA,
+  SeekPreRoll: 0x56BB,
+  Audio: 0xE1,
+  SamplingFrequency: 0xB5,
+  Channels: 0x9F,
+
+  Cluster: 0x1F43B675,
+  Timecode: 0xE7,
+  SimpleBlock: 0xA3,
+};
+
+export class WebMMuxer {
+  private chunks: Uint8Array[] = [];
+  private hasHeader = false;
+  private clusterTimestamp: number = 0; // 单位：毫秒
+  private sampleRate: number;
+  private frameDuration: number = 100; // 100ms per frame
+
+  constructor(sampleRate: number = 16000) {
+    this.sampleRate = sampleRate;
+  }
+
+  /**
+   * 写入 WebM 文件头（EBML Header + Segment Info + Track）
+   */
+  writeHeader(): void {
+    if (this.hasHeader) return;
+
+    const header = this.buildEBMLHeader();
+    const segmentInfo = this.buildSegmentInfo();
+    const tracks = this.buildOpusTracks();
+
+    // EBML Header
+    this.chunks.push(header);
+    // Segment (open-ended, use unknown size)
+    this.chunks.push(new Uint8Array([0x18, 0x53, 0x80, 0x67, 0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF]));
+    // SegmentInfo
+    this.chunks.push(segmentInfo);
+    // Tracks
+    this.chunks.push(tracks);
+
+    this.hasHeader = true;
+    this.clusterTimestamp = 0;
+  }
+
+  /**
+   * 写入 Opus 帧
+   * @param opusFrame Opus 编码后的数据
+   * @param durationMs 帧时长 (默认 100ms)
+   */
+  feedOpusFrame(opusFrame: Uint8Array, durationMs: number = this.frameDuration): void {
+    if (!this.hasHeader) {
+      this.writeHeader();
+    }
+
+    // 每帧都创建一个新 Cluster（与 MediaRecorder 行为一致）
+    const cluster = this.buildCluster(Math.round(this.clusterTimestamp), opusFrame);
+    this.chunks.push(cluster);
+    this.clusterTimestamp += durationMs;
+  }
+
+  /**
+   * 获取新增的 WebM 数据（自上次 flush 以来的数据）
+   * 返回后清空已返回的数据，避免重复发送
+   */
+  flush(): ArrayBuffer {
+    if (this.chunks.length === 0) {
+      return new ArrayBuffer(0);
+    }
+    
+    const totalLength = this.chunks.reduce((sum, chunk) => sum + chunk.length, 0);
+    const result = new Uint8Array(totalLength);
+    let offset = 0;
+    for (const chunk of this.chunks) {
+      result.set(chunk, offset);
+      offset += chunk.length;
+    }
+    
+    // 关键：清空已返回的数据，避免下次 flush 重复发送
+    this.chunks = [];
+    
+    return result.buffer;
+  }
+
+  /**
+   * 重置（新 speak 段开始时调用）
+   */
+  reset(): void {
+    this.chunks = [];
+    this.hasHeader = false;
+    this.clusterTimestamp = 0;
+  }
+
+  // ── EBML 编码辅助方法 ──
+
+  /**
+   * 编码 VINT（Variable-length Integer）
+   */
+  private encodeVINT(value: number, width?: number): Uint8Array {
+    if (width !== undefined) {
+      const bytes = new Uint8Array(width);
+      for (let i = width - 1; i >= 0; i--) {
+        bytes[i] = value & 0xFF;
+        value >>>= 8;
+      }
+      bytes[0] |= (1 << (8 - width));
+      return bytes;
+    }
+
+    // 自动选择最小宽度
+    if (value < 0x7F) {
+      return new Uint8Array([0x80 | value]);
+    } else if (value < 0x3FFF) {
+      return new Uint8Array([0x40 | (value >> 8), value & 0xFF]);
+    } else if (value < 0x1FFFFF) {
+      return new Uint8Array([0x20 | (value >> 16), (value >> 8) & 0xFF, value & 0xFF]);
+    } else if (value < 0x0FFFFFFF) {
+      return new Uint8Array([0x10 | (value >> 24), (value >> 16) & 0xFF, (value >> 8) & 0xFF, value & 0xFF]);
+    }
+    throw new Error(`VINT value too large: ${value}`);
+  }
+
+  /**
+   * 编码 EBML Element ID
+   */
+  private encodeID(id: number): Uint8Array {
+    if (id <= 0xFF) return new Uint8Array([id]);
+    if (id <= 0xFFFF) return new Uint8Array([id >> 8, id & 0xFF]);
+    if (id <= 0xFFFFFF) return new Uint8Array([id >> 16, (id >> 8) & 0xFF, id & 0xFF]);
+    return new Uint8Array([id >>> 24, (id >> 16) & 0xFF, (id >> 8) & 0xFF, id & 0xFF]);
+  }
+
+  /**
+   * 编码无符号整数
+   */
+  private encodeUint(value: number): Uint8Array {
+    if (value === 0) return new Uint8Array([0]);
+    const bytes: number[] = [];
+    let v = value;
+    while (v > 0) {
+      bytes.unshift(v & 0xFF);
+      v = Math.floor(v / 256);
+    }
+    return new Uint8Array(bytes);
+  }
+
+  /**
+   * 编码字符串
+   */
+  private encodeString(str: string): Uint8Array {
+    return new TextEncoder().encode(str);
+  }
+
+  /**
+   * 编码浮点数（32-bit IEEE 754）
+   */
+  private encodeFloat32(value: number): Uint8Array {
+    const buf = new ArrayBuffer(4);
+    new DataView(buf).setFloat32(0, value, false); // big-endian
+    return new Uint8Array(buf);
+  }
+
+  /**
+   * 编码 EBML Element = ID + Size + Data
+   */
+  private encodeElement(id: number, data: Uint8Array): Uint8Array {
+    const idBytes = this.encodeID(id);
+    const sizeBytes = this.encodeVINT(data.length);
+    const result = new Uint8Array(idBytes.length + sizeBytes.length + data.length);
+    result.set(idBytes, 0);
+    result.set(sizeBytes, idBytes.length);
+    result.set(data, idBytes.length + sizeBytes.length);
+    return result;
+  }
+
+  /**
+   * 编码 unsigned integer Element
+   */
+  private encodeUintElement(id: number, value: number): Uint8Array {
+    return this.encodeElement(id, this.encodeUint(value));
+  }
+
+  /**
+   * 编码 string Element
+   */
+  private encodeStringElement(id: number, value: string): Uint8Array {
+    return this.encodeElement(id, this.encodeString(value));
+  }
+
+  /**
+   * 编码 float32 Element
+   */
+  private encodeFloat32Element(id: number, value: number): Uint8Array {
+    return this.encodeElement(id, this.encodeFloat32(value));
+  }
+
+  /**
+   * 合并多个 Uint8Array
+   */
+  private concatArrays(arrays: Uint8Array[]): Uint8Array {
+    const totalLength = arrays.reduce((sum, arr) => sum + arr.length, 0);
+    const result = new Uint8Array(totalLength);
+    let offset = 0;
+    for (const arr of arrays) {
+      result.set(arr, offset);
+      offset += arr.length;
+    }
+    return result;
+  }
+
+  /**
+   * 构建 EBML Header
+   */
+  private buildEBMLHeader(): Uint8Array {
+    const children = this.concatArrays([
+      this.encodeUintElement(EBML_ID.EBMLVersion, 1),
+      this.encodeUintElement(EBML_ID.EBMLReadVersion, 1),
+      this.encodeUintElement(EBML_ID.EBMLMaxIDLength, 4),
+      this.encodeUintElement(EBML_ID.EBMLMaxSizeLength, 8),
+      this.encodeStringElement(EBML_ID.DocType, 'webm'),
+      // Opus tracks use CodecDelay and SeekPreRoll, which require WebM v4.
+      this.encodeUintElement(EBML_ID.DocTypeVersion, 4),
+      this.encodeUintElement(EBML_ID.DocTypeReadVersion, 2),
+    ]);
+    return this.encodeElement(EBML_ID.EBML, children);
+  }
+
+  /**
+   * 构建 SegmentInfo
+   */
+  private buildSegmentInfo(): Uint8Array {
+    const children = this.concatArrays([
+      this.encodeUintElement(EBML_ID.TimecodeScale, 1000000), // 1ms precision
+      this.encodeStringElement(EBML_ID.MuxingApp, 'xmov-avatar-sdk'),
+      this.encodeStringElement(EBML_ID.WritingApp, 'xmov-avatar-sdk'),
+    ]);
+    return this.encodeElement(EBML_ID.SegmentInfo, children);
+  }
+
+  /**
+   * 构建 Opus Audio Track
+   */
+  private buildOpusTracks(): Uint8Array {
+    // Opus CodecPrivate data (OpusHead)
+    // Version(1) + Channels(1) + PreSkip(2) + InputSampleRate(4) + OutputGain(2) + ChannelMappingFamily(1)
+    const opusHead = new Uint8Array([
+      0x4F, 0x70, 0x75, 0x73, 0x48, 0x65, 0x61, 0x64, // "OpusHead"
+      0x01,       // Version
+      0x01,       // Channel count (mono)
+      0x00, 0x00, // Pre-skip (0)
+      0x80, 0x3E, 0x00, 0x00, // Input sample rate (16000, little-endian)
+      0x00, 0x00, // Output gain (0)
+      0x00,       // Channel mapping family (0)
+    ]);
+
+    const audioChildren = this.concatArrays([
+      this.encodeFloat32Element(EBML_ID.SamplingFrequency, this.sampleRate),
+      this.encodeUintElement(EBML_ID.Channels, 1),
+    ]);
+
+    const trackEntryChildren = this.concatArrays([
+      this.encodeUintElement(EBML_ID.TrackNumber, 1),
+      this.encodeUintElement(EBML_ID.TrackUID, 1),
+      this.encodeUintElement(EBML_ID.TrackType, 2), // Audio
+      this.encodeUintElement(EBML_ID.FlagLacing, 0),
+      this.encodeStringElement(EBML_ID.Language, 'und'),
+      this.encodeStringElement(EBML_ID.CodecID, 'A_OPUS'),
+      this.encodeElement(EBML_ID.CodecPrivate, opusHead),
+      this.encodeUintElement(EBML_ID.CodecDelay, 0),
+      this.encodeUintElement(EBML_ID.SeekPreRoll, 80000000), // 80ms in ns
+      this.encodeElement(EBML_ID.Audio, audioChildren),
+    ]);
+
+    const trackEntry = this.encodeElement(EBML_ID.TrackEntry, trackEntryChildren);
+    return this.encodeElement(EBML_ID.Tracks, trackEntry);
+  }
+
+  /**
+   * 构建 Cluster + SimpleBlock
+   */
+  private buildCluster(timecodeMs: number, opusData: Uint8Array): Uint8Array {
+    // Timecode element
+    const timecode = this.encodeUintElement(EBML_ID.Timecode, timecodeMs);
+
+    // SimpleBlock: TrackNumber(VINT) + Timecode(2 bytes, signed, relative to cluster) + Flags(1 byte) + Data
+    const trackNumber = this.encodeVINT(1); // Track 1
+    const relativeTimecode = new Uint8Array([0x00, 0x00]); // 0ms relative
+    // WebCodecs Opus EncodedAudioChunk values are always key chunks.
+    const flags = new Uint8Array([0x80]);
+
+    const simpleBlockData = this.concatArrays([
+      trackNumber,
+      relativeTimecode,
+      flags,
+      opusData,
+    ]);
+    const simpleBlock = this.encodeElement(EBML_ID.SimpleBlock, simpleBlockData);
+
+    // Cluster = Timecode + SimpleBlock
+    const clusterChildren = this.concatArrays([timecode, simpleBlock]);
+    return this.encodeElement(EBML_ID.Cluster, clusterChildren);
+  }
+}

package/src/global.d.ts

@@ -0,0 +1,105 @@
+// src/global.d.ts
+// 1. 声明全局 protobuf 对象（适配 protobufjs UMD 格式）
+declare namespace globalThis {
+  const protobuf: {
+    roots: {
+      default: {
+        MeshData: {
+          create: (props?: any) => any;
+          encode: (msg: any, writer?: any) => any;
+          decode: (buf: Uint8Array | any, length?: number) => any;
+          verify: (msg: any) => string | null;
+          fromObject: (obj: any) => any;
+          toObject: (inst: any, options?: any) => any;
+          // 其他需要的方法可按需补充（参考你的 proto 编译文件）
+        };
+        JointData: typeof MeshData;
+        FaceFrameData: typeof MeshData;
+        FaceFrameDataList: typeof MeshData;
+      };
+    };
+  };
+}
+
+// 2. 扩展 Window 接口，添加 proxyProtobuf 属性
+interface Window {
+  proxyProtobuf: typeof protobuf.roots.default;
+}
+
+declare const VideoDecoder: any;
+
+// 或者，提供一个更具体的、虽然不完整的声明
+// 这可以提供一定的类型提示，比 'any' 要好
+declare class VideoDecoder {
+  constructor(options: {
+    output: (frame: any) => void;
+    error: (error: Error) => void;
+  });
+  configure(config: any): void;
+  decode(chunk: any): void;
+  close(): void;
+  static isConfigSupported(config: any): Promise<{ supported: boolean }>;
+}
+
+// WebCodecs Audio API 类型声明（用于 PCM 模式音频捕获）
+declare class AudioEncoder {
+  constructor(options: {
+    output: (chunk: EncodedAudioChunk, metadata?: any) => void;
+    error: (error: DOMException) => void;
+  });
+  configure(config: {
+    codec: string;
+    sampleRate: number;
+    numberOfChannels: number;
+    bitrate?: number;
+    opus?: {
+      format?: "opus" | "ogg";
+      frameDuration?: number;
+    };
+  }): void;
+  encode(data: AudioData): void;
+  flush(): Promise<void>;
+  close(): void;
+  static isConfigSupported(config: {
+    codec: string;
+    sampleRate: number;
+    numberOfChannels: number;
+    bitrate?: number;
+    opus?: {
+      format?: "opus" | "ogg";
+      frameDuration?: number;
+    };
+  }): Promise<{
+    supported: boolean;
+    config?: {
+      codec: string;
+      sampleRate: number;
+      numberOfChannels: number;
+      bitrate?: number;
+      opus?: {
+        format?: "opus" | "ogg";
+        frameDuration?: number;
+      };
+    };
+  }>;
+}
+
+declare class AudioData {
+  constructor(init: {
+    format: string;
+    sampleRate: number;
+    numberOfFrames: number;
+    numberOfChannels: number;
+    timestamp: number;
+    data: BufferSource;
+  });
+  close(): void;
+}
+
+declare class EncodedAudioChunk {
+  readonly type: string;
+  readonly timestamp: number;
+  readonly duration: number | null;
+  readonly byteLength: number;
+  copyTo(destination: BufferSource): void;
+}