@xmov/avatar 2.0.0-alpha.38

package/src/control/SaveAndDownload.js

@@ -0,0 +1,77 @@
+
+class SaveAndDownload {
+  constructor(fileName = "avatarData.json", enabled = false) {
+    this.fileName = fileName;
+    this.enabled = enabled; // 功能开关
+    this.data = {}; // 存储JSON数据
+    this.downloadButton = null;
+    if (enabled) {
+      this.initDownloadButton();
+    }
+  }
+
+  // 初始化下载按钮
+  initDownloadButton() {
+    if (!this.enabled) return;
+    
+    this.downloadButton = document.createElement("a");
+    this.downloadButton.style.position = "absolute";
+    this.downloadButton.style.left = "0";
+    this.downloadButton.style.bottom = "30px";
+    this.downloadButton.style.background = "#4CAF50";
+    this.downloadButton.style.color = "white";
+    this.downloadButton.style.fontSize = "12px";
+    this.downloadButton.style.padding = "10px";
+    this.downloadButton.style.borderRadius = "4px";
+    this.downloadButton.style.textDecoration = "none";
+    this.downloadButton.style.zIndex = "1000";
+    this.downloadButton.style.cursor = "pointer";
+    this.downloadButton.innerHTML = "下载数据提交开发回放";
+    this.downloadButton.addEventListener("click", () => {
+      this.download();
+    });
+    document.body.appendChild(this.downloadButton);
+  }
+
+  // 写入多个字段
+  writeFields(fields) {
+    if (!this.enabled) return;
+    Object.assign(this.data, fields);
+  }
+
+  // 追加多个项目到数组字段
+  appendMultipleToArray(fieldName, items) {
+    if (!this.enabled) return;
+    if (!this.data[fieldName]) {
+      this.data[fieldName] = [];
+    }
+    this.data[fieldName].push(...items);
+  }
+
+  // 下载文件
+  download() {
+    if (!this.enabled) return;
+    const jsonContent = this.generateJSONContent();
+    this.downloadFile(jsonContent, this.fileName, "application/json");
+  }
+
+  // 生成JSON文件内容
+  generateJSONContent() {
+    return JSON.stringify(this.data, null, 2);
+  }
+
+  // 下载文件的具体实现
+  downloadFile(content, fileName, contentType) {
+    if (!this.enabled) return;
+    const blob = new Blob([content], { type: contentType });
+    const url = URL.createObjectURL(blob);
+    
+    this.downloadButton.href = url;
+    this.downloadButton.download = fileName;
+    this.downloadButton.scrollIntoView({ behavior: "smooth", block: "center" });
+    
+    this.downloadButton.dataset.objectUrl = url;
+  }
+}
+
+export default SaveAndDownload;

package/src/control/StreamingClient.ts

@@ -0,0 +1,138 @@
+/**
+ * StreamingClient - 薄客户端模式下的 WebSocket 连接管理器
+ * 负责与 Streaming Server 建立连接，管理消息路由、心跳和重连。
+ */
+
+export interface StreamingClientOptions {
+  url: string;
+  onReady?: () => void;
+  onOffer?: (sdp: string) => void;
+  onIce?: (candidate: RTCIceCandidateInit) => void;
+  onEventProxy?: (eventType: string, data: any, serverFrame?: number) => void;
+  onError?: (error: string) => void;
+  onDisconnect?: () => void;
+}
+
+export class StreamingClient {
+  private ws: WebSocket | null = null;
+  private options: StreamingClientOptions;
+  private retryCount = 0;
+  private maxRetry = 27; // 3轮 × 9次
+  private retryTimer: number = -1;
+  private destroyed = false;
+  private connected = false;
+
+  constructor(options: StreamingClientOptions) {
+    this.options = options;
+  }
+
+  connect() {
+    if (this.destroyed) return;
+
+    try {
+      this.ws = new WebSocket(this.options.url);
+    } catch (e) {
+      console.error('[StreamingClient] 创建 WebSocket 失败:', e);
+      this.scheduleRetry();
+      return;
+    }
+
+    this.ws.onopen = () => {
+      this.connected = true;
+      this.retryCount = 0;
+      console.log('[StreamingClient] 已连接');
+      // 以 client 角色加入
+      this.send({ type: 'join', role: 'client' });
+    };
+
+    this.ws.onmessage = (e) => {
+      let msg: any;
+      try {
+        msg = JSON.parse(e.data);
+      } catch {
+        return;
+      }
+      this.handleMessage(msg);
+    };
+
+    this.ws.onclose = () => {
+      this.connected = false;
+      if (!this.destroyed) {
+        this.options.onDisconnect?.();
+        this.scheduleRetry();
+      }
+    };
+
+    this.ws.onerror = () => {
+      this.options.onError?.('WebSocket 连接错误');
+    };
+  }
+
+  private handleMessage(msg: any) {
+    switch (msg.type) {
+      case 'ready':
+        this.options.onReady?.();
+        break;
+      case 'offer':
+        this.options.onOffer?.(msg.sdp);
+        break;
+      case 'answer':
+        // Client 不发送 offer，不会收到 answer
+        break;
+      case 'ice':
+        this.options.onIce?.(msg.candidate);
+        break;
+      case 'event_proxy':
+        this.options.onEventProxy?.(msg.event_type, msg.data, msg.server_frame);
+        break;
+      case 'error':
+        this.options.onError?.(msg.message || '未知错误');
+        break;
+    }
+  }
+
+  send(msg: object) {
+    if (this.ws && this.ws.readyState === WebSocket.OPEN) {
+      this.ws.send(JSON.stringify(msg));
+    }
+  }
+
+  /**
+   * 发送 API 调用
+   */
+  sendApiCall(method: string, params: any[] = []) {
+    this.send({ type: 'api_call', method, params });
+  }
+
+  private scheduleRetry() {
+    if (this.destroyed) return;
+    if (this.retryCount >= this.maxRetry) {
+      console.warn('[StreamingClient] 重试次数已达上限');
+      this.options.onError?.('连接重试次数已达上限');
+      return;
+    }
+
+    const delay = this.retryCount === 0 ? 1000 : Math.pow(2, Math.min(this.retryCount, 6)) * 1000;
+    this.retryCount++;
+    this.retryTimer = window.setTimeout(() => {
+      this.connect();
+    }, delay);
+  }
+
+  isConnected() {
+    return this.connected;
+  }
+
+  destroy() {
+    this.destroyed = true;
+    if (this.retryTimer !== -1) {
+      clearTimeout(this.retryTimer);
+      this.retryTimer = -1;
+    }
+    if (this.ws) {
+      this.ws.close();
+      this.ws = null;
+    }
+    this.connected = false;
+  }
+}

package/src/control/VideoPlayer.ts

@@ -0,0 +1,306 @@
+/**
+ * VideoPlayer - 薄客户端模式下的 WebRTC 收流与播放管理
+ * 支持两种模式：
+ * - 普通模式：直接 <video> 播放
+ * - Alpha Mask 模式：canvas 渲染 + 双路流 alpha 合成（透明背景）
+ */
+
+import { StreamingClient } from './StreamingClient';
+
+export interface VideoPlayerOptions {
+  container: HTMLElement;
+  streamingClient: StreamingClient;
+  useAlphaMask?: boolean; // 是否启用 alpha mask 透明背景
+  onConnected?: () => void;
+  onDisconnected?: () => void;
+}
+
+export class VideoPlayer {
+  private video: HTMLVideoElement | null = null;
+  private canvas: HTMLCanvasElement | null = null;
+  private canvasCtx: CanvasRenderingContext2D | null = null;
+  private pc: RTCPeerConnection | null = null;
+  private stream: MediaStream | null = null;
+  private options: VideoPlayerOptions;
+  private destroyed = false;
+  private animFrameId = 0;
+  // alpha 处理用的临时 canvas
+  private tempCanvas: HTMLCanvasElement | null = null;
+  private tempCtx: CanvasRenderingContext2D | null = null;
+
+  constructor(options: VideoPlayerOptions) {
+    this.options = options;
+    if (options.useAlphaMask) {
+      this.createCanvasElement();
+    } else {
+      this.createVideoElement();
+    }
+  }
+
+  private createVideoElement() {
+    const video = document.createElement('video');
+    video.autoplay = true;
+    video.playsInline = true;
+    video.muted = true; // 先静音，等音频轨到达后取消
+    video.style.width = '100%';
+    video.style.height = '100%';
+    video.style.objectFit = 'contain';
+    video.style.background = 'transparent';
+    this.video = video;
+    this.options.container.appendChild(video);
+  }
+
+  private createCanvasElement() {
+    const canvas = document.createElement('canvas');
+    canvas.style.width = '100%';
+    canvas.style.height = '100%';
+    canvas.style.objectFit = 'contain';
+    canvas.style.background = 'transparent';
+    this.canvas = canvas;
+    this.canvasCtx = canvas.getContext('2d', { willReadFrequently: true });
+
+    // 用于 alpha 合成的临时 canvas
+    this.tempCanvas = document.createElement('canvas');
+    this.tempCtx = this.tempCanvas.getContext('2d', { willReadFrequently: true });
+
+    this.options.container.appendChild(canvas);
+
+    // 创建隐藏的 video 元素用于接收 WebRTC 流
+    // 注意：不能用 display:none，某些浏览器不解码。用绝对定位 + 1px 尺寸
+    const video = document.createElement('video');
+    video.autoplay = true;
+    video.playsInline = true;
+    video.muted = true;
+    video.style.position = 'absolute';
+    video.style.width = '1px';
+    video.style.height = '1px';
+    video.style.opacity = '0';
+    video.style.pointerEvents = 'none';
+    video.style.zIndex = '-1';
+    this.video = video;
+    this.options.container.appendChild(video);
+  }
+
+  /**
+   * 处理远端 SDP offer，创建 answer 并通过 StreamingClient 回传
+   */
+  async handleOffer(sdp: string) {
+    this.destroyPeer();
+
+    this.pc = new RTCPeerConnection({
+      iceServers: [
+        { urls: 'stun:stun.l.google.com:19302' },
+        { urls: 'stun:stun1.l.google.com:19302' },
+        // Open Relay 免费 TURN 服务器（跨网络 NAT 穿透）
+        { urls: 'turn:openrelay.metered.ca:80', username: 'openrelayproject', credential: 'openrelayproject' },
+        { urls: 'turn:openrelay.metered.ca:443', username: 'openrelayproject', credential: 'openrelayproject' },
+        { urls: 'turns:openrelay.metered.ca:443', username: 'openrelayproject', credential: 'openrelayproject' },
+      ]
+    });
+
+    this.pc.onicecandidate = (e) => {
+      if (e.candidate) {
+        this.options.streamingClient.send({
+          type: 'ice',
+          candidate: e.candidate.toJSON(),
+        });
+      }
+    };
+
+    this.pc.ontrack = (e) => {
+      console.log('[VideoPlayer] 收到轨道:', e.track.kind, e.track.id);
+      if (!this.stream) {
+        this.stream = new MediaStream();
+        if (this.video) {
+          this.video.srcObject = this.stream;
+        }
+      }
+      this.stream.addTrack(e.track);
+
+      // 音频轨到达后取消静音并尝试播放
+      if (e.track.kind === 'audio' && this.video) {
+        this.video.muted = false;
+        this.video.play().catch(() => {});
+      }
+
+      // 视频轨到达后，如果启用 alpha mask，开始渲染循环
+      if (e.track.kind === 'video' && this.options.useAlphaMask) {
+        console.log('[VideoPlayer] Alpha Mask 模式：启动 canvas 渲染循环');
+        this.startAlphaRendering();
+      }
+    };
+
+    this.pc.onconnectionstatechange = () => {
+      if (!this.pc) return;
+      console.log('[VideoPlayer] WebRTC 状态:', this.pc.connectionState);
+      if (this.pc.connectionState === 'connected') {
+        this.options.onConnected?.();
+      } else if (this.pc.connectionState === 'disconnected' || this.pc.connectionState === 'failed') {
+        this.options.onDisconnected?.();
+      }
+    };
+
+    await this.pc.setRemoteDescription(new RTCSessionDescription({ type: 'offer', sdp }));
+    const answer = await this.pc.createAnswer();
+    await this.pc.setLocalDescription(answer);
+
+    this.options.streamingClient.send({ type: 'answer', sdp: answer.sdp });
+  }
+
+  /**
+   * Alpha Mask 渲染循环
+   * 将双路流（RGB + Alpha）合成为带透明度的画面
+   */
+  private startAlphaRendering() {
+    if (!this.video || !this.canvas || !this.canvasCtx || !this.tempCanvas || !this.tempCtx) return;
+
+    const video = this.video;
+    const canvas = this.canvas;
+    const ctx = this.canvasCtx;
+    const tempCanvas = this.tempCanvas;
+    const tempCtx = this.tempCtx;
+
+    let lastWidth = 0;
+    let lastHeight = 0;
+    let frameCount = 0;
+    let debugInterval: ReturnType<typeof setInterval> | null = null;
+
+    // 启动调试日志：每 2 秒打印一次状态
+    debugInterval = setInterval(() => {
+      if (this.destroyed) {
+        if (debugInterval) clearInterval(debugInterval);
+        return;
+      }
+      console.log(`[VideoPlayer] Alpha 渲染调试: frames=${frameCount}, videoReadyState=${video.readyState}, videoSize=${video.videoWidth}x${video.videoHeight}, currentTime=${video.currentTime.toFixed(2)}s, paused=${video.paused}`);
+    }, 2000);
+
+    const render = () => {
+      if (this.destroyed) {
+        if (debugInterval) clearInterval(debugInterval);
+        return;
+      }
+
+      // 视频数据就绪
+      if (video.readyState >= 2 && video.videoWidth > 0) {
+        const vw = video.videoWidth;
+        const vh = video.videoHeight;
+
+        // 双路流：宽度是原始 2x，左半 RGB，右半 Alpha
+        const realW = vw / 2;
+        const realH = vh;
+
+        // 尺寸变化时更新 canvas 大小
+        if (vw !== lastWidth || vh !== lastHeight) {
+          canvas.width = realW;
+          canvas.height = realH;
+          tempCanvas.width = vw;  // 需要容纳完整视频帧（RGB + Alpha）
+          tempCanvas.height = realH;
+          lastWidth = vw;
+          lastHeight = vh;
+          console.log(`[VideoPlayer] Alpha Mask: 检测到双流 ${vw}x${vh} → 原始 ${realW}x${realH}`);
+        }
+
+        // 快照：先将整个视频帧绘制到临时 canvas，避免两次 drawImage(video) 之间视频切帧
+        tempCtx.clearRect(0, 0, realW * 2, realH);
+        tempCtx.drawImage(video, 0, 0);
+
+        // 1. 从快照绘制左半 RGB 到输出 canvas
+        ctx.clearRect(0, 0, realW, realH);
+        ctx.drawImage(tempCanvas, 0, 0, realW, realH, 0, 0, realW, realH);
+
+        // 2. 从同一快照读取右半 Alpha 灰度图
+        // 3. 读取像素数据并合成 alpha
+        try {
+          const rgbData = ctx.getImageData(0, 0, realW, realH);
+          const alphaData = tempCtx.getImageData(realW, 0, realW, realH);
+          const rgb = rgbData.data;
+          const alpha = alphaData.data;
+
+          // 首帧调试：检查像素数据
+          if (frameCount === 0) {
+            let nonZeroRGB = 0;
+            let nonZeroAlpha = 0;
+            const sampleSize = Math.min(1000, rgb.length / 4);
+            for (let i = 0; i < sampleSize * 4; i += 4) {
+              if (rgb[i] > 0 || rgb[i+1] > 0 || rgb[i+2] > 0) nonZeroRGB++;
+              if (alpha[i] > 0) nonZeroAlpha++;
+            }
+            console.log(`[VideoPlayer] 首帧像素检查: RGB非零=${nonZeroRGB}/${sampleSize}, Alpha非零=${nonZeroAlpha}/${sampleSize}`);
+            if (nonZeroRGB === 0) {
+              console.warn(`[VideoPlayer] 警告: RGB 全黑！视频可能没有内容`);
+            }
+          }
+
+          // 将 alpha 灰度（R 通道）写入 RGB 的 alpha 通道
+          for (let i = 0; i < rgb.length; i += 4) {
+            rgb[i + 3] = alpha[i]; // alpha[i] = R 通道 = 灰度值 = 原始 alpha
+          }
+
+          ctx.putImageData(rgbData, 0, 0);
+          frameCount++;
+        } catch (e) {
+          // getImageData/putImageData 可能因 tainted canvas 失败
+          // 降级为直接绘制（无透明背景）
+          console.warn(`[VideoPlayer] Alpha 合成失败，降级为直接绘制:`, e);
+          ctx.drawImage(video, 0, 0, realW, realH, 0, 0, realW, realH);
+          frameCount++;
+        }
+      }
+
+      this.animFrameId = requestAnimationFrame(render);
+    };
+
+    this.animFrameId = requestAnimationFrame(render);
+  }
+
+  /**
+   * 添加 ICE candidate
+   */
+  async addIceCandidate(candidate: RTCIceCandidateInit) {
+    if (this.pc) {
+      await this.pc.addIceCandidate(candidate).catch((e) => {
+        console.warn('[VideoPlayer] addIceCandidate 失败:', e);
+      });
+    }
+  }
+
+  getVideoElement(): HTMLVideoElement | null {
+    return this.video;
+  }
+
+  getCanvasElement(): HTMLCanvasElement | null {
+    return this.canvas;
+  }
+
+  isConnected(): boolean {
+    return this.pc?.connectionState === 'connected';
+  }
+
+  private destroyPeer() {
+    if (this.pc) {
+      this.pc.close();
+      this.pc = null;
+    }
+    this.stream = null;
+    if (this.animFrameId) {
+      cancelAnimationFrame(this.animFrameId);
+      this.animFrameId = 0;
+    }
+  }
+
+  destroy() {
+    this.destroyed = true;
+    this.destroyPeer();
+    if (this.video && this.video.parentNode) {
+      this.video.parentNode.removeChild(this.video);
+    }
+    if (this.canvas && this.canvas.parentNode) {
+      this.canvas.parentNode.removeChild(this.canvas);
+    }
+    this.video = null;
+    this.canvas = null;
+    this.canvasCtx = null;
+    this.tempCanvas = null;
+    this.tempCtx = null;
+  }
+}