agora-electron-sdk 4.5.2 → 4.5.3-dev.2

package/ts/Renderer/AgoraView.ts

@@ -12,6 +12,7 @@ const CHANNEL_ID_STRING = 'channel-id';
 const POSITION_STRING = 'position';
 const RENDERER_CONTENT_MODE_STRING = 'renderer-content-mode';
 const RENDERER_MIRROR_STRING = 'renderer-mirror';
+const ENABLE_ALPHA_MASK_STRING = 'enable-alpha-mask';
 
 const observedAttributes = [
   VIDEO_SOURCE_TYPE_STRING,
@@ -20,6 +21,7 @@ const observedAttributes = [
   POSITION_STRING,
   RENDERER_CONTENT_MODE_STRING,
   RENDERER_MIRROR_STRING,
+  ENABLE_ALPHA_MASK_STRING,
 ];
 
 declare global {
@@ -61,6 +63,10 @@ declare global {
      * The inline style of elements. See style .
      */
     'style': any;
+    /**
+     * Whether to enable alpha mask rendering: true : Enable alpha mask rendering. false : (Default) Disable alpha mask rendering. Alpha mask rendering can create images with transparent effects and extract portraits from videos. When used in combination with other methods, you can implement effects such as portrait-in-picture and watermarking.
+     */
+    'enable-alpha-mask': boolean;
   }
   namespace JSX {
     /**
@@ -163,13 +169,32 @@ export default class AgoraView extends HTMLElement {
     }
   }
 
+  get enableAlphaMask(): boolean {
+    return this.getAttribute(ENABLE_ALPHA_MASK_STRING) === 'true';
+  }
+
+  set enableAlphaMask(val) {
+    if (val) {
+      this.setAttribute(ENABLE_ALPHA_MASK_STRING, String(val));
+    } else {
+      this.removeAttribute(ENABLE_ALPHA_MASK_STRING);
+    }
+  }
+
   constructor() {
     super();
   }
 
   initializeRender = () => {
-    const { channelId, uid, sourceType, position, renderMode, renderMirror } =
-      this;
+    const {
+      channelId,
+      uid,
+      sourceType,
+      position,
+      renderMode,
+      renderMirror,
+      enableAlphaMask,
+    } = this;
     AgoraEnv.AgoraRendererManager?.addOrRemoveRenderer({
       sourceType,
       view: this,
@@ -181,17 +206,19 @@ export default class AgoraView extends HTMLElement {
         ? VideoMirrorModeType.VideoMirrorModeEnabled
         : VideoMirrorModeType.VideoMirrorModeDisabled,
       setupMode: VideoViewSetupMode.VideoViewSetupReplace,
+      enableAlphaMask,
     });
   };
 
   destroyRender = () => {
-    const { channelId, uid, sourceType, position } = this;
+    const { channelId, uid, sourceType, position, enableAlphaMask } = this;
     AgoraEnv.AgoraRendererManager?.removeRendererFromCache({
       channelId,
       position,
       uid,
       sourceType,
       view: this,
+      enableAlphaMask,
     });
   };
 
@@ -207,6 +234,7 @@ export default class AgoraView extends HTMLElement {
     const isSetRenderOption = [
       RENDERER_CONTENT_MODE_STRING,
       RENDERER_MIRROR_STRING,
+      ENABLE_ALPHA_MASK_STRING,
     ].includes(attrName);
 
     if (isSetRenderOption) {
@@ -216,6 +244,7 @@ export default class AgoraView extends HTMLElement {
         mirrorMode: this.renderMirror
           ? VideoMirrorModeType.VideoMirrorModeEnabled
           : VideoMirrorModeType.VideoMirrorModeDisabled,
+        enableAlphaMask: this.enableAlphaMask,
       });
       return;
     }

package/ts/Renderer/IRenderer.ts

@@ -11,8 +11,8 @@ export abstract class IRenderer {
   private _frameCount = 0;
   private _startTime: number | null = null;
 
-  public bind(element: HTMLElement) {
-    this.parentElement = element;
+  public bind(context: RendererContext) {
+    this.parentElement = context.view;
     this.container = document.createElement('div');
     Object.assign(this.container.style, {
       width: '100%',
@@ -22,7 +22,7 @@ export abstract class IRenderer {
       alignItems: 'center',
       overflow: 'hidden',
     });
-    this.parentElement.appendChild(this.container);
+    this.parentElement?.appendChild(this.container);
     this.canvas = document.createElement('canvas');
     this.canvas.style.display = 'none';
     this.container.appendChild(this.canvas);
@@ -45,6 +45,7 @@ export abstract class IRenderer {
   }
 
   public drawFrame(
+    uid: number,
     _videoFrame?: VideoFrame,
     _codecConfig?: CodecConfigInfo
   ): void {
@@ -64,6 +65,10 @@ export abstract class IRenderer {
       this.context.mirrorMode = context.mirrorMode;
       this.updateMirrorMode();
     }
+
+    if (this.context.enableAlphaMask !== context.enableAlphaMask) {
+      this.context.enableAlphaMask = context.enableAlphaMask;
+    }
     this.context = { ...this.context, ...context };
   }
 

package/ts/Renderer/IRendererCache.ts

@@ -1,3 +1,5 @@
+import { RtcConnection, VideoSourceType } from 'agora-electron-sdk';
+
 import { RendererCacheContext, RendererContext } from '../Types';
 
 import { IRenderer } from './IRenderer';
@@ -24,6 +26,16 @@ export function isUseConnection(context: RendererCacheContext): boolean {
 export abstract class IRendererCache {
   renderers: IRenderer[];
   cacheContext: RendererCacheContext;
+  callbackContext: { connection: RtcConnection; sourceType: VideoSourceType };
+
+  // 性能统计数据
+  public actualFps: number = 0;
+  public avgFrameTime: number = 0;
+  public maxFrameTime: number = 0;
+  public minFrameTime: number = 0;
+  public avgFrameInterval: number = 0;
+  public maxFrameInterval: number = 0;
+  public minFrameInterval: number = 0;
 
   constructor({
     channelId,
@@ -33,6 +45,7 @@ export abstract class IRendererCache {
     sourceType,
     localUid,
     position,
+    enableAlphaMask,
   }: RendererContext) {
     this.renderers = [];
     this.cacheContext = {
@@ -43,9 +56,21 @@ export abstract class IRendererCache {
       sourceType,
       localUid,
       position,
+      enableAlphaMask,
+    };
+    this.callbackContext = {
+      connection: { channelId, localUid },
+      sourceType: sourceType!,
     };
   }
 
+  public setCallbackContext(
+    connection: RtcConnection,
+    sourceType: VideoSourceType
+  ): void {
+    this.callbackContext = { connection, sourceType };
+  }
+
   public get key(): string {
     return generateRendererCacheKey(this.cacheContext);
   }
@@ -90,6 +115,14 @@ export abstract class IRendererCache {
     }
   }
 
+  public abstract fetchVideoFrame(): {
+    hasMoreFrame: boolean;
+    needRender: boolean;
+  };
+  public abstract renderFrame(): void;
+  public abstract startRendering(): void;
+  public abstract stopRendering(): void;
+
   public release(): void {
     this.removeRenderer();
   }

package/ts/Renderer/RendererCache.ts

@@ -11,19 +11,32 @@ export class RendererCache extends IRendererCache {
   private videoFrame: VideoFrame;
   private _enabled: boolean;
 
+  // 渲染相关属性
+  private _renderingFps: number = 0;
+  private _renderingTimer?: number;
+  private _isRendering: boolean = false;
+  private _lastRenderTime: number = 0;
+
+  // 性能监控相关属性
+  private _frameTimes: number[] = []; // 存储最近100帧的渲染时间
+  private _frameIntervals: number[] = []; // 存储最近100帧的帧间隔
+  private _lastFrameTimestamp: number = 0; // 上一帧的时间戳
+  private _statsInterval: number = 100; // 每100帧输出一次统计
+
   constructor(context: RendererContext) {
     super(context);
     this.videoFrame = {
       yBuffer: Buffer.alloc(0),
       uBuffer: Buffer.alloc(0),
       vBuffer: Buffer.alloc(0),
-      alphaBuffer: AgoraEnv.encodeAlpha ? Buffer.alloc(0) : undefined,
+      alphaBuffer: Buffer.alloc(0),
       width: 0,
       height: 0,
       yStride: 0,
       uStride: 0,
       vStride: 0,
       rotation: 0,
+      colorSpace: undefined,
     };
     this._enabled = false;
   }
@@ -39,6 +52,10 @@ export class RendererCache extends IRendererCache {
     if (this._enabled) return;
     AgoraElectronBridge.EnableVideoFrameCache(this.cacheContext);
     this._enabled = true;
+    if (AgoraEnv.AgoraRendererManager) {
+      this._renderingFps = AgoraEnv.AgoraRendererManager.renderingFps;
+    }
+    this.startRendering();
   }
 
   private disable() {
@@ -55,25 +72,30 @@ export class RendererCache extends IRendererCache {
     }
   }
 
-  override draw() {
-    let { ret, isNewFrame } = AgoraElectronBridge.GetVideoFrame(
+  override draw(): void {}
+
+  public fetchVideoFrame(): { hasMoreFrame: boolean; needRender: boolean } {
+    const renderAlpha = this.cacheContext.enableAlphaMask ?? false;
+    let needRender = false;
+    let { ret, hasMoreFrame } = AgoraElectronBridge.GetVideoFrame(
       this.cacheContext,
       this.videoFrame,
       {
-        encodeAlpha: AgoraEnv.encodeAlpha,
+        renderAlpha: renderAlpha,
       }
     );
 
     switch (ret) {
       case 0: // GET_VIDEO_FRAME_CACHE_RETURN_TYPE::OK = 0
         //
+        needRender = true;
         break;
       case 1: // GET_VIDEO_FRAME_CACHE_RETURN_TYPE::RESIZED = 1
         const { yStride, uStride, vStride, height } = this.videoFrame;
         this.videoFrame.yBuffer = Buffer.alloc(yStride! * height!);
         this.videoFrame.uBuffer = Buffer.alloc(uStride! * height!);
         this.videoFrame.vBuffer = Buffer.alloc(vStride! * height!);
-        if (AgoraEnv.encodeAlpha) {
+        if (renderAlpha) {
           this.videoFrame.alphaBuffer = Buffer.alloc(
             this.videoFrame.width! * this.videoFrame.height!
           );
@@ -83,26 +105,43 @@ export class RendererCache extends IRendererCache {
           this.cacheContext,
           this.videoFrame,
           {
-            encodeAlpha: AgoraEnv.encodeAlpha,
+            renderAlpha: renderAlpha,
           }
         );
         ret = result.ret;
-        isNewFrame = result.isNewFrame;
+        hasMoreFrame = result.hasMoreFrame;
+        needRender = false;
         break;
       case 2: // GET_VIDEO_FRAME_CACHE_RETURN_TYPE::NO_CACHE = 2
         logDebug('No renderer cache, please enable cache first');
-        return;
+        return { hasMoreFrame: false, needRender: false };
     }
 
-    if (!AgoraEnv.encodeAlpha) {
-      this.videoFrame.alphaBuffer = undefined;
+    if (!renderAlpha) {
+      if (
+        this.videoFrame.alphaBuffer &&
+        this.videoFrame.alphaBuffer.length > 0
+      ) {
+        this.videoFrame.alphaBuffer = Buffer.alloc(0);
+      }
+    } else {
+      if (
+        !this.videoFrame.alphaBuffer ||
+        this.videoFrame.alphaBuffer.length === 0
+      ) {
+        this.videoFrame.alphaBuffer = Buffer.alloc(
+          this.videoFrame.width! * this.videoFrame.height!
+        );
+      }
     }
 
-    if (isNewFrame) {
+    if (hasMoreFrame) {
       this.renderers.forEach((renderer) => {
-        renderer.drawFrame(this.videoFrame);
+        renderer.drawFrame(this.cacheContext.uid!, this.videoFrame);
       });
     }
+
+    return { hasMoreFrame, needRender };
   }
 
   override addRenderer(renderer: IRenderer): void {
@@ -110,6 +149,155 @@ export class RendererCache extends IRendererCache {
     this.shouldEnable();
   }
 
+  /**
+   * 渲染当前视频帧
+   * 此方法假设已经通过fetchVideoFrame获取了新帧
+   */
+  public renderFrame() {
+    if (this.renderers.length === 0) return;
+
+    // 记录当前时间戳，用于计算帧间隔
+    const now = performance.now();
+
+    // 计算帧间隔并记录
+    if (this._lastFrameTimestamp > 0) {
+      const interval = now - this._lastFrameTimestamp;
+      this._frameIntervals.push(interval);
+    }
+    this._lastFrameTimestamp = now;
+
+    // 使用高精度计数器测量渲染时间
+    const renderStartTime = performance.now();
+
+    // 执行渲染
+    this.renderers.forEach((renderer) => {
+      renderer.drawFrame(this.cacheContext.uid!, this.videoFrame);
+    });
+
+    // 计算渲染耗时
+    const renderEndTime = performance.now();
+    const frameTime = renderEndTime - renderStartTime;
+
+    // 更新帧时间统计
+    this._frameTimes.push(frameTime);
+
+    // 每statsInterval帧输出一次统计信息
+    if (this._frameTimes.length >= this._statsInterval) {
+      this._outputPerformanceStats();
+    }
+  }
+
+  /**
+   * 输出性能统计信息
+   * 包括帧时间和帧间隔的统计数据
+   */
+  private _outputPerformanceStats(): void {
+    if (this._frameTimes.length === 0) return;
+
+    // 计算帧时间统计
+    this.avgFrameTime =
+      this._frameTimes.reduce((a, b) => a + b, 0) / this._frameTimes.length;
+    this.maxFrameTime = Math.max(...this._frameTimes);
+    this.minFrameTime = Math.min(...this._frameTimes);
+
+    // 计算帧间隔统计
+    this.avgFrameInterval = 0;
+    this.maxFrameInterval = 0;
+    this.minFrameInterval = Number.MAX_VALUE;
+
+    if (this._frameIntervals.length > 0) {
+      this.avgFrameInterval =
+        this._frameIntervals.reduce((a, b) => a + b, 0) /
+        this._frameIntervals.length;
+      this.maxFrameInterval = Math.max(...this._frameIntervals);
+      this.minFrameInterval = Math.min(...this._frameIntervals);
+    }
+
+    // 计算实际帧率
+    this.actualFps =
+      this._frameIntervals.length > 0
+        ? 1000 / this.avgFrameInterval
+        : this._renderingFps;
+
+    // 输出帧时间统计
+    logDebug(
+      `[FPS_STATS][UID:${this.callbackContext.connection.localUid}] 帧时间统计(${this._frameTimes.length}帧):`,
+      `平均=${this.avgFrameTime.toFixed(2)}ms`,
+      `最大=${this.maxFrameTime.toFixed(2)}ms`,
+      `最小=${this.minFrameTime.toFixed(2)}ms`
+    );
+
+    // 输出帧间隔统计
+    if (this._frameIntervals.length > 0) {
+      logDebug(
+        `[FPS_STATS][UID:${this.callbackContext.connection.localUid}] 帧间隔统计(${this._frameIntervals.length}帧):`,
+        `实际帧率=${this.actualFps.toFixed(2)}fps`,
+        `目标帧率=${this._renderingFps}fps`,
+        `平均=${this.avgFrameInterval.toFixed(2)}ms`,
+        `最大=${this.maxFrameInterval.toFixed(2)}ms`,
+        `最小=${this.minFrameInterval.toFixed(2)}ms`
+      );
+    }
+
+    // 重置统计数据
+    this._frameTimes = [];
+    this._frameIntervals = [];
+  }
+
+  /**
+   * 开始独立渲染循环
+   */
+  public startRendering() {
+    if (this._renderingTimer || this._isRendering) return;
+
+    this._isRendering = true;
+
+    const renderingLooper = () => {
+      if (!this._isRendering) return;
+
+      const currentTime = performance.now();
+      const timeSinceLastRender = this._lastRenderTime
+        ? currentTime - this._lastRenderTime
+        : 0;
+      const frameInterval = 1000 / this._renderingFps;
+
+      // 如果距离上次渲染的时间不足一帧间隔，等待
+      if (this._lastRenderTime && timeSinceLastRender < frameInterval) {
+        const waitTime = frameInterval - timeSinceLastRender;
+        this._renderingTimer = window.setTimeout(renderingLooper, waitTime);
+        return;
+      }
+
+      // 记录当前时间作为本次循环的开始时间
+      this._lastRenderTime = currentTime;
+
+      // 获取第一帧并渲染
+      // 无论hasMoreFrame是true还是false，都渲染当前帧
+      // 因为fetchVideoFrame总是会获取一帧数据（如果有的话）
+      let { hasMoreFrame, needRender } = this.fetchVideoFrame();
+      if (needRender) {
+        this.renderFrame();
+      }
+
+      // 如果hasMoreFrame为true，表示还有更多帧需要获取
+      while (hasMoreFrame) {
+        // 获取下一帧
+        let { hasMoreFrame: nextHasMoreFrame, needRender } =
+          this.fetchVideoFrame();
+        hasMoreFrame = nextHasMoreFrame;
+        if (needRender) {
+          this.renderFrame();
+        }
+      }
+
+      // 安排下一帧
+      this._renderingTimer = window.setTimeout(renderingLooper, 0);
+    };
+
+    // 启动渲染循环
+    this._renderingTimer = window.setTimeout(renderingLooper, 0);
+  }
+
   /**
    * Remove the specified renderer if it is specified, otherwise remove all renderers
    */
@@ -118,6 +306,17 @@ export class RendererCache extends IRendererCache {
     this.shouldEnable();
   }
 
+  public stopRendering() {
+    if (!this._isRendering) return;
+
+    if (this._renderingTimer) {
+      window.clearTimeout(this._renderingTimer);
+      this._renderingTimer = undefined;
+    }
+
+    this._isRendering = false;
+  }
+
   public release(): void {
     super.release();
   }