agora-electron-sdk 4.5.1 → 4.5.2-dev.1

package/ts/Renderer/CapabilityManager.ts

@@ -0,0 +1,126 @@
+import semver from 'semver';
+
+import createAgoraRtcEngine from '../AgoraSdk';
+
+import {
+  GpuInfo,
+  VideoDecodeAcceleratorSupportedProfile,
+} from '../Decoder/gpu-utils';
+import { VideoCodecType } from '../Private/AgoraBase';
+import { IRtcEngineEx } from '../Private/IAgoraRtcEngineEx';
+import { ipcSend } from '../Private/ipc/renderer';
+
+import { IPCMessageType, VideoFallbackStrategy, codecMapping } from '../Types';
+import { AgoraEnv, logError, logInfo } from '../Utils';
+
+/**
+ * @ignore
+ */
+export class CapabilityManager {
+  gpuInfo: GpuInfo = new GpuInfo();
+  frameCodecMapping: {
+    [key in VideoCodecType]?: VideoDecodeAcceleratorSupportedProfile;
+  } = {};
+  webCodecsDecoderEnabled: boolean = AgoraEnv.enableWebCodecsDecoder;
+  private _engine: IRtcEngineEx;
+
+  setWebCodecsDecoderEnabled(enabled: boolean): void {
+    this.webCodecsDecoderEnabled = enabled;
+  }
+
+  constructor() {
+    this._engine = createAgoraRtcEngine();
+    if (AgoraEnv.enableWebCodecsDecoder) {
+      this.getGpuInfo(() => {
+        if (
+          AgoraEnv.videoFallbackStrategy ===
+          VideoFallbackStrategy.PerformancePriority
+        ) {
+          if (!this.isSupportedH265()) {
+            if (this.isSupportedH264()) {
+              this._engine.setParameters(
+                JSON.stringify({ 'che.video.h265_dec_enable': false })
+              );
+              logInfo(
+                'the videoFallbackStrategy is PerformancePriority, H265 is not supported, fallback to H264'
+              );
+            } else {
+              this.webCodecsDecoderEnabled = false;
+              logInfo(
+                'the videoFallbackStrategy is PerformancePriority, H264 and H265 are not supported, fallback to native decoder'
+              );
+            }
+          }
+        } else if (
+          AgoraEnv.videoFallbackStrategy ===
+          VideoFallbackStrategy.BandwidthPriority
+        ) {
+          if (!this.isSupportedH265()) {
+            this.webCodecsDecoderEnabled = false;
+            logInfo(
+              'the videoFallbackStrategy is BandwidthPriority, H265 is not supported, fallback to native decoder'
+            );
+          }
+        }
+      });
+    }
+  }
+
+  public getGpuInfo(callback?: () => void): void {
+    //getGpuInfo and videoDecoder is not supported in electron version < 22.0.0
+    //@ts-ignore
+    if (semver.lt(process.versions.electron, '22.0.0')) {
+      logError(
+        'WebCodecsDecoder is not supported in electron version < 22.0.0, please upgrade electron to 22.0.0 or later.'
+      );
+      return;
+    }
+    //@ts-ignore
+    if (process.type === 'renderer') {
+      ipcSend(IPCMessageType.AGORA_IPC_GET_GPU_INFO)
+        .then((result) => {
+          this.gpuInfo.videoDecodeAcceleratorSupportedProfile = result;
+          this.webCodecsDecoderEnabled = (AgoraEnv.enableWebCodecsDecoder &&
+            this.gpuInfo.videoDecodeAcceleratorSupportedProfile.length > 0)!;
+
+          result.forEach((profile: VideoDecodeAcceleratorSupportedProfile) => {
+            const match = codecMapping.find((item) =>
+              profile.codec.includes(item.profile)
+            );
+            if (match) {
+              //Normally, the range of compatible widths and heights should be the same under the same codec.
+              //there is no need to differentiate between different profiles. This could be optimized in the future.
+              this.frameCodecMapping[match.type] = {
+                codec: match.codec,
+                minWidth: profile.minWidth,
+                minHeight: profile.minHeight,
+                maxWidth: profile.maxWidth,
+                maxHeight: profile.maxHeight,
+              };
+            }
+          });
+          callback && callback();
+        })
+        .catch((error) => {
+          logError(
+            'Failed to get GPU info, please check if you are already import agora-electron-sdk in the main process.',
+            error
+          );
+        });
+    } else {
+      logError('This function only works in renderer process');
+    }
+  }
+
+  public isSupportedH264(): boolean {
+    return this.frameCodecMapping[VideoCodecType.VideoCodecH264] !== undefined;
+  }
+
+  public isSupportedH265(): boolean {
+    return this.frameCodecMapping[VideoCodecType.VideoCodecH265] !== undefined;
+  }
+
+  release(): void {
+    AgoraEnv.enableWebCodecsDecoder = false;
+  }
+}

package/ts/Renderer/IRenderer.ts

@@ -1,14 +1,15 @@
 import { VideoMirrorModeType } from '../Private/AgoraBase';
 import { RenderModeType, VideoFrame } from '../Private/AgoraMediaBase';
-import { RendererContext } from '../Types';
-
-type Context = Pick<RendererContext, 'renderMode' | 'mirrorMode'>;
+import { CodecConfigInfo, RendererContext, RendererType } from '../Types';
 
 export abstract class IRenderer {
   parentElement?: HTMLElement;
   container?: HTMLElement;
   canvas?: HTMLCanvasElement;
-  _context: Context = {};
+  rendererType: RendererType | undefined;
+  context: RendererContext = {};
+  private _frameCount = 0;
+  private _startTime: number | null = null;
 
   public bind(element: HTMLElement) {
     this.parentElement = element;
@@ -43,30 +44,30 @@ export abstract class IRenderer {
     this.parentElement = undefined;
   }
 
-  public drawFrame(_videoFrame?: VideoFrame): void {
+  public drawFrame(
+    _videoFrame?: VideoFrame,
+    _codecConfig?: CodecConfigInfo
+  ): void {
     if (!this.canvas) return;
     if (this.canvas.style.display !== '') {
       this.canvas.style.display = '';
     }
   }
 
-  public set context({ renderMode, mirrorMode }: Context) {
-    if (this.context.renderMode !== renderMode) {
-      this.context.renderMode = renderMode;
+  public setContext(context: RendererContext) {
+    if (this.context.renderMode !== context.renderMode) {
+      this.context.renderMode = context.renderMode;
       this.updateRenderMode();
     }
 
-    if (this.context.mirrorMode !== mirrorMode) {
-      this.context.mirrorMode = mirrorMode;
+    if (this.context.mirrorMode !== context.mirrorMode) {
+      this.context.mirrorMode = context.mirrorMode;
       this.updateMirrorMode();
     }
+    this.context = { ...this.context, ...context };
   }
 
-  public get context(): Context {
-    return this._context;
-  }
-
-  protected updateRenderMode() {
+  protected updateRenderMode(): void {
     if (!this.canvas || !this.container) return;
 
     const { clientWidth, clientHeight } = this.container;
@@ -76,9 +77,8 @@ export abstract class IRenderer {
     const canvasAspectRatio = width / height;
     const widthScale = clientWidth / width;
     const heightScale = clientHeight / height;
-
     const isHidden =
-      this.context?.renderMode === RenderModeType.RenderModeHidden;
+      this.context.renderMode === RenderModeType.RenderModeHidden;
 
     let scale = 1;
     // If container's aspect ratio is larger than canvas's aspect ratio
@@ -120,4 +120,39 @@ export abstract class IRenderer {
       );
     }
   }
+
+  public getFps(): number {
+    let fps = 0;
+    if (!this.context.enableFps || !this.container) {
+      return fps;
+    }
+    if (this._startTime == null) {
+      this._startTime = performance.now();
+    } else {
+      const elapsed = (performance.now() - this._startTime) / 1000;
+      fps = ++this._frameCount / elapsed;
+    }
+
+    let span = this.container.querySelector('span');
+    if (!span) {
+      span = document.createElement('span');
+
+      Object.assign(span.style, {
+        position: 'absolute',
+        bottom: '0',
+        left: '0',
+        zIndex: '10',
+        width: '55px',
+        background: '#fff',
+      });
+
+      this.container.style.position = 'relative';
+
+      this.container.appendChild(span);
+    }
+
+    span.innerText = `fps: ${fps.toFixed(0)}`;
+
+    return fps;
+  }
 }

package/ts/Renderer/IRendererCache.ts

@@ -0,0 +1,96 @@
+import { RendererCacheContext, RendererContext } from '../Types';
+
+import { IRenderer } from './IRenderer';
+
+/**
+ * @ignore
+ */
+export function generateRendererCacheKey({
+  channelId,
+  uid,
+  sourceType,
+}: RendererContext): string {
+  return `${channelId}_${uid}_${sourceType}`;
+}
+
+/**
+ * @ignore
+ */
+export function isUseConnection(context: RendererCacheContext): boolean {
+  // if RtcConnection is not undefined, then use connection
+  return !!context.channelId && context.localUid !== undefined;
+}
+
+export abstract class IRendererCache {
+  renderers: IRenderer[];
+  cacheContext: RendererCacheContext;
+
+  constructor({
+    channelId,
+    uid,
+    useWebCodecsDecoder,
+    enableFps,
+    sourceType,
+    localUid,
+    position,
+  }: RendererContext) {
+    this.renderers = [];
+    this.cacheContext = {
+      channelId,
+      uid,
+      useWebCodecsDecoder,
+      enableFps,
+      sourceType,
+      localUid,
+      position,
+    };
+  }
+
+  public get key(): string {
+    return generateRendererCacheKey(this.cacheContext);
+  }
+
+  public abstract draw(): void;
+
+  public findRenderer(view: Element): IRenderer | undefined {
+    return this.renderers.find((renderer) => renderer.parentElement === view);
+  }
+
+  public addRenderer(renderer: IRenderer): void {
+    this.renderers.push(renderer);
+  }
+
+  /**
+   * Remove the specified renderer if it is specified, otherwise remove all renderers
+   */
+  public removeRenderer(renderer?: IRenderer): void {
+    let start = 0;
+    let deleteCount = this.renderers.length;
+    if (renderer) {
+      start = this.renderers.indexOf(renderer);
+      if (start < 0) return;
+      deleteCount = 1;
+    }
+    this.renderers.splice(start, deleteCount).forEach((it) => it.unbind());
+  }
+
+  public setRendererContext(context: RendererContext): boolean {
+    if (context.view) {
+      const renderer = this.findRenderer(context.view);
+      if (renderer) {
+        renderer.setContext(context);
+        return true;
+      }
+      return false;
+    } else {
+      this.renderers.forEach((it) => {
+        it.setContext(context);
+      });
+      return this.renderers.length > 0;
+    }
+  }
+
+  public release(): void {
+    this.removeRenderer();
+  }
+}

package/ts/Renderer/RendererCache.ts

@@ -1,29 +1,23 @@
 import { VideoFrame } from '../Private/AgoraMediaBase';
-import { RendererCacheContext, RendererContext } from '../Types';
+import { AgoraElectronBridge } from '../Private/internal/IrisApiEngine';
+
+import { RendererContext } from '../Types';
 import { AgoraEnv, logDebug } from '../Utils';
 
 import { IRenderer } from './IRenderer';
+import { IRendererCache } from './IRendererCache';
 
-export function generateRendererCacheKey({
-  channelId,
-  uid,
-  sourceType,
-}: RendererContext): string {
-  return `${channelId}_${uid}_${sourceType}`;
-}
-
-export class RendererCache {
-  private _renderers: IRenderer[];
-  private _videoFrame: VideoFrame;
-  private _context: RendererCacheContext;
+export class RendererCache extends IRendererCache {
+  private videoFrame: VideoFrame;
   private _enabled: boolean;
 
-  constructor({ channelId, uid, sourceType, position }: RendererContext) {
-    this._renderers = [];
-    this._videoFrame = {
+  constructor(context: RendererContext) {
+    super(context);
+    this.videoFrame = {
       yBuffer: Buffer.alloc(0),
       uBuffer: Buffer.alloc(0),
       vBuffer: Buffer.alloc(0),
+      alphaBuffer: AgoraEnv.encodeAlpha ? Buffer.alloc(0) : undefined,
       width: 0,
       height: 0,
       yStride: 0,
@@ -31,33 +25,9 @@ export class RendererCache {
       vStride: 0,
       rotation: 0,
     };
-    this._context = { channelId, uid, sourceType, position };
     this._enabled = false;
   }
 
-  public get key(): string {
-    return generateRendererCacheKey(this._context);
-  }
-
-  public get renderers(): IRenderer[] {
-    return this._renderers;
-  }
-
-  public get videoFrame(): VideoFrame {
-    return this._videoFrame;
-  }
-
-  public get context(): RendererCacheContext {
-    return this._context;
-  }
-
-  /**
-   * @deprecated Use renderers instead
-   */
-  public get renders(): IRenderer[] {
-    return this.renderers;
-  }
-
   /**
    * @deprecated Use videoFrame instead
    */
@@ -65,19 +35,15 @@ export class RendererCache {
     return this.videoFrame;
   }
 
-  private get bridge() {
-    return AgoraEnv.AgoraElectronBridge;
-  }
-
   private enable() {
     if (this._enabled) return;
-    this.bridge.EnableVideoFrameCache(this._context);
+    AgoraElectronBridge.EnableVideoFrameCache(this.cacheContext);
     this._enabled = true;
   }
 
   private disable() {
     if (!this._enabled) return;
-    this.bridge.DisableVideoFrameCache(this._context);
+    AgoraElectronBridge.DisableVideoFrameCache(this.cacheContext);
     this._enabled = false;
   }
 
@@ -89,10 +55,13 @@ export class RendererCache {
     }
   }
 
-  public draw() {
-    let { ret, isNewFrame } = this.bridge.GetVideoFrame(
-      this.context,
-      this.videoFrame
+  override draw() {
+    let { ret, isNewFrame } = AgoraElectronBridge.GetVideoFrame(
+      this.cacheContext,
+      this.videoFrame,
+      {
+        encodeAlpha: AgoraEnv.encodeAlpha,
+      }
     );
 
     switch (ret) {
@@ -104,8 +73,19 @@ export class RendererCache {
         this.videoFrame.yBuffer = Buffer.alloc(yStride! * height!);
         this.videoFrame.uBuffer = Buffer.alloc(uStride! * height!);
         this.videoFrame.vBuffer = Buffer.alloc(vStride! * height!);
-
-        const result = this.bridge.GetVideoFrame(this.context, this.videoFrame);
+        if (AgoraEnv.encodeAlpha) {
+          this.videoFrame.alphaBuffer = Buffer.alloc(
+            this.videoFrame.width! * this.videoFrame.height!
+          );
+        }
+
+        const result = AgoraElectronBridge.GetVideoFrame(
+          this.cacheContext,
+          this.videoFrame,
+          {
+            encodeAlpha: AgoraEnv.encodeAlpha,
+          }
+        );
         ret = result.ret;
         isNewFrame = result.isNewFrame;
         break;
@@ -114,6 +94,10 @@ export class RendererCache {
         return;
     }
 
+    if (!AgoraEnv.encodeAlpha) {
+      this.videoFrame.alphaBuffer = undefined;
+    }
+
     if (isNewFrame) {
       this.renderers.forEach((renderer) => {
         renderer.drawFrame(this.videoFrame);
@@ -121,47 +105,20 @@ export class RendererCache {
     }
   }
 
-  public findRenderer(view: Element): IRenderer | undefined {
-    return this._renderers.find((renderer) => renderer.parentElement === view);
-  }
-
-  public addRenderer(renderer: IRenderer): void {
-    this._renderers.push(renderer);
+  override addRenderer(renderer: IRenderer): void {
+    super.addRenderer(renderer);
     this.shouldEnable();
   }
 
   /**
    * Remove the specified renderer if it is specified, otherwise remove all renderers
    */
-  public removeRenderer(renderer?: IRenderer): void {
-    let start = 0;
-    let deleteCount = this._renderers.length;
-    if (renderer) {
-      start = this._renderers.indexOf(renderer);
-      if (start < 0) return;
-      deleteCount = 1;
-    }
-    this._renderers.splice(start, deleteCount).forEach((it) => it.unbind());
+  override removeRenderer(renderer?: IRenderer): void {
+    super.removeRenderer(renderer);
     this.shouldEnable();
   }
 
-  public setRendererContext({
-    view,
-    renderMode,
-    mirrorMode,
-  }: RendererContext): boolean {
-    if (view) {
-      const renderer = this.findRenderer(view);
-      if (renderer) {
-        renderer.context = { renderMode, mirrorMode };
-        return true;
-      }
-      return false;
-    } else {
-      this._renderers.forEach((it) => {
-        it.context = { renderMode, mirrorMode };
-      });
-      return this._renderers.length > 0;
-    }
+  public release(): void {
+    super.release();
   }
 }