agora-electron-sdk 0.0.0-dev.1

package/ts/Renderer/CapabilityManager.ts

@@ -0,0 +1,126 @@
+import semver from 'semver';
+
+import createAgoraRtcEngine from '../AgoraSdk';
+
+import {
+  GpuInfo,
+  VideoDecodeAcceleratorSupportedProfile,
+} from '../Decoder/gpu-utils';
+import { VideoCodecType } from '../Private/AgoraBase';
+import { IRtcEngineEx } from '../Private/IAgoraRtcEngineEx';
+import { ipcSend } from '../Private/ipc/renderer';
+
+import { IPCMessageType, VideoFallbackStrategy, codecMapping } from '../Types';
+import { AgoraEnv, logError, logInfo } from '../Utils';
+
+/**
+ * @ignore
+ */
+export class CapabilityManager {
+  gpuInfo: GpuInfo = new GpuInfo();
+  frameCodecMapping: {
+    [key in VideoCodecType]?: VideoDecodeAcceleratorSupportedProfile;
+  } = {};
+  webCodecsDecoderEnabled: boolean = AgoraEnv.enableWebCodecsDecoder;
+  private _engine: IRtcEngineEx;
+
+  setWebCodecsDecoderEnabled(enabled: boolean): void {
+    this.webCodecsDecoderEnabled = enabled;
+  }
+
+  constructor() {
+    this._engine = createAgoraRtcEngine();
+    if (AgoraEnv.enableWebCodecsDecoder) {
+      this.getGpuInfo(() => {
+        if (
+          AgoraEnv.videoFallbackStrategy ===
+          VideoFallbackStrategy.PerformancePriority
+        ) {
+          if (!this.isSupportedH265()) {
+            if (this.isSupportedH264()) {
+              this._engine.setParameters(
+                JSON.stringify({ 'che.video.h265_dec_enable': false })
+              );
+              logInfo(
+                'the videoFallbackStrategy is PerformancePriority, H265 is not supported, fallback to H264'
+              );
+            } else {
+              this.webCodecsDecoderEnabled = false;
+              logInfo(
+                'the videoFallbackStrategy is PerformancePriority, H264 and H265 are not supported, fallback to native decoder'
+              );
+            }
+          }
+        } else if (
+          AgoraEnv.videoFallbackStrategy ===
+          VideoFallbackStrategy.BandwidthPriority
+        ) {
+          if (!this.isSupportedH265()) {
+            this.webCodecsDecoderEnabled = false;
+            logInfo(
+              'the videoFallbackStrategy is BandwidthPriority, H265 is not supported, fallback to native decoder'
+            );
+          }
+        }
+      });
+    }
+  }
+
+  public getGpuInfo(callback?: () => void): void {
+    //getGpuInfo and videoDecoder is not supported in electron version < 22.0.0
+    //@ts-ignore
+    if (semver.lt(process.versions.electron, '22.0.0')) {
+      logError(
+        'WebCodecsDecoder is not supported in electron version < 22.0.0, please upgrade electron to 22.0.0 or later.'
+      );
+      return;
+    }
+    //@ts-ignore
+    if (process.type === 'renderer') {
+      ipcSend(IPCMessageType.AGORA_IPC_GET_GPU_INFO)
+        .then((result) => {
+          this.gpuInfo.videoDecodeAcceleratorSupportedProfile = result;
+          this.webCodecsDecoderEnabled = (AgoraEnv.enableWebCodecsDecoder &&
+            this.gpuInfo.videoDecodeAcceleratorSupportedProfile.length > 0)!;
+
+          result.forEach((profile: VideoDecodeAcceleratorSupportedProfile) => {
+            const match = codecMapping.find((item) =>
+              profile.codec.includes(item.profile)
+            );
+            if (match) {
+              //Normally, the range of compatible widths and heights should be the same under the same codec.
+              //there is no need to differentiate between different profiles. This could be optimized in the future.
+              this.frameCodecMapping[match.type] = {
+                codec: match.codec,
+                minWidth: profile.minWidth,
+                minHeight: profile.minHeight,
+                maxWidth: profile.maxWidth,
+                maxHeight: profile.maxHeight,
+              };
+            }
+          });
+          callback && callback();
+        })
+        .catch((error) => {
+          logError(
+            'Failed to get GPU info, please check if you are already import agora-electron-sdk in the main process.',
+            error
+          );
+        });
+    } else {
+      logError('This function only works in renderer process');
+    }
+  }
+
+  public isSupportedH264(): boolean {
+    return this.frameCodecMapping[VideoCodecType.VideoCodecH264] !== undefined;
+  }
+
+  public isSupportedH265(): boolean {
+    return this.frameCodecMapping[VideoCodecType.VideoCodecH265] !== undefined;
+  }
+
+  release(): void {
+    AgoraEnv.enableWebCodecsDecoder = false;
+  }
+}

package/ts/Renderer/IRenderer.ts

@@ -0,0 +1,158 @@
+import { VideoMirrorModeType } from '../Private/AgoraBase';
+import { RenderModeType, VideoFrame } from '../Private/AgoraMediaBase';
+import { CodecConfigInfo, RendererContext, RendererType } from '../Types';
+
+export abstract class IRenderer {
+  parentElement?: HTMLElement;
+  container?: HTMLElement;
+  canvas?: HTMLCanvasElement;
+  rendererType: RendererType | undefined;
+  context: RendererContext = {};
+  private _frameCount = 0;
+  private _startTime: number | null = null;
+
+  public bind(element: HTMLElement) {
+    this.parentElement = element;
+    this.container = document.createElement('div');
+    Object.assign(this.container.style, {
+      width: '100%',
+      height: '100%',
+      display: 'flex',
+      justifyContent: 'center',
+      alignItems: 'center',
+      overflow: 'hidden',
+    });
+    this.parentElement.appendChild(this.container);
+    this.canvas = document.createElement('canvas');
+    this.canvas.style.display = 'none';
+    this.container.appendChild(this.canvas);
+  }
+
+  public unbind() {
+    if (this.container && this.canvas?.parentNode === this.container) {
+      this.container.removeChild(this.canvas);
+    }
+    if (
+      this.parentElement &&
+      this.container?.parentNode === this.parentElement
+    ) {
+      this.parentElement.removeChild(this.container);
+    }
+
+    this.canvas = undefined;
+    this.container = undefined;
+    this.parentElement = undefined;
+  }
+
+  public drawFrame(
+    _videoFrame?: VideoFrame,
+    _codecConfig?: CodecConfigInfo
+  ): void {
+    if (!this.canvas) return;
+    if (this.canvas.style.display !== '') {
+      this.canvas.style.display = '';
+    }
+  }
+
+  public setContext(context: RendererContext) {
+    if (this.context.renderMode !== context.renderMode) {
+      this.context.renderMode = context.renderMode;
+      this.updateRenderMode();
+    }
+
+    if (this.context.mirrorMode !== context.mirrorMode) {
+      this.context.mirrorMode = context.mirrorMode;
+      this.updateMirrorMode();
+    }
+    this.context = { ...this.context, ...context };
+  }
+
+  protected updateRenderMode(): void {
+    if (!this.canvas || !this.container) return;
+
+    const { clientWidth, clientHeight } = this.container;
+    const { width, height } = this.canvas;
+
+    const containerAspectRatio = clientWidth / clientHeight;
+    const canvasAspectRatio = width / height;
+    const widthScale = clientWidth / width;
+    const heightScale = clientHeight / height;
+    const isHidden =
+      this.context.renderMode === RenderModeType.RenderModeHidden;
+
+    let scale = 1;
+    // If container's aspect ratio is larger than canvas's aspect ratio
+    if (containerAspectRatio > canvasAspectRatio) {
+      // Scale canvas to fit container's width on hidden mode
+      // Scale canvas to fit container's height on fit mode
+      scale = isHidden ? widthScale : heightScale;
+    } else {
+      // Scale canvas to fit container's height on hidden mode
+      // Scale canvas to fit container's width on fit mode
+      scale = isHidden ? heightScale : widthScale;
+    }
+    this.canvas.style.transform = `scale(${scale})`;
+  }
+
+  protected updateMirrorMode(): void {
+    if (!this.parentElement) return;
+
+    Object.assign(this.parentElement.style, {
+      transform:
+        this.context.mirrorMode === VideoMirrorModeType.VideoMirrorModeEnabled
+          ? 'rotateY(180deg)'
+          : '',
+    });
+  }
+
+  protected rotateCanvas({ width, height, rotation }: VideoFrame): void {
+    if (!this.canvas) return;
+
+    if (rotation === 0 || rotation === 180) {
+      this.canvas.width = width!;
+      this.canvas.height = height!;
+    } else if (rotation === 90 || rotation === 270) {
+      this.canvas.height = width!;
+      this.canvas.width = height!;
+    } else {
+      throw new Error(
+        `Invalid rotation: ${rotation}, only 0, 90, 180, 270 are supported`
+      );
+    }
+  }
+
+  public getFps(): number {
+    let fps = 0;
+    if (!this.context.enableFps || !this.container) {
+      return fps;
+    }
+    if (this._startTime == null) {
+      this._startTime = performance.now();
+    } else {
+      const elapsed = (performance.now() - this._startTime) / 1000;
+      fps = ++this._frameCount / elapsed;
+    }
+
+    let span = this.container.querySelector('span');
+    if (!span) {
+      span = document.createElement('span');
+
+      Object.assign(span.style, {
+        position: 'absolute',
+        bottom: '0',
+        left: '0',
+        zIndex: '10',
+        width: '55px',
+        background: '#fff',
+      });
+
+      this.container.style.position = 'relative';
+
+      this.container.appendChild(span);
+    }
+
+    span.innerText = `fps: ${fps.toFixed(0)}`;
+
+    return fps;
+  }
+}

package/ts/Renderer/IRendererCache.ts

@@ -0,0 +1,96 @@
+import { RendererCacheContext, RendererContext } from '../Types';
+
+import { IRenderer } from './IRenderer';
+
+/**
+ * @ignore
+ */
+export function generateRendererCacheKey({
+  channelId,
+  uid,
+  sourceType,
+}: RendererContext): string {
+  return `${channelId}_${uid}_${sourceType}`;
+}
+
+/**
+ * @ignore
+ */
+export function isUseConnection(context: RendererCacheContext): boolean {
+  // if RtcConnection is not undefined, then use connection
+  return !!context.channelId && context.localUid !== undefined;
+}
+
+export abstract class IRendererCache {
+  renderers: IRenderer[];
+  cacheContext: RendererCacheContext;
+
+  constructor({
+    channelId,
+    uid,
+    useWebCodecsDecoder,
+    enableFps,
+    sourceType,
+    localUid,
+    position,
+  }: RendererContext) {
+    this.renderers = [];
+    this.cacheContext = {
+      channelId,
+      uid,
+      useWebCodecsDecoder,
+      enableFps,
+      sourceType,
+      localUid,
+      position,
+    };
+  }
+
+  public get key(): string {
+    return generateRendererCacheKey(this.cacheContext);
+  }
+
+  public abstract draw(): void;
+
+  public findRenderer(view: Element): IRenderer | undefined {
+    return this.renderers.find((renderer) => renderer.parentElement === view);
+  }
+
+  public addRenderer(renderer: IRenderer): void {
+    this.renderers.push(renderer);
+  }
+
+  /**
+   * Remove the specified renderer if it is specified, otherwise remove all renderers
+   */
+  public removeRenderer(renderer?: IRenderer): void {
+    let start = 0;
+    let deleteCount = this.renderers.length;
+    if (renderer) {
+      start = this.renderers.indexOf(renderer);
+      if (start < 0) return;
+      deleteCount = 1;
+    }
+    this.renderers.splice(start, deleteCount).forEach((it) => it.unbind());
+  }
+
+  public setRendererContext(context: RendererContext): boolean {
+    if (context.view) {
+      const renderer = this.findRenderer(context.view);
+      if (renderer) {
+        renderer.setContext(context);
+        return true;
+      }
+      return false;
+    } else {
+      this.renderers.forEach((it) => {
+        it.setContext(context);
+      });
+      return this.renderers.length > 0;
+    }
+  }
+
+  public release(): void {
+    this.removeRenderer();
+  }
+}

package/ts/Renderer/RendererCache.ts

@@ -0,0 +1,124 @@
+import { VideoFrame } from '../Private/AgoraMediaBase';
+import { AgoraElectronBridge } from '../Private/internal/IrisApiEngine';
+
+import { RendererContext } from '../Types';
+import { AgoraEnv, logDebug } from '../Utils';
+
+import { IRenderer } from './IRenderer';
+import { IRendererCache } from './IRendererCache';
+
+export class RendererCache extends IRendererCache {
+  private videoFrame: VideoFrame;
+  private _enabled: boolean;
+
+  constructor(context: RendererContext) {
+    super(context);
+    this.videoFrame = {
+      yBuffer: Buffer.alloc(0),
+      uBuffer: Buffer.alloc(0),
+      vBuffer: Buffer.alloc(0),
+      alphaBuffer: AgoraEnv.encodeAlpha ? Buffer.alloc(0) : undefined,
+      width: 0,
+      height: 0,
+      yStride: 0,
+      uStride: 0,
+      vStride: 0,
+      rotation: 0,
+    };
+    this._enabled = false;
+  }
+
+  /**
+   * @deprecated Use videoFrame instead
+   */
+  public get shareVideoFrame(): VideoFrame | undefined {
+    return this.videoFrame;
+  }
+
+  private enable() {
+    if (this._enabled) return;
+    AgoraElectronBridge.EnableVideoFrameCache(this.cacheContext);
+    this._enabled = true;
+  }
+
+  private disable() {
+    if (!this._enabled) return;
+    AgoraElectronBridge.DisableVideoFrameCache(this.cacheContext);
+    this._enabled = false;
+  }
+
+  private shouldEnable() {
+    if (this.renderers.length > 0) {
+      this.enable();
+    } else {
+      this.disable();
+    }
+  }
+
+  override draw() {
+    let { ret, isNewFrame } = AgoraElectronBridge.GetVideoFrame(
+      this.cacheContext,
+      this.videoFrame,
+      {
+        encodeAlpha: AgoraEnv.encodeAlpha,
+      }
+    );
+
+    switch (ret) {
+      case 0: // GET_VIDEO_FRAME_CACHE_RETURN_TYPE::OK = 0
+        //
+        break;
+      case 1: // GET_VIDEO_FRAME_CACHE_RETURN_TYPE::RESIZED = 1
+        const { yStride, uStride, vStride, height } = this.videoFrame;
+        this.videoFrame.yBuffer = Buffer.alloc(yStride! * height!);
+        this.videoFrame.uBuffer = Buffer.alloc(uStride! * height!);
+        this.videoFrame.vBuffer = Buffer.alloc(vStride! * height!);
+        if (AgoraEnv.encodeAlpha) {
+          this.videoFrame.alphaBuffer = Buffer.alloc(
+            this.videoFrame.width! * this.videoFrame.height!
+          );
+        }
+
+        const result = AgoraElectronBridge.GetVideoFrame(
+          this.cacheContext,
+          this.videoFrame,
+          {
+            encodeAlpha: AgoraEnv.encodeAlpha,
+          }
+        );
+        ret = result.ret;
+        isNewFrame = result.isNewFrame;
+        break;
+      case 2: // GET_VIDEO_FRAME_CACHE_RETURN_TYPE::NO_CACHE = 2
+        logDebug('No renderer cache, please enable cache first');
+        return;
+    }
+
+    if (!AgoraEnv.encodeAlpha) {
+      this.videoFrame.alphaBuffer = undefined;
+    }
+
+    if (isNewFrame) {
+      this.renderers.forEach((renderer) => {
+        renderer.drawFrame(this.videoFrame);
+      });
+    }
+  }
+
+  override addRenderer(renderer: IRenderer): void {
+    super.addRenderer(renderer);
+    this.shouldEnable();
+  }
+
+  /**
+   * Remove the specified renderer if it is specified, otherwise remove all renderers
+   */
+  override removeRenderer(renderer?: IRenderer): void {
+    super.removeRenderer(renderer);
+    this.shouldEnable();
+  }
+
+  public release(): void {
+    super.release();
+  }
+}