agora-electron-sdk 0.0.0-dev.1

package/ts/Renderer/RendererManager.ts

@@ -0,0 +1,400 @@
+import createAgoraRtcEngine from '../AgoraSdk';
+import {
+  VideoMirrorModeType,
+  VideoStreamType,
+  VideoViewSetupMode,
+} from '../Private/AgoraBase';
+import {
+  RenderModeType,
+  VideoModulePosition,
+  VideoSourceType,
+} from '../Private/AgoraMediaBase';
+import {
+  RendererCacheContext,
+  RendererCacheType,
+  RendererContext,
+  RendererType,
+} from '../Types';
+import { AgoraEnv, isSupportWebGL, logDebug } from '../Utils';
+
+import { IRenderer } from './IRenderer';
+import { generateRendererCacheKey, isUseConnection } from './IRendererCache';
+import { RendererCache } from './RendererCache';
+import { WebCodecsRenderer } from './WebCodecsRenderer';
+import { WebCodecsRendererCache } from './WebCodecsRendererCache';
+import { WebGLFallback, WebGLRenderer } from './WebGLRenderer';
+import { YUVCanvasRenderer } from './YUVCanvasRenderer';
+
+/**
+ * @ignore
+ */
+export class RendererManager {
+  /**
+   * @ignore
+   */
+  private renderingFps: number;
+  /**
+   * @ignore
+   */
+  private _currentFrameCount: number;
+  /**
+   * @ignore
+   */
+  private _previousFirstFrameTime: number;
+  /**
+   * @ignore
+   */
+  private _renderingTimer?: number;
+  /**
+   * @ignore
+   */
+  private _rendererCaches: RendererCacheType[];
+  /**
+   * @ignore
+   */
+  private _context: RendererContext;
+
+  /**
+   * @ignore
+   */
+  private rendererType: RendererType;
+
+  /**
+   * Currently, the remote video frame is observed in the pre-renderer position and you can not change it.
+   * the local video frame is observed in the pre-encoder position by default, and you can change it.
+   * @ignore
+   */
+  private defaultObservedFramePosition: number =
+    VideoModulePosition.PositionPreRenderer |
+    VideoModulePosition.PositionPreEncoder;
+
+  constructor() {
+    this.renderingFps = 15;
+    this._currentFrameCount = 0;
+    this._previousFirstFrameTime = 0;
+    this._rendererCaches = [];
+    this._context = {
+      renderMode: RenderModeType.RenderModeHidden,
+      mirrorMode: VideoMirrorModeType.VideoMirrorModeDisabled,
+    };
+    this.rendererType = isSupportWebGL()
+      ? RendererType.WEBGL
+      : RendererType.SOFTWARE;
+  }
+
+  public setRenderingFps(fps: number) {
+    this.renderingFps = fps;
+    if (this._renderingTimer) {
+      this.stopRendering();
+      this.startRendering();
+    }
+  }
+
+  public set defaultChannelId(channelId: string) {
+    this._context.channelId = channelId;
+  }
+
+  public get defaultChannelId(): string {
+    return this._context.channelId ?? '';
+  }
+
+  public get defaultRenderMode(): RenderModeType {
+    return this._context.renderMode!;
+  }
+
+  public get defaultMirrorMode(): VideoMirrorModeType {
+    return this._context.mirrorMode!;
+  }
+
+  public release(): void {
+    this.stopRendering();
+    this.clearRendererCache();
+  }
+
+  private presetRendererContext(context: RendererContext): RendererContext {
+    //this is for preset default value
+    context.renderMode = context.renderMode || this.defaultRenderMode;
+    context.mirrorMode = context.mirrorMode || this.defaultMirrorMode;
+    context.useWebCodecsDecoder = context.useWebCodecsDecoder || false;
+    context.enableFps = context.enableFps || false;
+    context.position = context.position || this.defaultObservedFramePosition;
+
+    if (!AgoraEnv.CapabilityManager?.webCodecsDecoderEnabled) {
+      context.useWebCodecsDecoder = false;
+    }
+
+    switch (context.sourceType) {
+      case VideoSourceType.VideoSourceRemote:
+        if (context.uid === undefined) {
+          throw new Error('uid is required');
+        }
+        context.channelId = context.channelId ?? this.defaultChannelId;
+        break;
+      case VideoSourceType.VideoSourceMediaPlayer:
+        if (context.mediaPlayerId === undefined) {
+          throw new Error('mediaPlayerId is required');
+        }
+        context.channelId = '';
+        context.uid = context.mediaPlayerId;
+        break;
+      case undefined:
+        if (context.uid) {
+          context.sourceType = VideoSourceType.VideoSourceRemote;
+        }
+        break;
+      default:
+        context.channelId = '';
+        context.uid = 0;
+        break;
+    }
+    return context;
+  }
+
+  public addOrRemoveRenderer(
+    context: RendererContext
+  ): RendererCacheType | undefined {
+    // To be compatible with the old API
+    let { setupMode = VideoViewSetupMode.VideoViewSetupAdd } = context;
+    if (!context.view) setupMode = VideoViewSetupMode.VideoViewSetupRemove;
+    switch (setupMode) {
+      case VideoViewSetupMode.VideoViewSetupAdd:
+        return this.addRendererToCache(context);
+      case VideoViewSetupMode.VideoViewSetupRemove:
+        this.removeRendererFromCache(context);
+        return undefined;
+      case VideoViewSetupMode.VideoViewSetupReplace:
+        this.removeRendererFromCache(context);
+        return this.addRendererToCache(context);
+    }
+  }
+
+  private addRendererToCache(
+    context: RendererContext
+  ): RendererCacheType | undefined {
+    const checkedContext = this.presetRendererContext(context);
+
+    if (!checkedContext.view) return undefined;
+
+    if (this.findRenderer(checkedContext.view)) {
+      throw new Error('You have already added this view to the renderer');
+    }
+
+    let rendererCache = this.getRendererCache(checkedContext);
+    if (!rendererCache) {
+      if (context.useWebCodecsDecoder) {
+        rendererCache = new WebCodecsRendererCache(checkedContext);
+      } else {
+        rendererCache = new RendererCache(checkedContext);
+      }
+      this._rendererCaches.push(rendererCache);
+    }
+    rendererCache.addRenderer(this.createRenderer(checkedContext));
+    if (!context.useWebCodecsDecoder) {
+      this.startRendering();
+    }
+    return rendererCache;
+  }
+
+  public removeRendererFromCache(context: RendererContext): void {
+    const checkedContext = this.presetRendererContext(context);
+
+    const rendererCache = this.getRendererCache(checkedContext);
+    if (!rendererCache) return;
+    if (checkedContext.view) {
+      const renderer = rendererCache.findRenderer(checkedContext.view);
+      if (!renderer) return;
+      rendererCache.removeRenderer(renderer);
+    } else {
+      rendererCache.removeRenderer();
+    }
+    if (rendererCache.renderers.length === 0) {
+      rendererCache.release();
+      this._rendererCaches.splice(
+        this._rendererCaches.indexOf(rendererCache),
+        1
+      );
+    }
+  }
+
+  public clearRendererCache(): void {
+    for (const rendererCache of this._rendererCaches) {
+      rendererCache.release();
+    }
+    this._rendererCaches.splice(0);
+  }
+
+  public getRendererCache(
+    context: RendererContext
+  ): RendererCacheType | undefined {
+    return this._rendererCaches.find(
+      (cache) => cache.key === generateRendererCacheKey(context)
+    );
+  }
+
+  public getRenderers(context: RendererContext): IRenderer[] {
+    return this.getRendererCache(context)?.renderers || [];
+  }
+
+  public findRenderer(view: Element): IRenderer | undefined {
+    for (const rendererCache of this._rendererCaches) {
+      const renderer = rendererCache.findRenderer(view);
+      if (renderer) return renderer;
+    }
+    return undefined;
+  }
+
+  protected createRenderer(
+    context: RendererContext,
+    rendererType: RendererType = this.rendererType
+  ): IRenderer {
+    let renderer: IRenderer;
+    switch (rendererType) {
+      case RendererType.WEBGL:
+        if (context.useWebCodecsDecoder) {
+          renderer = new WebCodecsRenderer();
+        } else {
+          renderer = new WebGLRenderer(
+            this.handleWebGLFallback(context).bind(this)
+          );
+          renderer.bind(context.view);
+        }
+        break;
+      case RendererType.SOFTWARE:
+        renderer = new YUVCanvasRenderer();
+        renderer.bind(context.view);
+        break;
+      default:
+        throw new Error('Unknown renderer type');
+    }
+
+    renderer.setContext(context);
+    return renderer;
+  }
+
+  public startRendering(): void {
+    if (this._renderingTimer) return;
+
+    const renderingLooper = () => {
+      if (this._previousFirstFrameTime === 0) {
+        // Get the current time as the time of the first frame of per second
+        this._previousFirstFrameTime = performance.now();
+        // Reset the frame count
+        this._currentFrameCount = 0;
+      }
+
+      // Increase the frame count
+      ++this._currentFrameCount;
+
+      // Get the current time
+      const currentFrameTime = performance.now();
+      // Calculate the time difference between the current frame and the previous frame
+      const deltaTime = currentFrameTime - this._previousFirstFrameTime;
+      // Calculate the expected time of the current frame
+      const expectedTime = (this._currentFrameCount * 1000) / this.renderingFps;
+      logDebug(
+        new Date().toLocaleTimeString(),
+        'currentFrameCount',
+        this._currentFrameCount,
+        'expectedTime',
+        expectedTime,
+        'deltaTime',
+        deltaTime
+      );
+
+      if (this._rendererCaches.length === 0) {
+        // If there is no renderer, stop rendering
+        this.stopRendering();
+        return;
+      }
+
+      // Render all renderers that do not use WebCodecs
+      for (const rendererCache of this._rendererCaches.filter(
+        (cache) => cache instanceof RendererCache
+      )) {
+        this.doRendering(rendererCache);
+      }
+
+      if (this._currentFrameCount >= this.renderingFps) {
+        this._previousFirstFrameTime = 0;
+      }
+
+      if (deltaTime < expectedTime) {
+        // If the time difference between the current frame and the previous frame is less than the expected time, then wait for the difference
+        this._renderingTimer = window.setTimeout(
+          renderingLooper,
+          expectedTime - deltaTime
+        );
+      } else {
+        // If the time difference between the current frame and the previous frame is greater than the expected time, then render immediately
+        renderingLooper();
+      }
+    };
+    renderingLooper();
+  }
+
+  public doRendering(rendererCache: RendererCacheType): void {
+    rendererCache.draw();
+  }
+
+  private handleWebGLFallback(context: RendererContext): WebGLFallback {
+    return (renderer: WebGLRenderer) => {
+      const renderers = this.getRenderers(context);
+      renderer.unbind();
+      const newRenderer = this.createRenderer(context, RendererType.SOFTWARE);
+      renderers.splice(renderers.indexOf(renderer), 1, newRenderer);
+    };
+  }
+
+  public handleWebCodecsFallback(context: RendererCacheContext): void {
+    let engine = createAgoraRtcEngine();
+    engine.getMediaEngine().unregisterVideoEncodedFrameObserver({});
+    if (context.uid) {
+      if (isUseConnection(context)) {
+        engine.setRemoteVideoSubscriptionOptionsEx(
+          context.uid,
+          {
+            type: VideoStreamType.VideoStreamHigh,
+            encodedFrameOnly: false,
+          },
+          {
+            channelId: context.channelId,
+            localUid: context.localUid,
+          }
+        );
+      } else {
+        engine.setRemoteVideoSubscriptionOptions(context.uid, {
+          type: VideoStreamType.VideoStreamHigh,
+          encodedFrameOnly: false,
+        });
+      }
+    }
+    AgoraEnv.enableWebCodecsDecoder = false;
+    AgoraEnv.CapabilityManager?.setWebCodecsDecoderEnabled(false);
+    let renderers = this.getRenderers(context);
+    for (let renderer of renderers) {
+      this.addOrRemoveRenderer({
+        ...renderer.context,
+        setupMode: VideoViewSetupMode.VideoViewSetupReplace,
+      });
+    }
+  }
+
+  public stopRendering(): void {
+    if (this._renderingTimer) {
+      window.clearTimeout(this._renderingTimer);
+      this._renderingTimer = undefined;
+    }
+  }
+
+  public setRendererContext(context: RendererContext): boolean {
+    const checkedContext = this.presetRendererContext(context);
+
+    for (const rendererCache of this._rendererCaches) {
+      const result = rendererCache.setRendererContext(checkedContext);
+      if (result) {
+        return true;
+      }
+    }
+    return false;
+  }
+}

package/ts/Renderer/WebCodecsRenderer/index.ts

@@ -0,0 +1,145 @@
+import { VideoFrame } from '../../Private/AgoraMediaBase';
+import { CodecConfigInfo, RendererType } from '../../Types';
+import { getContextByCanvas } from '../../Utils';
+import { IRenderer } from '../IRenderer';
+
+export class WebCodecsRenderer extends IRenderer {
+  gl?: WebGLRenderingContext | WebGL2RenderingContext | null;
+  // eslint-disable-next-line auto-import/auto-import
+  offscreenCanvas: OffscreenCanvas | undefined;
+
+  constructor() {
+    super();
+    this.rendererType = RendererType.WEBCODECSRENDERER;
+  }
+
+  static vertexShaderSource = `
+    attribute vec2 xy;
+    varying highp vec2 uv;
+    void main(void) {
+      gl_Position = vec4(xy, 0.0, 1.0);
+      // Map vertex coordinates (-1 to +1) to UV coordinates (0 to 1).
+      // UV coordinates are Y-flipped relative to vertex coordinates.
+      uv = vec2((1.0 + xy.x) / 2.0, (1.0 - xy.y) / 2.0);
+    }
+  `;
+  static fragmentShaderSource = `
+    varying highp vec2 uv;
+    uniform sampler2D texture;
+    void main(void) {
+      gl_FragColor = texture2D(texture, uv);
+    }
+  `;
+
+  bind(element: HTMLElement) {
+    super.bind(element);
+    if (!this.canvas) return;
+    this.offscreenCanvas = this.canvas.transferControlToOffscreen();
+    this.gl = getContextByCanvas(this.offscreenCanvas);
+    if (!this.gl) return;
+    const vertexShader = this.gl.createShader(this.gl.VERTEX_SHADER);
+    if (!vertexShader) return;
+    this.gl.shaderSource(vertexShader, WebCodecsRenderer.vertexShaderSource);
+    this.gl.compileShader(vertexShader);
+    if (!this.gl.getShaderParameter(vertexShader, this.gl.COMPILE_STATUS)) {
+      throw this.gl.getShaderInfoLog(vertexShader);
+    }
+    const fragmentShader = this.gl.createShader(this.gl.FRAGMENT_SHADER);
+    if (!fragmentShader) return;
+    this.gl.shaderSource(
+      fragmentShader,
+      WebCodecsRenderer.fragmentShaderSource
+    );
+    this.gl.compileShader(fragmentShader);
+    if (!this.gl.getShaderParameter(fragmentShader, this.gl.COMPILE_STATUS)) {
+      throw this.gl.getShaderInfoLog(fragmentShader);
+    }
+    const shaderProgram = this.gl.createProgram();
+    if (!shaderProgram) return;
+    this.gl.attachShader(shaderProgram, vertexShader);
+    this.gl.attachShader(shaderProgram, fragmentShader);
+    this.gl.linkProgram(shaderProgram);
+    if (!this.gl.getProgramParameter(shaderProgram, this.gl.LINK_STATUS)) {
+      throw this.gl.getProgramInfoLog(shaderProgram);
+    }
+    this.gl.useProgram(shaderProgram);
+    // Vertex coordinates, clockwise from bottom-left.
+    const vertexBuffer = this.gl.createBuffer();
+    this.gl.bindBuffer(this.gl.ARRAY_BUFFER, vertexBuffer);
+    this.gl.bufferData(
+      this.gl.ARRAY_BUFFER,
+      new Float32Array([-1.0, -1.0, -1.0, +1.0, +1.0, +1.0, +1.0, -1.0]),
+      this.gl.STATIC_DRAW
+    );
+    const xyLocation = this.gl.getAttribLocation(shaderProgram, 'xy');
+    this.gl.vertexAttribPointer(xyLocation, 2, this.gl.FLOAT, false, 0, 0);
+    this.gl.enableVertexAttribArray(xyLocation);
+    // Create one texture to upload frames to.
+    const texture = this.gl.createTexture();
+    this.gl.bindTexture(this.gl.TEXTURE_2D, texture);
+    this.gl.texParameteri(
+      this.gl.TEXTURE_2D,
+      this.gl.TEXTURE_MAG_FILTER,
+      this.gl.NEAREST
+    );
+    this.gl.texParameteri(
+      this.gl.TEXTURE_2D,
+      this.gl.TEXTURE_MIN_FILTER,
+      this.gl.NEAREST
+    );
+    this.gl.texParameteri(
+      this.gl.TEXTURE_2D,
+      this.gl.TEXTURE_WRAP_S,
+      this.gl.CLAMP_TO_EDGE
+    );
+    this.gl.texParameteri(
+      this.gl.TEXTURE_2D,
+      this.gl.TEXTURE_WRAP_T,
+      this.gl.CLAMP_TO_EDGE
+    );
+  }
+
+  drawFrame(frame: any, _codecConfig: CodecConfigInfo) {
+    if (!this.offscreenCanvas || !frame) return;
+
+    this.offscreenCanvas.width = _codecConfig.codedWidth!;
+    this.offscreenCanvas.height = _codecConfig.codedHeight!;
+
+    this.updateRenderMode();
+    this.rotateCanvas({
+      rotation: _codecConfig.rotation,
+    });
+    if (!this.gl) return;
+
+    if (this.gl) {
+      // Upload the frame.
+      this.gl.texImage2D(
+        this.gl.TEXTURE_2D,
+        0,
+        this.gl.RGBA,
+        this.gl.RGBA,
+        this.gl.UNSIGNED_BYTE,
+        frame
+      );
+      frame.close();
+      // Configure and clear the drawing area.
+      this.gl.viewport(
+        0,
+        0,
+        this.gl.drawingBufferWidth,
+        this.gl.drawingBufferHeight
+      );
+      this.gl.clearColor(0.0, 0.0, 0.0, 0.0);
+      this.gl.clear(this.gl.COLOR_BUFFER_BIT);
+      // Draw the frame.
+      this.gl.drawArrays(this.gl.TRIANGLE_FAN, 0, 4);
+    }
+    super.drawFrame();
+    this.getFps();
+  }
+
+  protected override rotateCanvas({ rotation }: VideoFrame) {
+    if (!this.offscreenCanvas || !this.canvas) return;
+    this.canvas.style.transform += ` rotateZ(${rotation}deg)`;
+  }
+}

package/ts/Renderer/WebCodecsRendererCache.ts

@@ -0,0 +1,137 @@
+import createAgoraRtcEngine from '../AgoraSdk';
+import { WebCodecsDecoder } from '../Decoder/index';
+import { EncodedVideoFrameInfo, VideoStreamType } from '../Private/AgoraBase';
+import { IRtcEngineEx } from '../Private/IAgoraRtcEngineEx';
+import { AgoraElectronBridge } from '../Private/internal/IrisApiEngine';
+
+import { RendererContext, RendererType } from '../Types';
+import { AgoraEnv, logInfo } from '../Utils';
+
+import { IRendererCache, isUseConnection } from './IRendererCache';
+import { WebCodecsRenderer } from './WebCodecsRenderer/index';
+
+export class WebCodecsRendererCache extends IRendererCache {
+  private _decoder?: WebCodecsDecoder | null;
+  private _engine?: IRtcEngineEx;
+  private _firstFrame = true;
+
+  constructor(context: RendererContext) {
+    super(context);
+    this._engine = createAgoraRtcEngine();
+    this._decoder = new WebCodecsDecoder(
+      this.renderers as WebCodecsRenderer[],
+      this.onDecoderError.bind(this),
+      context
+    );
+    this.draw();
+  }
+
+  onDecoderError(e: any) {
+    logInfo('webCodecsDecoder decode failed, fallback to native decoder', e);
+    AgoraEnv.AgoraRendererManager?.handleWebCodecsFallback(this.cacheContext);
+  }
+
+  onEncodedVideoFrameReceived(...[data, buffer]: any) {
+    let _data: any;
+    try {
+      _data = JSON.parse(data) ?? {};
+    } catch (e) {
+      _data = {};
+    }
+    if (
+      Object.keys(_data).length === 0 ||
+      !this._decoder ||
+      this.cacheContext.uid !== _data.uid
+    )
+      return;
+    if (this._firstFrame) {
+      for (let renderer of this.renderers) {
+        if (renderer.rendererType !== RendererType.WEBCODECSRENDERER) {
+          continue;
+        }
+        renderer.bind(renderer.context.view);
+      }
+
+      try {
+        this._decoder.decoderConfigure(_data.videoEncodedFrameInfo);
+      } catch (error: any) {
+        logInfo(error);
+        return;
+      }
+      this._firstFrame = false;
+    }
+    if (this.shouldFallback(_data.videoEncodedFrameInfo)) {
+      AgoraEnv.AgoraRendererManager?.handleWebCodecsFallback(this.cacheContext);
+    } else {
+      this._decoder.decodeFrame(
+        buffer,
+        _data.videoEncodedFrameInfo,
+        new Date().getTime()
+      );
+    }
+  }
+
+  public draw() {
+    if (isUseConnection(this.cacheContext)) {
+      this._engine?.setRemoteVideoSubscriptionOptionsEx(
+        this.cacheContext.uid!,
+        {
+          type: VideoStreamType.VideoStreamHigh,
+          encodedFrameOnly: true,
+        },
+        {
+          channelId: this.cacheContext.channelId,
+          localUid: this.cacheContext.localUid,
+        }
+      );
+    } else {
+      this._engine?.setRemoteVideoSubscriptionOptions(this.cacheContext.uid!, {
+        type: VideoStreamType.VideoStreamHigh,
+        encodedFrameOnly: true,
+      });
+    }
+    AgoraElectronBridge.OnEvent(
+      `call_back_with_encoded_video_frame_${this.cacheContext.uid}`,
+      (...params: any) => {
+        try {
+          this.onEncodedVideoFrameReceived(...params);
+        } catch (e) {
+          console.error(e);
+        }
+      }
+    );
+  }
+
+  public shouldFallback(frameInfo: EncodedVideoFrameInfo): boolean {
+    let shouldFallback = false;
+    if (!frameInfo.codecType) {
+      shouldFallback = true;
+      logInfo('codecType is not supported, fallback to native decoder');
+    } else {
+      const mapping =
+        AgoraEnv.CapabilityManager?.frameCodecMapping[frameInfo.codecType];
+      if (mapping === undefined) {
+        shouldFallback = true;
+        logInfo('codecType is not supported, fallback to native decoder');
+      } else if (
+        mapping.minWidth >= frameInfo.width! &&
+        mapping.minHeight >= frameInfo.height! &&
+        mapping.maxWidth <= frameInfo.width! &&
+        mapping.maxHeight <= frameInfo.height!
+      ) {
+        shouldFallback = true;
+        logInfo('frame size is not supported, fallback to native decoder');
+      }
+    }
+    return shouldFallback;
+  }
+
+  public release(): void {
+    AgoraElectronBridge.UnEvent(
+      `call_back_with_encoded_video_frame_${this.cacheContext.uid}`
+    );
+    this._decoder?.release();
+    this._decoder = null;
+    super.release();
+  }
+}