agora-electron-sdk 4.5.1 → 4.5.2-dev.2

package/ts/Renderer/RendererManager.ts

@@ -1,110 +1,400 @@
+import createAgoraRtcEngine from '../AgoraSdk';
+import {
+  VideoMirrorModeType,
+  VideoStreamType,
+  VideoViewSetupMode,
+} from '../Private/AgoraBase';
+import {
+  RenderModeType,
+  VideoModulePosition,
+  VideoSourceType,
+} from '../Private/AgoraMediaBase';
 import {
-  RENDER_MODE,
   RendererCacheContext,
+  RendererCacheType,
   RendererContext,
   RendererType,
 } from '../Types';
-import { isSupportWebGL } from '../Utils';
+import { AgoraEnv, isSupportWebGL, logDebug } from '../Utils';
 
 import { IRenderer } from './IRenderer';
-import { IRendererManager } from './IRendererManager';
+import { generateRendererCacheKey, isUseConnection } from './IRendererCache';
 import { RendererCache } from './RendererCache';
+import { WebCodecsRenderer } from './WebCodecsRenderer';
+import { WebCodecsRendererCache } from './WebCodecsRendererCache';
 import { WebGLFallback, WebGLRenderer } from './WebGLRenderer';
 import { YUVCanvasRenderer } from './YUVCanvasRenderer';
 
 /**
  * @ignore
  */
-export class RendererManager extends IRendererManager {
+export class RendererManager {
+  /**
+   * @ignore
+   */
+  private renderingFps: number;
+  /**
+   * @ignore
+   */
+  private _currentFrameCount: number;
+  /**
+   * @ignore
+   */
+  private _previousFirstFrameTime: number;
+  /**
+   * @ignore
+   */
+  private _renderingTimer?: number;
   /**
    * @ignore
    */
-  private _rendererType: RendererType;
+  private _rendererCaches: RendererCacheType[];
+  /**
+   * @ignore
+   */
+  private _context: RendererContext;
 
-  public set rendererType(rendererType: RendererType) {
-    if (this._rendererType !== rendererType) {
-      this._rendererType = rendererType;
-    }
-  }
+  /**
+   * @ignore
+   */
+  private rendererType: RendererType;
 
-  public get rendererType(): RendererType {
-    return this._rendererType;
-  }
+  /**
+   * Currently, the remote video frame is observed in the pre-renderer position and you can not change it.
+   * the local video frame is observed in the pre-encoder position by default, and you can change it.
+   * @ignore
+   */
+  private defaultObservedFramePosition: number =
+    VideoModulePosition.PositionPreRenderer |
+    VideoModulePosition.PositionPreEncoder;
 
   constructor() {
-    super();
-    this._rendererType = isSupportWebGL()
+    this.renderingFps = 15;
+    this._currentFrameCount = 0;
+    this._previousFirstFrameTime = 0;
+    this._rendererCaches = [];
+    this._context = {
+      renderMode: RenderModeType.RenderModeHidden,
+      mirrorMode: VideoMirrorModeType.VideoMirrorModeDisabled,
+    };
+    this.rendererType = isSupportWebGL()
       ? RendererType.WEBGL
       : RendererType.SOFTWARE;
   }
 
-  /**
-   * @deprecated Use rendererType instead
-   */
-  public setRenderMode(mode: RENDER_MODE) {
-    this.rendererType = mode;
+  public setRenderingFps(fps: number) {
+    this.renderingFps = fps;
+    if (this._renderingTimer) {
+      this.stopRendering();
+      this.startRendering();
+    }
   }
 
-  /**
-   * @deprecated Use renderingFps instead
-   */
-  public setFPS(fps: number) {
-    this.renderingFps = fps;
+  public set defaultChannelId(channelId: string) {
+    this._context.channelId = channelId;
   }
 
-  /**
-   * @deprecated Use getRendererCache instead
-   */
-  public getRender(context: RendererCacheContext): RendererCache | undefined {
-    return this.getRendererCache(context);
+  public get defaultChannelId(): string {
+    return this._context.channelId ?? '';
   }
 
-  protected override createRenderer(
-    context: RendererContext,
-    rendererType?: RendererType
-  ): IRenderer {
-    if (rendererType === undefined) {
-      rendererType = this.rendererType;
+  public get defaultRenderMode(): RenderModeType {
+    return this._context.renderMode!;
+  }
+
+  public get defaultMirrorMode(): VideoMirrorModeType {
+    return this._context.mirrorMode!;
+  }
+
+  public release(): void {
+    this.stopRendering();
+    this.clearRendererCache();
+  }
+
+  private presetRendererContext(context: RendererContext): RendererContext {
+    //this is for preset default value
+    context.renderMode = context.renderMode || this.defaultRenderMode;
+    context.mirrorMode = context.mirrorMode || this.defaultMirrorMode;
+    context.useWebCodecsDecoder = context.useWebCodecsDecoder || false;
+    context.enableFps = context.enableFps || false;
+    context.position = context.position || this.defaultObservedFramePosition;
+
+    if (!AgoraEnv.CapabilityManager?.webCodecsDecoderEnabled) {
+      context.useWebCodecsDecoder = false;
     }
 
+    switch (context.sourceType) {
+      case VideoSourceType.VideoSourceRemote:
+        if (context.uid === undefined) {
+          throw new Error('uid is required');
+        }
+        context.channelId = context.channelId ?? this.defaultChannelId;
+        break;
+      case VideoSourceType.VideoSourceMediaPlayer:
+        if (context.mediaPlayerId === undefined) {
+          throw new Error('mediaPlayerId is required');
+        }
+        context.channelId = '';
+        context.uid = context.mediaPlayerId;
+        break;
+      case undefined:
+        if (context.uid) {
+          context.sourceType = VideoSourceType.VideoSourceRemote;
+        }
+        break;
+      default:
+        context.channelId = '';
+        context.uid = 0;
+        break;
+    }
+    return context;
+  }
+
+  public addOrRemoveRenderer(
+    context: RendererContext
+  ): RendererCacheType | undefined {
+    // To be compatible with the old API
+    let { setupMode = VideoViewSetupMode.VideoViewSetupAdd } = context;
+    if (!context.view) setupMode = VideoViewSetupMode.VideoViewSetupRemove;
+    switch (setupMode) {
+      case VideoViewSetupMode.VideoViewSetupAdd:
+        return this.addRendererToCache(context);
+      case VideoViewSetupMode.VideoViewSetupRemove:
+        this.removeRendererFromCache(context);
+        return undefined;
+      case VideoViewSetupMode.VideoViewSetupReplace:
+        this.removeRendererFromCache(context);
+        return this.addRendererToCache(context);
+    }
+  }
+
+  private addRendererToCache(
+    context: RendererContext
+  ): RendererCacheType | undefined {
+    const checkedContext = this.presetRendererContext(context);
+
+    if (!checkedContext.view) return undefined;
+
+    if (this.findRenderer(checkedContext.view)) {
+      throw new Error('You have already added this view to the renderer');
+    }
+
+    let rendererCache = this.getRendererCache(checkedContext);
+    if (!rendererCache) {
+      if (context.useWebCodecsDecoder) {
+        rendererCache = new WebCodecsRendererCache(checkedContext);
+      } else {
+        rendererCache = new RendererCache(checkedContext);
+      }
+      this._rendererCaches.push(rendererCache);
+    }
+    rendererCache.addRenderer(this.createRenderer(checkedContext));
+    if (!context.useWebCodecsDecoder) {
+      this.startRendering();
+    }
+    return rendererCache;
+  }
+
+  public removeRendererFromCache(context: RendererContext): void {
+    const checkedContext = this.presetRendererContext(context);
+
+    const rendererCache = this.getRendererCache(checkedContext);
+    if (!rendererCache) return;
+    if (checkedContext.view) {
+      const renderer = rendererCache.findRenderer(checkedContext.view);
+      if (!renderer) return;
+      rendererCache.removeRenderer(renderer);
+    } else {
+      rendererCache.removeRenderer();
+    }
+    if (rendererCache.renderers.length === 0) {
+      rendererCache.release();
+      this._rendererCaches.splice(
+        this._rendererCaches.indexOf(rendererCache),
+        1
+      );
+    }
+  }
+
+  public clearRendererCache(): void {
+    for (const rendererCache of this._rendererCaches) {
+      rendererCache.release();
+    }
+    this._rendererCaches.splice(0);
+  }
+
+  public getRendererCache(
+    context: RendererContext
+  ): RendererCacheType | undefined {
+    return this._rendererCaches.find(
+      (cache) => cache.key === generateRendererCacheKey(context)
+    );
+  }
+
+  public getRenderers(context: RendererContext): IRenderer[] {
+    return this.getRendererCache(context)?.renderers || [];
+  }
+
+  public findRenderer(view: Element): IRenderer | undefined {
+    for (const rendererCache of this._rendererCaches) {
+      const renderer = rendererCache.findRenderer(view);
+      if (renderer) return renderer;
+    }
+    return undefined;
+  }
+
+  protected createRenderer(
+    context: RendererContext,
+    rendererType: RendererType = this.rendererType
+  ): IRenderer {
     let renderer: IRenderer;
     switch (rendererType) {
       case RendererType.WEBGL:
-        renderer = new WebGLRenderer(
-          this.handleWebGLFallback(context).bind(this)
-        );
+        if (context.useWebCodecsDecoder) {
+          renderer = new WebCodecsRenderer();
+        } else {
+          renderer = new WebGLRenderer(
+            this.handleWebGLFallback(context).bind(this)
+          );
+          renderer.bind(context.view);
+        }
         break;
       case RendererType.SOFTWARE:
         renderer = new YUVCanvasRenderer();
+        renderer.bind(context.view);
         break;
       default:
         throw new Error('Unknown renderer type');
     }
 
-    renderer.bind(context.view);
-    renderer.context = {
-      renderMode: context.renderMode,
-      mirrorMode: context.mirrorMode,
-    };
+    renderer.setContext(context);
     return renderer;
   }
 
-  public override doRendering(rendererCache: RendererCache): void {
+  public startRendering(): void {
+    if (this._renderingTimer) return;
+
+    const renderingLooper = () => {
+      if (this._previousFirstFrameTime === 0) {
+        // Get the current time as the time of the first frame of per second
+        this._previousFirstFrameTime = performance.now();
+        // Reset the frame count
+        this._currentFrameCount = 0;
+      }
+
+      // Increase the frame count
+      ++this._currentFrameCount;
+
+      // Get the current time
+      const currentFrameTime = performance.now();
+      // Calculate the time difference between the current frame and the previous frame
+      const deltaTime = currentFrameTime - this._previousFirstFrameTime;
+      // Calculate the expected time of the current frame
+      const expectedTime = (this._currentFrameCount * 1000) / this.renderingFps;
+      logDebug(
+        new Date().toLocaleTimeString(),
+        'currentFrameCount',
+        this._currentFrameCount,
+        'expectedTime',
+        expectedTime,
+        'deltaTime',
+        deltaTime
+      );
+
+      if (this._rendererCaches.length === 0) {
+        // If there is no renderer, stop rendering
+        this.stopRendering();
+        return;
+      }
+
+      // Render all renderers that do not use WebCodecs
+      for (const rendererCache of this._rendererCaches.filter(
+        (cache) => cache instanceof RendererCache
+      )) {
+        this.doRendering(rendererCache);
+      }
+
+      if (this._currentFrameCount >= this.renderingFps) {
+        this._previousFirstFrameTime = 0;
+      }
+
+      if (deltaTime < expectedTime) {
+        // If the time difference between the current frame and the previous frame is less than the expected time, then wait for the difference
+        this._renderingTimer = window.setTimeout(
+          renderingLooper,
+          expectedTime - deltaTime
+        );
+      } else {
+        // If the time difference between the current frame and the previous frame is greater than the expected time, then render immediately
+        renderingLooper();
+      }
+    };
+    renderingLooper();
+  }
+
+  public doRendering(rendererCache: RendererCacheType): void {
     rendererCache.draw();
   }
 
   private handleWebGLFallback(context: RendererContext): WebGLFallback {
     return (renderer: WebGLRenderer) => {
-      const {
-        context: { renderMode, mirrorMode },
-      } = renderer;
       const renderers = this.getRenderers(context);
       renderer.unbind();
-      const newRenderer = this.createRenderer(
-        { ...context, renderMode, mirrorMode },
-        RendererType.SOFTWARE
-      );
+      const newRenderer = this.createRenderer(context, RendererType.SOFTWARE);
       renderers.splice(renderers.indexOf(renderer), 1, newRenderer);
     };
   }
+
+  public handleWebCodecsFallback(context: RendererCacheContext): void {
+    let engine = createAgoraRtcEngine();
+    engine.getMediaEngine().unregisterVideoEncodedFrameObserver({});
+    if (context.uid) {
+      if (isUseConnection(context)) {
+        engine.setRemoteVideoSubscriptionOptionsEx(
+          context.uid,
+          {
+            type: VideoStreamType.VideoStreamHigh,
+            encodedFrameOnly: false,
+          },
+          {
+            channelId: context.channelId,
+            localUid: context.localUid,
+          }
+        );
+      } else {
+        engine.setRemoteVideoSubscriptionOptions(context.uid, {
+          type: VideoStreamType.VideoStreamHigh,
+          encodedFrameOnly: false,
+        });
+      }
+    }
+    AgoraEnv.enableWebCodecsDecoder = false;
+    AgoraEnv.CapabilityManager?.setWebCodecsDecoderEnabled(false);
+    let renderers = this.getRenderers(context);
+    for (let renderer of renderers) {
+      this.addOrRemoveRenderer({
+        ...renderer.context,
+        setupMode: VideoViewSetupMode.VideoViewSetupReplace,
+      });
+    }
+  }
+
+  public stopRendering(): void {
+    if (this._renderingTimer) {
+      window.clearTimeout(this._renderingTimer);
+      this._renderingTimer = undefined;
+    }
+  }
+
+  public setRendererContext(context: RendererContext): boolean {
+    const checkedContext = this.presetRendererContext(context);
+
+    for (const rendererCache of this._rendererCaches) {
+      const result = rendererCache.setRendererContext(checkedContext);
+      if (result) {
+        return true;
+      }
+    }
+    return false;
+  }
 }

package/ts/Renderer/WebCodecsRenderer/index.ts

@@ -0,0 +1,145 @@
+import { VideoFrame } from '../../Private/AgoraMediaBase';
+import { CodecConfigInfo, RendererType } from '../../Types';
+import { getContextByCanvas } from '../../Utils';
+import { IRenderer } from '../IRenderer';
+
+export class WebCodecsRenderer extends IRenderer {
+  gl?: WebGLRenderingContext | WebGL2RenderingContext | null;
+  // eslint-disable-next-line auto-import/auto-import
+  offscreenCanvas: OffscreenCanvas | undefined;
+
+  constructor() {
+    super();
+    this.rendererType = RendererType.WEBCODECSRENDERER;
+  }
+
+  static vertexShaderSource = `
+    attribute vec2 xy;
+    varying highp vec2 uv;
+    void main(void) {
+      gl_Position = vec4(xy, 0.0, 1.0);
+      // Map vertex coordinates (-1 to +1) to UV coordinates (0 to 1).
+      // UV coordinates are Y-flipped relative to vertex coordinates.
+      uv = vec2((1.0 + xy.x) / 2.0, (1.0 - xy.y) / 2.0);
+    }
+  `;
+  static fragmentShaderSource = `
+    varying highp vec2 uv;
+    uniform sampler2D texture;
+    void main(void) {
+      gl_FragColor = texture2D(texture, uv);
+    }
+  `;
+
+  bind(element: HTMLElement) {
+    super.bind(element);
+    if (!this.canvas) return;
+    this.offscreenCanvas = this.canvas.transferControlToOffscreen();
+    this.gl = getContextByCanvas(this.offscreenCanvas);
+    if (!this.gl) return;
+    const vertexShader = this.gl.createShader(this.gl.VERTEX_SHADER);
+    if (!vertexShader) return;
+    this.gl.shaderSource(vertexShader, WebCodecsRenderer.vertexShaderSource);
+    this.gl.compileShader(vertexShader);
+    if (!this.gl.getShaderParameter(vertexShader, this.gl.COMPILE_STATUS)) {
+      throw this.gl.getShaderInfoLog(vertexShader);
+    }
+    const fragmentShader = this.gl.createShader(this.gl.FRAGMENT_SHADER);
+    if (!fragmentShader) return;
+    this.gl.shaderSource(
+      fragmentShader,
+      WebCodecsRenderer.fragmentShaderSource
+    );
+    this.gl.compileShader(fragmentShader);
+    if (!this.gl.getShaderParameter(fragmentShader, this.gl.COMPILE_STATUS)) {
+      throw this.gl.getShaderInfoLog(fragmentShader);
+    }
+    const shaderProgram = this.gl.createProgram();
+    if (!shaderProgram) return;
+    this.gl.attachShader(shaderProgram, vertexShader);
+    this.gl.attachShader(shaderProgram, fragmentShader);
+    this.gl.linkProgram(shaderProgram);
+    if (!this.gl.getProgramParameter(shaderProgram, this.gl.LINK_STATUS)) {
+      throw this.gl.getProgramInfoLog(shaderProgram);
+    }
+    this.gl.useProgram(shaderProgram);
+    // Vertex coordinates, clockwise from bottom-left.
+    const vertexBuffer = this.gl.createBuffer();
+    this.gl.bindBuffer(this.gl.ARRAY_BUFFER, vertexBuffer);
+    this.gl.bufferData(
+      this.gl.ARRAY_BUFFER,
+      new Float32Array([-1.0, -1.0, -1.0, +1.0, +1.0, +1.0, +1.0, -1.0]),
+      this.gl.STATIC_DRAW
+    );
+    const xyLocation = this.gl.getAttribLocation(shaderProgram, 'xy');
+    this.gl.vertexAttribPointer(xyLocation, 2, this.gl.FLOAT, false, 0, 0);
+    this.gl.enableVertexAttribArray(xyLocation);
+    // Create one texture to upload frames to.
+    const texture = this.gl.createTexture();
+    this.gl.bindTexture(this.gl.TEXTURE_2D, texture);
+    this.gl.texParameteri(
+      this.gl.TEXTURE_2D,
+      this.gl.TEXTURE_MAG_FILTER,
+      this.gl.NEAREST
+    );
+    this.gl.texParameteri(
+      this.gl.TEXTURE_2D,
+      this.gl.TEXTURE_MIN_FILTER,
+      this.gl.NEAREST
+    );
+    this.gl.texParameteri(
+      this.gl.TEXTURE_2D,
+      this.gl.TEXTURE_WRAP_S,
+      this.gl.CLAMP_TO_EDGE
+    );
+    this.gl.texParameteri(
+      this.gl.TEXTURE_2D,
+      this.gl.TEXTURE_WRAP_T,
+      this.gl.CLAMP_TO_EDGE
+    );
+  }
+
+  drawFrame(frame: any, _codecConfig: CodecConfigInfo) {
+    if (!this.offscreenCanvas || !frame) return;
+
+    this.offscreenCanvas.width = _codecConfig.codedWidth!;
+    this.offscreenCanvas.height = _codecConfig.codedHeight!;
+
+    this.updateRenderMode();
+    this.rotateCanvas({
+      rotation: _codecConfig.rotation,
+    });
+    if (!this.gl) return;
+
+    if (this.gl) {
+      // Upload the frame.
+      this.gl.texImage2D(
+        this.gl.TEXTURE_2D,
+        0,
+        this.gl.RGBA,
+        this.gl.RGBA,
+        this.gl.UNSIGNED_BYTE,
+        frame
+      );
+      frame.close();
+      // Configure and clear the drawing area.
+      this.gl.viewport(
+        0,
+        0,
+        this.gl.drawingBufferWidth,
+        this.gl.drawingBufferHeight
+      );
+      this.gl.clearColor(0.0, 0.0, 0.0, 0.0);
+      this.gl.clear(this.gl.COLOR_BUFFER_BIT);
+      // Draw the frame.
+      this.gl.drawArrays(this.gl.TRIANGLE_FAN, 0, 4);
+    }
+    super.drawFrame();
+    this.getFps();
+  }
+
+  protected override rotateCanvas({ rotation }: VideoFrame) {
+    if (!this.offscreenCanvas || !this.canvas) return;
+    this.canvas.style.transform += ` rotateZ(${rotation}deg)`;
+  }
+}