agora-electron-sdk 4.5.1 → 4.5.2-dev.2

package/ts/Renderer/WebCodecsRendererCache.ts

@@ -0,0 +1,137 @@
+import createAgoraRtcEngine from '../AgoraSdk';
+import { WebCodecsDecoder } from '../Decoder/index';
+import { EncodedVideoFrameInfo, VideoStreamType } from '../Private/AgoraBase';
+import { IRtcEngineEx } from '../Private/IAgoraRtcEngineEx';
+import { AgoraElectronBridge } from '../Private/internal/IrisApiEngine';
+
+import { RendererContext, RendererType } from '../Types';
+import { AgoraEnv, logInfo } from '../Utils';
+
+import { IRendererCache, isUseConnection } from './IRendererCache';
+import { WebCodecsRenderer } from './WebCodecsRenderer/index';
+
+export class WebCodecsRendererCache extends IRendererCache {
+  private _decoder?: WebCodecsDecoder | null;
+  private _engine?: IRtcEngineEx;
+  private _firstFrame = true;
+
+  constructor(context: RendererContext) {
+    super(context);
+    this._engine = createAgoraRtcEngine();
+    this._decoder = new WebCodecsDecoder(
+      this.renderers as WebCodecsRenderer[],
+      this.onDecoderError.bind(this),
+      context
+    );
+    this.draw();
+  }
+
+  onDecoderError(e: any) {
+    logInfo('webCodecsDecoder decode failed, fallback to native decoder', e);
+    AgoraEnv.AgoraRendererManager?.handleWebCodecsFallback(this.cacheContext);
+  }
+
+  onEncodedVideoFrameReceived(...[data, buffer]: any) {
+    let _data: any;
+    try {
+      _data = JSON.parse(data) ?? {};
+    } catch (e) {
+      _data = {};
+    }
+    if (
+      Object.keys(_data).length === 0 ||
+      !this._decoder ||
+      this.cacheContext.uid !== _data.uid
+    )
+      return;
+    if (this._firstFrame) {
+      for (let renderer of this.renderers) {
+        if (renderer.rendererType !== RendererType.WEBCODECSRENDERER) {
+          continue;
+        }
+        renderer.bind(renderer.context.view);
+      }
+
+      try {
+        this._decoder.decoderConfigure(_data.videoEncodedFrameInfo);
+      } catch (error: any) {
+        logInfo(error);
+        return;
+      }
+      this._firstFrame = false;
+    }
+    if (this.shouldFallback(_data.videoEncodedFrameInfo)) {
+      AgoraEnv.AgoraRendererManager?.handleWebCodecsFallback(this.cacheContext);
+    } else {
+      this._decoder.decodeFrame(
+        buffer,
+        _data.videoEncodedFrameInfo,
+        new Date().getTime()
+      );
+    }
+  }
+
+  public draw() {
+    if (isUseConnection(this.cacheContext)) {
+      this._engine?.setRemoteVideoSubscriptionOptionsEx(
+        this.cacheContext.uid!,
+        {
+          type: VideoStreamType.VideoStreamHigh,
+          encodedFrameOnly: true,
+        },
+        {
+          channelId: this.cacheContext.channelId,
+          localUid: this.cacheContext.localUid,
+        }
+      );
+    } else {
+      this._engine?.setRemoteVideoSubscriptionOptions(this.cacheContext.uid!, {
+        type: VideoStreamType.VideoStreamHigh,
+        encodedFrameOnly: true,
+      });
+    }
+    AgoraElectronBridge.OnEvent(
+      `call_back_with_encoded_video_frame_${this.cacheContext.uid}`,
+      (...params: any) => {
+        try {
+          this.onEncodedVideoFrameReceived(...params);
+        } catch (e) {
+          console.error(e);
+        }
+      }
+    );
+  }
+
+  public shouldFallback(frameInfo: EncodedVideoFrameInfo): boolean {
+    let shouldFallback = false;
+    if (!frameInfo.codecType) {
+      shouldFallback = true;
+      logInfo('codecType is not supported, fallback to native decoder');
+    } else {
+      const mapping =
+        AgoraEnv.CapabilityManager?.frameCodecMapping[frameInfo.codecType];
+      if (mapping === undefined) {
+        shouldFallback = true;
+        logInfo('codecType is not supported, fallback to native decoder');
+      } else if (
+        mapping.minWidth >= frameInfo.width! &&
+        mapping.minHeight >= frameInfo.height! &&
+        mapping.maxWidth <= frameInfo.width! &&
+        mapping.maxHeight <= frameInfo.height!
+      ) {
+        shouldFallback = true;
+        logInfo('frame size is not supported, fallback to native decoder');
+      }
+    }
+    return shouldFallback;
+  }
+
+  public release(): void {
+    AgoraElectronBridge.UnEvent(
+      `call_back_with_encoded_video_frame_${this.cacheContext.uid}`
+    );
+    this._decoder?.release();
+    this._decoder = null;
+    super.release();
+  }
+}

package/ts/Renderer/WebGLRenderer/index.ts

@@ -1,5 +1,6 @@
 import { VideoFrame } from '../../Private/AgoraMediaBase';
-import { logWarn } from '../../Utils';
+import { RendererType } from '../../Types';
+import { AgoraEnv, logWarn } from '../../Utils';
 import { IRenderer } from '../IRenderer';
 
 export type WebGLFallback = (renderer: WebGLRenderer, error: Error) => void;
@@ -7,59 +8,72 @@ export type WebGLFallback = (renderer: WebGLRenderer, error: Error) => void;
 const createProgramFromSources =
   require('./webgl-utils').createProgramFromSources;
 
-const vertexShaderSource =
-  'attribute vec2 a_position;' +
-  'attribute vec2 a_texCoord;' +
-  'uniform vec2 u_resolution;' +
-  'varying vec2 v_texCoord;' +
-  'void main() {' +
-  'vec2 zeroToOne = a_position / u_resolution;' +
-  '   vec2 zeroToTwo = zeroToOne * 2.0;' +
-  '   vec2 clipSpace = zeroToTwo - 1.0;' +
-  '   gl_Position = vec4(clipSpace * vec2(1, -1), 0, 1);' +
-  'v_texCoord = a_texCoord;' +
-  '}';
-const yuvShaderSource =
-  'precision mediump float;' +
-  'uniform sampler2D Ytex;' +
-  'uniform sampler2D Utex;' +
-  'uniform sampler2D Vtex;' +
-  'varying vec2 v_texCoord;' +
-  'void main(void) {' +
-  '  float nx,ny,r,g,b,y,u,v;' +
-  '  mediump vec4 txl,ux,vx;' +
-  '  nx=v_texCoord[0];' +
-  '  ny=v_texCoord[1];' +
-  '  y=texture2D(Ytex,vec2(nx,ny)).r;' +
-  '  u=texture2D(Utex,vec2(nx,ny)).r;' +
-  '  v=texture2D(Vtex,vec2(nx,ny)).r;' +
-  '  y=1.1643*(y-0.0625);' +
-  '  u=u-0.5;' +
-  '  v=v-0.5;' +
-  '  r=y+1.5958*v;' +
-  '  g=y-0.39173*u-0.81290*v;' +
-  '  b=y+2.017*u;' +
-  '  gl_FragColor=vec4(r,g,b,1.0);' +
-  '}';
+const vertexShaderSource = `
+  attribute vec2 a_position;
+  attribute vec2 a_texCoord;
+  uniform vec2 u_resolution;
+  varying vec2 v_texCoord;
+  void main() {
+    vec2 zeroToOne = a_position / u_resolution;
+    vec2 zeroToTwo = zeroToOne * 2.0;
+    vec2 clipSpace = zeroToTwo - 1.0;
+    gl_Position = vec4(clipSpace * vec2(1, -1), 0, 1);
+    v_texCoord = a_texCoord;
+  }`;
+const yuvShaderSource = `
+  precision mediump float;
+  uniform sampler2D Ytex;
+  uniform sampler2D Utex;
+  uniform sampler2D Vtex;
+  uniform sampler2D Atex;
+  uniform bool hasAlpha;
+  varying vec2 v_texCoord;
+  void main(void) {
+    float nx,ny,r,g,b,y,u,v,a;
+    mediump vec4 txl,ux,vx;
+    nx=v_texCoord[0];
+    ny=v_texCoord[1];
+    y=texture2D(Ytex,vec2(nx,ny)).r;
+    u=texture2D(Utex,vec2(nx,ny)).r;
+    v=texture2D(Vtex,vec2(nx,ny)).r;
+    if (hasAlpha) {
+      a=texture2D(Atex,vec2(nx,ny)).r;
+    } else {
+      a=1.0;
+    }
+    y=1.1643*(y-0.0625);
+    u=u-0.5;
+    v=v-0.5;
+    r=y+1.5958*v;
+    g=y-0.39173*u-0.81290*v;
+    b=y+2.017*u;
+    gl_FragColor=vec4(r,g,b,a);
+  }`;
 
 export class WebGLRenderer extends IRenderer {
-  gl?: WebGLRenderingContext | WebGL2RenderingContext | null;
-  program?: WebGLProgram;
+  gl: WebGLRenderingContext | WebGL2RenderingContext | null;
+  program: WebGLProgram | null;
   positionLocation?: number;
   texCoordLocation?: number;
   yTexture: WebGLTexture | null;
   uTexture: WebGLTexture | null;
   vTexture: WebGLTexture | null;
+  aTexture: WebGLTexture | null;
+  hasAlpha: WebGLUniformLocation | null;
   texCoordBuffer: WebGLBuffer | null;
   surfaceBuffer: WebGLBuffer | null;
   fallback?: WebGLFallback;
 
   constructor(fallback?: WebGLFallback) {
     super();
-    this.gl = undefined;
+    this.gl = null;
+    this.rendererType = RendererType.WEBGL;
+    this.program = null;
     this.yTexture = null;
     this.uTexture = null;
     this.vTexture = null;
+    this.aTexture = null;
+    this.hasAlpha = null;
     this.texCoordBuffer = null;
     this.surfaceBuffer = null;
     this.fallback = fallback;
@@ -87,10 +101,10 @@ export class WebGLRenderer extends IRenderer {
         const context = this.canvas?.getContext(contextName, {
           depth: true,
           stencil: true,
-          alpha: false,
+          alpha: true,
           antialias: false,
           premultipliedAlpha: true,
-          preserveDrawingBuffer: true,
+          preserveDrawingBuffer: !AgoraEnv.encodeAlpha,
           powerPreference: 'default',
           failIfMajorPerformanceCaveat: false,
         });
@@ -112,11 +126,15 @@ export class WebGLRenderer extends IRenderer {
     }
 
     // Set clear color to black, fully opaque
-    this.gl.clearColor(0.0, 0.0, 0.0, 1.0);
+    this.gl.clearColor(0.0, 0.0, 0.0, 0.0);
     // Enable depth testing
     this.gl.enable(this.gl.DEPTH_TEST);
     // Near things obscure far things
     this.gl.depthFunc(this.gl.LEQUAL);
+    // Enable blending
+    this.gl.enable(this.gl.BLEND);
+    // Set blending function
+    this.gl.blendFunc(this.gl.SRC_ALPHA, this.gl.ONE_MINUS_SRC_ALPHA);
     // Clear the color as well as the depth buffer.
     this.gl.clear(
       this.gl.COLOR_BUFFER_BIT |
@@ -147,7 +165,7 @@ export class WebGLRenderer extends IRenderer {
     );
 
     this.releaseTextures();
-    this.gl = undefined;
+    this.gl = null;
 
     super.unbind();
   }
@@ -162,6 +180,7 @@ export class WebGLRenderer extends IRenderer {
     uBuffer,
     vBuffer,
     rotation,
+    alphaBuffer,
   }: VideoFrame) {
     this.rotateCanvas({ width, height, rotation });
     this.updateRenderMode();
@@ -206,57 +225,81 @@ export class WebGLRenderer extends IRenderer {
 
     this.gl.pixelStorei(this.gl.UNPACK_ALIGNMENT, 1);
 
-    this.gl.activeTexture(this.gl.TEXTURE0);
-    this.gl.bindTexture(this.gl.TEXTURE_2D, this.yTexture);
-    this.gl.texImage2D(
-      this.gl.TEXTURE_2D,
-      0,
-      this.gl.LUMINANCE,
-      // Should use xWidth instead of width here (yStide)
-      xWidth,
-      height!,
-      0,
-      this.gl.LUMINANCE,
-      this.gl.UNSIGNED_BYTE,
-      yBuffer!
-    );
+    type TextureInfo = {
+      texture: WebGLTexture | null;
+      stride: number;
+      height: number;
+      pixels: ArrayBufferView | null;
+    };
 
-    this.gl.activeTexture(this.gl.TEXTURE1);
-    this.gl.bindTexture(this.gl.TEXTURE_2D, this.uTexture);
-    this.gl.texImage2D(
-      this.gl.TEXTURE_2D,
-      0,
-      this.gl.LUMINANCE,
-      uStride!,
-      height! / 2,
-      0,
-      this.gl.LUMINANCE,
-      this.gl.UNSIGNED_BYTE,
-      uBuffer!
-    );
+    const activeTexture = (
+      textureIndex: number,
+      { texture, stride, height, pixels }: TextureInfo
+    ) => {
+      if (!this.gl) return;
 
-    this.gl.activeTexture(this.gl.TEXTURE2);
-    this.gl.bindTexture(this.gl.TEXTURE_2D, this.vTexture);
-    this.gl.texImage2D(
-      this.gl.TEXTURE_2D,
-      0,
-      this.gl.LUMINANCE,
-      vStride!,
-      height! / 2,
-      0,
-      this.gl.LUMINANCE,
-      this.gl.UNSIGNED_BYTE,
-      vBuffer!
-    );
+      this.gl.activeTexture(textureIndex);
+      this.gl.bindTexture(this.gl.TEXTURE_2D, texture);
+      this.gl.texImage2D(
+        this.gl.TEXTURE_2D,
+        0,
+        this.gl.LUMINANCE,
+        stride,
+        height,
+        0,
+        this.gl.LUMINANCE,
+        this.gl.UNSIGNED_BYTE,
+        pixels
+      );
+    };
+
+    const textures: Record<number, TextureInfo> = {
+      [this.gl.TEXTURE0]: {
+        texture: this.yTexture,
+        stride: yStride!,
+        height: height!,
+        pixels: yBuffer!,
+      },
+      [this.gl.TEXTURE1]: {
+        texture: this.uTexture,
+        stride: uStride!,
+        height: height! / 2,
+        pixels: uBuffer!,
+      },
+      [this.gl.TEXTURE2]: {
+        texture: this.vTexture,
+        stride: vStride!,
+        height: height! / 2,
+        pixels: vBuffer!,
+      },
+    };
+    if (alphaBuffer) {
+      textures[this.gl.TEXTURE3] = {
+        texture: this.aTexture,
+        stride: width!,
+        height: height!,
+        pixels: alphaBuffer,
+      };
+      this.gl.uniform1i(this.hasAlpha, 1);
+    } else {
+      this.gl.uniform1i(this.hasAlpha, 0);
+    }
+
+    for (const textureIndex in textures) {
+      if (textures.hasOwnProperty(textureIndex)) {
+        activeTexture(+textureIndex, textures[textureIndex]!);
+      }
+    }
 
     this.gl.drawArrays(this.gl.TRIANGLES, 0, 6);
+
     super.drawFrame();
   }
 
   protected override rotateCanvas({ width, height, rotation }: VideoFrame) {
     super.rotateCanvas({ width, height, rotation });
 
-    if (!this.gl) return;
+    if (!this.gl || !this.program) return;
 
     this.gl.viewport(0, 0, width!, height!);
 
@@ -328,34 +371,40 @@ export class WebGLRenderer extends IRenderer {
     );
 
     const resolutionLocation = this.gl.getUniformLocation(
-      this.program!,
+      this.program,
       'u_resolution'
     );
     this.gl.uniform2f(resolutionLocation, width!, height!);
   }
 
   private initTextures() {
-    if (!this.gl) return;
+    if (!this.gl || !this.program) return;
 
     this.positionLocation = this.gl.getAttribLocation(
-      this.program!,
+      this.program,
       'a_position'
     );
     this.texCoordLocation = this.gl.getAttribLocation(
-      this.program!,
+      this.program,
       'a_texCoord'
     );
 
+    this.hasAlpha = this.gl.getUniformLocation(this.program, 'hasAlpha');
+
     this.surfaceBuffer = this.gl.createBuffer();
     this.texCoordBuffer = this.gl.createBuffer();
 
-    const createTexture = (textureIndex: number) => {
-      if (!this.gl) return null;
+    const createTexture = (
+      texture: number,
+      textureIndex: number,
+      textureName: string
+    ) => {
+      if (!this.gl || !this.program) return null;
 
       // Create a texture.
-      this.gl.activeTexture(textureIndex);
-      const texture = this.gl.createTexture();
-      this.gl.bindTexture(this.gl.TEXTURE_2D, texture);
+      this.gl.activeTexture(texture);
+      const textureObj = this.gl.createTexture();
+      this.gl.bindTexture(this.gl.TEXTURE_2D, textureObj);
       // Set the parameters so we can render any size
       this.gl.texParameteri(
         this.gl.TEXTURE_2D,
@@ -377,26 +426,23 @@ export class WebGLRenderer extends IRenderer {
         this.gl.TEXTURE_MAG_FILTER,
         this.gl.NEAREST
       );
-      return texture;
-    };
-
-    this.yTexture = createTexture(this.gl.TEXTURE0);
-    this.uTexture = createTexture(this.gl.TEXTURE1);
-    this.vTexture = createTexture(this.gl.TEXTURE2);
 
-    const y = this.gl.getUniformLocation(this.program!, 'Ytex');
-    this.gl.uniform1i(y, 0); /* Bind Ytex to texture unit 0 */
-
-    const u = this.gl.getUniformLocation(this.program!, 'Utex');
-    this.gl.uniform1i(u, 1); /* Bind Utex to texture unit 1 */
+      this.gl.uniform1i(
+        this.gl.getUniformLocation(this.program, textureName),
+        textureIndex
+      ); /* Bind Ytex to texture unit index */
+      return textureObj;
+    };
 
-    const v = this.gl.getUniformLocation(this.program!, 'Vtex');
-    this.gl.uniform1i(v, 2); /* Bind Vtex to texture unit 2 */
+    this.yTexture = createTexture(this.gl.TEXTURE0, 0, 'Ytex');
+    this.uTexture = createTexture(this.gl.TEXTURE1, 1, 'Utex');
+    this.vTexture = createTexture(this.gl.TEXTURE2, 2, 'Vtex');
+    this.aTexture = createTexture(this.gl.TEXTURE3, 3, 'Atex');
   }
 
   private releaseTextures() {
-    this.gl?.deleteProgram(this.program!);
-    this.program = undefined;
+    this.gl?.deleteProgram(this.program);
+    this.program = null;
 
     this.positionLocation = undefined;
     this.texCoordLocation = undefined;

package/ts/Renderer/YUVCanvasRenderer/index.ts

@@ -24,35 +24,37 @@ export class YUVCanvasRenderer extends IRenderer {
     uBuffer,
     vBuffer,
     rotation,
+    alphaBuffer,
   }: VideoFrame) {
     this.rotateCanvas({ width, height, rotation });
     this.updateRenderMode();
 
     if (!this.frameSink) return;
 
-    this.frameSink.drawFrame(
-      YUVBuffer.frame(
-        YUVBuffer.format({
-          width,
-          height,
-          chromaWidth: width! / 2,
-          chromaHeight: height! / 2,
-          cropLeft: yStride! - width!,
-        }),
-        {
-          bytes: yBuffer,
-          stride: yStride,
-        },
-        {
-          bytes: uBuffer,
-          stride: uStride,
-        },
-        {
-          bytes: vBuffer,
-          stride: vStride,
-        }
-      )
+    const frame = YUVBuffer.frame(
+      YUVBuffer.format({
+        width,
+        height,
+        chromaWidth: width! / 2,
+        chromaHeight: height! / 2,
+        cropLeft: yStride! - width!,
+      }),
+      {
+        bytes: yBuffer,
+        stride: yStride,
+      },
+      {
+        bytes: uBuffer,
+        stride: uStride,
+      },
+      {
+        bytes: vBuffer,
+        stride: vStride,
+      }
     );
+    frame.a = alphaBuffer;
+    this.frameSink.drawFrame(frame);
+
     super.drawFrame();
   }
 

package/ts/Renderer/index.ts

@@ -1,2 +1 @@
 export * from './IRenderer';
-export * from './IRendererManager';