agora-electron-sdk 0.0.0-dev.1

package/js/Renderer/WebCodecsRenderer/index.js

@@ -0,0 +1,109 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WebCodecsRenderer = void 0;
+const Types_1 = require("../../Types");
+const Utils_1 = require("../../Utils");
+const IRenderer_1 = require("../IRenderer");
+class WebCodecsRenderer extends IRenderer_1.IRenderer {
+    constructor() {
+        super();
+        this.rendererType = Types_1.RendererType.WEBCODECSRENDERER;
+    }
+    bind(element) {
+        super.bind(element);
+        if (!this.canvas)
+            return;
+        this.offscreenCanvas = this.canvas.transferControlToOffscreen();
+        this.gl = (0, Utils_1.getContextByCanvas)(this.offscreenCanvas);
+        if (!this.gl)
+            return;
+        const vertexShader = this.gl.createShader(this.gl.VERTEX_SHADER);
+        if (!vertexShader)
+            return;
+        this.gl.shaderSource(vertexShader, WebCodecsRenderer.vertexShaderSource);
+        this.gl.compileShader(vertexShader);
+        if (!this.gl.getShaderParameter(vertexShader, this.gl.COMPILE_STATUS)) {
+            throw this.gl.getShaderInfoLog(vertexShader);
+        }
+        const fragmentShader = this.gl.createShader(this.gl.FRAGMENT_SHADER);
+        if (!fragmentShader)
+            return;
+        this.gl.shaderSource(fragmentShader, WebCodecsRenderer.fragmentShaderSource);
+        this.gl.compileShader(fragmentShader);
+        if (!this.gl.getShaderParameter(fragmentShader, this.gl.COMPILE_STATUS)) {
+            throw this.gl.getShaderInfoLog(fragmentShader);
+        }
+        const shaderProgram = this.gl.createProgram();
+        if (!shaderProgram)
+            return;
+        this.gl.attachShader(shaderProgram, vertexShader);
+        this.gl.attachShader(shaderProgram, fragmentShader);
+        this.gl.linkProgram(shaderProgram);
+        if (!this.gl.getProgramParameter(shaderProgram, this.gl.LINK_STATUS)) {
+            throw this.gl.getProgramInfoLog(shaderProgram);
+        }
+        this.gl.useProgram(shaderProgram);
+        // Vertex coordinates, clockwise from bottom-left.
+        const vertexBuffer = this.gl.createBuffer();
+        this.gl.bindBuffer(this.gl.ARRAY_BUFFER, vertexBuffer);
+        this.gl.bufferData(this.gl.ARRAY_BUFFER, new Float32Array([-1.0, -1.0, -1.0, +1.0, +1.0, +1.0, +1.0, -1.0]), this.gl.STATIC_DRAW);
+        const xyLocation = this.gl.getAttribLocation(shaderProgram, 'xy');
+        this.gl.vertexAttribPointer(xyLocation, 2, this.gl.FLOAT, false, 0, 0);
+        this.gl.enableVertexAttribArray(xyLocation);
+        // Create one texture to upload frames to.
+        const texture = this.gl.createTexture();
+        this.gl.bindTexture(this.gl.TEXTURE_2D, texture);
+        this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MAG_FILTER, this.gl.NEAREST);
+        this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MIN_FILTER, this.gl.NEAREST);
+        this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_S, this.gl.CLAMP_TO_EDGE);
+        this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_T, this.gl.CLAMP_TO_EDGE);
+    }
+    drawFrame(frame, _codecConfig) {
+        if (!this.offscreenCanvas || !frame)
+            return;
+        this.offscreenCanvas.width = _codecConfig.codedWidth;
+        this.offscreenCanvas.height = _codecConfig.codedHeight;
+        this.updateRenderMode();
+        this.rotateCanvas({
+            rotation: _codecConfig.rotation,
+        });
+        if (!this.gl)
+            return;
+        if (this.gl) {
+            // Upload the frame.
+            this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, this.gl.RGBA, this.gl.UNSIGNED_BYTE, frame);
+            frame.close();
+            // Configure and clear the drawing area.
+            this.gl.viewport(0, 0, this.gl.drawingBufferWidth, this.gl.drawingBufferHeight);
+            this.gl.clearColor(0.0, 0.0, 0.0, 0.0);
+            this.gl.clear(this.gl.COLOR_BUFFER_BIT);
+            // Draw the frame.
+            this.gl.drawArrays(this.gl.TRIANGLE_FAN, 0, 4);
+        }
+        super.drawFrame();
+        this.getFps();
+    }
+    rotateCanvas({ rotation }) {
+        if (!this.offscreenCanvas || !this.canvas)
+            return;
+        this.canvas.style.transform += ` rotateZ(${rotation}deg)`;
+    }
+}
+exports.WebCodecsRenderer = WebCodecsRenderer;
+WebCodecsRenderer.vertexShaderSource = `
+    attribute vec2 xy;
+    varying highp vec2 uv;
+    void main(void) {
+      gl_Position = vec4(xy, 0.0, 1.0);
+      // Map vertex coordinates (-1 to +1) to UV coordinates (0 to 1).
+      // UV coordinates are Y-flipped relative to vertex coordinates.
+      uv = vec2((1.0 + xy.x) / 2.0, (1.0 - xy.y) / 2.0);
+    }
+  `;
+WebCodecsRenderer.fragmentShaderSource = `
+    varying highp vec2 uv;
+    uniform sampler2D texture;
+    void main(void) {
+      gl_FragColor = texture2D(texture, uv);
+    }
+  `;

package/js/Renderer/WebCodecsRendererCache.js

@@ -0,0 +1,115 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WebCodecsRendererCache = void 0;
+const AgoraSdk_1 = __importDefault(require("../AgoraSdk"));
+const index_1 = require("../Decoder/index");
+const AgoraBase_1 = require("../Private/AgoraBase");
+const IrisApiEngine_1 = require("../Private/internal/IrisApiEngine");
+const Types_1 = require("../Types");
+const Utils_1 = require("../Utils");
+const IRendererCache_1 = require("./IRendererCache");
+class WebCodecsRendererCache extends IRendererCache_1.IRendererCache {
+    constructor(context) {
+        super(context);
+        this._firstFrame = true;
+        this._engine = (0, AgoraSdk_1.default)();
+        this._decoder = new index_1.WebCodecsDecoder(this.renderers, this.onDecoderError.bind(this), context);
+        this.draw();
+    }
+    onDecoderError(e) {
+        (0, Utils_1.logInfo)('webCodecsDecoder decode failed, fallback to native decoder', e);
+        Utils_1.AgoraEnv.AgoraRendererManager?.handleWebCodecsFallback(this.cacheContext);
+    }
+    onEncodedVideoFrameReceived(...[data, buffer]) {
+        let _data;
+        try {
+            _data = JSON.parse(data) ?? {};
+        }
+        catch (e) {
+            _data = {};
+        }
+        if (Object.keys(_data).length === 0 ||
+            !this._decoder ||
+            this.cacheContext.uid !== _data.uid)
+            return;
+        if (this._firstFrame) {
+            for (let renderer of this.renderers) {
+                if (renderer.rendererType !== Types_1.RendererType.WEBCODECSRENDERER) {
+                    continue;
+                }
+                renderer.bind(renderer.context.view);
+            }
+            try {
+                this._decoder.decoderConfigure(_data.videoEncodedFrameInfo);
+            }
+            catch (error) {
+                (0, Utils_1.logInfo)(error);
+                return;
+            }
+            this._firstFrame = false;
+        }
+        if (this.shouldFallback(_data.videoEncodedFrameInfo)) {
+            Utils_1.AgoraEnv.AgoraRendererManager?.handleWebCodecsFallback(this.cacheContext);
+        }
+        else {
+            this._decoder.decodeFrame(buffer, _data.videoEncodedFrameInfo, new Date().getTime());
+        }
+    }
+    draw() {
+        if ((0, IRendererCache_1.isUseConnection)(this.cacheContext)) {
+            this._engine?.setRemoteVideoSubscriptionOptionsEx(this.cacheContext.uid, {
+                type: AgoraBase_1.VideoStreamType.VideoStreamHigh,
+                encodedFrameOnly: true,
+            }, {
+                channelId: this.cacheContext.channelId,
+                localUid: this.cacheContext.localUid,
+            });
+        }
+        else {
+            this._engine?.setRemoteVideoSubscriptionOptions(this.cacheContext.uid, {
+                type: AgoraBase_1.VideoStreamType.VideoStreamHigh,
+                encodedFrameOnly: true,
+            });
+        }
+        IrisApiEngine_1.AgoraElectronBridge.OnEvent(`call_back_with_encoded_video_frame_${this.cacheContext.uid}`, (...params) => {
+            try {
+                this.onEncodedVideoFrameReceived(...params);
+            }
+            catch (e) {
+                console.error(e);
+            }
+        });
+    }
+    shouldFallback(frameInfo) {
+        let shouldFallback = false;
+        if (!frameInfo.codecType) {
+            shouldFallback = true;
+            (0, Utils_1.logInfo)('codecType is not supported, fallback to native decoder');
+        }
+        else {
+            const mapping = Utils_1.AgoraEnv.CapabilityManager?.frameCodecMapping[frameInfo.codecType];
+            if (mapping === undefined) {
+                shouldFallback = true;
+                (0, Utils_1.logInfo)('codecType is not supported, fallback to native decoder');
+            }
+            else if (mapping.minWidth >= frameInfo.width &&
+                mapping.minHeight >= frameInfo.height &&
+                mapping.maxWidth <= frameInfo.width &&
+                mapping.maxHeight <= frameInfo.height) {
+                shouldFallback = true;
+                (0, Utils_1.logInfo)('frame size is not supported, fallback to native decoder');
+            }
+        }
+        return shouldFallback;
+    }
+    release() {
+        IrisApiEngine_1.AgoraElectronBridge.UnEvent(`call_back_with_encoded_video_frame_${this.cacheContext.uid}`);
+        this._decoder?.release();
+        this._decoder = null;
+        super.release();
+    }
+}
+exports.WebCodecsRendererCache = WebCodecsRendererCache;

package/js/Renderer/WebGLRenderer/index.js

@@ -0,0 +1,314 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WebGLRenderer = void 0;
+const Types_1 = require("../../Types");
+const Utils_1 = require("../../Utils");
+const IRenderer_1 = require("../IRenderer");
+const createProgramFromSources = require('./webgl-utils').createProgramFromSources;
+const vertexShaderSource = `
+  attribute vec2 a_position;
+  attribute vec2 a_texCoord;
+  uniform vec2 u_resolution;
+  varying vec2 v_texCoord;
+  void main() {
+    vec2 zeroToOne = a_position / u_resolution;
+    vec2 zeroToTwo = zeroToOne * 2.0;
+    vec2 clipSpace = zeroToTwo - 1.0;
+    gl_Position = vec4(clipSpace * vec2(1, -1), 0, 1);
+    v_texCoord = a_texCoord;
+  }`;
+const yuvShaderSource = `
+  precision mediump float;
+  uniform sampler2D Ytex;
+  uniform sampler2D Utex;
+  uniform sampler2D Vtex;
+  uniform sampler2D Atex;
+  uniform bool hasAlpha;
+  varying vec2 v_texCoord;
+  void main(void) {
+    float nx,ny,r,g,b,y,u,v,a;
+    mediump vec4 txl,ux,vx;
+    nx=v_texCoord[0];
+    ny=v_texCoord[1];
+    y=texture2D(Ytex,vec2(nx,ny)).r;
+    u=texture2D(Utex,vec2(nx,ny)).r;
+    v=texture2D(Vtex,vec2(nx,ny)).r;
+    if (hasAlpha) {
+      a=texture2D(Atex,vec2(nx,ny)).r;
+    } else {
+      a=1.0;
+    }
+    y=1.1643*(y-0.0625);
+    u=u-0.5;
+    v=v-0.5;
+    r=y+1.5958*v;
+    g=y-0.39173*u-0.81290*v;
+    b=y+2.017*u;
+    gl_FragColor=vec4(r,g,b,a);
+  }`;
+class WebGLRenderer extends IRenderer_1.IRenderer {
+    constructor(fallback) {
+        super();
+        this.handleContextLost = (event) => {
+            event.preventDefault();
+            (0, Utils_1.logWarn)('webglcontextlost', event);
+            this.releaseTextures();
+            this.fallback?.call(null, this, new Error('Browser not support! No WebGL detected.'));
+        };
+        this.handleContextRestored = (event) => {
+            event.preventDefault();
+            (0, Utils_1.logWarn)('webglcontextrestored', event);
+            // Setup GLSL program
+            this.program = createProgramFromSources(this.gl, [
+                vertexShaderSource,
+                yuvShaderSource,
+            ]);
+            this.gl?.useProgram(this.program);
+            this.initTextures();
+        };
+        this.gl = null;
+        this.rendererType = Types_1.RendererType.WEBGL;
+        this.program = null;
+        this.yTexture = null;
+        this.uTexture = null;
+        this.vTexture = null;
+        this.aTexture = null;
+        this.hasAlpha = null;
+        this.texCoordBuffer = null;
+        this.surfaceBuffer = null;
+        this.fallback = fallback;
+    }
+    bind(view) {
+        super.bind(view);
+        this.canvas?.addEventListener('webglcontextlost', this.handleContextLost, false);
+        this.canvas?.addEventListener('webglcontextrestored', this.handleContextRestored, false);
+        const getContext = (contextNames = ['webgl2', 'webgl', 'experimental-webgl']) => {
+            for (let i = 0; i < contextNames.length; i++) {
+                const contextName = contextNames[i];
+                const context = this.canvas?.getContext(contextName, {
+                    depth: true,
+                    stencil: true,
+                    alpha: true,
+                    antialias: false,
+                    premultipliedAlpha: true,
+                    preserveDrawingBuffer: !Utils_1.AgoraEnv.encodeAlpha,
+                    powerPreference: 'default',
+                    failIfMajorPerformanceCaveat: false,
+                });
+                if (context) {
+                    return context;
+                }
+            }
+            return null;
+        };
+        this.gl ?? (this.gl = getContext());
+        if (!this.gl) {
+            this.fallback?.call(null, this, new Error('Browser not support! No WebGL detected.'));
+            return;
+        }
+        // Set clear color to black, fully opaque
+        this.gl.clearColor(0.0, 0.0, 0.0, 0.0);
+        // Enable depth testing
+        this.gl.enable(this.gl.DEPTH_TEST);
+        // Near things obscure far things
+        this.gl.depthFunc(this.gl.LEQUAL);
+        // Enable blending
+        this.gl.enable(this.gl.BLEND);
+        // Set blending function
+        this.gl.blendFunc(this.gl.SRC_ALPHA, this.gl.ONE_MINUS_SRC_ALPHA);
+        // Clear the color as well as the depth buffer.
+        this.gl.clear(this.gl.COLOR_BUFFER_BIT |
+            this.gl.DEPTH_BUFFER_BIT |
+            this.gl.STENCIL_BUFFER_BIT);
+        // Setup GLSL program
+        this.program = createProgramFromSources(this.gl, [
+            vertexShaderSource,
+            yuvShaderSource,
+        ]);
+        this.gl.useProgram(this.program);
+        this.initTextures();
+    }
+    unbind() {
+        this.canvas?.removeEventListener('webglcontextlost', this.handleContextLost, false);
+        this.canvas?.removeEventListener('webglcontextrestored', this.handleContextRestored, false);
+        this.releaseTextures();
+        this.gl = null;
+        super.unbind();
+    }
+    drawFrame({ width, height, yStride, uStride, vStride, yBuffer, uBuffer, vBuffer, rotation, alphaBuffer, }) {
+        this.rotateCanvas({ width, height, rotation });
+        this.updateRenderMode();
+        if (!this.gl || !this.program)
+            return;
+        const left = 0, top = 0, right = yStride - width, bottom = 0;
+        this.gl.bindBuffer(this.gl.ARRAY_BUFFER, this.texCoordBuffer);
+        const xWidth = width + left + right;
+        const xHeight = height + top + bottom;
+        this.gl.bufferData(this.gl.ARRAY_BUFFER, new Float32Array([
+            left / xWidth,
+            bottom / xHeight,
+            1 - right / xWidth,
+            bottom / xHeight,
+            left / xWidth,
+            1 - top / xHeight,
+            left / xWidth,
+            1 - top / xHeight,
+            1 - right / xWidth,
+            bottom / xHeight,
+            1 - right / xWidth,
+            1 - top / xHeight,
+        ]), this.gl.STATIC_DRAW);
+        this.gl.enableVertexAttribArray(this.texCoordLocation);
+        this.gl.vertexAttribPointer(this.texCoordLocation, 2, this.gl.FLOAT, false, 0, 0);
+        this.gl.pixelStorei(this.gl.UNPACK_ALIGNMENT, 1);
+        const activeTexture = (textureIndex, { texture, stride, height, pixels }) => {
+            if (!this.gl)
+                return;
+            this.gl.activeTexture(textureIndex);
+            this.gl.bindTexture(this.gl.TEXTURE_2D, texture);
+            this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.LUMINANCE, stride, height, 0, this.gl.LUMINANCE, this.gl.UNSIGNED_BYTE, pixels);
+        };
+        const textures = {
+            [this.gl.TEXTURE0]: {
+                texture: this.yTexture,
+                stride: yStride,
+                height: height,
+                pixels: yBuffer,
+            },
+            [this.gl.TEXTURE1]: {
+                texture: this.uTexture,
+                stride: uStride,
+                height: height / 2,
+                pixels: uBuffer,
+            },
+            [this.gl.TEXTURE2]: {
+                texture: this.vTexture,
+                stride: vStride,
+                height: height / 2,
+                pixels: vBuffer,
+            },
+        };
+        if (alphaBuffer) {
+            textures[this.gl.TEXTURE3] = {
+                texture: this.aTexture,
+                stride: width,
+                height: height,
+                pixels: alphaBuffer,
+            };
+            this.gl.uniform1i(this.hasAlpha, 1);
+        }
+        else {
+            this.gl.uniform1i(this.hasAlpha, 0);
+        }
+        for (const textureIndex in textures) {
+            if (textures.hasOwnProperty(textureIndex)) {
+                activeTexture(+textureIndex, textures[textureIndex]);
+            }
+        }
+        this.gl.drawArrays(this.gl.TRIANGLES, 0, 6);
+        super.drawFrame();
+    }
+    rotateCanvas({ width, height, rotation }) {
+        super.rotateCanvas({ width, height, rotation });
+        if (!this.gl || !this.program)
+            return;
+        this.gl.viewport(0, 0, width, height);
+        this.gl.bindBuffer(this.gl.ARRAY_BUFFER, this.surfaceBuffer);
+        this.gl.enableVertexAttribArray(this.positionLocation);
+        this.gl.vertexAttribPointer(this.positionLocation, 2, this.gl.FLOAT, false, 0, 0);
+        // 4 vertex, 1(x1,y1), 2(x2,y1), 3(x2,y2), 4(x1,y2)
+        // 0: 1,2,4/4,2,3
+        // 90: 2,3,1/1,3,4
+        // 180: 3,4,2/2,4,1
+        // 270: 4,1,3/3,1,2
+        const p1 = { x: 0, y: 0 };
+        const p2 = { x: width, y: 0 };
+        const p3 = { x: width, y: height };
+        const p4 = { x: 0, y: height };
+        let pp1 = p1, pp2 = p2, pp3 = p3, pp4 = p4;
+        switch (rotation) {
+            case 0:
+                break;
+            case 90:
+                pp1 = p2;
+                pp2 = p3;
+                pp3 = p4;
+                pp4 = p1;
+                break;
+            case 180:
+                pp1 = p3;
+                pp2 = p4;
+                pp3 = p1;
+                pp4 = p2;
+                break;
+            case 270:
+                pp1 = p4;
+                pp2 = p1;
+                pp3 = p2;
+                pp4 = p3;
+                break;
+            default:
+        }
+        this.gl.bufferData(this.gl.ARRAY_BUFFER, new Float32Array([
+            pp1.x,
+            pp1.y,
+            pp2.x,
+            pp2.y,
+            pp4.x,
+            pp4.y,
+            pp4.x,
+            pp4.y,
+            pp2.x,
+            pp2.y,
+            pp3.x,
+            pp3.y,
+        ]), this.gl.STATIC_DRAW);
+        const resolutionLocation = this.gl.getUniformLocation(this.program, 'u_resolution');
+        this.gl.uniform2f(resolutionLocation, width, height);
+    }
+    initTextures() {
+        if (!this.gl || !this.program)
+            return;
+        this.positionLocation = this.gl.getAttribLocation(this.program, 'a_position');
+        this.texCoordLocation = this.gl.getAttribLocation(this.program, 'a_texCoord');
+        this.hasAlpha = this.gl.getUniformLocation(this.program, 'hasAlpha');
+        this.surfaceBuffer = this.gl.createBuffer();
+        this.texCoordBuffer = this.gl.createBuffer();
+        const createTexture = (texture, textureIndex, textureName) => {
+            if (!this.gl || !this.program)
+                return null;
+            // Create a texture.
+            this.gl.activeTexture(texture);
+            const textureObj = this.gl.createTexture();
+            this.gl.bindTexture(this.gl.TEXTURE_2D, textureObj);
+            // Set the parameters so we can render any size
+            this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_S, this.gl.CLAMP_TO_EDGE);
+            this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_T, this.gl.CLAMP_TO_EDGE);
+            this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MIN_FILTER, this.gl.NEAREST);
+            this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MAG_FILTER, this.gl.NEAREST);
+            this.gl.uniform1i(this.gl.getUniformLocation(this.program, textureName), textureIndex); /* Bind Ytex to texture unit index */
+            return textureObj;
+        };
+        this.yTexture = createTexture(this.gl.TEXTURE0, 0, 'Ytex');
+        this.uTexture = createTexture(this.gl.TEXTURE1, 1, 'Utex');
+        this.vTexture = createTexture(this.gl.TEXTURE2, 2, 'Vtex');
+        this.aTexture = createTexture(this.gl.TEXTURE3, 3, 'Atex');
+    }
+    releaseTextures() {
+        this.gl?.deleteProgram(this.program);
+        this.program = null;
+        this.positionLocation = undefined;
+        this.texCoordLocation = undefined;
+        this.gl?.deleteTexture(this.yTexture);
+        this.gl?.deleteTexture(this.uTexture);
+        this.gl?.deleteTexture(this.vTexture);
+        this.yTexture = null;
+        this.uTexture = null;
+        this.vTexture = null;
+        this.gl?.deleteBuffer(this.texCoordBuffer);
+        this.gl?.deleteBuffer(this.surfaceBuffer);
+        this.texCoordBuffer = null;
+        this.surfaceBuffer = null;
+    }
+}
+exports.WebGLRenderer = WebGLRenderer;