agora-electron-sdk 4.5.1 → 4.5.2-dev.2

package/js/Renderer/RendererManager.js

@@ -1,80 +1,301 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.RendererManager = void 0;
+const AgoraSdk_1 = __importDefault(require("../AgoraSdk"));
+const AgoraBase_1 = require("../Private/AgoraBase");
+const AgoraMediaBase_1 = require("../Private/AgoraMediaBase");
 const Types_1 = require("../Types");
 const Utils_1 = require("../Utils");
-const IRendererManager_1 = require("./IRendererManager");
+const IRendererCache_1 = require("./IRendererCache");
+const RendererCache_1 = require("./RendererCache");
+const WebCodecsRenderer_1 = require("./WebCodecsRenderer");
+const WebCodecsRendererCache_1 = require("./WebCodecsRendererCache");
 const WebGLRenderer_1 = require("./WebGLRenderer");
 const YUVCanvasRenderer_1 = require("./YUVCanvasRenderer");
 /**
  * @ignore
  */
-class RendererManager extends IRendererManager_1.IRendererManager {
-    set rendererType(rendererType) {
-        if (this._rendererType !== rendererType) {
-            this._rendererType = rendererType;
-        }
-    }
-    get rendererType() {
-        return this._rendererType;
-    }
+class RendererManager {
     constructor() {
-        super();
-        this._rendererType = (0, Utils_1.isSupportWebGL)()
+        /**
+         * Currently, the remote video frame is observed in the pre-renderer position and you can not change it.
+         * the local video frame is observed in the pre-encoder position by default, and you can change it.
+         * @ignore
+         */
+        this.defaultObservedFramePosition = AgoraMediaBase_1.VideoModulePosition.PositionPreRenderer |
+            AgoraMediaBase_1.VideoModulePosition.PositionPreEncoder;
+        this.renderingFps = 15;
+        this._currentFrameCount = 0;
+        this._previousFirstFrameTime = 0;
+        this._rendererCaches = [];
+        this._context = {
+            renderMode: AgoraMediaBase_1.RenderModeType.RenderModeHidden,
+            mirrorMode: AgoraBase_1.VideoMirrorModeType.VideoMirrorModeDisabled,
+        };
+        this.rendererType = (0, Utils_1.isSupportWebGL)()
             ? Types_1.RendererType.WEBGL
             : Types_1.RendererType.SOFTWARE;
     }
-    /**
-     * @deprecated Use rendererType instead
-     */
-    setRenderMode(mode) {
-        this.rendererType = mode;
-    }
-    /**
-     * @deprecated Use renderingFps instead
-     */
-    setFPS(fps) {
+    setRenderingFps(fps) {
         this.renderingFps = fps;
+        if (this._renderingTimer) {
+            this.stopRendering();
+            this.startRendering();
+        }
+    }
+    set defaultChannelId(channelId) {
+        this._context.channelId = channelId;
+    }
+    get defaultChannelId() {
+        return this._context.channelId ?? '';
+    }
+    get defaultRenderMode() {
+        return this._context.renderMode;
+    }
+    get defaultMirrorMode() {
+        return this._context.mirrorMode;
+    }
+    release() {
+        this.stopRendering();
+        this.clearRendererCache();
     }
-    /**
-     * @deprecated Use getRendererCache instead
-     */
-    getRender(context) {
-        return this.getRendererCache(context);
+    presetRendererContext(context) {
+        //this is for preset default value
+        context.renderMode = context.renderMode || this.defaultRenderMode;
+        context.mirrorMode = context.mirrorMode || this.defaultMirrorMode;
+        context.useWebCodecsDecoder = context.useWebCodecsDecoder || false;
+        context.enableFps = context.enableFps || false;
+        context.position = context.position || this.defaultObservedFramePosition;
+        if (!Utils_1.AgoraEnv.CapabilityManager?.webCodecsDecoderEnabled) {
+            context.useWebCodecsDecoder = false;
+        }
+        switch (context.sourceType) {
+            case AgoraMediaBase_1.VideoSourceType.VideoSourceRemote:
+                if (context.uid === undefined) {
+                    throw new Error('uid is required');
+                }
+                context.channelId = context.channelId ?? this.defaultChannelId;
+                break;
+            case AgoraMediaBase_1.VideoSourceType.VideoSourceMediaPlayer:
+                if (context.mediaPlayerId === undefined) {
+                    throw new Error('mediaPlayerId is required');
+                }
+                context.channelId = '';
+                context.uid = context.mediaPlayerId;
+                break;
+            case undefined:
+                if (context.uid) {
+                    context.sourceType = AgoraMediaBase_1.VideoSourceType.VideoSourceRemote;
+                }
+                break;
+            default:
+                context.channelId = '';
+                context.uid = 0;
+                break;
+        }
+        return context;
+    }
+    addOrRemoveRenderer(context) {
+        // To be compatible with the old API
+        let { setupMode = AgoraBase_1.VideoViewSetupMode.VideoViewSetupAdd } = context;
+        if (!context.view)
+            setupMode = AgoraBase_1.VideoViewSetupMode.VideoViewSetupRemove;
+        switch (setupMode) {
+            case AgoraBase_1.VideoViewSetupMode.VideoViewSetupAdd:
+                return this.addRendererToCache(context);
+            case AgoraBase_1.VideoViewSetupMode.VideoViewSetupRemove:
+                this.removeRendererFromCache(context);
+                return undefined;
+            case AgoraBase_1.VideoViewSetupMode.VideoViewSetupReplace:
+                this.removeRendererFromCache(context);
+                return this.addRendererToCache(context);
+        }
     }
-    createRenderer(context, rendererType) {
-        if (rendererType === undefined) {
-            rendererType = this.rendererType;
+    addRendererToCache(context) {
+        const checkedContext = this.presetRendererContext(context);
+        if (!checkedContext.view)
+            return undefined;
+        if (this.findRenderer(checkedContext.view)) {
+            throw new Error('You have already added this view to the renderer');
+        }
+        let rendererCache = this.getRendererCache(checkedContext);
+        if (!rendererCache) {
+            if (context.useWebCodecsDecoder) {
+                rendererCache = new WebCodecsRendererCache_1.WebCodecsRendererCache(checkedContext);
+            }
+            else {
+                rendererCache = new RendererCache_1.RendererCache(checkedContext);
+            }
+            this._rendererCaches.push(rendererCache);
+        }
+        rendererCache.addRenderer(this.createRenderer(checkedContext));
+        if (!context.useWebCodecsDecoder) {
+            this.startRendering();
         }
+        return rendererCache;
+    }
+    removeRendererFromCache(context) {
+        const checkedContext = this.presetRendererContext(context);
+        const rendererCache = this.getRendererCache(checkedContext);
+        if (!rendererCache)
+            return;
+        if (checkedContext.view) {
+            const renderer = rendererCache.findRenderer(checkedContext.view);
+            if (!renderer)
+                return;
+            rendererCache.removeRenderer(renderer);
+        }
+        else {
+            rendererCache.removeRenderer();
+        }
+        if (rendererCache.renderers.length === 0) {
+            rendererCache.release();
+            this._rendererCaches.splice(this._rendererCaches.indexOf(rendererCache), 1);
+        }
+    }
+    clearRendererCache() {
+        for (const rendererCache of this._rendererCaches) {
+            rendererCache.release();
+        }
+        this._rendererCaches.splice(0);
+    }
+    getRendererCache(context) {
+        return this._rendererCaches.find((cache) => cache.key === (0, IRendererCache_1.generateRendererCacheKey)(context));
+    }
+    getRenderers(context) {
+        return this.getRendererCache(context)?.renderers || [];
+    }
+    findRenderer(view) {
+        for (const rendererCache of this._rendererCaches) {
+            const renderer = rendererCache.findRenderer(view);
+            if (renderer)
+                return renderer;
+        }
+        return undefined;
+    }
+    createRenderer(context, rendererType = this.rendererType) {
         let renderer;
         switch (rendererType) {
             case Types_1.RendererType.WEBGL:
-                renderer = new WebGLRenderer_1.WebGLRenderer(this.handleWebGLFallback(context).bind(this));
+                if (context.useWebCodecsDecoder) {
+                    renderer = new WebCodecsRenderer_1.WebCodecsRenderer();
+                }
+                else {
+                    renderer = new WebGLRenderer_1.WebGLRenderer(this.handleWebGLFallback(context).bind(this));
+                    renderer.bind(context.view);
+                }
                 break;
             case Types_1.RendererType.SOFTWARE:
                 renderer = new YUVCanvasRenderer_1.YUVCanvasRenderer();
+                renderer.bind(context.view);
                 break;
             default:
                 throw new Error('Unknown renderer type');
         }
-        renderer.bind(context.view);
-        renderer.context = {
-            renderMode: context.renderMode,
-            mirrorMode: context.mirrorMode,
-        };
+        renderer.setContext(context);
         return renderer;
     }
+    startRendering() {
+        if (this._renderingTimer)
+            return;
+        const renderingLooper = () => {
+            if (this._previousFirstFrameTime === 0) {
+                // Get the current time as the time of the first frame of per second
+                this._previousFirstFrameTime = performance.now();
+                // Reset the frame count
+                this._currentFrameCount = 0;
+            }
+            // Increase the frame count
+            ++this._currentFrameCount;
+            // Get the current time
+            const currentFrameTime = performance.now();
+            // Calculate the time difference between the current frame and the previous frame
+            const deltaTime = currentFrameTime - this._previousFirstFrameTime;
+            // Calculate the expected time of the current frame
+            const expectedTime = (this._currentFrameCount * 1000) / this.renderingFps;
+            (0, Utils_1.logDebug)(new Date().toLocaleTimeString(), 'currentFrameCount', this._currentFrameCount, 'expectedTime', expectedTime, 'deltaTime', deltaTime);
+            if (this._rendererCaches.length === 0) {
+                // If there is no renderer, stop rendering
+                this.stopRendering();
+                return;
+            }
+            // Render all renderers that do not use WebCodecs
+            for (const rendererCache of this._rendererCaches.filter((cache) => cache instanceof RendererCache_1.RendererCache)) {
+                this.doRendering(rendererCache);
+            }
+            if (this._currentFrameCount >= this.renderingFps) {
+                this._previousFirstFrameTime = 0;
+            }
+            if (deltaTime < expectedTime) {
+                // If the time difference between the current frame and the previous frame is less than the expected time, then wait for the difference
+                this._renderingTimer = window.setTimeout(renderingLooper, expectedTime - deltaTime);
+            }
+            else {
+                // If the time difference between the current frame and the previous frame is greater than the expected time, then render immediately
+                renderingLooper();
+            }
+        };
+        renderingLooper();
+    }
     doRendering(rendererCache) {
         rendererCache.draw();
     }
     handleWebGLFallback(context) {
         return (renderer) => {
-            const { context: { renderMode, mirrorMode }, } = renderer;
             const renderers = this.getRenderers(context);
             renderer.unbind();
-            const newRenderer = this.createRenderer({ ...context, renderMode, mirrorMode }, Types_1.RendererType.SOFTWARE);
+            const newRenderer = this.createRenderer(context, Types_1.RendererType.SOFTWARE);
             renderers.splice(renderers.indexOf(renderer), 1, newRenderer);
         };
     }
+    handleWebCodecsFallback(context) {
+        let engine = (0, AgoraSdk_1.default)();
+        engine.getMediaEngine().unregisterVideoEncodedFrameObserver({});
+        if (context.uid) {
+            if ((0, IRendererCache_1.isUseConnection)(context)) {
+                engine.setRemoteVideoSubscriptionOptionsEx(context.uid, {
+                    type: AgoraBase_1.VideoStreamType.VideoStreamHigh,
+                    encodedFrameOnly: false,
+                }, {
+                    channelId: context.channelId,
+                    localUid: context.localUid,
+                });
+            }
+            else {
+                engine.setRemoteVideoSubscriptionOptions(context.uid, {
+                    type: AgoraBase_1.VideoStreamType.VideoStreamHigh,
+                    encodedFrameOnly: false,
+                });
+            }
+        }
+        Utils_1.AgoraEnv.enableWebCodecsDecoder = false;
+        Utils_1.AgoraEnv.CapabilityManager?.setWebCodecsDecoderEnabled(false);
+        let renderers = this.getRenderers(context);
+        for (let renderer of renderers) {
+            this.addOrRemoveRenderer({
+                ...renderer.context,
+                setupMode: AgoraBase_1.VideoViewSetupMode.VideoViewSetupReplace,
+            });
+        }
+    }
+    stopRendering() {
+        if (this._renderingTimer) {
+            window.clearTimeout(this._renderingTimer);
+            this._renderingTimer = undefined;
+        }
+    }
+    setRendererContext(context) {
+        const checkedContext = this.presetRendererContext(context);
+        for (const rendererCache of this._rendererCaches) {
+            const result = rendererCache.setRendererContext(checkedContext);
+            if (result) {
+                return true;
+            }
+        }
+        return false;
+    }
 }
 exports.RendererManager = RendererManager;

package/js/Renderer/WebCodecsRenderer/index.js

@@ -0,0 +1,109 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WebCodecsRenderer = void 0;
+const Types_1 = require("../../Types");
+const Utils_1 = require("../../Utils");
+const IRenderer_1 = require("../IRenderer");
+class WebCodecsRenderer extends IRenderer_1.IRenderer {
+    constructor() {
+        super();
+        this.rendererType = Types_1.RendererType.WEBCODECSRENDERER;
+    }
+    bind(element) {
+        super.bind(element);
+        if (!this.canvas)
+            return;
+        this.offscreenCanvas = this.canvas.transferControlToOffscreen();
+        this.gl = (0, Utils_1.getContextByCanvas)(this.offscreenCanvas);
+        if (!this.gl)
+            return;
+        const vertexShader = this.gl.createShader(this.gl.VERTEX_SHADER);
+        if (!vertexShader)
+            return;
+        this.gl.shaderSource(vertexShader, WebCodecsRenderer.vertexShaderSource);
+        this.gl.compileShader(vertexShader);
+        if (!this.gl.getShaderParameter(vertexShader, this.gl.COMPILE_STATUS)) {
+            throw this.gl.getShaderInfoLog(vertexShader);
+        }
+        const fragmentShader = this.gl.createShader(this.gl.FRAGMENT_SHADER);
+        if (!fragmentShader)
+            return;
+        this.gl.shaderSource(fragmentShader, WebCodecsRenderer.fragmentShaderSource);
+        this.gl.compileShader(fragmentShader);
+        if (!this.gl.getShaderParameter(fragmentShader, this.gl.COMPILE_STATUS)) {
+            throw this.gl.getShaderInfoLog(fragmentShader);
+        }
+        const shaderProgram = this.gl.createProgram();
+        if (!shaderProgram)
+            return;
+        this.gl.attachShader(shaderProgram, vertexShader);
+        this.gl.attachShader(shaderProgram, fragmentShader);
+        this.gl.linkProgram(shaderProgram);
+        if (!this.gl.getProgramParameter(shaderProgram, this.gl.LINK_STATUS)) {
+            throw this.gl.getProgramInfoLog(shaderProgram);
+        }
+        this.gl.useProgram(shaderProgram);
+        // Vertex coordinates, clockwise from bottom-left.
+        const vertexBuffer = this.gl.createBuffer();
+        this.gl.bindBuffer(this.gl.ARRAY_BUFFER, vertexBuffer);
+        this.gl.bufferData(this.gl.ARRAY_BUFFER, new Float32Array([-1.0, -1.0, -1.0, +1.0, +1.0, +1.0, +1.0, -1.0]), this.gl.STATIC_DRAW);
+        const xyLocation = this.gl.getAttribLocation(shaderProgram, 'xy');
+        this.gl.vertexAttribPointer(xyLocation, 2, this.gl.FLOAT, false, 0, 0);
+        this.gl.enableVertexAttribArray(xyLocation);
+        // Create one texture to upload frames to.
+        const texture = this.gl.createTexture();
+        this.gl.bindTexture(this.gl.TEXTURE_2D, texture);
+        this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MAG_FILTER, this.gl.NEAREST);
+        this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MIN_FILTER, this.gl.NEAREST);
+        this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_S, this.gl.CLAMP_TO_EDGE);
+        this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_T, this.gl.CLAMP_TO_EDGE);
+    }
+    drawFrame(frame, _codecConfig) {
+        if (!this.offscreenCanvas || !frame)
+            return;
+        this.offscreenCanvas.width = _codecConfig.codedWidth;
+        this.offscreenCanvas.height = _codecConfig.codedHeight;
+        this.updateRenderMode();
+        this.rotateCanvas({
+            rotation: _codecConfig.rotation,
+        });
+        if (!this.gl)
+            return;
+        if (this.gl) {
+            // Upload the frame.
+            this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, this.gl.RGBA, this.gl.UNSIGNED_BYTE, frame);
+            frame.close();
+            // Configure and clear the drawing area.
+            this.gl.viewport(0, 0, this.gl.drawingBufferWidth, this.gl.drawingBufferHeight);
+            this.gl.clearColor(0.0, 0.0, 0.0, 0.0);
+            this.gl.clear(this.gl.COLOR_BUFFER_BIT);
+            // Draw the frame.
+            this.gl.drawArrays(this.gl.TRIANGLE_FAN, 0, 4);
+        }
+        super.drawFrame();
+        this.getFps();
+    }
+    rotateCanvas({ rotation }) {
+        if (!this.offscreenCanvas || !this.canvas)
+            return;
+        this.canvas.style.transform += ` rotateZ(${rotation}deg)`;
+    }
+}
+exports.WebCodecsRenderer = WebCodecsRenderer;
+WebCodecsRenderer.vertexShaderSource = `
+    attribute vec2 xy;
+    varying highp vec2 uv;
+    void main(void) {
+      gl_Position = vec4(xy, 0.0, 1.0);
+      // Map vertex coordinates (-1 to +1) to UV coordinates (0 to 1).
+      // UV coordinates are Y-flipped relative to vertex coordinates.
+      uv = vec2((1.0 + xy.x) / 2.0, (1.0 - xy.y) / 2.0);
+    }
+  `;
+WebCodecsRenderer.fragmentShaderSource = `
+    varying highp vec2 uv;
+    uniform sampler2D texture;
+    void main(void) {
+      gl_FragColor = texture2D(texture, uv);
+    }
+  `;

package/js/Renderer/WebCodecsRendererCache.js

@@ -0,0 +1,115 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WebCodecsRendererCache = void 0;
+const AgoraSdk_1 = __importDefault(require("../AgoraSdk"));
+const index_1 = require("../Decoder/index");
+const AgoraBase_1 = require("../Private/AgoraBase");
+const IrisApiEngine_1 = require("../Private/internal/IrisApiEngine");
+const Types_1 = require("../Types");
+const Utils_1 = require("../Utils");
+const IRendererCache_1 = require("./IRendererCache");
+class WebCodecsRendererCache extends IRendererCache_1.IRendererCache {
+    constructor(context) {
+        super(context);
+        this._firstFrame = true;
+        this._engine = (0, AgoraSdk_1.default)();
+        this._decoder = new index_1.WebCodecsDecoder(this.renderers, this.onDecoderError.bind(this), context);
+        this.draw();
+    }
+    onDecoderError(e) {
+        (0, Utils_1.logInfo)('webCodecsDecoder decode failed, fallback to native decoder', e);
+        Utils_1.AgoraEnv.AgoraRendererManager?.handleWebCodecsFallback(this.cacheContext);
+    }
+    onEncodedVideoFrameReceived(...[data, buffer]) {
+        let _data;
+        try {
+            _data = JSON.parse(data) ?? {};
+        }
+        catch (e) {
+            _data = {};
+        }
+        if (Object.keys(_data).length === 0 ||
+            !this._decoder ||
+            this.cacheContext.uid !== _data.uid)
+            return;
+        if (this._firstFrame) {
+            for (let renderer of this.renderers) {
+                if (renderer.rendererType !== Types_1.RendererType.WEBCODECSRENDERER) {
+                    continue;
+                }
+                renderer.bind(renderer.context.view);
+            }
+            try {
+                this._decoder.decoderConfigure(_data.videoEncodedFrameInfo);
+            }
+            catch (error) {
+                (0, Utils_1.logInfo)(error);
+                return;
+            }
+            this._firstFrame = false;
+        }
+        if (this.shouldFallback(_data.videoEncodedFrameInfo)) {
+            Utils_1.AgoraEnv.AgoraRendererManager?.handleWebCodecsFallback(this.cacheContext);
+        }
+        else {
+            this._decoder.decodeFrame(buffer, _data.videoEncodedFrameInfo, new Date().getTime());
+        }
+    }
+    draw() {
+        if ((0, IRendererCache_1.isUseConnection)(this.cacheContext)) {
+            this._engine?.setRemoteVideoSubscriptionOptionsEx(this.cacheContext.uid, {
+                type: AgoraBase_1.VideoStreamType.VideoStreamHigh,
+                encodedFrameOnly: true,
+            }, {
+                channelId: this.cacheContext.channelId,
+                localUid: this.cacheContext.localUid,
+            });
+        }
+        else {
+            this._engine?.setRemoteVideoSubscriptionOptions(this.cacheContext.uid, {
+                type: AgoraBase_1.VideoStreamType.VideoStreamHigh,
+                encodedFrameOnly: true,
+            });
+        }
+        IrisApiEngine_1.AgoraElectronBridge.OnEvent(`call_back_with_encoded_video_frame_${this.cacheContext.uid}`, (...params) => {
+            try {
+                this.onEncodedVideoFrameReceived(...params);
+            }
+            catch (e) {
+                console.error(e);
+            }
+        });
+    }
+    shouldFallback(frameInfo) {
+        let shouldFallback = false;
+        if (!frameInfo.codecType) {
+            shouldFallback = true;
+            (0, Utils_1.logInfo)('codecType is not supported, fallback to native decoder');
+        }
+        else {
+            const mapping = Utils_1.AgoraEnv.CapabilityManager?.frameCodecMapping[frameInfo.codecType];
+            if (mapping === undefined) {
+                shouldFallback = true;
+                (0, Utils_1.logInfo)('codecType is not supported, fallback to native decoder');
+            }
+            else if (mapping.minWidth >= frameInfo.width &&
+                mapping.minHeight >= frameInfo.height &&
+                mapping.maxWidth <= frameInfo.width &&
+                mapping.maxHeight <= frameInfo.height) {
+                shouldFallback = true;
+                (0, Utils_1.logInfo)('frame size is not supported, fallback to native decoder');
+            }
+        }
+        return shouldFallback;
+    }
+    release() {
+        IrisApiEngine_1.AgoraElectronBridge.UnEvent(`call_back_with_encoded_video_frame_${this.cacheContext.uid}`);
+        this._decoder?.release();
+        this._decoder = null;
+        super.release();
+    }
+}
+exports.WebCodecsRendererCache = WebCodecsRendererCache;