agora-electron-sdk 0.0.0-dev.1

package/ts/Decoder/gpu-utils.ts

@@ -0,0 +1,92 @@
+//@ts-ignore
+import { BrowserWindow } from 'electron';
+
+/**
+ * @ignore
+ */
+
+export type VideoDecodeAcceleratorSupportedProfile = {
+  codec: string;
+  minWidth: number;
+  maxWidth: number;
+  minHeight: number;
+  maxHeight: number;
+};
+
+/**
+ * @ignore
+ */
+export class GpuInfo {
+  videoDecodeAcceleratorSupportedProfile: VideoDecodeAcceleratorSupportedProfile[] =
+    [];
+}
+
+/**
+ * @ignore
+ */
+export const getGpuInfoInternal = (callback: any): void => {
+  //@ts-ignore
+  if (process.type === 'renderer') {
+    console.error('getGpuInfoInternal should be called in main process');
+    return;
+  }
+  const gpuPage = new BrowserWindow({
+    show: false,
+    webPreferences: { offscreen: true },
+  });
+  gpuPage.loadURL('chrome://gpu');
+  let executeJavaScriptText =
+    `` +
+    `let videoAccelerationInfo = [];` +
+    `let nodeList = document.querySelector('info-view')?.shadowRoot?.querySelector('#video-acceleration-info info-view-table')?.shadowRoot?.querySelectorAll('#info-view-table info-view-table-row') || [];` +
+    `for (node of nodeList) {` +
+    `  videoAccelerationInfo.push({` +
+    `     title: node.shadowRoot.querySelector('#title')?.innerText,` +
+    `     value: node.shadowRoot.querySelector('#value')?.innerText,` +
+    ` })` +
+    `}` +
+    `JSON.stringify(videoAccelerationInfo)`;
+  gpuPage.webContents
+    .executeJavaScript(executeJavaScriptText)
+    .then((result: string) => {
+      if (!result) {
+        console.error(
+          'Failed to get GPU info, chrome://gpu is not available in this environment.'
+        );
+      }
+      let filterResult: { title: string; value: string }[] = JSON.parse(
+        result
+      ).filter((item: any) => {
+        return item.title.indexOf('Decode') !== -1;
+      });
+      let convertResult: VideoDecodeAcceleratorSupportedProfile[] = [];
+      const resolutionPattern = /(\d+)x(\d+) to (\d+)x(\d+)/;
+      for (const profile of filterResult) {
+        const match = profile.value.match(resolutionPattern);
+        if (!match) {
+          continue;
+        }
+
+        const [_resolution, minWidth, minHeight, maxWidth, maxHeight] = match;
+
+        convertResult.push({
+          codec: profile.title,
+          minWidth: minWidth ? Number(minWidth) : 0,
+          maxWidth: maxWidth ? Number(maxWidth) : 0,
+          minHeight: minHeight ? Number(minHeight) : 0,
+          maxHeight: maxHeight ? Number(maxHeight) : 0,
+        });
+      }
+      typeof callback === 'function' && callback(convertResult);
+    })
+    .catch((error: any) => {
+      console.error(
+        'Failed to get GPU info, please import agora-electron-sdk in main process',
+        error
+      );
+      typeof callback === 'function' && callback(error);
+    })
+    .finally(() => {
+      gpuPage.close();
+    });
+};

package/ts/Decoder/index.ts

@@ -0,0 +1,206 @@
+import { EncodedVideoFrameInfo, VideoFrameType } from '../Private/AgoraBase';
+
+import { WebCodecsRenderer } from '../Renderer/WebCodecsRenderer/index';
+import { CodecConfigInfo, RendererCacheContext, RendererType } from '../Types';
+import { AgoraEnv, logDebug, logInfo } from '../Utils';
+
+const frameTypeMapping = {
+  [VideoFrameType.VideoFrameTypeDeltaFrame]: 'delta',
+  [VideoFrameType.VideoFrameTypeKeyFrame]: 'key',
+  [VideoFrameType.VideoFrameTypeDroppableFrame]: 'delta', // this is a workaround for the issue that the frameType is not correct
+};
+
+export class WebCodecsDecoder {
+  private _decoder: VideoDecoder;
+  private renderers: WebCodecsRenderer[] = [];
+  private _cacheContext: RendererCacheContext;
+  // eslint-disable-next-line auto-import/auto-import
+  private pendingFrame: VideoFrame | null = null;
+  private _currentCodecConfig: CodecConfigInfo | null = null;
+
+  private _base_ts = 0;
+  private _base_ts_ntp = 1;
+  private _last_ts_ntp = 1;
+  private _decode_retry_count = 0;
+
+  constructor(
+    renders: WebCodecsRenderer[],
+    onError: (e: any) => void,
+    context: RendererCacheContext
+  ) {
+    this.renderers = renders;
+    this._cacheContext = context;
+    this._decoder = new VideoDecoder({
+      // @ts-ignore
+      output: this._output.bind(this),
+      error: (e) => {
+        onError(e);
+      },
+    });
+  }
+
+  // eslint-disable-next-line auto-import/auto-import
+  _output(frame: VideoFrame) {
+    // Schedule the frame to be rendered.
+    this._renderFrame(frame);
+  }
+
+  // eslint-disable-next-line auto-import/auto-import
+  private _renderFrame(frame: VideoFrame) {
+    if (!this.pendingFrame) {
+      // Schedule rendering in the next animation frame.
+      // eslint-disable-next-line auto-import/auto-import
+      requestAnimationFrame(this.renderAnimationFrame.bind(this));
+    } else {
+      // Close the current pending frame before replacing it.
+      this.pendingFrame.close();
+    }
+    // Set or replace the pending frame.
+    this.pendingFrame = frame;
+  }
+
+  renderAnimationFrame() {
+    for (let renderer of this.renderers) {
+      if (renderer.rendererType !== RendererType.WEBCODECSRENDERER) {
+        continue;
+      }
+      renderer.drawFrame(this.pendingFrame, this._currentCodecConfig!);
+      this.pendingFrame = null;
+    }
+  }
+
+  decoderConfigure(frameInfo: EncodedVideoFrameInfo) {
+    this.pendingFrame = null;
+    // @ts-ignore
+    let codec =
+      AgoraEnv.CapabilityManager?.frameCodecMapping[frameInfo.codecType!]
+        ?.codec;
+    if (!codec) {
+      AgoraEnv.AgoraRendererManager?.handleWebCodecsFallback(
+        this._cacheContext
+      );
+      throw new Error(
+        `codec ${frameInfo.codecType} is not in frameCodecMapping,failed to configure decoder, fallback to native decoder`
+      );
+    }
+    this._currentCodecConfig = {
+      codecType: frameInfo.codecType,
+      codedWidth: frameInfo.width,
+      codedHeight: frameInfo.height,
+      rotation: frameInfo.rotation,
+    };
+    this._decoder!.configure({
+      codec: codec,
+      codedWidth: frameInfo.width,
+      codedHeight: frameInfo.height,
+    });
+    logInfo(
+      `configure decoder: codedWidth: ${frameInfo.width}, codedHeight: ${frameInfo.height},codec: ${codec}`
+    );
+  }
+
+  updateTimestamps(ts: number) {
+    if (this._base_ts !== 0) {
+      if (ts > this._base_ts) {
+        this._last_ts_ntp =
+          this._base_ts_ntp + Math.floor(((ts - this._base_ts) * 1000) / 90);
+      } else {
+        this._base_ts = ts;
+        this._last_ts_ntp++;
+        this._base_ts_ntp = this._last_ts_ntp;
+      }
+    } else {
+      this._base_ts = ts;
+      this._last_ts_ntp = 1;
+    }
+  }
+
+  handleCodecIsChanged(frameInfo: EncodedVideoFrameInfo) {
+    if (
+      this._currentCodecConfig?.codecType !== frameInfo.codecType ||
+      this._currentCodecConfig?.codedWidth !== frameInfo.width ||
+      this._currentCodecConfig?.codedHeight !== frameInfo.height ||
+      this._currentCodecConfig?.rotation !== frameInfo.rotation
+    ) {
+      logInfo('frameInfo has changed, reconfigure decoder');
+      this._decoder.reset();
+      this.decoderConfigure(frameInfo);
+    }
+  }
+
+  // @ts-ignore
+  decodeFrame(
+    imageBuffer: Uint8Array,
+    frameInfo: EncodedVideoFrameInfo,
+    ts: number
+  ) {
+    try {
+      this.handleCodecIsChanged(frameInfo);
+    } catch (error: any) {
+      logInfo(error);
+      return;
+    }
+
+    if (!imageBuffer) {
+      logDebug('imageBuffer is empty, skip decode frame');
+      return;
+    }
+
+    let frameType: string | undefined;
+    if (frameInfo.frameType !== undefined) {
+      // @ts-ignore
+      frameType = frameTypeMapping[frameInfo.frameType];
+    }
+    if (!frameType) {
+      logDebug('frameType is not in frameTypeMapping, skip decode frame');
+      return;
+    }
+
+    this.updateTimestamps(ts);
+
+    try {
+      this._decoder.decode(
+        new EncodedVideoChunk({
+          data: imageBuffer,
+          timestamp: this._last_ts_ntp,
+          // @ts-ignore
+          type: frameType,
+          // @ts-ignore
+          transfer: [imageBuffer.buffer],
+        })
+      );
+      this._decode_retry_count = 0;
+    } catch (e) {
+      /// There are some cases that the decoder failed to decode the frame, we will retry for a few times
+      /// If the decoder still failed to decode the frame, we will fallback to native decoder
+      /// The retry count is defined in AgoraEnv.maxDecodeRetryCount
+      /// The retry count will be reset to 0 when the decoder successfully decode a frame
+      if (this._decode_retry_count >= AgoraEnv.maxDecodeRetryCount) {
+        AgoraEnv.AgoraRendererManager?.handleWebCodecsFallback(
+          this._cacheContext
+        );
+        throw new Error(
+          `failed to decode frame over ${this._decode_retry_count} times, fallback to native decoder`
+        );
+      }
+      this._decode_retry_count++;
+    }
+  }
+
+  reset() {
+    this._base_ts = 0;
+    this._base_ts_ntp = 1;
+    this._last_ts_ntp = 1;
+    this._decoder.reset();
+  }
+
+  release() {
+    try {
+      if (this.pendingFrame) {
+        this.pendingFrame.close();
+      }
+      this._decoder.close();
+    } catch (e) {}
+    this.pendingFrame = null;
+  }
+}