agora-appbuilder-core 4.0.0 → 4.0.1

package/template/bridge/rtc/webNg/RtcEngine.ts

@@ -25,14 +25,17 @@ import AgoraRTC, {
   ClientRoleOptions,
   CameraVideoTrackInitConfig,
   MicrophoneAudioTrackInitConfig,
+  IMicrophoneAudioTrack,
 } from 'agora-rtc-sdk-ng';
 import type {
   RtcEngineEvents,
   Subscription,
 } from 'react-native-agora/lib/typescript/src/common/RtcEvents';
+
+import {IRtcEngine} from 'react-native-agora';
 import {VideoProfile} from '../quality';
-import {ChannelProfile, ClientRole} from '../../../agora-rn-uikit';
-import {role, mode} from './Types';
+import {ChannelProfileType, ClientRoleType} from '../../../agora-rn-uikit';
+import {role, mode, RtcEngineContext} from './Types';
 import {LOG_ENABLED, GEO_FENCING} from '../../../config.json';
 import {Platform} from 'react-native';
 import isMobileOrTablet from '../../../src/utils/isMobileOrTablet';
@@ -137,6 +140,19 @@ export enum RnEncryptionEnum {
    * @since v3.3.1
    */
   AES256GCM = 6,
+  /**
+   * 7: 128-bit GCM encryption, GCM mode.
+   *
+   * @since v3.4.5
+   */
+
+  AES128GCM2 = 7,
+  /**
+   * 8: 256-bit AES encryption, GCM mode.
+   *
+   * @since v3.4.5
+   */
+  AES256GCM2 = 8,
 }
 
 export enum VideoStreamType {
@@ -156,6 +172,7 @@ interface RemoteStream {
   audio?: IRemoteAudioTrack;
   video?: IRemoteVideoTrack;
 }
+
 if ($config.GEO_FENCING) {
   AgoraRTC.setArea({
     areaCode: AREAS.GLOBAL,
@@ -177,14 +194,15 @@ export default class RtcEngine {
   public client: any | IAgoraRTCClient;
   public screenClient: any | IAgoraRTCClient;
   public eventsMap = new Map<string, callbackType>([
-    ['UserJoined', () => null],
-    ['UserOffline', () => null],
-    ['JoinChannelSuccess', () => null],
-    ['ScreenshareStopped', () => null],
-    ['RemoteAudioStateChanged', () => null],
-    ['RemoteVideoStateChanged', () => null],
-    ['NetworkQuality', () => null],
-    ['ActiveSpeaker', () => null],
+    ['onUserJoined', () => null],
+    ['onUserOffline', () => null],
+    ['onJoinChannelSuccess', () => null],
+    ['onScreenshareStopped', () => null],
+    ['onRemoteAudioStateChanged', () => null],
+    ['onRemoteVideoStateChanged', () => null],
+    ['onNetworkQuality', () => null],
+    ['onActiveSpeaker', () => null],
+    ['onStreamMessage', () => null],
   ]);
   public localStream: LocalStream = {};
   public screenStream: ScreenStream = {};
@@ -202,23 +220,27 @@ export default class RtcEngine {
   private muteLocalVideoMutex = false;
   private muteLocalAudioMutex = false;
   private speakerDeviceId = '';
+  private usersVolumeLevel = [];
   // Create channel profile and set it here
 
-  // Create channel profile and set it here
-
-  constructor(appId: string) {
+  initialize(context: RtcEngineContext) {
+    const {appId} = context;
     this.appId = appId;
-    // this.AgoraRTC = AgoraRTC;
   }
 
-  static async create(appId: string): Promise<RtcEngine> {
-    let engine = new RtcEngine(appId);
-    window.engine = engine;
-    return engine;
+  getRemoteVideoStats(id: string) {
+    try {
+      const data = this.client.getRemoteVideoStats();
+      return data && data[id] ? data[id] : null;
+    } catch (error) {
+      console.log(`ERROR: on getRemoteVideoStats - ` + error?.message);
+      return null;
+    }
   }
 
   async setVideoProfile(profile: VideoProfile): Promise<void> {
     this.videoProfile = profile;
+    this.localStream?.video?.setEncoderConfiguration(profile);
   }
 
   async enableAudio(): Promise<void> {
@@ -240,7 +262,10 @@ export default class RtcEngine {
     }
   }
 
-  async enableVideo(): Promise<void> {
+  async enableVideo(
+    preferredCameraId?: string,
+    preferredMicrophoneId?: string,
+  ): Promise<void> {
     /**
      * Issue: Backgrounding the browser or app causes the audio streaming to be cut off.
      * Impact: All browsers and apps that use WKWebView on iOS 15.x, such as Safari and Chrome.
@@ -252,14 +277,16 @@ export default class RtcEngine {
 
     const audioConfig: MicrophoneAudioTrackInitConfig = {
       bypassWebAudio: Platform.OS == 'web' && isMobileOrTablet(),
-      // microphoneId: this.audioDeviceId,
+      microphoneId: preferredMicrophoneId,
     };
     const videoConfig: CameraVideoTrackInitConfig = {
       encoderConfig: this.videoProfile,
-      // cameraId: this.videoDeviceId,
+      cameraId: preferredCameraId,
     };
     try {
       let [localAudio, localVideo] =
+        // If preferred devices are not present, the createTrack call will fallover to
+        // the catch block below.
         await AgoraRTC.createMicrophoneAndCameraTracks(
           audioConfig,
           videoConfig,
@@ -277,9 +304,15 @@ export default class RtcEngine {
     } catch (e) {
       let audioError = false;
       let videoError = false;
-      try {
-        let localAudio = await AgoraRTC.createMicrophoneAudioTrack(audioConfig);
 
+      try {
+        let localAudio: IMicrophoneAudioTrack;
+        try {
+          localAudio = await AgoraRTC.createMicrophoneAudioTrack(audioConfig);
+        } catch (e) {
+          videoConfig.microphoneId = '';
+          localAudio = await AgoraRTC.createMicrophoneAudioTrack(audioConfig);
+        }
         this.localStream.audio = localAudio;
         this.audioDeviceId = localAudio
           ?.getMediaStreamTrack()
@@ -288,8 +321,41 @@ export default class RtcEngine {
       } catch (error) {
         audioError = error;
       }
+
       try {
-        let localVideo = await AgoraRTC.createCameraVideoTrack(videoConfig);
+        let localVideo: ICameraVideoTrack;
+        try {
+          localVideo = await AgoraRTC.createCameraVideoTrack(videoConfig);
+        } catch (e) {
+          videoConfig.cameraId = '';
+          try {
+            localVideo = await AgoraRTC.createCameraVideoTrack(videoConfig);
+          } catch (e) {
+            console.log(
+              '[RTCEngineBridge]: Provided cameraId and default camera failed, trying other available devices',
+            );
+            const devices = await navigator.mediaDevices.enumerateDevices();
+            for (let device of devices) {
+              if (device.kind === 'videoinput') {
+                videoConfig.cameraId = device.deviceId;
+                try {
+                  localVideo = await AgoraRTC.createCameraVideoTrack(
+                    videoConfig,
+                  );
+                  break;
+                } catch (e) {
+                  videoError = e;
+                  console.log(
+                    '[RTCEngineBridge]:',
+                    'Camera not available with deviceId' + device,
+                    'Reason: ',
+                    e,
+                  );
+                }
+              }
+            }
+          }
+        }
         this.localStream.video = localVideo;
         this.videoDeviceId = localVideo
           ?.getMediaStreamTrack()
@@ -298,6 +364,7 @@ export default class RtcEngine {
       } catch (error) {
         videoError = error;
       }
+
       e.status = {audioError, videoError};
       throw e;
       // if (audioError && videoError) throw e;
@@ -340,7 +407,7 @@ export default class RtcEngine {
 
           if (this.isPublished === false) {
             this.isPublished = true;
-            (this.eventsMap.get('JoinChannelSuccess') as callbackType)();
+            (this.eventsMap.get('onJoinChannelSuccess') as callbackType)();
           }
         }
       } catch (e) {
@@ -353,19 +420,21 @@ export default class RtcEngine {
   async joinChannel(
     token: string,
     channelName: string,
-    optionalInfo: string,
     optionalUid: number,
+    _optionalInfo: {},
   ): Promise<void> {
     // TODO create agora client here
-    this.client.on('user-joined', (user) => {
-      (this.eventsMap.get('UserJoined') as callbackType)(user.uid);
-      (this.eventsMap.get('RemoteVideoStateChanged') as callbackType)(
+    this.client.on('user-joined', user => {
+      (this.eventsMap.get('onUserJoined') as callbackType)({}, user.uid);
+      (this.eventsMap.get('onRemoteVideoStateChanged') as callbackType)(
+        {},
         user.uid,
         0,
         0,
         0,
       );
-      (this.eventsMap.get('RemoteAudioStateChanged') as callbackType)(
+      (this.eventsMap.get('onRemoteAudioStateChanged') as callbackType)(
+        {},
         user.uid,
         0,
         0,
@@ -373,18 +442,19 @@ export default class RtcEngine {
       );
     });
 
-    this.client.on('user-left', (user) => {
+    this.client.on('user-left', user => {
       const uid = user.uid;
       if (this.remoteStreams.has(uid)) {
         this.remoteStreams.delete(uid);
       }
-      (this.eventsMap.get('UserOffline') as callbackType)(uid);
+      (this.eventsMap.get('onUserOffline') as callbackType)({}, uid);
       // (this.eventsMap.get('UserJoined') as callbackType)(uid);
     });
     this.client.on('user-published', async (user, mediaType) => {
       // Initiate the subscription
       if (this.inScreenshare && user.uid === this.screenClient.uid) {
-        (this.eventsMap.get('RemoteVideoStateChanged') as callbackType)(
+        (this.eventsMap.get('onRemoteVideoStateChanged') as callbackType)(
+          {},
           user.uid,
           2,
           0,
@@ -408,7 +478,8 @@ export default class RtcEngine {
             .get(user.uid)
             ?.audio?.setPlaybackDevice(this.speakerDeviceId);
         }
-        (this.eventsMap.get('RemoteAudioStateChanged') as callbackType)(
+        (this.eventsMap.get('onRemoteAudioStateChanged') as callbackType)(
+          {},
           user.uid,
           2,
           0,
@@ -422,7 +493,8 @@ export default class RtcEngine {
           ...this.remoteStreams.get(user.uid),
           video: videoTrack,
         });
-        (this.eventsMap.get('RemoteVideoStateChanged') as callbackType)(
+        (this.eventsMap.get('onRemoteVideoStateChanged') as callbackType)(
+          {},
           user.uid,
           2,
           0,
@@ -434,7 +506,8 @@ export default class RtcEngine {
       if (mediaType === 'audio') {
         const {audio, ...rest} = this.remoteStreams.get(user.uid);
         this.remoteStreams.set(user.uid, rest);
-        (this.eventsMap.get('RemoteAudioStateChanged') as callbackType)(
+        (this.eventsMap.get('onRemoteAudioStateChanged') as callbackType)(
+          {},
           user.uid,
           0,
           0,
@@ -443,7 +516,8 @@ export default class RtcEngine {
       } else {
         const {video, ...rest} = this.remoteStreams.get(user.uid);
         this.remoteStreams.set(user.uid, rest);
-        (this.eventsMap.get('RemoteVideoStateChanged') as callbackType)(
+        (this.eventsMap.get('onRemoteVideoStateChanged') as callbackType)(
+          {},
           user.uid,
           0,
           0,
@@ -452,7 +526,10 @@ export default class RtcEngine {
       }
     });
 
-    this.client.on('volume-indicator', (volumes) => {
+    this.client.on('volume-indicator', volumes => {
+      this.usersVolumeLevel = volumes;
+      /**
+       * old active speaker logic
       const highestvolumeObj = volumes.reduce(
         (highestVolume, volume, index) => {
           if (highestVolume === null) {
@@ -463,7 +540,7 @@ export default class RtcEngine {
             }
             return highestVolume;
           }
-          // console.log(`${index} UID ${volume.uid} Level ${volume.level}`);
+          //console.log(`${index} UID ${volume.uid} Level ${volume.level}`);
         },
         null,
       );
@@ -480,6 +557,7 @@ export default class RtcEngine {
         activeSpeakerCallBack(activeSpeakerUid);
         this.activeSpeakerUid = activeSpeakerUid;
       }
+       */
     });
 
     // this.client.on('stream-fallback', (evt))
@@ -491,28 +569,38 @@ export default class RtcEngine {
       'network-quality',
       async ({downlinkNetworkQuality, uplinkNetworkQuality}) => {
         const networkQualityIndicatorCallback = this.eventsMap.get(
-          'NetworkQuality',
+          'onNetworkQuality',
         ) as callbackType;
 
         networkQualityIndicatorCallback(
+          {},
           0,
-          downlinkNetworkQuality,
           uplinkNetworkQuality,
+          downlinkNetworkQuality,
         );
 
         const remoteUserNetworkQualities =
           this.client.getRemoteNetworkQuality();
 
-        Object.keys(remoteUserNetworkQualities).forEach((uid) => {
+        Object.keys(remoteUserNetworkQualities).forEach(uid => {
           networkQualityIndicatorCallback(
+            {},
             uid,
-            remoteUserNetworkQualities[uid].downlinkNetworkQuality,
             remoteUserNetworkQualities[uid].uplinkNetworkQuality,
+            remoteUserNetworkQualities[uid].downlinkNetworkQuality,
           );
         });
       },
     );
 
+    /* Recieve Captions  */
+    this.client.on('stream-message', (uid: UID, payload: UInt8Array) => {
+      console.log(
+        `stt-web: onStreamMessageCallback uid:${uid} , payload:${payload}`,
+      );
+      (this.eventsMap.get('onStreamMessage') as callbackType)(uid, payload);
+    });
+
     await this.client.join(
       this.appId,
       channelName,
@@ -525,6 +613,10 @@ export default class RtcEngine {
     console.log('enabling screen sleep');
   }
 
+  getUsersVolumeLevel() {
+    return this.usersVolumeLevel;
+  }
+
   async leaveChannel(): Promise<void> {
     this.client.leave();
     this.remoteStreams.forEach((stream, uid, map) => {
@@ -540,17 +632,19 @@ export default class RtcEngine {
     listener: RtcEngineEvents[EventType],
   ): Subscription {
     if (
-      event === 'UserJoined' ||
-      event === 'UserOffline' ||
-      event === 'JoinChannelSuccess' ||
-      event === 'ScreenshareStopped' ||
-      event === 'RemoteAudioStateChanged' ||
-      event === 'RemoteVideoStateChanged' ||
-      event === 'NetworkQuality' ||
-      event === 'ActiveSpeaker'
+      event === 'onUserJoined' ||
+      event === 'onUserOffline' ||
+      event === 'onJoinChannelSuccess' ||
+      event === 'onScreenshareStopped' ||
+      event === 'onRemoteAudioStateChanged' ||
+      event === 'onRemoteVideoStateChanged' ||
+      event === 'onNetworkQuality' ||
+      event === 'onActiveSpeaker' ||
+      event === 'onStreamMessage'
     ) {
       this.eventsMap.set(event, listener as callbackType);
     }
+
     return {
       remove: () => {
         console.log(
@@ -651,17 +745,21 @@ export default class RtcEngine {
     return devices;
   }
 
-  async setChannelProfile(profile: ChannelProfile): Promise<void> {
+  async setChannelProfile(profile: ChannelProfileType): Promise<void> {
     try {
       this.client = AgoraRTC.createClient({
-        codec: 'vp8',
+        codec: 'vp9',
         mode:
-          profile === ChannelProfile.LiveBroadcasting ? mode.live : mode.rtc,
+          profile === ChannelProfileType.ChannelProfileLiveBroadcasting
+            ? mode.live
+            : mode.rtc,
       });
       this.screenClient = AgoraRTC.createClient({
-        codec: 'vp8',
+        codec: 'vp9',
         mode:
-          profile === ChannelProfile.LiveBroadcasting ? mode.live : mode.rtc,
+          profile === ChannelProfileType.ChannelProfileLiveBroadcasting
+            ? mode.live
+            : mode.rtc,
       });
     } catch (e) {
       throw e;
@@ -669,11 +767,11 @@ export default class RtcEngine {
   }
 
   async setClientRole(
-    clientRole: ClientRole,
+    clientRole: ClientRoleType,
     options?: ClientRoleOptions,
   ): Promise<void> {
     try {
-      if (clientRole == ClientRole.Audience) {
+      if (clientRole == ClientRoleType.ClientRoleAudience) {
         if (this.isJoined) {
           // Unpublish the streams when role is changed to Audience
           await this.client.unpublish();
@@ -683,7 +781,7 @@ export default class RtcEngine {
         }
         await this.client.setClientRole(role.audience, options);
         await this.screenClient.setClientRole(role.audience, options);
-      } else if (clientRole == ClientRole.Broadcaster) {
+      } else if (clientRole == ClientRoleType.ClientRoleBroadcaster) {
         await this.client.setClientRole(role.host);
         await this.screenClient.setClientRole(role.host);
       }
@@ -730,11 +828,11 @@ export default class RtcEngine {
 
   async changeSpeaker(speakerId, callback, error) {
     try {
-      this.speakerDeviceId = speakerId;
       // setting sepeaker for all remote stream (previously joined users)
       this.remoteStreams?.forEach((stream, uid, map) => {
         stream?.audio?.setPlaybackDevice(speakerId);
       });
+      this.speakerDeviceId = speakerId;
       callback(speakerId);
     } catch (e) {
       error(e);
@@ -762,7 +860,7 @@ export default class RtcEngine {
 
   // Bug in implementation !!!
   async setRemoteSubscribeFallbackOption(option: 0 | 1 | 2) {
-    this.streams.forEach((stream) => {
+    this.streams.forEach(stream => {
       this.client.setStreamFallbackOption(stream, option);
     });
     Promise.resolve();
@@ -787,6 +885,17 @@ export default class RtcEngine {
           break;
         case RnEncryptionEnum.SM4128ECB:
           mode = 'sm4-128-ecb';
+          break;
+        case RnEncryptionEnum.AES256GCM:
+          mode = 'aes-256-gcm';
+          break;
+        case RnEncryptionEnum.AES128GCM2:
+          mode = 'aes-128-gcm2';
+          break;
+        case RnEncryptionEnum.AES256GCM2:
+          mode = 'aes-256-gcm2';
+          break;
+
         default:
           mode = 'none';
       }
@@ -801,14 +910,25 @@ export default class RtcEngine {
     config: {
       encryptionMode: RnEncryptionEnum;
       encryptionKey: string;
+      encryptionKdfSalt: string;
     },
   ): Promise<void> {
     let mode: EncryptionMode;
     mode = this.getEncryptionMode(enabled, config?.encryptionMode);
     try {
       await Promise.all([
-        this.client.setEncryptionConfig(mode, config.encryptionKey),
-        this.screenClient.setEncryptionConfig(mode, config.encryptionKey),
+        this.client.setEncryptionConfig(
+          mode,
+          config.encryptionKey,
+          config.encryptionKdfSalt,
+          true, // encryptDataStream
+        ),
+        this.screenClient.setEncryptionConfig(
+          mode,
+          config.encryptionKey,
+          config.encryptionKdfSalt,
+          true, // encryptDataStream
+        ),
       ]);
     } catch (e) {
       throw e;
@@ -835,13 +955,18 @@ export default class RtcEngine {
     console.error('Please use enableEncryption instead');
   }
 
-  async destroy(): Promise<void> {
+  // async removeAllListeners<EventType extends keyof RtcEngineEvents>(event: EventType) {
+  //   this.client.removeAllListeners(eventName);
+  // }
+
+  async release(): Promise<void> {
     if (this.inScreenshare) {
-      (this.eventsMap.get('UserOffline') as callbackType)(
+      (this.eventsMap.get('onUserOffline') as callbackType)(
+        {},
         this.screenClient.uid,
       );
       this.screenClient.leave();
-      (this.eventsMap.get('ScreenshareStopped') as callbackType)();
+      (this.eventsMap.get('onScreenshareStopped') as callbackType)();
     }
     this.eventsMap.forEach((callback, event, map) => {
       this.client.off(event, callback);
@@ -892,6 +1017,7 @@ export default class RtcEngine {
     encryption: {
       screenKey: string;
       mode: RnEncryptionEnum;
+      salt: string;
     },
     config: ScreenVideoTrackInitConfig = {},
     audio: 'enable' | 'disable' | 'auto' = 'auto',
@@ -912,6 +1038,8 @@ export default class RtcEngine {
             await this.screenClient.setEncryptionConfig(
               mode,
               encryption.screenKey,
+              encryption.salt,
+              true, // encryptDataStream
             );
           } catch (e) {
             console.log('e: Encryption for screenshare failed', e);
@@ -948,7 +1076,8 @@ export default class RtcEngine {
       );
 
       this.screenStream.video.on('track-ended', () => {
-        (this.eventsMap.get('UserOffline') as callbackType)(
+        (this.eventsMap.get('onUserOffline') as callbackType)(
+          {},
           this.screenClient.uid,
         );
 
@@ -958,15 +1087,16 @@ export default class RtcEngine {
         this.screenStream.video?.close();
         this.screenStream = {};
 
-        (this.eventsMap.get('ScreenshareStopped') as callbackType)();
+        (this.eventsMap.get('onScreenshareStopped') as callbackType)();
         this.inScreenshare = false;
       });
     } else {
-      (this.eventsMap.get('UserOffline') as callbackType)(
+      (this.eventsMap.get('onUserOffline') as callbackType)(
+        {},
         this.screenClient.uid,
       );
       this.screenClient.leave();
-      (this.eventsMap.get('ScreenshareStopped') as callbackType)();
+      (this.eventsMap.get('onScreenshareStopped') as callbackType)();
       try {
         this.screenStream.audio?.close();
         this.screenStream.video?.close();

package/template/bridge/rtc/webNg/{SurfaceView.tsx → RtcSurfaceView.tsx}

@@ -13,57 +13,51 @@
 import {ILocalVideoTrack, IRemoteVideoTrack} from 'agora-rtc-sdk-ng';
 import React, {useEffect} from 'react';
 import {StyleProp, StyleSheet, ViewProps, ViewStyle} from 'react-native';
-import {VideoMirrorMode, VideoRenderMode} from 'react-native-agora';
+import {RenderModeType} from './Types';
 
 export interface RtcSurfaceViewProps extends ViewProps {
-  zOrderMediaOverlay?: boolean;
-  zOrderOnTop?: boolean;
-  renderMode?: any;
-  channelId?: string;
-  mirrorMode?: any;
-}
-export interface RtcUidProps {
-  uid: number;
+  canvas: {
+    renderMode?: RenderModeType;
+    uid?: number;
+  };
 }
+
 export interface StyleProps {
   style?: StyleProp<ViewStyle>;
 }
 
-interface SurfaceViewInterface
-  extends RtcSurfaceViewProps,
-    RtcUidProps,
-    StyleProps {}
-
-const SurfaceView = (props: SurfaceViewInterface) => {
-  //   console.log('Surface View props', props);
+interface SurfaceViewInterface extends RtcSurfaceViewProps, StyleProps {}
 
+const RtcSurfaceView = (props: SurfaceViewInterface) => {
+  console.log('Rtc Surface View props', props);
+  const {uid, renderMode} = props.canvas;
   const stream: ILocalVideoTrack | IRemoteVideoTrack =
-    props.uid === 0
+    uid === 0
       ? window.engine.localStream.video
-      : props.uid === 1
+      : uid === 1
       ? window.engine.screenStream.video
-      : window.engine.remoteStreams.get(props.uid)?.video;
+      : window.engine.remoteStreams.get(uid)?.video;
   // console.log(props, window.engine, stream);
   useEffect(
     function () {
       if (stream?.play) {
-        if (props.renderMode === 2) {
-          stream.play(String(props.uid), {fit: 'contain'});
+        if (renderMode === RenderModeType.RenderModeFit) {
+          stream.play(String(uid), {fit: 'contain'});
         } else {
-          stream.play(String(props.uid));
+          stream.play(String(uid));
         }
       }
       return () => {
-        console.log(`unmounting stream ${props.uid}`, stream);
+        console.log(`unmounting stream ${uid}`, stream);
         stream && stream.stop();
       };
     },
-    [props.uid, props.renderMode, stream],
+    [uid, renderMode, stream],
   );
 
   return stream ? (
     <div
-      id={String(props.uid)}
+      id={String(uid)}
       className={'video-container'}
       style={{...style.full, ...(props.style as Object), overflow: 'hidden'}}
     />
@@ -78,4 +72,4 @@ const style = StyleSheet.create({
   },
 });
 
-export default SurfaceView;
+export default RtcSurfaceView;