agora-appbuilder-core 4.0.0-api.5 → 4.0.0-api.6

package/template/bridge/rtc/webNg/RtcEngine.ts

@@ -25,6 +25,7 @@ import AgoraRTC, {
   ClientRoleOptions,
   CameraVideoTrackInitConfig,
   MicrophoneAudioTrackInitConfig,
+  IMicrophoneAudioTrack,
 } from 'agora-rtc-sdk-ng';
 import type {
   RtcEngineEvents,
@@ -137,6 +138,19 @@ export enum RnEncryptionEnum {
    * @since v3.3.1
    */
   AES256GCM = 6,
+  /**
+   * 7: 128-bit GCM encryption, GCM mode.
+   *
+   * @since v3.4.5
+   */
+
+  AES128GCM2 = 7,
+  /**
+   * 8: 256-bit AES encryption, GCM mode.
+   *
+   * @since v3.4.5
+   */
+  AES256GCM2 = 8,
 }
 
 export enum VideoStreamType {
@@ -185,6 +199,7 @@ export default class RtcEngine {
     ['RemoteVideoStateChanged', () => null],
     ['NetworkQuality', () => null],
     ['ActiveSpeaker', () => null],
+    ['StreamMessage', () => null],
   ]);
   public localStream: LocalStream = {};
   public screenStream: ScreenStream = {};
@@ -202,6 +217,7 @@ export default class RtcEngine {
   private muteLocalVideoMutex = false;
   private muteLocalAudioMutex = false;
   private speakerDeviceId = '';
+  private usersVolumeLevel = [];
   // Create channel profile and set it here
 
   // Create channel profile and set it here
@@ -217,8 +233,19 @@ export default class RtcEngine {
     return engine;
   }
 
+  getRemoteVideoStats(id: string) {
+    try {
+      const data = this.client.getRemoteVideoStats();
+      return data && data[id] ? data[id] : null;
+    } catch (error) {
+      console.log(`ERROR: on getRemoteVideoStats - ` + error?.message);
+      return null;
+    }
+  }
+
   async setVideoProfile(profile: VideoProfile): Promise<void> {
     this.videoProfile = profile;
+    this.localStream?.video?.setEncoderConfiguration(profile);
   }
 
   async enableAudio(): Promise<void> {
@@ -240,7 +267,10 @@ export default class RtcEngine {
     }
   }
 
-  async enableVideo(): Promise<void> {
+  async enableVideo(
+    preferredCameraId?: string,
+    preferredMicrophoneId?: string,
+  ): Promise<void> {
     /**
      * Issue: Backgrounding the browser or app causes the audio streaming to be cut off.
      * Impact: All browsers and apps that use WKWebView on iOS 15.x, such as Safari and Chrome.
@@ -252,14 +282,16 @@ export default class RtcEngine {
 
     const audioConfig: MicrophoneAudioTrackInitConfig = {
       bypassWebAudio: Platform.OS == 'web' && isMobileOrTablet(),
-      // microphoneId: this.audioDeviceId,
+      microphoneId: preferredMicrophoneId,
     };
     const videoConfig: CameraVideoTrackInitConfig = {
       encoderConfig: this.videoProfile,
-      // cameraId: this.videoDeviceId,
+      cameraId: preferredCameraId,
     };
     try {
       let [localAudio, localVideo] =
+        // If preferred devices are not present, the createTrack call will fallover to
+        // the catch block below.
         await AgoraRTC.createMicrophoneAndCameraTracks(
           audioConfig,
           videoConfig,
@@ -277,9 +309,15 @@ export default class RtcEngine {
     } catch (e) {
       let audioError = false;
       let videoError = false;
-      try {
-        let localAudio = await AgoraRTC.createMicrophoneAudioTrack(audioConfig);
 
+      try {
+        let localAudio: IMicrophoneAudioTrack;
+        try {
+          localAudio = await AgoraRTC.createMicrophoneAudioTrack(audioConfig);
+        } catch (e) {
+          videoConfig.microphoneId = '';
+          localAudio = await AgoraRTC.createMicrophoneAudioTrack(audioConfig);
+        }
         this.localStream.audio = localAudio;
         this.audioDeviceId = localAudio
           ?.getMediaStreamTrack()
@@ -288,8 +326,41 @@ export default class RtcEngine {
       } catch (error) {
         audioError = error;
       }
+
       try {
-        let localVideo = await AgoraRTC.createCameraVideoTrack(videoConfig);
+        let localVideo: ICameraVideoTrack;
+        try {
+          localVideo = await AgoraRTC.createCameraVideoTrack(videoConfig);
+        } catch (e) {
+          videoConfig.cameraId = '';
+          try {
+            localVideo = await AgoraRTC.createCameraVideoTrack(videoConfig);
+          } catch (e) {
+            console.log(
+              '[RTCEngineBridge]: Provided cameraId and default camera failed, trying other available devices',
+            );
+            const devices = await navigator.mediaDevices.enumerateDevices();
+            for (let device of devices) {
+              if (device.kind === 'videoinput') {
+                videoConfig.cameraId = device.deviceId;
+                try {
+                  localVideo = await AgoraRTC.createCameraVideoTrack(
+                    videoConfig,
+                  );
+                  break;
+                } catch (e) {
+                  videoError = e;
+                  console.log(
+                    '[RTCEngineBridge]:',
+                    'Camera not available with deviceId' + device,
+                    'Reason: ',
+                    e,
+                  );
+                }
+              }
+            }
+          }
+        }
         this.localStream.video = localVideo;
         this.videoDeviceId = localVideo
           ?.getMediaStreamTrack()
@@ -298,6 +369,7 @@ export default class RtcEngine {
       } catch (error) {
         videoError = error;
       }
+
       e.status = {audioError, videoError};
       throw e;
       // if (audioError && videoError) throw e;
@@ -357,7 +429,7 @@ export default class RtcEngine {
     optionalUid: number,
   ): Promise<void> {
     // TODO create agora client here
-    this.client.on('user-joined', (user) => {
+    this.client.on('user-joined', user => {
       (this.eventsMap.get('UserJoined') as callbackType)(user.uid);
       (this.eventsMap.get('RemoteVideoStateChanged') as callbackType)(
         user.uid,
@@ -373,7 +445,7 @@ export default class RtcEngine {
       );
     });
 
-    this.client.on('user-left', (user) => {
+    this.client.on('user-left', user => {
       const uid = user.uid;
       if (this.remoteStreams.has(uid)) {
         this.remoteStreams.delete(uid);
@@ -452,7 +524,10 @@ export default class RtcEngine {
       }
     });
 
-    this.client.on('volume-indicator', (volumes) => {
+    this.client.on('volume-indicator', volumes => {
+      this.usersVolumeLevel = volumes;
+      /**
+       * old active speaker logic
       const highestvolumeObj = volumes.reduce(
         (highestVolume, volume, index) => {
           if (highestVolume === null) {
@@ -463,7 +538,7 @@ export default class RtcEngine {
             }
             return highestVolume;
           }
-          // console.log(`${index} UID ${volume.uid} Level ${volume.level}`);
+          //console.log(`${index} UID ${volume.uid} Level ${volume.level}`);
         },
         null,
       );
@@ -480,6 +555,7 @@ export default class RtcEngine {
         activeSpeakerCallBack(activeSpeakerUid);
         this.activeSpeakerUid = activeSpeakerUid;
       }
+       */
     });
 
     // this.client.on('stream-fallback', (evt))
@@ -503,7 +579,7 @@ export default class RtcEngine {
         const remoteUserNetworkQualities =
           this.client.getRemoteNetworkQuality();
 
-        Object.keys(remoteUserNetworkQualities).forEach((uid) => {
+        Object.keys(remoteUserNetworkQualities).forEach(uid => {
           networkQualityIndicatorCallback(
             uid,
             remoteUserNetworkQualities[uid].downlinkNetworkQuality,
@@ -513,6 +589,14 @@ export default class RtcEngine {
       },
     );
 
+    /* Recieve Captions  */
+    this.client.on('stream-message', (uid: UID, payload: UInt8Array) => {
+      console.log(
+        `stt-web: onStreamMessageCallback uid:${uid} , payload:${payload}`,
+      );
+      (this.eventsMap.get('StreamMessage') as callbackType)(uid, payload);
+    });
+
     await this.client.join(
       this.appId,
       channelName,
@@ -525,6 +609,10 @@ export default class RtcEngine {
     console.log('enabling screen sleep');
   }
 
+  getUsersVolumeLevel() {
+    return this.usersVolumeLevel;
+  }
+
   async leaveChannel(): Promise<void> {
     this.client.leave();
     this.remoteStreams.forEach((stream, uid, map) => {
@@ -547,10 +635,12 @@ export default class RtcEngine {
       event === 'RemoteAudioStateChanged' ||
       event === 'RemoteVideoStateChanged' ||
       event === 'NetworkQuality' ||
-      event === 'ActiveSpeaker'
+      event === 'ActiveSpeaker' ||
+      event === 'StreamMessage'
     ) {
       this.eventsMap.set(event, listener as callbackType);
     }
+
     return {
       remove: () => {
         console.log(
@@ -730,11 +820,11 @@ export default class RtcEngine {
 
   async changeSpeaker(speakerId, callback, error) {
     try {
-      this.speakerDeviceId = speakerId;
       // setting sepeaker for all remote stream (previously joined users)
       this.remoteStreams?.forEach((stream, uid, map) => {
         stream?.audio?.setPlaybackDevice(speakerId);
       });
+      this.speakerDeviceId = speakerId;
       callback(speakerId);
     } catch (e) {
       error(e);
@@ -762,7 +852,7 @@ export default class RtcEngine {
 
   // Bug in implementation !!!
   async setRemoteSubscribeFallbackOption(option: 0 | 1 | 2) {
-    this.streams.forEach((stream) => {
+    this.streams.forEach(stream => {
       this.client.setStreamFallbackOption(stream, option);
     });
     Promise.resolve();
@@ -787,6 +877,17 @@ export default class RtcEngine {
           break;
         case RnEncryptionEnum.SM4128ECB:
           mode = 'sm4-128-ecb';
+          break;
+        case RnEncryptionEnum.AES256GCM:
+          mode = 'aes-256-gcm';
+          break;
+        case RnEncryptionEnum.AES128GCM2:
+          mode = 'aes-128-gcm2';
+          break;
+        case RnEncryptionEnum.AES256GCM2:
+          mode = 'aes-256-gcm2';
+          break;
+
         default:
           mode = 'none';
       }
@@ -801,14 +902,25 @@ export default class RtcEngine {
     config: {
       encryptionMode: RnEncryptionEnum;
       encryptionKey: string;
+      encryptionKdfSalt: string;
     },
   ): Promise<void> {
     let mode: EncryptionMode;
     mode = this.getEncryptionMode(enabled, config?.encryptionMode);
     try {
       await Promise.all([
-        this.client.setEncryptionConfig(mode, config.encryptionKey),
-        this.screenClient.setEncryptionConfig(mode, config.encryptionKey),
+        this.client.setEncryptionConfig(
+          mode,
+          config.encryptionKey,
+          config.encryptionKdfSalt,
+          true, // encryptDataStream
+        ),
+        this.screenClient.setEncryptionConfig(
+          mode,
+          config.encryptionKey,
+          config.encryptionKdfSalt,
+          true, // encryptDataStream
+        ),
       ]);
     } catch (e) {
       throw e;
@@ -892,6 +1004,7 @@ export default class RtcEngine {
     encryption: {
       screenKey: string;
       mode: RnEncryptionEnum;
+      salt: string;
     },
     config: ScreenVideoTrackInitConfig = {},
     audio: 'enable' | 'disable' | 'auto' = 'auto',
@@ -912,6 +1025,8 @@ export default class RtcEngine {
             await this.screenClient.setEncryptionConfig(
               mode,
               encryption.screenKey,
+              encryption.salt,
+              true, // encryptDataStream
             );
           } catch (e) {
             console.log('e: Encryption for screenshare failed', e);

package/template/bridge/rtc/webNg/index.ts

@@ -28,5 +28,76 @@ export const VideoRenderMode = {...Types.VideoRenderMode};
 export const VideoMirrorMode = {...Types.VideoRenderMode};
 
 export {RnEncryptionEnum as EncryptionMode} from './RtcEngine';
-export {AREAS as AreaCode} from './RtcEngine'
+export {AREAS as AreaCode} from './RtcEngine';
+export type VideoEncoderConfiguration = any;
+export declare enum AudioScenario {
+  /**
+   * 0: Default audio scenario.
+   *
+   * **Note**
+   *  If you run the iOS app on an M1 Mac, due to the hardware differences between M1 Macs, iPhones, and iPads,
+   * the default audio scenario of the Agora iOS SDK is the same as that of the Agora macOS SDK.
+   */
+  Default = 0,
+  /**
+   * 1: Entertainment scenario where users need to frequently switch the user role.
+   */
+  ChatRoomEntertainment = 1,
+  /**
+   * 2: Education scenario where users want smoothness and stability.
+   */
+  Education = 2,
+  /**
+   * 3: High-quality audio chatroom scenario where hosts mainly play music.
+   */
+  GameStreaming = 3,
+  /**
+   * 4: Showroom scenario where a single host wants high-quality audio.
+   */
+  ShowRoom = 4,
+  /**
+   * 5: Gaming scenario for group chat that only contains the human voice.
+   */
+  ChatRoomGaming = 5,
+  /**
+   * IoT (Internet of Things) scenario where users use IoT devices with low power consumption.
+   *
+   * @since v3.2.0.
+   */
+  IOT = 6,
+  /**
+   * Meeting scenario that mainly contains the human voice.
+   *
+   * @since v3.2.0.
+   */
+  MEETING = 8,
+}
+export declare enum AudioProfile {
+  /**
+   * 0: Default audio profile.
+   * - In the `Communication` profile: A sample rate of 32 KHz, audio encoding, mono, and a bitrate of up to 18 Kbps.
+   * - In the `LiveBroadcasting` profile: A sample rate of 48 KHz, music encoding, mono, and a bitrate of up to 64 Kbps.
+   */
+  Default = 0,
+  /**
+   * 1: A sample rate of 32 KHz, audio encoding, mono, and a bitrate of up to 18 Kbps.
+   */
+  SpeechStandard = 1,
+  /**
+   * 2: A sample rate of 48 KHz, music encoding, mono, and a bitrate of up to 64 Kbps.
+   */
+  MusicStandard = 2,
+  /**
+   * 3: A sample rate of 48 KHz, music encoding, stereo, and a bitrate of up to 80 Kbps.
+   */
+  MusicStandardStereo = 3,
+  /**
+   * 4: A sample rate of 48 KHz, music encoding, mono, and a bitrate of up to 96 Kbps.
+   */
+  MusicHighQuality = 4,
+  /**
+   * 5: A sample rate of 48 KHz, music encoding, stereo, and a bitrate of up to 128 Kbps.
+   */
+  MusicHighQualityStereo = 5,
+}
 export default RtcEngine;

package/template/bridge/rtm/web/index.ts

@@ -18,10 +18,10 @@ import {
 } from 'agora-react-native-rtm/lib/typescript/src';
 import {RtmClientEvents} from 'agora-react-native-rtm/lib/typescript/src/RtmEngine';
 import AgoraRTM, {VERSION} from 'agora-rtm-sdk';
+import RtmClient from 'agora-react-native-rtm';
 // export {RtmAttribute}
 //
-interface RtmAttributePlaceholder {
-};
+interface RtmAttributePlaceholder {}
 export {RtmAttributePlaceholder as RtmAttribute};
 
 type callbackType = (args?: any) => void;
@@ -99,6 +99,8 @@ export default class RtmEngine {
     this.appId = APP_ID;
     this.client = AgoraRTM.createInstance(this.appId);
 
+    window.rtmClient = this.client;
+
     this.client.on('ConnectionStateChanged', (state, reason) => {
       this.clientEventsMap.get('connectionStateChanged')({state, reason});
     });
@@ -174,7 +176,7 @@ export default class RtmEngine {
   }
 
   async logout(): Promise<any> {
-    return this.client.logout();
+    return await this.client.logout();
   }
 
   async joinChannel(channelId: string): Promise<any> {

package/template/configTransform.js

@@ -12,7 +12,22 @@
 let jsonFile = require('./config.json');
 let PREFIX = '$config';
 let config = {};
-Object.keys(jsonFile).map((key) => {
+Object.keys(jsonFile).map(key => {
   config[`${PREFIX}.${key}`] = jsonFile[key];
 });
+
+//find any missing config
+const defaultConfig = require('./defaultConfig');
+const filteredArray = Object.keys(defaultConfig).filter(
+  value => !Object.keys(jsonFile).includes(value),
+);
+
+//add missing config with default value
+if (filteredArray && filteredArray?.length) {
+  //console.error('config.json missing ', filteredArray);
+  filteredArray.forEach(i => {
+    config[`${PREFIX}.${i}`] = defaultConfig[i];
+  });
+}
+
 module.exports = config;

package/template/customization-api/action-library.ts

@@ -8,11 +8,3 @@ export {default as useRemoteEndcall} from '../src/utils/useRemoteEndCall';
 export {default as useSwitchCamera} from '../src/utils/useSwitchCamera';
 
 export {controlMessageEnum} from '../src/components/ChatContext';
-//audio/video toggle state
-export {ToggleState} from '../agora-rn-uikit/src/Contexts/PropsContext';
-
-export {
-  useChatUIControls,
-  ChatType,
-} from '../src/components/chat-ui/useChatUIControls';
-export type {ChatUIControlsInterface} from '../src/components/chat-ui/useChatUIControls';

package/template/customization-api/app-state.ts

@@ -26,8 +26,8 @@ export {useLocalUserInfo} from '../src/app-state/useLocalUserInfo';
  * UI contexts
  */
 // commented for v1 release
-//export {usePreCall} from '../src/components/precall/usePreCall';
-//export type {PreCallContextInterface} from '../src/components/precall/usePreCall';
+export {usePreCall} from '../src/components/precall/usePreCall';
+export type {PreCallContextInterface} from '../src/components/precall/usePreCall';
 export {useLayout} from '../src/utils/useLayout';
 export type {LayoutContextInterface} from '../src/utils/useLayout';
 // commented for v1 release
@@ -47,5 +47,14 @@ export type {messageInterface} from '../src/app-state/useMessages';
 export {SidePanelType} from '../src/subComponents/SidePanelEnum';
 export {useSidePanel} from '../src/utils/useSidePanel';
 
+export {useNoiseSupression} from '../src/app-state/useNoiseSupression';
+export {useVideoQuality} from '../src/app-state/useVideoQuality';
+
 //hook used to get/set username
 export {default as useUserName} from '../src/utils/useUserName';
+
+export {
+  useChatUIControls,
+  ChatType,
+} from '../src/components/chat-ui/useChatUIControls';
+export type {ChatUIControlsInterface} from '../src/components/chat-ui/useChatUIControls';

package/template/customization-api/customEvents.ts

@@ -1,9 +1,14 @@
 // 1. Import Events
 import Events from '../src/rtm-events-api/Events';
-import {EventSource, PersistanceLevel} from '../src/rtm-events-api';
+import {
+  EventSource,
+  PersistanceLevel,
+  EventCallback,
+} from '../src/rtm-events-api';
 
 // 2. Initialize with source "fpe"
 const customEvents = new Events(EventSource.fpe);
 
 // 3. export
 export {customEvents, PersistanceLevel};
+export type {EventCallback};

package/template/customization-api/index.ts

@@ -24,3 +24,7 @@ export * from './customEvents';
 export * from './sub-components';
 export * from './typeDefinition';
 export * from './utils';
+export * from './types';
+
+//TODO: hari remove later - used for simple-practice demo
+export * from './temp';

package/template/customization-api/sub-components.ts

@@ -1,7 +1,7 @@
 //Common components
 export {default as TextInput} from '../src/atoms/TextInput';
 //Icons
-export {Icons} from '../agora-rn-uikit';
+//export {Icons} from '../agora-rn-uikit';
 
 // commented for v1 release
 //create screen
@@ -13,7 +13,7 @@ export {Icons} from '../agora-rn-uikit';
 
 // commented for v1 release
 //Precall components
-//export * from '../src/components/precall/index';
+export * from '../src/components/precall/index';
 
 //video call components
 export {
@@ -33,6 +33,8 @@ export {default as NetworkQualityPill} from '../src/subComponents/NetworkQuality
 //videocall screen
 export {default as VideocallScreen} from '../src/pages/video-call/VideoCallScreen';
 export {default as PrecallScreen} from '../src/components/Precall';
+export {default as VBPanel} from '../src/components/virtual-background/VBPanel';
+export {WhiteboardListener} from '../src/components/Controls';
 
 // commented for v1 release
 //Settings screen
@@ -44,11 +46,12 @@ export {default as PrecallScreen} from '../src/components/Precall';
 //export {default as LanguageSelector} from '../src/subComponents/LanguageSelector';
 
 //Sidepanel buttons
-export {SidePanelButtonsArray} from '../src/subComponents/SidePanelButtons';
+//export {SidePanelButtonsArray} from '../src/subComponents/SidePanelButtons';
 export {
   MaxVideoView as UiKitMaxVideoView,
   ClientRole as UikitClientRole,
   ChannelProfile as UikitChannelProfile,
+  PropsContext as UikitPropsContext,
 } from '../agora-rn-uikit';
 export {default as Toolbar} from '../src/atoms/Toolbar';
 export {default as ToolbarItem} from '../src/atoms/ToolbarItem';

package/template/customization-api/temp.ts

@@ -0,0 +1,50 @@
+//exporting for simple-practice demo
+
+import VideoRenderer from '../src/pages/video-call/VideoRenderer';
+import {DispatchContext} from '../agora-rn-uikit';
+import IconButton from '../src/atoms/IconButton';
+import WhiteboardView from '../src/components/whiteboard/WhiteboardView';
+import {
+  useWhiteboard,
+  whiteboardContext,
+} from '../src/components/whiteboard/WhiteboardConfigure';
+import {useVideoCall} from '../src/components/useVideoCall';
+import {useScreenshare} from '../src/subComponents/screenshare/useScreenshare';
+import EndcallPopup from '../src/subComponents/EndcallPopup';
+import useSTTAPI from '../src/subComponents/caption/useSTTAPI';
+import {useCaption} from '../src/subComponents/caption/useCaption';
+import useMuteToggleLocal, {
+  MUTE_LOCAL_TYPE,
+} from '../src/utils/useMuteToggleLocal';
+import {RoomPhase} from 'white-web-sdk';
+import {useScreenContext} from '../src/components/contexts/ScreenShareContext';
+import {filterObject} from '../src/utils/index';
+import {useToggleWhiteboard} from '../src/components/Controls';
+import {ShowInputURL} from '../src/components/Share';
+import useRemoteMute, {MUTE_REMOTE_TYPE} from '../src/utils/useRemoteMute';
+import getCustomRoute from '../src/utils/getCustomRoute';
+import endCallEveryOne from '../src/utils/endCallEveryOne';
+export {
+  VideoRenderer,
+  DispatchContext,
+  IconButton,
+  WhiteboardView,
+  whiteboardContext,
+  useVideoCall,
+  useScreenshare,
+  EndcallPopup,
+  useSTTAPI,
+  useCaption,
+  useWhiteboard,
+  useMuteToggleLocal,
+  MUTE_LOCAL_TYPE,
+  RoomPhase,
+  useScreenContext,
+  filterObject,
+  useToggleWhiteboard,
+  ShowInputURL,
+  useRemoteMute,
+  MUTE_REMOTE_TYPE,
+  getCustomRoute,
+  endCallEveryOne,
+};