@byteplus/rtc 4.61.2 → 4.62.0

package/index.d.ts

@@ -66,6 +66,8 @@ declare const Config: {
 	SEI_TIME_OUT: number;
 	SEI_COUNT_FPS: number;
 	PRE_ICE: boolean;
+	STATS_LOOP_INTERVAL: number;
+	HIDDEN_STATS: boolean;
 };
 declare const _default: VERTC;
 declare enum AudioMixingError {
@@ -1281,7 +1283,8 @@ export declare enum MirrorMode {
 }
 
 /** 
- * @brief The local mirror type. You cannot use mirror effect during transmission and encoding due to the restrictions of web explorer.
+ * @brief Whether to enable the mirror effect when rendering the video.
+ * You cannot use mirror effect during transmission and encoding due to the restrictions of web explorer.
  */
 export declare enum MirrorType {
 	
@@ -1291,7 +1294,7 @@ export declare enum MirrorType {
 	MIRROR_TYPE_NONE = 0,
 	
 	/** 
-	 * @brief Enable mirror effect for the local video only. RTC SDK will not add mirror effect on the video to encode and transmit to the other
+	 * @brief Enable mirror effect for the videos.
 	 */
 	MIRROR_TYPE_RENDER = 1
 }
@@ -1741,6 +1744,27 @@ export declare enum VideoSourceType {
 }
 export declare type RTCExtension = any;
 
+/** 
+ * @brief Audio frames struct
+ */
+export interface AudioFrameData {
+	
+	/** 
+	 * @brief Audio data in PCM format
+	 */
+	channels: Float32Array[];
+	
+	/** 
+	 * @brief The number of channels
+	 */
+	channelCount: number;
+	
+	/** 
+	 * @brief Sample rate in Hz.
+	 */
+	sampleRate: number;
+}
+
 /** 
  * Event of the state change of audio mixing
  */
@@ -2167,6 +2191,21 @@ export interface IAudioMixingManager {
 	 */
 	setAudioMixingPosition: (id: number, position: number) => void;
 	
+	/** 
+	 * @brief Toggle the callbacks carrying the mixing audio frames.
+	 * @param mixingId Audio from microphone or device audio.
+	 * You will get the callback carrying the audio data as soon as the audio mixing starts.
+	 * @param callback The callback function to receive the audio frames. An empty value indicates disabling the callback.
+	 * When the user disconnects, the audio data is empty.
+	 * @param frameSize The number of samples per frame. The interval between callbacks increases as the frameSize gets larger.
+	 * It can be set to 256, 512, 1024, 2048, 4096, 8192, or 16384. It defaults to 4096. We recommend use the default value unless the application where the audio frames to be used requires a specific size.
+	 * @note + You can call this method after the engine is created. And you will receive the audio frame once mixing audio starts.
+	 * + Leave the room will not void the setting.
+	 * @group Audio Mixing
+	 * @order 45
+	 */
+	setAudioFrameCallback: (id: number, callback?: (data: AudioFrameData) => void, frameSize?: number) => void;
+	
 	/** 
 	 * @brief Start PCM mixing.
 	 *        To mix multiple PCM audio data, call this API multiple times with different ids.
@@ -2545,7 +2584,7 @@ export interface IEngineEvents {
 	[EngineEventsTypes.onLocalAudioPropertiesReport]: (event: LocalAudioPropertiesInfo[]) => void;
 	
 	/** 
-	 * @brief After calling [enableAudioPropertiesReport](docs-104478#enableaudiopropertiesreport), you will periodically receive this callback for the information about the subscribed remote audio streams.
+	 * @brief After calling [enableAudioPropertiesReport](docs-104478#enableaudiopropertiesreport), you will receive the cycling callbacks, each providing instant information about the subscribed remote audio streams.
 	 *        The remote audio streams includes the microphone audio and screen audio collected using the RTC SDK internal mechanism/custom mechanism.
 	 * @param event Information about remote audio.
 	 * @group Audio Management
@@ -2779,7 +2818,7 @@ export interface IEngineEvents {
 		/** 
 		 * @brief Error message.
 		 */
-		message?: String;
+		message?: string;
 	}) => void;
 	
 	/** 
@@ -2807,7 +2846,7 @@ export interface IEngineEvents {
 		/** 
 		 * @brief Error message.
 		 */
-		message?: String;
+		message?: string;
 	}) => void;
 	
 	/** 
@@ -3411,7 +3450,7 @@ export interface IRTCEngine extends EnhancedEventEmitter<IEngineEvents> {
 	 * @param streamIndex Video stream properties.
 	 * @param videoPlayerOption View information and rendering mode. Since V4.61, call this API repeatedly with different playerIds to add the stream to different players.
 	 * Since V4.56, you can set the rotation of the stream via the `rotation` property.
-	 * @notes You should bind the view after the user joins a room. This setting will sustain when the user leaves the room.
+	 * @notes You should bind the view after the user joins a room. This setting will sustain until the user leaves the room.
 	 * @group Video Management
 	 * @order 21
 	 */
@@ -3425,6 +3464,17 @@ export interface IRTCEngine extends EnhancedEventEmitter<IEngineEvents> {
 	 */
 	setLocalVideoMirrorType: (mirrorType: MirrorType) => void;
 	
+	/** 
+	 * @brief When using internal rendering, enable mirroring for the remote stream.
+	 * @param userId ID of the remote user
+	 * @param streamIndex Mainstream or screen-sharing stream.
+	 * @param mirrorType The mirror type for the remote stream.
+	 * @notes + In addition, you must also call [setRemoteVideoPlayer](docs-104478#rtcengine-setremotevideoplayer) to mirror the remote stream in the specified player.
+	 * @group Video Management
+	 * @order 32
+	 */
+	setRemoteVideoMirrorType: (userId: string, streamIndex: StreamIndex, mirrorType: MirrorType) => void;
+	
 	/** 
 	 * @brief Sets the speaker or the earpiece as the audio playback device. The audio playback device defaults to the speaker.
 	 * @param deviceId ID of the audio playback device. Call [enumerateDevices](#enumeratedevices) to get device ID.
@@ -3474,17 +3524,17 @@ export interface IRTCEngine extends EnhancedEventEmitter<IEngineEvents> {
 	 * @param userId The user ID. The empty value indicates the local stream.
 	 * @return If the specified stream does not exist, the volume values are `0`.
 	 * + linear volume. The value is in linear relation to the original volume. The higher the value, the higher the volume. The range is [0,255]. <br>
-	 * &ensp; - [0, 25]: Silence
-	 * &ensp; - [26, 75]: Low volume
-	 * &ensp; - [76, 204]: Medium volume
-	 * &ensp; - [205, 255]: High volume
+	 * &ensp; - \[0, 25]: Silence
+	 * &ensp; - \[26, 75]: Low volume
+	 * &ensp; - \[76, 204]: Medium volume
+	 * &ensp; - \[205, 255]: High volume
 	 * + non-linear volume in dB. The value is in proportion to the log value of the original volume. You can use the value to recognize the Active Speaker in the room. The range is [-127, 0].
-	 * &ensp; - [-127, -60]: Silence
-	 * &ensp; - [-59, -40]: Low volume
-	 * &ensp; - [-39, -20]: Medium volume
-	 * &ensp; - [-19, 0]: High volume
+	 * &ensp; - \[-127, -60]: Silence
+	 * &ensp; - \[-59, -40]: Low volume
+	 * &ensp; - \[-39, -20]: Medium volume
+	 * &ensp; - \[-19, 0]: High volume
 	 * @notes + The interval for repeatedly calling this method should be at least 100 milliseconds.
-	 * + You can also get volume infor via [onLocalAudioPropertiesReport](docs-104479#onlocalaudiopropertiesreport) and [onRemoteAudioPropertiesReport](docs-104479#onremoteaudiopropertiesreport).
+	 * + You can also get the audio volume via [onLocalAudioPropertiesReport](docs-104479#onlocalaudiopropertiesreport) and [onRemoteAudioPropertiesReport](docs-104479#onremoteaudiopropertiesreport).
 	 * @group Audio Management
 	 * @order 27
 	 */
@@ -3493,6 +3543,23 @@ export interface IRTCEngine extends EnhancedEventEmitter<IEngineEvents> {
 		nonlinearVolume: number;
 	};
 	
+	/** 
+	 * @brief Toggle the callbacks carrying the audio frames.
+	 * @param streamIndex Audio from microphone or device audio.
+	 * @param userId ID of the remote user who published the audio. An empty value indicates the local audio.
+	 * When the Audio seclection enabled, only the local audio frames are available.
+	 * You will get the callback carrying the audio data as soon as the user starts publish audio.
+	 * @param callback The callback function to receive the audio frames. An empty value indicates disabling the callback.
+	 * When the user disconnects, the audio data is zero amplitude.
+	 * @param frameSize The number of samples per frame. The interval between callbacks increases as the frameSize gets larger.
+	 * It can be set to 256, 512, 1024, 2048, 4096, 8192, or 16384. It defaults to 4096. We recommend use the default value unless the application where the audio frames to be used requires a specific size.
+	 * @note + You can call this method after the engine is created. And you will receive the audio frame once the local audio starts capture, or the remote audio arrives.
+	 * + Neither rejoining the room nor republishing the stream will void the setting.
+	 * * @group Custom Stream Processing
+	 * @order 40
+	 */
+	setAudioFrameCallback: (streamIndex: StreamIndex, userId?: string, callback?: (data: AudioFrameData) => void, frameSize?: number) => void;
+	
 	/** 
 	 * @brief Pause receiving remote media streams.
 	 * @param mediaType Media stream type subscribed to.
@@ -3669,7 +3736,7 @@ export interface IRTCEngine extends EnhancedEventEmitter<IEngineEvents> {
 	 * @brief Starts pushing media streams to CDN and sets the relevant configurations.
 	 * @param transcode Configurations to be set when pushing media streams to CDN.
 	 * @return
-	 * @notes + After calling this API, you will receive the pushing a mixed stream to CDN events via [onStreamMixingEvent](104479#onstreammixingevent).
+	 * @notes + After calling this API, you will receive the pushing a mixed stream to CDN events via [onStreamMixingEvent](docs-104479#onstreammixingevent).
 	 *        + If you have subscribed to the push-to-CDN server callback in the [console](https://console.byteplus.com/rtc/cloudRTC?tab=callback), calling this API will receive a [TranscodeStarted](https://docs.byteplus.com/en/byteplus-rtc/docs/75125#transcodestarted) server callback notification. When calling this API repeatedly, subsequent calls to this API will trigger both [TranscodeStarted](https://docs.byteplus.com/en/byteplus-rtc/docs/75125#transcodestarted) and [TranscodeUpdated](https://docs.byteplus.com/en/byteplus-rtc/docs/75125#transcodeupdated) callbacks.
 	 *        + Call [stopLiveTranscoding](#stoplivetranscoding) to stop pushing mixed streams to CDN.
 	 * @group Push to CDN
@@ -4190,7 +4257,7 @@ export interface IRTCEngine extends EnhancedEventEmitter<IEngineEvents> {
 	 * @brief Take a snapshot of the local video.
 	 * @param streamIndex The specified video stream.
 	 * @param playerId Player ID. An empty value indicates the default player. If the default player does not exist by then, the call will not take effect.
-	 * @notes + The snapshot is taken with all video effects on, including rotation, mirroring, virtual background.
+	 * @notes + The snapshot is taken with all video effects on, including the beauty effect, virtual background and mirroring.
 	 *        + You can take the snapshot either using SDK internal video capture or customized capture.
 	 * @return `INVOKED_BEFORE_CAPTURE`: No such stream. Please make sure that the specified stream has been published.
 	 * @group Video Snapshot
@@ -4275,7 +4342,7 @@ export interface IRTCEngine extends EnhancedEventEmitter<IEngineEvents> {
 	
 	/** 
 	 * @hidden
-	*/
+	 */
 	connect: () => Promise<void>;
 	
 	/** 
@@ -4396,6 +4463,52 @@ export interface IRTCEngine extends EnhancedEventEmitter<IEngineEvents> {
 	 * @order 12
 	 */
 	setAudioSelectionConfig: (audioSelectionPriority: AudioSelectionPriority) => Promise<void>;
+	
+	/** 
+	 * @brief Gets the statistics of subscribed video streams and network states.
+	 * @return Include the camera video and the screen recording.
+	 * @notes The statistics obtained through this method within 1 second remain unchanged.
+	 * @group Audio & Video Transport
+	 * @order 66
+	 */
+	getRemoteVideoStats: () => Record<string, RemoteUserVideoStats[]>;
+	
+	/** 
+	 * @brief Gets the statistics of subscribed audio streams and network states.
+	 * @return Include the microphone audio and the device audio.
+	 * @notes The statistics obtained through this method within 1 second remain unchanged.
+	 * @group Audio & Video Transport
+	 * @order 67
+	 */
+	getRemoteAudioStats: () => Record<string, RemoteUserAudioStats[]>;
+	
+	/** 
+	 * @brief Gets the statistics of local video streams and network states.
+	 * @return Include the camera video and the screen recording.
+	 * @notes The statistics obtained through this method within 1 second remain unchanged.
+	 * @group Audio & Video Transport
+	 * @order 68
+	 */
+	getLocalVideoStats: () => LocalUserVideoStats;
+	
+	/** 
+	 * @brief Gets the statistics of local audio streams and network states.
+	 * @return Include the microphone audio and the device audio.
+	 * @notes The statistics obtained through this method within 1 second remain unchanged.
+	 * @group Audio & Video Transport
+	 * @order 69
+	 */
+	getLocalAudioStats: () => LocalUserAudioStats;
+	
+	/** 
+	 * @hidden for internal use
+	 */
+	getPublicVideoStats: () => Record<string, RemoteVideoStats>;
+	
+	/** 
+	 * @hidden for internal use
+	 */
+	getPublicAudioStats: () => Record<string, RemoteAudioStats>;
 }
 
 /** 
@@ -4578,7 +4691,7 @@ export interface InitPublishState {
 	video: boolean;
 	
 	/** 
-	 * @brief Computer audio
+	 * @brief Device audio
 	 */
 	screenAudio: boolean;
 	
@@ -4890,6 +5003,38 @@ export interface LocalStreamStats {
 	rxQuality?: number;
 }
 
+/** 
+ * State of the local media stream
+ */
+export interface LocalUserAudioStats {
+	
+	/** 
+	 * @brief State of the microphone audio stream
+	 */
+	mainAudioStats: LocalAudioStats;
+	
+	/** 
+	 * @brief State of the device audio stream
+	 */
+	screenAudioStats: LocalAudioStats;
+}
+
+/** 
+ * Statistics of local video streams and network states in the cycle of 1s.
+ */
+export interface LocalUserVideoStats {
+	
+	/** 
+	 * @brief State of the camera video stream.
+	 */
+	mainVideoStats: LocalVideoStats;
+	
+	/** 
+	 * @brief State of the screen recording stream.
+	 */
+	screenVideoStats: LocalVideoStats;
+}
+
 /** 
  * Local video states
  */
@@ -5431,6 +5576,38 @@ export interface RemoteStreamSwitch {
 	reason: FallbackOrRecoverReason;
 }
 
+/** 
+ * Statistics of subscribed audio streams and network states in the cycle of 1s.
+ */
+export interface RemoteUserAudioStats {
+	
+	/** 
+	 * @brief State of the microphone audio stream
+	 */
+	mainAudioStats: RemoteAudioStats;
+	
+	/** 
+	 * @brief State of the device audio stream
+	 */
+	screenAudioStats: RemoteAudioStats;
+}
+
+/** 
+ * Statistics of subscribed video streams and network states in the cycle of 1s.
+ */
+export interface RemoteUserVideoStats {
+	
+	/** 
+	 * @brief State of the camera video stream.
+	 */
+	mainVideoStats: RemoteVideoStats;
+	
+	/** 
+	 * @brief State of the screen recording stream.
+	 */
+	screenVideoStats: RemoteVideoStats;
+}
+
 /** 
  * @brief Configuration of remote video stream
  */
@@ -5697,6 +5874,7 @@ export interface SetParamInterface {
 	SEI_COUNT_FPS: number;
 	/** 是否开启提前建连 */
 	PRE_ICE: boolean;
+	STATS_LOOP_INTERVAL: number;
 }
 
 /** 
@@ -5973,7 +6151,7 @@ export interface VideoEncoderConfig {
 export interface VideoPlayerOption {
 	
 	/** 
-	 * @brief DOM
+	 * @brief Render window, as the parent element for video rendering. Video's `htmlVideoElement` will be attached under this parent element for rendering, and the width and height of the rendered video will be based on `renderDom`, and is determined whether to completely fill the entire area of the parent element depending on the rendering mode.
 	 */
 	renderDom?: string | HTMLElement;