@agora-js/media 4.20.0 → 4.20.1-100

package/rtc-sdk-media_en.d.ts

@@ -279,6 +279,8 @@ export declare class AudioProcessorDestination extends EventEmitter implements I
     }): void;
 }
 
+declare type AudioRenderMode = "stereo" | "mono" | "surround";
+
 declare abstract class AudioSource extends EventEmitter {
     outputNode: GainNode;
     outputTrack?: MediaStreamTrack;
@@ -646,6 +648,7 @@ export declare interface CustomAudioTrackInitConfig {
      * > Firefox does not support setting the audio encoding rate.
      */
     encoderConfig?: AudioEncoderConfiguration | AudioEncoderConfigurationPreset;
+    channelCount?: number;
 }
 
 /**
@@ -896,6 +899,18 @@ declare function event_beauty_effect_overload(): void;
  */
 declare function event_first_frame_decoded(): void;
 
+/**
+ * **Since**
+ * <br>&emsp;&emsp;&emsp;*4.20.1*
+ *
+ * Parsing SEI data from the H.264 video stream triggers this event, which returns the SEI data.
+ *
+ * @param sei SEI data in Uint8Array
+ * @asMemberOf ILocalVideoTrack
+ * @event
+ */
+declare function event_sei_received(sei: Uint8Array): void;
+
 /**
  * Occurs when the state of processing the audio buffer in [BufferSourceAudioTrack]{@link IBufferSourceAudioTrack} changes.
  *
@@ -999,6 +1014,14 @@ declare function event_video_element_visible_status(data?: CheckVideoVisibleResu
  */
 declare function event_video_element_visible_status_2(data?: CheckVideoVisibleResult): void;
 
+/**
+ * Occurs when the video state changes.
+ *
+ * @event
+ * @asMemberOf IRemoteTrack
+ */
+declare function event_video_state_changed(videoState: VideoState): void;
+
 export declare interface ExtendedVideoFrameCallbackMetadata extends VideoFrameCallbackMetadata {
     timestamp: DOMHighResTimeStamp;
 }
@@ -1042,7 +1065,7 @@ export declare function getAudioContext(): AudioContext;
 
 export declare function getAudioEncoderConfiguration(profile: AudioEncoderConfigurationPreset | AudioEncoderConfiguration): AudioEncoderConfiguration;
 
-export declare function getBitrateFromResAndFps(width: number, height: number, fps: number, bitrateAdapterType: "STANDARD_BITRATE" | "COMPATIABLE_BITRATE"): [number, number] | void;
+export declare function getBitrateConstrainRange(width: number, height: number, frameRate: number, bitrateMin?: number, bitrateMax?: number): DoubleRange;
 
 export declare function getCompatibility(): AgoraRTCCompatibility;
 
@@ -1317,7 +1340,6 @@ export declare interface ILocalAudioTrack extends ILocalTrack {
      *
      * > Note:
      * > - As of v4.7.0, this method no longer takes effect. Use [IRemoteAudioTrack.setPlaybackDevice]{@link IRemoteAudioTrack.setPlaybackDevice} instead.
-     * > - This method supports Chrome on desktop devices only. Other browsers throw a '`NOT_SUPPORTED` error when calling this method.
      *
      * Sets the playback device (speaker) for the remote audio stream.
      *
@@ -1330,6 +1352,8 @@ export declare interface ILocalAudioTrack extends ILocalTrack {
      * **DEPRECATED** from v4.1.0. Use [AgoraRTCClient.getLocalVideoStats]{@link IAgoraRTCClient.getLocalVideoStats} and [AgoraRTCClient.getLocalAudioStats]{@link IAgoraRTCClient.getLocalAudioStats} instead.
      */
     getStats(): LocalAudioTrackStats;
+    selectChannels(channels: number[], mode: AudioRenderMode): void;
+    getVolumeLevels(channels?: number[]): number[];
     /**
      * Inserts a `Processor` to the local audio track.
      *
@@ -1424,7 +1448,7 @@ export declare interface ILocalTrack extends ITrack {
      *
      * If the track is published, a successful call of `setMuted(true)` triggers the [user-unpublished]{@link IAgoraRTCClient.event_user_unpublished} event on the remote client, and a successful call of `setMuted(false)` triggers the [user-published]{@link IAgoraRTCClient.event_user_published} event.
      *
-     * > - Calling `setMuted(true)` does not stop capturing audio or video and takes shorter time to take effect than [[setEnabled]]. For details, see [What are the differences between setEnabled and setMuted?](https://docs.agora.io/en/Interactive%20Broadcast/faq/differences_between_setenabled_and_setmuted).
+     * > - Calling `setMuted(true)` does not stop capturing audio or video and takes shorter time to take effect than [[setEnabled]]. For details, see [What are the differences between setEnabled and setMuted?](https://docs.agora.io/en/interactive-live-streaming/develop/product-workflow?platform=web#setenabled-and-setmuted).
      * > - Do not call `setEnabled` and `setMuted` together.
      *
      * @param muted Whether to stop sending the media data of the track:
@@ -1469,7 +1493,7 @@ export declare interface ILocalVideoTrack extends ILocalTrack {
      */
     on(event: "transceiver-updated", listener: typeof event_transceiver_updated): void;
     /**
-     * Plays a local video track on the web page.
+     * Plays a remote video track on the web page.
      *
      * @param element Specifies a DOM element. The SDK will create a `<video>` element under the specified DOM element to play the video track. You can specify a DOM element in either of the following ways:
      * - `string`: Specify the ID of the DOM element.
@@ -1636,6 +1660,12 @@ export declare interface ILocalVideoTrack extends ILocalTrack {
      * @param config The video encoder configurations. You can pass either [[VideoEncoderConfigurationPreset]] or a customized [[VideoEncoderConfiguration]] object.
      */
     setEncoderConfiguration(config: VideoEncoderConfiguration | VideoEncoderConfigurationPreset): Promise<void>;
+    /**
+     * Add the SEI data to the H.264 video stream.
+     *
+     * @param config SEI data.
+     */
+    sendSeiData(sei: Uint8Array): void;
 }
 
 export declare interface ImageTypedData {
@@ -1677,6 +1707,18 @@ export declare interface IMicrophoneAudioTrack extends ILocalAudioTrack {
     setEnabled(enabled: boolean): Promise<void>;
 }
 
+export declare function interceptLocalAudioFrame(sender: RTCRtpSender): Promise<void>;
+
+export declare function interceptLocalVideoFrame(sender: RTCRtpSender, localVideoTrack: LocalVideoTrack): Promise<void>;
+
+export declare function interceptRemoteAudioFrame(receiver: RTCRtpReceiver): Promise<void>;
+
+export declare function interceptRemoteVideoFrame(receiver: RTCRtpReceiver, options?: InterceptVideoFrameOptions): Promise<void>;
+
+declare interface InterceptVideoFrameOptions {
+    onSei?: (sei: Uint8Array) => void;
+}
+
 /**
  * `RemoteAudioTrack` is the basic interface for the remote audio track.
  *
@@ -1701,7 +1743,7 @@ export declare interface IRemoteAudioTrack extends IRemoteTrack {
      *
      * Sets the audio playback device, for example, the speaker.
      *
-     * > This method supports Chrome on desktop devices only. Other browsers throw a `NOT_SUPPORTED` error when calling this method.
+     * > This method supports Chrome and Edge on desktop devices only. Other browsers throw a `NOT_SUPPORTED` error when calling this method.
      * @param deviceId Device ID, which can be retrieved by calling [[getPlaybackDevices]].
      */
     setPlaybackDevice(deviceId: string): Promise<void>;
@@ -1743,6 +1785,8 @@ export declare interface IRemoteAudioTrack extends IRemoteTrack {
      * Usually a user with audio level above 0.6 is a speaking user.
      */
     getVolumeLevel(): number;
+    selectChannels(channels: number[], mode: AudioRenderMode): void;
+    getVolumeLevels(channels?: number[]): number[];
     /**
      * **Since**
      * <br>&emsp;&emsp;&emsp;*4.13.0*
@@ -1820,6 +1864,11 @@ export declare interface IRemoteTrack extends ITrack {
  * You can get create a remote video track by the [AgoraRTCRemoteUser.videoTrack]{@link IAgoraRTCRemoteUser.videoTrack} object after calling [subscribe]{@link IAgoraRTCClient.subscribe}.
  */
 export declare interface IRemoteVideoTrack extends IRemoteTrack {
+    /**
+     * @param event The event name.
+     * @param listener See [video-state-changed]{@link event_video_state_changed}.
+     */
+    on(event: "video-state-changed", listener: typeof event_video_state_changed): void;
     /**
      * @param event The event name.
      * @param listener See [first-frame-decoded]{@link event_first_frame_decoded}.
@@ -1836,6 +1885,12 @@ export declare interface IRemoteVideoTrack extends IRemoteTrack {
      * @param listener See [IRemoteTrack.transceiver-updated]{@link event_transceiver_updated_2}.
      */
     on(event: "transceiver-updated", listener: typeof event_transceiver_updated_2): void;
+    /**
+     * Adds an event listener.
+     * @param event The event name.
+     * @param listener See [ILocalTrack.sei-received]{@link event_sei_received}.
+     */
+    on(event: "sei-received", listener: typeof event_sei_received): void;
     /**
      * Gets the statistics of a remote video track.
      *
@@ -1975,23 +2030,25 @@ export declare class LocalAudioTrack extends LocalTrack implements ILocalAudioTr
     readonly trackMediaType: "audio" | "video";
     _encoderConfig?: AudioEncoderConfiguration;
     _trackSource: AudioTrackSource | FakeTrackSource;
-    get _source(): AudioTrackSource | FakeTrackSource;
+    channelCount?: number;
+    private transformer?;
+    get _source(): AudioTrackSource;
     set _source(source: AudioTrackSource | FakeTrackSource);
     _enabled: boolean;
     private _volume;
     _useAudioElement: boolean;
     _bypassWebAudio: boolean;
     protected processor?: IAudioProcessor;
-    protected _processorContext: AudioProcessorContext;
+    protected _processorContext: AudioProcessorContext | undefined;
     protected get processorContext(): AudioProcessorContext;
-    protected set processorContext(ctx: AudioProcessorContext);
-    _processorDestination: AudioProcessorDestination;
+    _processorDestination: AudioProcessorDestination | undefined;
     get processorDestination(): AudioProcessorDestination;
-    set processorDestination(des: AudioProcessorDestination);
     protected _getOriginVolumeLevel: boolean;
     get isPlaying(): boolean;
     get __className__(): string;
-    constructor(track: MediaStreamTrack, encoderConfig?: AudioEncoderConfiguration, trackId?: string, getOriginVolumeLevel?: boolean, deferWebAudio?: boolean);
+    constructor(track: MediaStreamTrack, encoderConfig?: AudioEncoderConfiguration, trackId?: string, getOriginVolumeLevel?: boolean, channelCount?: number);
+    selectChannels(channels: number[], mode?: AudioRenderMode): void;
+    getVolumeLevels(channels?: number[]): number[];
     setVolume(volume: number): void;
     getVolumeLevel(): number;
     setPlaybackDevice(deviceId: string): Promise<void>;
@@ -2010,7 +2067,10 @@ export declare class LocalAudioTrack extends LocalTrack implements ILocalAudioTr
     unpipe(): void;
     private bindProcessorDestinationEvents;
     private unbindProcessorDestinationEvents;
-    private unbindProcessorContextEvents;
+    protected bindProcessorContextEvents(processorContext: AudioProcessorContext): void;
+    protected unbindProcessorContextEvents(processorContext: AudioProcessorContext): void;
+    private initWebAudio;
+    private initProcessor;
 }
 
 /**
@@ -2083,9 +2143,9 @@ export declare abstract class LocalTrack extends Track implements ILocalTrack {
     _isClosed: boolean;
     protected _enabledMutex: PromiseMutex;
     protected processor?: IBaseProcessor;
-    protected abstract _processorContext: IProcessorContext;
-    protected get processorContext(): IProcessorContext;
-    protected set processorContext(ctx: IProcessorContext);
+    protected abstract _processorContext: IProcessorContext | undefined;
+    protected get processorContext(): IProcessorContext | undefined;
+    abstract processorDestination: IBaseProcessor;
     constructor(track: MediaStreamTrack, trackId?: string);
     abstract getStats(): LocalVideoTrackStats | LocalAudioTrackStats;
     abstract setMuted(enabled: boolean): Promise<void>;
@@ -2164,6 +2224,7 @@ export declare class LocalVideoTrack extends LocalTrack implements ILocalVideoTr
     clone(config?: VideoEncoderConfiguration | VideoEncoderConfigurationPreset, cloneTrack?: boolean): LocalVideoTrack;
     replaceTrack(track: MediaStreamTrack, stopOldTrack: boolean): Promise<void>;
     startMonitorStats(): void;
+    sendSeiData(sei: Uint8Array): never;
     private bindProcessorDestinationEvents;
     private unbindProcessorDestinationEvents;
     private unbindProcessorContextEvents;
@@ -2334,7 +2395,8 @@ export declare class MicrophoneAudioTrack extends LocalAudioTrack implements IMi
     close(): void;
     protected onTrackEnded(): void;
     renewMediaStreamTrack(newConstraints?: MediaTrackConstraints): Promise<void>;
-    private bindProcessorContextEvents;
+    protected bindProcessorContextEvents(processorContext: AudioProcessorContext): void;
+    protected unbindProcessorContextEvents(processorContext: AudioProcessorContext): void;
 }
 
 /**
@@ -2427,15 +2489,19 @@ export declare class RemoteAudioTrack extends RemoteTrack implements IRemoteAudi
     readonly trackMediaType: "audio" | "video";
     _source: AudioTrackSource | FakeTrackSource;
     _useAudioElement: boolean;
+    channelCount?: number;
     private _volume;
     protected processorContext: AudioProcessorContext;
     processorDestination: AudioProcessorDestination;
     private _played;
     private _bypassWebAudio;
+    private transformer?;
     get isPlaying(): boolean;
     get __className__(): string;
-    constructor(track: MediaStreamTrack, userId: UID, uintId: number, store: SDKStore);
+    constructor(track: MediaStreamTrack, userId: UID, uintId: number, store: SDKStore, channelCount?: number);
     setAudioFrameCallback(callback: null | ((buffer: AudioBuffer) => void), frameSize?: number): void;
+    selectChannels(channels: number[], mode?: AudioRenderMode): void;
+    getVolumeLevels(channels?: number[]): number[];
     setVolume(volume: number): void;
     setPlaybackDevice(deviceId: string): Promise<void>;
     getVolumeLevel(): number;
@@ -2593,7 +2659,8 @@ export declare abstract class RemoteTrack extends Track implements IRemoteTrack
 
 export declare enum RemoteTrackEvents {
     FIRST_FRAME_DECODED = "first-frame-decoded",
-    VIDEO_ELEMENT_VISIBLE_STATUS = "video-element-visible-status"
+    VIDEO_ELEMENT_VISIBLE_STATUS = "video-element-visible-status",
+    VIDEO_STATE_CHANGED = "video-state-changed"
 }
 
 export declare class RemoteVideoTrack extends RemoteTrack implements IRemoteVideoTrack {
@@ -2621,6 +2688,7 @@ export declare class RemoteVideoTrack extends RemoteTrack implements IRemoteVide
     private bindProcessorDestinationEvents;
     private unbindProcessorDestinationEvents;
     _destroy(): void;
+    _onSei(sei: Uint8Array): void;
 }
 
 /**
@@ -2998,7 +3066,9 @@ export declare abstract class Track extends EventEmitter implements ITrack {
 }
 
 export declare enum TrackEvents {
-    TRANSCEIVER_UPDATED = "transceiver-updated"
+    TRANSCEIVER_UPDATED = "transceiver-updated",
+    SEI_TO_SEND = "sei-to-send",
+    SEI_RECEIVED = "sei-received"
 }
 
 export declare enum TrackHint {
@@ -3010,6 +3080,7 @@ export declare enum TrackHint {
 export declare enum TrackInternalEvent {
     NEED_RENEGOTIATE = "@need_renegotiate",
     NEED_REPLACE_TRACK = "@need_replace_track",
+    NEED_REPLACE_MIXING_TRACK = "@need_replace_mixing_track",
     NEED_CLOSE = "@need_close",
     NEED_ENABLE_TRACK = "@need_enable_track",
     NEED_DISABLE_TRACK = "@need_disable_track",
@@ -3141,6 +3212,7 @@ declare class VideoPlayer {
     trackId: string;
     config: PlayerConfig;
     onFirstVideoFrameDecoded?: () => void;
+    onVideoStateChanged?: (state: VideoState) => void;
     freezeTimeCounterList: number[];
     renderFreezeAccTime: number;
     isKeepLastFrame: boolean;
@@ -3154,7 +3226,11 @@ declare class VideoPlayer {
     protected cacheVideoElement?: HTMLVideoElement;
     get videoElementStatus(): MediaElementStatus;
     set videoElementStatus(status: MediaElementStatus);
+    get videoState(): VideoState;
+    set videoState(state: VideoState);
+    private _videoState;
     private videoElementCheckInterval?;
+    private videoElementFreezeTimeout?;
     private _videoElementStatus;
     private isGettingVideoDimensions;
     constructor(config: PlayerConfig);
@@ -3245,6 +3321,28 @@ export declare class VideoProcessorDestination extends EventEmitter implements I
     reset(): void;
 }
 
+/**
+ * The state of the video stream.
+ */
+export declare enum VideoState {
+    /**
+     * 0: The initial state of the video.
+     */
+    VideoStateStopped = 0,
+    /**
+     * 1: The local user has received the first video packet.
+     */
+    VideoStateStarting = 1,
+    /**
+     * 2: The video stream is being decoded and played normally.
+     */
+    VideoStateDecoding = 2,
+    /**
+     * 3: The video stream is frozen.
+     */
+    VideoStateFrozen = 3
+}
+
 declare class VisibilityWatcher extends EventEmitter {
     private _lastHiddenTime;
     private _lastVisibleTime;