agora-electron-sdk 4.3.0 → 4.3.1-dev.1

package/ts/Private/AgoraMediaBase.ts

@@ -73,6 +73,10 @@ export enum VideoSourceType {
    * 14: The fourth screen.
    */
   VideoSourceScreenFourth = 14,
+  /**
+   * @ignore
+   */
+  VideoSourceSpeechDriven = 15,
   /**
    * 100: An unknown video source.
    */
@@ -112,11 +116,11 @@ export enum AudioRoute {
    */
   RouteBluetoothDeviceHfp = 5,
   /**
-   * 7: The audio route is a USB peripheral device. (For macOS only)
+   * 6: The audio route is a USB peripheral device. (For macOS only)
    */
   RouteUsb = 6,
   /**
-   * 6: The audio route is an HDMI peripheral device. (For macOS only)
+   * 7: The audio route is an HDMI peripheral device. (For macOS only)
    */
   RouteHdmi = 7,
   /**
@@ -204,7 +208,7 @@ export enum MediaSourceType {
    */
   SecondaryScreenSource = 5,
   /**
-   * 6. Custom video source.
+   * 6: Custom video source.
    */
   CustomVideoSource = 6,
   /**
@@ -231,6 +235,10 @@ export enum MediaSourceType {
    * @ignore
    */
   TranscodedVideoSource = 12,
+  /**
+   * @ignore
+   */
+  SpeechDrivenVideoSource = 13,
   /**
    * 100: Unknown media source.
    */
@@ -449,6 +457,10 @@ export enum VideoPixelFormat {
    * @ignore
    */
   VideoTextureId3d11texture2d = 17,
+  /**
+   * @ignore
+   */
+  VideoPixelI010 = 18,
 }
 
 /**
@@ -611,6 +623,10 @@ export class ExternalVideoFrame {
    * @ignore
    */
   alphaBuffer?: Uint8Array;
+  /**
+   * @ignore
+   */
+  fillAlphaBuffer?: boolean;
   /**
    * This parameter only applies to video data in Windows Texture format. It represents an index of an ID3D11Texture2D texture object used by the video frame in the ID3D11Texture2D array.
    */
@@ -636,15 +652,15 @@ export class VideoFrame {
    */
   height?: number;
   /**
-   * For YUV data, the line span of the Y buffer; for RGBA data, the total data length.
+   * For YUV data, the line span of the Y buffer; for RGBA data, the total data length. When dealing with video data, it is necessary to process the offset between each line of pixel data based on this parameter, otherwise it may result in image distortion.
    */
   yStride?: number;
   /**
-   * For YUV data, the line span of the U buffer; for RGBA data, the value is 0.
+   * For YUV data, the line span of the U buffer; for RGBA data, the value is 0. When dealing with video data, it is necessary to process the offset between each line of pixel data based on this parameter, otherwise it may result in image distortion.
    */
   uStride?: number;
   /**
-   * For YUV data, the line span of the V buffer; for RGBA data, the value is 0.
+   * For YUV data, the line span of the V buffer; for RGBA data, the value is 0. When dealing with video data, it is necessary to process the offset between each line of pixel data based on this parameter, otherwise it may result in image distortion.
    */
   vStride?: number;
   /**
@@ -664,7 +680,7 @@ export class VideoFrame {
    */
   rotation?: number;
   /**
-   * The Unix timestamp (ms) when the video frame is rendered. This timestamp can be used to guide the rendering of the video frame. It is required.
+   * The Unix timestamp (ms) when the video frame is rendered. This timestamp can be used to guide the rendering of the video frame. This parameter is required.
    */
   renderTimeMs?: number;
   /**
@@ -782,7 +798,7 @@ export class AudioFrame {
    */
   samplesPerChannel?: number;
   /**
-   * The number of bytes per sample. The number of bytes per audio sample, which is usually 16-bit (2-byte).
+   * The number of bytes per sample. For PCM, this parameter is generally set to 16 bits (2 bytes).
    */
   bytesPerSample?: BytesPerSample;
   /**
@@ -815,6 +831,10 @@ export class AudioFrame {
    * @ignore
    */
   audioTrackNumber?: number;
+  /**
+   * @ignore
+   */
+  rtpTimestamp?: number;
 }
 
 /**
@@ -891,9 +911,6 @@ export interface IAudioFrameObserverBase {
    *
    * @param channelId The channel ID.
    * @param audioFrame The raw audio data. See AudioFrame.
-   *
-   * @returns
-   * Without practical meaning.
    */
   onRecordAudioFrame?(channelId: string, audioFrame: AudioFrame): void;
 
@@ -904,9 +921,6 @@ export interface IAudioFrameObserverBase {
    *
    * @param channelId The channel ID.
    * @param audioFrame The raw audio data. See AudioFrame.
-   *
-   * @returns
-   * Without practical meaning.
    */
   onPlaybackAudioFrame?(channelId: string, audioFrame: AudioFrame): void;
 
@@ -917,9 +931,6 @@ export interface IAudioFrameObserverBase {
    *
    * @param channelId The channel ID.
    * @param audioFrame The raw audio data. See AudioFrame.
-   *
-   * @returns
-   * Without practical meaning.
    */
   onMixedAudioFrame?(channelId: string, audioFrame: AudioFrame): void;
 
@@ -929,9 +940,6 @@ export interface IAudioFrameObserverBase {
    * In order to ensure that the obtained in-ear audio data meets the expectations, Agora recommends that you set the in-ear monitoring-ear audio data format as follows: After calling setEarMonitoringAudioFrameParameters to set the audio data format and registerAudioFrameObserver to register the audio frame observer object, the SDK calculates the sampling interval according to the parameters set in the methods, and triggers the onEarMonitoringAudioFrame callback according to the sampling interval.
    *
    * @param audioFrame The raw audio data. See AudioFrame.
-   *
-   * @returns
-   * Without practical meaning.
    */
   onEarMonitoringAudioFrame?(audioFrame: AudioFrame): void;
 }
@@ -943,12 +951,11 @@ export interface IAudioFrameObserver extends IAudioFrameObserverBase {
   /**
    * Retrieves the audio frame of a specified user before mixing.
    *
+   * Due to framework limitations, this callback does not support sending processed audio data back to the SDK.
+   *
    * @param channelId The channel ID.
    * @param uid The user ID of the specified user.
    * @param audioFrame The raw audio data. See AudioFrame.
-   *
-   * @returns
-   * Without practical meaning.
    */
   onPlaybackAudioFrameBeforeMixing?(
     channelId: string,
@@ -995,9 +1002,6 @@ export interface IAudioSpectrumObserver {
    * After successfully calling registerAudioSpectrumObserver to implement the onLocalAudioSpectrum callback in IAudioSpectrumObserver and calling enableAudioSpectrumMonitor to enable audio spectrum monitoring, the SDK will trigger the callback as the time interval you set to report the received remote audio data spectrum.
    *
    * @param data The audio spectrum data of the local user. See AudioSpectrumData.
-   *
-   * @returns
-   * Whether the spectrum data is received: true : Spectrum data is received. false : No spectrum data is received.
    */
   onLocalAudioSpectrum?(data: AudioSpectrumData): void;
 
@@ -1008,9 +1012,6 @@ export interface IAudioSpectrumObserver {
    *
    * @param spectrums The audio spectrum information of the remote user, see UserAudioSpectrumInfo. The number of arrays is the number of remote users monitored by the SDK. If the array is null, it means that no audio spectrum of remote users is detected.
    * @param spectrumNumber The number of remote users.
-   *
-   * @returns
-   * Whether the spectrum data is received: true : Spectrum data is received. false : No spectrum data is received.
    */
   onRemoteAudioSpectrum?(
     spectrums: UserAudioSpectrumInfo[],
@@ -1031,9 +1032,6 @@ export interface IVideoEncodedFrameObserver {
    * @param imageBuffer The encoded video image buffer.
    * @param length The data length of the video image.
    * @param videoEncodedFrameInfo For the information of the encoded video frame, see EncodedVideoFrameInfo.
-   *
-   * @returns
-   * Without practical meaning.
    */
   onEncodedVideoFrameReceived?(
     uid: number,
@@ -1064,18 +1062,12 @@ export interface IVideoFrameObserver {
   /**
    * Occurs each time the SDK receives a video frame captured by local devices.
    *
-   * After you successfully register the video frame observer, the SDK triggers this callback each time it receives a video frame. In this callback, you can get the video data captured by local devices. You can then pre-process the data according to your scenarios. Once the pre-processing is complete, you can directly modify videoFrame in this callback, and set the return value to true to send the modified video data to the SDK.
-   *  The video data that this callback gets has not been pre-processed such as watermarking, cropping, and rotating.
-   *  If the video data type you get is RGBA, the SDK does not support processing the data of the alpha channel.
+   * You can get raw video data collected by the local device through this callback.
    *
    * @param sourceType Video source types, including cameras, screens, or media player. See VideoSourceType.
    * @param videoFrame The video frame. See VideoFrame. The default value of the video frame data format obtained through this callback is as follows:
    *  macOS: I420 or CVPixelBufferRef
    *  Windows: YUV420
-   *
-   * @returns
-   * When the video processing mode is ProcessModeReadOnly : true : Reserved for future use. false : Reserved for future use.
-   *  When the video processing mode is ProcessModeReadWrite : true : Sets the SDK to receive the video frame. false : Sets the SDK to discard the video frame.
    */
   onCaptureVideoFrame?(
     sourceType: VideoSourceType,
@@ -1086,16 +1078,13 @@ export interface IVideoFrameObserver {
    * Occurs each time the SDK receives a video frame before encoding.
    *
    * After you successfully register the video frame observer, the SDK triggers this callback each time it receives a video frame. In this callback, you can get the video data before encoding and then process the data according to your particular scenarios. After processing, you can send the processed video data back to the SDK in this callback.
+   *  Due to framework limitations, this callback does not support sending processed video data back to the SDK.
    *  The video data that this callback gets has been preprocessed, with its content cropped and rotated, and the image enhanced.
    *
    * @param sourceType The type of the video source. See VideoSourceType.
    * @param videoFrame The video frame. See VideoFrame. The default value of the video frame data format obtained through this callback is as follows:
    *  macOS: I420 or CVPixelBufferRef
    *  Windows: YUV420
-   *
-   * @returns
-   * When the video processing mode is ProcessModeReadOnly : true : Reserved for future use. false : Reserved for future use.
-   *  When the video processing mode is ProcessModeReadWrite : true : Sets the SDK to receive the video frame. false : Sets the SDK to discard the video frame.
    */
   onPreEncodeVideoFrame?(
     sourceType: VideoSourceType,
@@ -1112,16 +1101,13 @@ export interface IVideoFrameObserver {
    *
    * After you successfully register the video frame observer, the SDK triggers this callback each time it receives a video frame. In this callback, you can get the video data sent from the remote end before rendering, and then process it according to the particular scenarios.
    *  If the video data type you get is RGBA, the SDK does not support processing the data of the alpha channel.
+   *  Due to framework limitations, this callback does not support sending processed video data back to the SDK.
    *
    * @param channelId The channel ID.
    * @param remoteUid The user ID of the remote user who sends the current video frame.
    * @param videoFrame The video frame. See VideoFrame. The default value of the video frame data format obtained through this callback is as follows:
    *  macOS: I420 or CVPixelBufferRef
    *  Windows: YUV420
-   *
-   * @returns
-   * When the video processing mode is ProcessModeReadOnly : true : Reserved for future use. false : Reserved for future use.
-   *  When the video processing mode is ProcessModeReadWrite : true : Sets the SDK to receive the video frame. false : Sets the SDK to discard the video frame.
    */
   onRenderVideoFrame?(
     channelId: string,
@@ -1247,6 +1233,50 @@ export class MediaRecorderConfiguration {
   recorderInfoUpdateInterval?: number;
 }
 
+/**
+ * Facial information observer.
+ *
+ * You can call registerFaceInfoObserver to register or unregister the IFaceInfoObserver object.
+ */
+export interface IFaceInfoObserver {
+  /**
+   * Occurs when the facial information processed by speech driven extension is received.
+   *
+   * @param outFaceInfo Output parameter, the JSON string of the facial information processed by the voice driver plugin, including the following fields:
+   *  faces: Object sequence. The collection of facial information, with each face corresponding to an object.
+   *  blendshapes: Object. The collection of face capture coefficients, named according to ARkit standards, with each key-value pair representing a blendshape coefficient. The blendshape coefficient is a floating point number with a range of [0.0, 1.0].
+   *  rotation: Object sequence. The rotation of the head, which includes the following three key-value pairs, with values as floating point numbers ranging from -180.0 to 180.0:
+   *  pitch: Head pitch angle. A positve value means looking down, while a negative value means looking up.
+   *  yaw: Head yaw angle. A positve value means turning left, while a negative value means turning right.
+   *  roll: Head roll angle. A positve value means tilting to the right, while a negative value means tilting to the left.
+   *  timestamp: String. The timestamp of the output result, in milliseconds. Here is an example of JSON:
+   * {
+   *  "faces":[{
+   *  "blendshapes":{
+   *  "eyeBlinkLeft":0.9, "eyeLookDownLeft":0.0, "eyeLookInLeft":0.0, "eyeLookOutLeft":0.0, "eyeLookUpLeft":0.0,
+   *  "eyeSquintLeft":0.0, "eyeWideLeft":0.0, "eyeBlinkRight":0.0, "eyeLookDownRight":0.0, "eyeLookInRight":0.0,
+   *  "eyeLookOutRight":0.0, "eyeLookUpRight":0.0, "eyeSquintRight":0.0, "eyeWideRight":0.0, "jawForward":0.0,
+   *  "jawLeft":0.0, "jawRight":0.0, "jawOpen":0.0, "mouthClose":0.0, "mouthFunnel":0.0, "mouthPucker":0.0,
+   *  "mouthLeft":0.0, "mouthRight":0.0, "mouthSmileLeft":0.0, "mouthSmileRight":0.0, "mouthFrownLeft":0.0,
+   *  "mouthFrownRight":0.0, "mouthDimpleLeft":0.0, "mouthDimpleRight":0.0, "mouthStretchLeft":0.0, "mouthStretchRight":0.0,
+   *  "mouthRollLower":0.0, "mouthRollUpper":0.0, "mouthShrugLower":0.0, "mouthShrugUpper":0.0, "mouthPressLeft":0.0,
+   *  "mouthPressRight":0.0, "mouthLowerDownLeft":0.0, "mouthLowerDownRight":0.0, "mouthUpperUpLeft":0.0, "mouthUpperUpRight":0.0,
+   *  "browDownLeft":0.0, "browDownRight":0.0, "browInnerUp":0.0, "browOuterUpLeft":0.0, "browOuterUpRight":0.0,
+   *  "cheekPuff":0.0, "cheekSquintLeft":0.0, "cheekSquintRight":0.0, "noseSneerLeft":0.0, "noseSneerRight":0.0,
+   *  "tongueOut":0.0
+   *  },
+   *  "rotation":{"pitch":30.0, "yaw":25.5, "roll":-15.5},
+   *
+   *  }],
+   *  "timestamp":"654879876546"
+   * }
+   *
+   * @returns
+   * true : Facial information JSON parsing successful. false : Facial information JSON parsing failed.
+   */
+  onFaceInfo?(outFaceInfo: string): void;
+}
+
 /**
  * @ignore
  */

package/ts/Private/IAgoraLog.ts

@@ -69,7 +69,7 @@ export enum LogFilterType {
  */
 export class LogConfig {
   /**
-   * The complete path of the log files. Ensure that the path for the log file exists and is writable. You can use this parameter to rename the log files. The default path is:
+   * The complete path of the log files. Agora recommends using the default log directory. If you need to modify the default directory, ensure that the directory you specify exists and is writable. The default log directory is:
    *  macOS:
    *  If Sandbox is enabled: App Sandbox/Library/Logs/agorasdk.log. For example, /Users/<username>/Library/Containers/<AppBundleIdentifier>/Data/Library/Logs/agorasdk.log.
    *  If Sandbox is disabled: ~/Library/Logs/agorasdk.log

package/ts/Private/IAgoraMediaEngine.ts

@@ -10,6 +10,7 @@ import {
   ExternalVideoFrame,
   ExternalVideoSourceType,
   IAudioFrameObserver,
+  IFaceInfoObserver,
   IVideoEncodedFrameObserver,
   IVideoFrameObserver,
 } from './AgoraMediaBase';
@@ -98,9 +99,28 @@ export abstract class IMediaEngine {
     observer: IVideoEncodedFrameObserver
   ): number;
 
+  /**
+   * Registers a facial information observer.
+   *
+   * You can call this method to register the onFaceInfo callback to receive the facial information processed by Agora speech driven extension. When calling this method to register a facial information observer, you can register callbacks in the IFaceInfoObserver class as needed. After successfully registering the facial information observer, the SDK triggers the callback you have registered when it captures the facial information converted by the speech driven extension.
+   *  Ensure that you call this method before joining a channel.
+   *  Before calling this method, you need to make sure that the speech driven extension has been enabled by calling enableExtension.
+   *
+   * @param observer Facial information observer, see IFaceInfoObserver.
+   *
+   * @returns
+   * 0: Success.
+   *  < 0: Failure.
+   */
+  abstract registerFaceInfoObserver(observer: IFaceInfoObserver): number;
+
   /**
    * Pushes the external audio frame.
    *
+   * Before calling this method to push external audio data, perform the following steps:
+   *  Call createCustomAudioTrack to create a custom audio track and get the audio track ID.
+   *  Call joinChannel to join the channel. In ChannelMediaOptions, set publishCustomAduioTrackId to the audio track ID that you want to publish, and set publishCustomAudioTrack to true.
+   *
    * @param frame The external audio frame. See AudioFrame.
    * @param trackId The audio track ID. If you want to publish a custom external audio source, set this parameter to the ID of the corresponding custom audio track you want to publish.
    *
@@ -113,13 +133,12 @@ export abstract class IMediaEngine {
   /**
    * Pulls the remote audio data.
    *
-   * Before calling this method, you need to call setExternalAudioSink to notify the app to enable and set the external rendering. After a successful call of this method, the app pulls the decoded and mixed audio data for playback.
-   *  This method only supports pulling data from custom audio source. If you need to pull the data captured by the SDK, do not call this method.
+   * Before calling this method, call setExternalAudioSink (enabled : true) to notify the app to enable and set the external audio rendering. After a successful call of this method, the app pulls the decoded and mixed audio data for playback.
    *  Call this method after joining a channel.
-   *  Once you enable the external audio sink, the app will not retrieve any audio data from the onPlaybackAudioFrame callback.
-   *  The difference between this method and the onPlaybackAudioFrame callback is as follows:
+   *  Both this method and onPlaybackAudioFrame callback can be used to get audio data after remote mixing. Note that after calling setExternalAudioSink to enable external audio rendering, the app no longer receives data from the onPlaybackAudioFrame callback. Therefore, you should choose between this method and the onPlaybackAudioFrame callback based on your actual business requirements. The specific distinctions between them are as follows:
+   *  After calling this method, the app automatically pulls the audio data from the SDK. By setting the audio data parameters, the SDK adjusts the frame buffer to help the app handle latency, effectively avoiding audio playback jitter.
    *  The SDK sends the audio data to the app through the onPlaybackAudioFrame callback. Any delay in processing the audio frames may result in audio jitter.
-   *  After a successful method call, the app automatically pulls the audio data from the SDK. After setting the audio data parameters, the SDK adjusts the frame buffer and avoids problems caused by jitter in the external audio playback.
+   *  This method is only used for retrieving audio data after remote mixing. If you need to get audio data from different audio processing stages such as capture and playback, you can register the corresponding callbacks by calling registerAudioFrameObserver.
    *
    * @returns
    * The AudioFrame instance, if the method call succeeds.
@@ -302,4 +321,15 @@ export abstract class IMediaEngine {
   abstract unregisterVideoEncodedFrameObserver(
     observer: IVideoEncodedFrameObserver
   ): number;
+
+  /**
+   * Unregisters a facial information observer.
+   *
+   * @param observer Facial information observer, see IFaceInfoObserver.
+   *
+   * @returns
+   * 0: Success.
+   *  < 0: Failure.
+   */
+  abstract unregisterFaceInfoObserver(observer: IFaceInfoObserver): number;
 }

package/ts/Private/IAgoraMediaPlayer.ts

@@ -205,7 +205,16 @@ export abstract class IMediaPlayer {
   abstract selectAudioTrack(index: number): number;
 
   /**
-   * @ignore
+   * Selects the audio tracks that you want to play on your local device and publish to the channel respectively.
+   *
+   * You can call this method to determine the audio track to be played on your local device and published to the channel. Before calling this method, you need to open the media file with the openWithMediaSource method and set enableMultiAudioTrack in MediaSource as true.
+   *
+   * @param playoutTrackIndex The index of audio tracks for local playback. You can obtain the index through getStreamInfo.
+   * @param publishTrackIndex The index of audio tracks to be published in the channel. You can obtain the index through getStreamInfo.
+   *
+   * @returns
+   * 0: Success.
+   *  < 0: Failure.
    */
   abstract selectMultiAudioTrack(
     playoutTrackIndex: number,