@stream-io/video-client 1.32.0 → 1.33.1

package/src/gen/coordinator/index.ts

@@ -86,6 +86,12 @@ export interface AudioSettingsRequest {
    * @memberof AudioSettingsRequest
    */
   default_device: AudioSettingsRequestDefaultDeviceEnum;
+  /**
+   *
+   * @type {boolean}
+   * @memberof AudioSettingsRequest
+   */
+  hifi_audio_enabled?: boolean;
   /**
    *
    * @type {boolean}
@@ -146,6 +152,12 @@ export interface AudioSettingsResponse {
    * @memberof AudioSettingsResponse
    */
   default_device: AudioSettingsResponseDefaultDeviceEnum;
+  /**
+   *
+   * @type {boolean}
+   * @memberof AudioSettingsResponse
+   */
+  hifi_audio_enabled?: boolean;
   /**
    *
    * @type {boolean}
@@ -408,6 +420,24 @@ export interface CallClosedCaption {
    * @memberof CallClosedCaption
    */
   end_time: string;
+  /**
+   *
+   * @type {string}
+   * @memberof CallClosedCaption
+   */
+  id: string;
+  /**
+   *
+   * @type {string}
+   * @memberof CallClosedCaption
+   */
+  language: string;
+  /**
+   *
+   * @type {string}
+   * @memberof CallClosedCaption
+   */
+  service?: string;
   /**
    *
    * @type {string}
@@ -426,6 +456,12 @@ export interface CallClosedCaption {
    * @memberof CallClosedCaption
    */
   text: string;
+  /**
+   *
+   * @type {boolean}
+   * @memberof CallClosedCaption
+   */
+  translated: boolean;
   /**
    *
    * @type {UserResponse}
@@ -917,6 +953,18 @@ export interface CallIngressResponse {
    * @memberof CallIngressResponse
    */
   rtmp: RTMPIngress;
+  /**
+   *
+   * @type {SRTIngress}
+   * @memberof CallIngressResponse
+   */
+  srt: SRTIngress;
+  /**
+   *
+   * @type {WHIPIngress}
+   * @memberof CallIngressResponse
+   */
+  whip: WHIPIngress;
 }
 /**
  * This event is sent when a call is started. Clients receiving this event should start the call.
@@ -1791,6 +1839,12 @@ export interface CallResponse {
    * @memberof CallResponse
    */
   transcribing: boolean;
+  /**
+   *
+   * @type {boolean}
+   * @memberof CallResponse
+   */
+  translating: boolean;
   /**
    * The type of call
    * @type {string}
@@ -2099,6 +2153,12 @@ export interface CallSessionParticipantLeftEvent {
    * @memberof CallSessionParticipantLeftEvent
    */
   participant: CallParticipantResponse;
+  /**
+   * The reason why the participant left the session
+   * @type {string}
+   * @memberof CallSessionParticipantLeftEvent
+   */
+  reason?: string;
   /**
    * Call session ID
    * @type {string}
@@ -2732,7 +2792,7 @@ export interface CallUpdatedEvent {
    */
   created_at: string;
   /**
-   * The type of event: "call.ended" in this case
+   * The type of event: "call.updated" in this case
    * @type {string}
    * @memberof CallUpdatedEvent
    */
@@ -2835,6 +2895,12 @@ export interface CallUserMutedEvent {
    * @memberof CallUserMutedEvent
    */
   muted_user_ids: Array<string>;
+  /**
+   *
+   * @type {string}
+   * @memberof CallUserMutedEvent
+   */
+  reason: string;
   /**
    * The type of event: "call.user_muted" in this case
    * @type {string}
@@ -3787,6 +3853,12 @@ export interface GetCallReportResponse {
    * @memberof GetCallReportResponse
    */
   report: ReportResponse;
+  /**
+   *
+   * @type {CallSessionResponse}
+   * @memberof GetCallReportResponse
+   */
+  session?: CallSessionResponse;
   /**
    *
    * @type {string}
@@ -4928,6 +5000,7 @@ export const OwnCapability = {
   REMOVE_CALL_MEMBER: 'remove-call-member',
   SCREENSHARE: 'screenshare',
   SEND_AUDIO: 'send-audio',
+  SEND_CLOSED_CAPTIONS_CALL: 'send-closed-captions-call',
   SEND_VIDEO: 'send-video',
   START_BROADCAST_CALL: 'start-broadcast-call',
   START_CLOSED_CAPTIONS_CALL: 'start-closed-captions-call',
@@ -6184,6 +6257,19 @@ export interface SFUResponse {
    */
   ws_endpoint: string;
 }
+/**
+ *
+ * @export
+ * @interface SRTIngress
+ */
+export interface SRTIngress {
+  /**
+   *
+   * @type {string}
+   * @memberof SRTIngress
+   */
+  address: string;
+}
 /**
  *
  * @export
@@ -6349,6 +6435,25 @@ export interface SortParamRequest {
    */
   field?: string;
 }
+/**
+ *
+ * @export
+ * @interface SpeechSegmentConfig
+ */
+export interface SpeechSegmentConfig {
+  /**
+   *
+   * @type {number}
+   * @memberof SpeechSegmentConfig
+   */
+  max_speech_caption_ms?: number;
+  /**
+   *
+   * @type {number}
+   * @memberof SpeechSegmentConfig
+   */
+  silence_duration_ms?: number;
+}
 /**
  *
  * @export
@@ -6373,6 +6478,12 @@ export interface StartClosedCaptionsRequest {
    * @memberof StartClosedCaptionsRequest
    */
   language?: StartClosedCaptionsRequestLanguageEnum;
+  /**
+   *
+   * @type {SpeechSegmentConfig}
+   * @memberof StartClosedCaptionsRequest
+   */
+  speech_segment_config?: SpeechSegmentConfig;
 }
 
 /**
@@ -6937,7 +7048,19 @@ export interface TranscriptionSettingsRequest {
    * @type {string}
    * @memberof TranscriptionSettingsRequest
    */
-  mode: TranscriptionSettingsRequestModeEnum;
+  mode?: TranscriptionSettingsRequestModeEnum;
+  /**
+   *
+   * @type {SpeechSegmentConfig}
+   * @memberof TranscriptionSettingsRequest
+   */
+  speech_segment_config?: SpeechSegmentConfig;
+  /**
+   *
+   * @type {TranslationSettings}
+   * @memberof TranscriptionSettingsRequest
+   */
+  translation?: TranslationSettings;
 }
 
 /**
@@ -7031,6 +7154,18 @@ export interface TranscriptionSettingsResponse {
    * @memberof TranscriptionSettingsResponse
    */
   mode: TranscriptionSettingsResponseModeEnum;
+  /**
+   *
+   * @type {SpeechSegmentConfig}
+   * @memberof TranscriptionSettingsResponse
+   */
+  speech_segment_config?: SpeechSegmentConfig;
+  /**
+   *
+   * @type {TranslationSettings}
+   * @memberof TranscriptionSettingsResponse
+   */
+  translation?: TranslationSettings;
 }
 
 /**
@@ -7100,6 +7235,25 @@ export const TranscriptionSettingsResponseModeEnum = {
 export type TranscriptionSettingsResponseModeEnum =
   (typeof TranscriptionSettingsResponseModeEnum)[keyof typeof TranscriptionSettingsResponseModeEnum];
 
+/**
+ *
+ * @export
+ * @interface TranslationSettings
+ */
+export interface TranslationSettings {
+  /**
+   *
+   * @type {boolean}
+   * @memberof TranslationSettings
+   */
+  enabled?: boolean;
+  /**
+   *
+   * @type {Array<string>}
+   * @memberof TranslationSettings
+   */
+  languages?: Array<string>;
+}
 /**
  * UnblockUserRequest is the payload for unblocking a user.
  * @export
@@ -7925,6 +8079,19 @@ export const VideoSettingsResponseCameraFacingEnum = {
 export type VideoSettingsResponseCameraFacingEnum =
   (typeof VideoSettingsResponseCameraFacingEnum)[keyof typeof VideoSettingsResponseCameraFacingEnum];
 
+/**
+ *
+ * @export
+ * @interface WHIPIngress
+ */
+export interface WHIPIngress {
+  /**
+   * URL for a new whip input, every time a new link is created
+   * @type {string}
+   * @memberof WHIPIngress
+   */
+  address: string;
+}
 /**
  * Websocket auth message
  * @export

package/src/gen/video/sfu/models/models.ts

@@ -301,6 +301,12 @@ export interface PublishOption {
    * @generated from protobuf field: bool use_single_layer = 9;
    */
   useSingleLayer: boolean;
+  /**
+   * Audio bitrate profiles for different audio quality profiles.
+   *
+   * @generated from protobuf field: repeated stream.video.sfu.models.AudioBitrate audio_bitrate_profiles = 10;
+   */
+  audioBitrateProfiles: AudioBitrate[];
 }
 /**
  * @generated from protobuf message stream.video.sfu.models.Codec
@@ -344,6 +350,19 @@ export interface ICETrickle {
    */
   sessionId: string;
 }
+/**
+ * @generated from protobuf message stream.video.sfu.models.AudioBitrate
+ */
+export interface AudioBitrate {
+  /**
+   * @generated from protobuf field: stream.video.sfu.models.AudioBitrateProfile profile = 1;
+   */
+  profile: AudioBitrateProfile;
+  /**
+   * @generated from protobuf field: int32 bitrate = 2;
+   */
+  bitrate: number;
+}
 /**
  * @generated from protobuf message stream.video.sfu.models.TrackInfo
  */
@@ -794,6 +813,23 @@ export enum ParticipantSource {
    */
   SRT = 5,
 }
+/**
+ * @generated from protobuf enum stream.video.sfu.models.AudioBitrateProfile
+ */
+export enum AudioBitrateProfile {
+  /**
+   * @generated from protobuf enum value: AUDIO_BITRATE_PROFILE_VOICE_STANDARD_UNSPECIFIED = 0;
+   */
+  VOICE_STANDARD_UNSPECIFIED = 0,
+  /**
+   * @generated from protobuf enum value: AUDIO_BITRATE_PROFILE_VOICE_HIGH_QUALITY = 1;
+   */
+  VOICE_HIGH_QUALITY = 1,
+  /**
+   * @generated from protobuf enum value: AUDIO_BITRATE_PROFILE_MUSIC_HIGH_QUALITY = 2;
+   */
+  MUSIC_HIGH_QUALITY = 2,
+}
 /**
  * @generated from protobuf enum stream.video.sfu.models.ErrorCode
  */
@@ -1398,6 +1434,13 @@ class PublishOption$Type extends MessageType<PublishOption> {
         kind: 'scalar',
         T: 8 /*ScalarType.BOOL*/,
       },
+      {
+        no: 10,
+        name: 'audio_bitrate_profiles',
+        kind: 'message',
+        repeat: 2 /*RepeatType.UNPACKED*/,
+        T: () => AudioBitrate,
+      },
     ]);
   }
 }
@@ -1461,6 +1504,28 @@ class ICETrickle$Type extends MessageType<ICETrickle> {
  */
 export const ICETrickle = new ICETrickle$Type();
 // @generated message type with reflection information, may provide speed optimized methods
+class AudioBitrate$Type extends MessageType<AudioBitrate> {
+  constructor() {
+    super('stream.video.sfu.models.AudioBitrate', [
+      {
+        no: 1,
+        name: 'profile',
+        kind: 'enum',
+        T: () => [
+          'stream.video.sfu.models.AudioBitrateProfile',
+          AudioBitrateProfile,
+          'AUDIO_BITRATE_PROFILE_',
+        ],
+      },
+      { no: 2, name: 'bitrate', kind: 'scalar', T: 5 /*ScalarType.INT32*/ },
+    ]);
+  }
+}
+/**
+ * @generated MessageType for protobuf message stream.video.sfu.models.AudioBitrate
+ */
+export const AudioBitrate = new AudioBitrate$Type();
+// @generated message type with reflection information, may provide speed optimized methods
 class TrackInfo$Type extends MessageType<TrackInfo> {
   constructor() {
     super('stream.video.sfu.models.TrackInfo', [

package/src/rtc/BasePeerConnection.ts

@@ -16,22 +16,7 @@ import { StreamSfuClient } from '../StreamSfuClient';
 import { AllSfuEvents, Dispatcher } from './Dispatcher';
 import { withoutConcurrency } from '../helpers/concurrency';
 import { StatsTracer, Tracer, traceRTCPeerConnection } from '../stats';
-
-export type OnReconnectionNeeded = (
-  kind: WebsocketReconnectStrategy,
-  reason: string,
-) => void;
-
-export type BasePeerConnectionOpts = {
-  sfuClient: StreamSfuClient;
-  state: CallState;
-  connectionConfig?: RTCConfiguration;
-  dispatcher: Dispatcher;
-  onReconnectionNeeded?: OnReconnectionNeeded;
-  tag: string;
-  enableTracing: boolean;
-  iceRestartDelay?: number;
-};
+import { BasePeerConnectionOpts, OnReconnectionNeeded } from './types';
 
 /**
  * A base class for the `Publisher` and `Subscriber` classes.

package/src/rtc/Publisher.ts

@@ -1,7 +1,9 @@
+import { BasePeerConnection } from './BasePeerConnection';
 import {
-  BasePeerConnection,
-  BasePeerConnectionOpts,
-} from './BasePeerConnection';
+  PublishBundle,
+  PublisherConstructorOpts,
+  TrackPublishOptions,
+} from './types';
 import { NegotiationError } from './NegotiationError';
 import { TransceiverCache } from './TransceiverCache';
 import {
@@ -12,20 +14,17 @@ import {
 } from '../gen/video/sfu/models/models';
 import { VideoSender } from '../gen/video/sfu/event/events';
 import {
+  computeAudioLayers,
   computeVideoLayers,
   toSvcEncodings,
   toVideoLayers,
-} from './videoLayers';
+} from './layers';
 import { isSvcCodec } from './codecs';
 import { isAudioTrackType } from './helpers/tracks';
 import { extractMid } from './helpers/sdp';
 import { withoutConcurrency } from '../helpers/concurrency';
 import { isReactNative } from '../helpers/platforms';
 
-export type PublisherConstructorOpts = BasePeerConnectionOpts & {
-  publishOptions: PublishOption[];
-};
-
 /**
  * The `Publisher` is responsible for publishing/unpublishing media streams to/from the SFU
  *
@@ -77,8 +76,13 @@ export class Publisher extends BasePeerConnection {
    *
    * @param track the track to publish.
    * @param trackType the track type to publish.
+   * @param options the publish options to use.
    */
-  publish = async (track: MediaStreamTrack, trackType: TrackType) => {
+  publish = async (
+    track: MediaStreamTrack,
+    trackType: TrackType,
+    options: TrackPublishOptions = {},
+  ) => {
     if (!this.publishOptions.some((o) => o.trackType === trackType)) {
       throw new Error(`No publish options found for ${TrackType[trackType]}`);
     }
@@ -90,12 +94,17 @@ export class Publisher extends BasePeerConnection {
       // appear in the SDP in multiple transceivers
       const trackToPublish = this.cloneTrack(track);
 
-      const transceiver = this.transceiverCache.get(publishOption);
+      const { transceiver } = this.transceiverCache.get(publishOption) || {};
       if (!transceiver) {
-        await this.addTransceiver(trackToPublish, publishOption);
+        await this.addTransceiver(trackToPublish, publishOption, options);
       } else {
         const previousTrack = transceiver.sender.track;
-        await this.updateTransceiver(transceiver, trackToPublish, trackType);
+        await this.updateTransceiver(
+          transceiver,
+          trackToPublish,
+          trackType,
+          options,
+        );
         if (!isReactNative()) {
           this.stopTrack(previousTrack);
         }
@@ -109,11 +118,14 @@ export class Publisher extends BasePeerConnection {
   private addTransceiver = async (
     track: MediaStreamTrack,
     publishOption: PublishOption,
+    options: TrackPublishOptions,
   ) => {
-    const videoEncodings = computeVideoLayers(track, publishOption);
+    const encodings = isAudioTrackType(publishOption.trackType)
+      ? computeAudioLayers(publishOption, options)
+      : computeVideoLayers(track, publishOption);
     const sendEncodings = isSvcCodec(publishOption.codec?.name)
-      ? toSvcEncodings(videoEncodings)
-      : videoEncodings;
+      ? toSvcEncodings(encodings)
+      : encodings;
     const transceiver = this.pc.addTransceiver(track, {
       direction: 'sendonly',
       sendEncodings,
@@ -125,7 +137,7 @@ export class Publisher extends BasePeerConnection {
 
     const trackType = publishOption.trackType;
     this.logger('debug', `Added ${TrackType[trackType]} transceiver`);
-    this.transceiverCache.add(publishOption, transceiver);
+    this.transceiverCache.add({ publishOption, transceiver, options });
     this.trackIdToTrackType.set(track.id, trackType);
 
     await this.negotiate();
@@ -138,11 +150,44 @@ export class Publisher extends BasePeerConnection {
     transceiver: RTCRtpTransceiver,
     track: MediaStreamTrack | null,
     trackType: TrackType,
+    options: TrackPublishOptions = {},
   ) => {
     const sender = transceiver.sender;
     if (sender.track) this.trackIdToTrackType.delete(sender.track.id);
     await sender.replaceTrack(track);
     if (track) this.trackIdToTrackType.set(track.id, trackType);
+    if (isAudioTrackType(trackType)) {
+      await this.updateAudioPublishOptions(trackType, options);
+    }
+  };
+
+  /**
+   * Updates the publish options for the given track type.
+   */
+  private updateAudioPublishOptions = async (
+    trackType: TrackType,
+    options: TrackPublishOptions,
+  ) => {
+    for (const publishOption of this.publishOptions) {
+      if (publishOption.trackType !== trackType) continue;
+      const bundle = this.transceiverCache.get(publishOption);
+      if (!bundle) continue;
+
+      const { transceiver, options: current } = bundle;
+      if (current.audioBitrateProfile !== options.audioBitrateProfile) {
+        const encodings = computeAudioLayers(publishOption, options);
+        if (encodings && encodings.length > 0) {
+          const params = transceiver.sender.getParameters();
+          const [currentEncoding] = params.encodings;
+          const [targetEncoding] = encodings;
+          if (currentEncoding.maxBitrate !== targetEncoding.maxBitrate) {
+            currentEncoding.maxBitrate = targetEncoding.maxBitrate;
+          }
+          await transceiver.sender.setParameters(params);
+        }
+      }
+      this.transceiverCache.update(publishOption, { options });
+    }
   };
 
   /**
@@ -160,12 +205,12 @@ export class Publisher extends BasePeerConnection {
           !!i.transceiver.sender.track &&
           i.publishOption.trackType === trackType,
       );
-      if (!item || !item.transceiver) continue;
+      if (!item) continue;
 
       // take the track from the existing transceiver for the same track type,
       // clone it and publish it with the new publish options
       const track = this.cloneTrack(item.transceiver.sender.track!);
-      await this.addTransceiver(track, publishOption);
+      await this.addTransceiver(track, publishOption, item.options);
     }
 
     // stop publishing with options not required anymore -> [vp9]
@@ -372,11 +417,8 @@ export class Publisher extends BasePeerConnection {
   getAnnouncedTracks = (sdp: string | undefined): TrackInfo[] => {
     const trackInfos: TrackInfo[] = [];
     for (const bundle of this.transceiverCache.items()) {
-      const { transceiver, publishOption } = bundle;
-      const track = transceiver.sender.track;
-      if (!track) continue;
-
-      trackInfos.push(this.toTrackInfo(transceiver, publishOption, sdp));
+      if (!bundle.transceiver.sender.track) continue;
+      trackInfos.push(this.toTrackInfo(bundle, sdp));
     }
     return trackInfos;
   };
@@ -390,10 +432,9 @@ export class Publisher extends BasePeerConnection {
     const sdp = this.pc.localDescription?.sdp;
     const trackInfos: TrackInfo[] = [];
     for (const publishOption of this.publishOptions) {
-      const transceiver = this.transceiverCache.get(publishOption);
-      if (!transceiver || !transceiver.sender.track) continue;
-
-      trackInfos.push(this.toTrackInfo(transceiver, publishOption, sdp));
+      const bundle = this.transceiverCache.get(publishOption);
+      if (!bundle || !bundle.transceiver.sender.track) continue;
+      trackInfos.push(this.toTrackInfo(bundle, sdp));
     }
     return trackInfos;
   };
@@ -402,10 +443,10 @@ export class Publisher extends BasePeerConnection {
    * Converts the given transceiver to a `TrackInfo` object.
    */
   private toTrackInfo = (
-    transceiver: RTCRtpTransceiver,
-    publishOption: PublishOption,
+    bundle: PublishBundle,
     sdp: string | undefined,
   ): TrackInfo => {
+    const { transceiver, publishOption } = bundle;
     const track = transceiver.sender.track!;
     const isTrackLive = track.readyState === 'live';
     const layers = isTrackLive
@@ -414,16 +455,18 @@ export class Publisher extends BasePeerConnection {
     this.transceiverCache.setLayers(publishOption, layers);
 
     const isAudioTrack = isAudioTrackType(publishOption.trackType);
-    const isStereo = isAudioTrack && track.getSettings().channelCount === 2;
     const transceiverIndex = this.transceiverCache.indexOf(transceiver);
     const audioSettings = this.state.settings?.audio;
+    const stereo =
+      publishOption.trackType === TrackType.SCREEN_SHARE_AUDIO ||
+      (isAudioTrack && !!audioSettings?.hifi_audio_enabled);
 
     return {
       trackId: track.id,
       layers: toVideoLayers(layers),
       trackType: publishOption.trackType,
       mid: extractMid(transceiver, transceiverIndex, sdp),
-      stereo: isStereo,
+      stereo,
       dtx: isAudioTrack && !!audioSettings?.opus_dtx_enabled,
       red: isAudioTrack && !!audioSettings?.redundant_coding_enabled,
       muted: !isTrackLive,

package/src/rtc/Subscriber.ts

@@ -1,7 +1,5 @@
-import {
-  BasePeerConnection,
-  BasePeerConnectionOpts,
-} from './BasePeerConnection';
+import { BasePeerConnection } from './BasePeerConnection';
+import { BasePeerConnectionOpts } from './types';
 import { NegotiationError } from './NegotiationError';
 import { PeerType } from '../gen/video/sfu/models/models';
 import { SubscriberOffer } from '../gen/video/sfu/event/events';