@stream-io/video-client 1.13.1 → 1.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. package/CHANGELOG.md +14 -0
  2. package/dist/index.browser.es.js +1704 -1762
  3. package/dist/index.browser.es.js.map +1 -1
  4. package/dist/index.cjs.js +1706 -1780
  5. package/dist/index.cjs.js.map +1 -1
  6. package/dist/index.es.js +1704 -1762
  7. package/dist/index.es.js.map +1 -1
  8. package/dist/src/Call.d.ts +61 -30
  9. package/dist/src/StreamSfuClient.d.ts +4 -5
  10. package/dist/src/devices/CameraManager.d.ts +5 -8
  11. package/dist/src/devices/InputMediaDeviceManager.d.ts +5 -5
  12. package/dist/src/devices/MicrophoneManager.d.ts +7 -2
  13. package/dist/src/devices/ScreenShareManager.d.ts +1 -2
  14. package/dist/src/gen/coordinator/index.d.ts +904 -515
  15. package/dist/src/gen/video/sfu/event/events.d.ts +38 -19
  16. package/dist/src/gen/video/sfu/models/models.d.ts +76 -9
  17. package/dist/src/helpers/array.d.ts +7 -0
  18. package/dist/src/permissions/PermissionsContext.d.ts +6 -0
  19. package/dist/src/rtc/BasePeerConnection.d.ts +90 -0
  20. package/dist/src/rtc/Dispatcher.d.ts +0 -1
  21. package/dist/src/rtc/IceTrickleBuffer.d.ts +3 -2
  22. package/dist/src/rtc/Publisher.d.ts +32 -86
  23. package/dist/src/rtc/Subscriber.d.ts +4 -56
  24. package/dist/src/rtc/TransceiverCache.d.ts +55 -0
  25. package/dist/src/rtc/codecs.d.ts +1 -15
  26. package/dist/src/rtc/helpers/sdp.d.ts +8 -0
  27. package/dist/src/rtc/helpers/tracks.d.ts +1 -0
  28. package/dist/src/rtc/index.d.ts +3 -0
  29. package/dist/src/rtc/videoLayers.d.ts +11 -25
  30. package/dist/src/stats/{stateStoreStatsReporter.d.ts → CallStateStatsReporter.d.ts} +5 -1
  31. package/dist/src/stats/SfuStatsReporter.d.ts +4 -2
  32. package/dist/src/stats/index.d.ts +1 -1
  33. package/dist/src/stats/types.d.ts +8 -0
  34. package/dist/src/store/CallState.d.ts +47 -5
  35. package/dist/src/store/rxUtils.d.ts +15 -1
  36. package/dist/src/types.d.ts +26 -22
  37. package/package.json +1 -1
  38. package/src/Call.ts +310 -271
  39. package/src/StreamSfuClient.ts +9 -14
  40. package/src/StreamVideoClient.ts +1 -1
  41. package/src/__tests__/Call.publishing.test.ts +306 -0
  42. package/src/devices/CameraManager.ts +33 -16
  43. package/src/devices/InputMediaDeviceManager.ts +36 -27
  44. package/src/devices/MicrophoneManager.ts +29 -8
  45. package/src/devices/ScreenShareManager.ts +6 -8
  46. package/src/devices/__tests__/CameraManager.test.ts +111 -14
  47. package/src/devices/__tests__/InputMediaDeviceManager.test.ts +4 -4
  48. package/src/devices/__tests__/MicrophoneManager.test.ts +59 -21
  49. package/src/devices/__tests__/ScreenShareManager.test.ts +5 -5
  50. package/src/devices/__tests__/mocks.ts +1 -0
  51. package/src/events/__tests__/internal.test.ts +132 -0
  52. package/src/events/__tests__/mutes.test.ts +0 -3
  53. package/src/events/__tests__/speaker.test.ts +92 -0
  54. package/src/events/participant.ts +3 -4
  55. package/src/gen/coordinator/index.ts +902 -514
  56. package/src/gen/video/sfu/event/events.ts +91 -30
  57. package/src/gen/video/sfu/models/models.ts +105 -13
  58. package/src/helpers/array.ts +14 -0
  59. package/src/permissions/PermissionsContext.ts +22 -0
  60. package/src/permissions/__tests__/PermissionsContext.test.ts +40 -0
  61. package/src/rpc/__tests__/createClient.test.ts +38 -0
  62. package/src/rpc/createClient.ts +11 -5
  63. package/src/rtc/BasePeerConnection.ts +240 -0
  64. package/src/rtc/Dispatcher.ts +0 -9
  65. package/src/rtc/IceTrickleBuffer.ts +24 -4
  66. package/src/rtc/Publisher.ts +210 -528
  67. package/src/rtc/Subscriber.ts +26 -200
  68. package/src/rtc/TransceiverCache.ts +120 -0
  69. package/src/rtc/__tests__/Publisher.test.ts +407 -210
  70. package/src/rtc/__tests__/Subscriber.test.ts +88 -36
  71. package/src/rtc/__tests__/mocks/webrtc.mocks.ts +22 -2
  72. package/src/rtc/__tests__/videoLayers.test.ts +161 -54
  73. package/src/rtc/codecs.ts +1 -131
  74. package/src/rtc/helpers/__tests__/rtcConfiguration.test.ts +34 -0
  75. package/src/rtc/helpers/__tests__/sdp.test.ts +59 -0
  76. package/src/rtc/helpers/sdp.ts +30 -0
  77. package/src/rtc/helpers/tracks.ts +3 -0
  78. package/src/rtc/index.ts +4 -0
  79. package/src/rtc/videoLayers.ts +68 -76
  80. package/src/stats/{stateStoreStatsReporter.ts → CallStateStatsReporter.ts} +58 -27
  81. package/src/stats/SfuStatsReporter.ts +31 -3
  82. package/src/stats/index.ts +1 -1
  83. package/src/stats/types.ts +12 -0
  84. package/src/store/CallState.ts +115 -5
  85. package/src/store/__tests__/CallState.test.ts +101 -0
  86. package/src/store/rxUtils.ts +23 -1
  87. package/src/types.ts +27 -22
  88. package/dist/src/helpers/sdp-munging.d.ts +0 -24
  89. package/dist/src/rtc/bitrateLookup.d.ts +0 -2
  90. package/dist/src/rtc/helpers/iceCandidate.d.ts +0 -2
  91. package/src/helpers/__tests__/hq-audio-sdp.ts +0 -332
  92. package/src/helpers/__tests__/sdp-munging.test.ts +0 -283
  93. package/src/helpers/sdp-munging.ts +0 -265
  94. package/src/rtc/__tests__/bitrateLookup.test.ts +0 -12
  95. package/src/rtc/__tests__/codecs.test.ts +0 -145
  96. package/src/rtc/bitrateLookup.ts +0 -61
  97. package/src/rtc/helpers/iceCandidate.ts +0 -16
  98. /package/dist/src/{compatibility.d.ts → helpers/compatibility.d.ts} +0 -0
  99. /package/src/{compatibility.ts → helpers/compatibility.ts} +0 -0
package/dist/index.es.js CHANGED
@@ -4,9 +4,9 @@ import { ServiceType, stackIntercept, RpcError } from '@protobuf-ts/runtime-rpc'
4
4
  import axios from 'axios';
5
5
  export { AxiosError } from 'axios';
6
6
  import { TwirpFetchTransport, TwirpErrorCode } from '@protobuf-ts/twirp-transport';
7
- import { UAParser } from 'ua-parser-js';
8
7
  import { ReplaySubject, combineLatest, BehaviorSubject, map, shareReplay, distinctUntilChanged, takeWhile, distinctUntilKeyChanged, fromEventPattern, startWith, concatMap, from, fromEvent, debounceTime, merge, pairwise, of } from 'rxjs';
9
- import * as SDP from 'sdp-transform';
8
+ import { parse } from 'sdp-transform';
9
+ import { UAParser } from 'ua-parser-js';
10
10
  import https from 'https';
11
11
 
12
12
  /* tslint:disable */
@@ -57,6 +57,48 @@ const ChannelConfigWithInfoBlocklistBehaviorEnum = {
57
57
  BLOCK: 'block',
58
58
  SHADOW_BLOCK: 'shadow_block',
59
59
  };
60
+ /**
61
+ * All possibility of string to use
62
+ * @export
63
+ */
64
+ const ChannelOwnCapability = {
65
+ BAN_CHANNEL_MEMBERS: 'ban-channel-members',
66
+ CAST_POLL_VOTE: 'cast-poll-vote',
67
+ CONNECT_EVENTS: 'connect-events',
68
+ CREATE_ATTACHMENT: 'create-attachment',
69
+ CREATE_CALL: 'create-call',
70
+ DELETE_ANY_MESSAGE: 'delete-any-message',
71
+ DELETE_CHANNEL: 'delete-channel',
72
+ DELETE_OWN_MESSAGE: 'delete-own-message',
73
+ FLAG_MESSAGE: 'flag-message',
74
+ FREEZE_CHANNEL: 'freeze-channel',
75
+ JOIN_CALL: 'join-call',
76
+ JOIN_CHANNEL: 'join-channel',
77
+ LEAVE_CHANNEL: 'leave-channel',
78
+ MUTE_CHANNEL: 'mute-channel',
79
+ PIN_MESSAGE: 'pin-message',
80
+ QUERY_POLL_VOTES: 'query-poll-votes',
81
+ QUOTE_MESSAGE: 'quote-message',
82
+ READ_EVENTS: 'read-events',
83
+ SEARCH_MESSAGES: 'search-messages',
84
+ SEND_CUSTOM_EVENTS: 'send-custom-events',
85
+ SEND_LINKS: 'send-links',
86
+ SEND_MESSAGE: 'send-message',
87
+ SEND_POLL: 'send-poll',
88
+ SEND_REACTION: 'send-reaction',
89
+ SEND_REPLY: 'send-reply',
90
+ SEND_TYPING_EVENTS: 'send-typing-events',
91
+ SET_CHANNEL_COOLDOWN: 'set-channel-cooldown',
92
+ SKIP_SLOW_MODE: 'skip-slow-mode',
93
+ SLOW_MODE: 'slow-mode',
94
+ TYPING_EVENTS: 'typing-events',
95
+ UPDATE_ANY_MESSAGE: 'update-any-message',
96
+ UPDATE_CHANNEL: 'update-channel',
97
+ UPDATE_CHANNEL_MEMBERS: 'update-channel-members',
98
+ UPDATE_OWN_MESSAGE: 'update-own-message',
99
+ UPDATE_THREAD: 'update-thread',
100
+ UPLOAD_FILE: 'upload-file',
101
+ };
60
102
  /**
61
103
  * @export
62
104
  */
@@ -96,9 +138,11 @@ const OwnCapability = {
96
138
  SEND_AUDIO: 'send-audio',
97
139
  SEND_VIDEO: 'send-video',
98
140
  START_BROADCAST_CALL: 'start-broadcast-call',
141
+ START_CLOSED_CAPTIONS_CALL: 'start-closed-captions-call',
99
142
  START_RECORD_CALL: 'start-record-call',
100
143
  START_TRANSCRIPTION_CALL: 'start-transcription-call',
101
144
  STOP_BROADCAST_CALL: 'stop-broadcast-call',
145
+ STOP_CLOSED_CAPTIONS_CALL: 'stop-closed-captions-call',
102
146
  STOP_RECORD_CALL: 'stop-record-call',
103
147
  STOP_TRANSCRIPTION_CALL: 'stop-transcription-call',
104
148
  UPDATE_CALL: 'update-call',
@@ -129,6 +173,14 @@ const RecordSettingsRequestQualityEnum = {
129
173
  PORTRAIT_1080X1920: 'portrait-1080x1920',
130
174
  PORTRAIT_1440X2560: 'portrait-1440x2560',
131
175
  };
176
+ /**
177
+ * @export
178
+ */
179
+ const TranscriptionSettingsRequestClosedCaptionModeEnum = {
180
+ AVAILABLE: 'available',
181
+ DISABLED: 'disabled',
182
+ AUTO_ON: 'auto-on',
183
+ };
132
184
  /**
133
185
  * @export
134
186
  */
@@ -137,6 +189,14 @@ const TranscriptionSettingsRequestModeEnum = {
137
189
  DISABLED: 'disabled',
138
190
  AUTO_ON: 'auto-on',
139
191
  };
192
+ /**
193
+ * @export
194
+ */
195
+ const TranscriptionSettingsResponseClosedCaptionModeEnum = {
196
+ AVAILABLE: 'available',
197
+ DISABLED: 'disabled',
198
+ AUTO_ON: 'auto-on',
199
+ };
140
200
  /**
141
201
  * @export
142
202
  */
@@ -1165,23 +1225,33 @@ class VideoLayer$Type extends MessageType {
1165
1225
  */
1166
1226
  const VideoLayer = new VideoLayer$Type();
1167
1227
  // @generated message type with reflection information, may provide speed optimized methods
1168
- class PublishOptions$Type extends MessageType {
1228
+ class SubscribeOption$Type extends MessageType {
1169
1229
  constructor() {
1170
- super('stream.video.sfu.models.PublishOptions', [
1230
+ super('stream.video.sfu.models.SubscribeOption', [
1171
1231
  {
1172
1232
  no: 1,
1233
+ name: 'track_type',
1234
+ kind: 'enum',
1235
+ T: () => [
1236
+ 'stream.video.sfu.models.TrackType',
1237
+ TrackType,
1238
+ 'TRACK_TYPE_',
1239
+ ],
1240
+ },
1241
+ {
1242
+ no: 2,
1173
1243
  name: 'codecs',
1174
1244
  kind: 'message',
1175
1245
  repeat: 1 /*RepeatType.PACKED*/,
1176
- T: () => PublishOption,
1246
+ T: () => Codec,
1177
1247
  },
1178
1248
  ]);
1179
1249
  }
1180
1250
  }
1181
1251
  /**
1182
- * @generated MessageType for protobuf message stream.video.sfu.models.PublishOptions
1252
+ * @generated MessageType for protobuf message stream.video.sfu.models.SubscribeOption
1183
1253
  */
1184
- const PublishOptions = new PublishOptions$Type();
1254
+ const SubscribeOption = new SubscribeOption$Type();
1185
1255
  // @generated message type with reflection information, may provide speed optimized methods
1186
1256
  class PublishOption$Type extends MessageType {
1187
1257
  constructor() {
@@ -1211,6 +1281,13 @@ class PublishOption$Type extends MessageType {
1211
1281
  kind: 'scalar',
1212
1282
  T: 5 /*ScalarType.INT32*/,
1213
1283
  },
1284
+ {
1285
+ no: 7,
1286
+ name: 'video_dimension',
1287
+ kind: 'message',
1288
+ T: () => VideoDimension,
1289
+ },
1290
+ { no: 8, name: 'id', kind: 'scalar', T: 5 /*ScalarType.INT32*/ },
1214
1291
  ]);
1215
1292
  }
1216
1293
  }
@@ -1223,7 +1300,7 @@ class Codec$Type extends MessageType {
1223
1300
  constructor() {
1224
1301
  super('stream.video.sfu.models.Codec', [
1225
1302
  {
1226
- no: 11,
1303
+ no: 16,
1227
1304
  name: 'payload_type',
1228
1305
  kind: 'scalar',
1229
1306
  T: 13 /*ScalarType.UINT32*/,
@@ -1236,7 +1313,7 @@ class Codec$Type extends MessageType {
1236
1313
  T: 13 /*ScalarType.UINT32*/,
1237
1314
  },
1238
1315
  {
1239
- no: 13,
1316
+ no: 15,
1240
1317
  name: 'encoding_parameters',
1241
1318
  kind: 'scalar',
1242
1319
  T: 9 /*ScalarType.STRING*/,
@@ -1300,6 +1377,13 @@ class TrackInfo$Type extends MessageType {
1300
1377
  { no: 8, name: 'stereo', kind: 'scalar', T: 8 /*ScalarType.BOOL*/ },
1301
1378
  { no: 9, name: 'red', kind: 'scalar', T: 8 /*ScalarType.BOOL*/ },
1302
1379
  { no: 10, name: 'muted', kind: 'scalar', T: 8 /*ScalarType.BOOL*/ },
1380
+ { no: 11, name: 'codec', kind: 'message', T: () => Codec },
1381
+ {
1382
+ no: 12,
1383
+ name: 'publish_option_id',
1384
+ kind: 'scalar',
1385
+ T: 5 /*ScalarType.INT32*/,
1386
+ },
1303
1387
  ]);
1304
1388
  }
1305
1389
  }
@@ -1573,10 +1657,10 @@ var models = /*#__PURE__*/Object.freeze({
1573
1657
  get PeerType () { return PeerType; },
1574
1658
  Pin: Pin,
1575
1659
  PublishOption: PublishOption,
1576
- PublishOptions: PublishOptions,
1577
1660
  Sdk: Sdk,
1578
1661
  get SdkType () { return SdkType; },
1579
1662
  StreamQuality: StreamQuality,
1663
+ SubscribeOption: SubscribeOption,
1580
1664
  TrackInfo: TrackInfo,
1581
1665
  get TrackType () { return TrackType; },
1582
1666
  get TrackUnpublishReason () { return TrackUnpublishReason; },
@@ -2206,13 +2290,6 @@ class SfuEvent$Type extends MessageType {
2206
2290
  oneof: 'eventPayload',
2207
2291
  T: () => ParticipantMigrationComplete,
2208
2292
  },
2209
- {
2210
- no: 26,
2211
- name: 'codec_negotiation_complete',
2212
- kind: 'message',
2213
- oneof: 'eventPayload',
2214
- T: () => CodecNegotiationComplete,
2215
- },
2216
2293
  {
2217
2294
  no: 27,
2218
2295
  name: 'change_publish_options',
@@ -2233,10 +2310,12 @@ class ChangePublishOptions$Type extends MessageType {
2233
2310
  super('stream.video.sfu.event.ChangePublishOptions', [
2234
2311
  {
2235
2312
  no: 1,
2236
- name: 'publish_option',
2313
+ name: 'publish_options',
2237
2314
  kind: 'message',
2315
+ repeat: 1 /*RepeatType.PACKED*/,
2238
2316
  T: () => PublishOption,
2239
2317
  },
2318
+ { no: 2, name: 'reason', kind: 'scalar', T: 9 /*ScalarType.STRING*/ },
2240
2319
  ]);
2241
2320
  }
2242
2321
  }
@@ -2245,15 +2324,15 @@ class ChangePublishOptions$Type extends MessageType {
2245
2324
  */
2246
2325
  const ChangePublishOptions = new ChangePublishOptions$Type();
2247
2326
  // @generated message type with reflection information, may provide speed optimized methods
2248
- class CodecNegotiationComplete$Type extends MessageType {
2327
+ class ChangePublishOptionsComplete$Type extends MessageType {
2249
2328
  constructor() {
2250
- super('stream.video.sfu.event.CodecNegotiationComplete', []);
2329
+ super('stream.video.sfu.event.ChangePublishOptionsComplete', []);
2251
2330
  }
2252
2331
  }
2253
2332
  /**
2254
- * @generated MessageType for protobuf message stream.video.sfu.event.CodecNegotiationComplete
2333
+ * @generated MessageType for protobuf message stream.video.sfu.event.ChangePublishOptionsComplete
2255
2334
  */
2256
- const CodecNegotiationComplete = new CodecNegotiationComplete$Type();
2335
+ const ChangePublishOptionsComplete = new ChangePublishOptionsComplete$Type();
2257
2336
  // @generated message type with reflection information, may provide speed optimized methods
2258
2337
  class ParticipantMigrationComplete$Type extends MessageType {
2259
2338
  constructor() {
@@ -2511,6 +2590,20 @@ class JoinRequest$Type extends MessageType {
2511
2590
  kind: 'message',
2512
2591
  T: () => ReconnectDetails,
2513
2592
  },
2593
+ {
2594
+ no: 9,
2595
+ name: 'preferred_publish_options',
2596
+ kind: 'message',
2597
+ repeat: 1 /*RepeatType.PACKED*/,
2598
+ T: () => PublishOption,
2599
+ },
2600
+ {
2601
+ no: 10,
2602
+ name: 'preferred_subscribe_options',
2603
+ kind: 'message',
2604
+ repeat: 1 /*RepeatType.PACKED*/,
2605
+ T: () => SubscribeOption,
2606
+ },
2514
2607
  ]);
2515
2608
  }
2516
2609
  }
@@ -2618,7 +2711,8 @@ class JoinResponse$Type extends MessageType {
2618
2711
  no: 4,
2619
2712
  name: 'publish_options',
2620
2713
  kind: 'message',
2621
- T: () => PublishOptions,
2714
+ repeat: 1 /*RepeatType.PACKED*/,
2715
+ T: () => PublishOption,
2622
2716
  },
2623
2717
  ]);
2624
2718
  }
@@ -2783,6 +2877,22 @@ class AudioSender$Type extends MessageType {
2783
2877
  constructor() {
2784
2878
  super('stream.video.sfu.event.AudioSender', [
2785
2879
  { no: 2, name: 'codec', kind: 'message', T: () => Codec },
2880
+ {
2881
+ no: 3,
2882
+ name: 'track_type',
2883
+ kind: 'enum',
2884
+ T: () => [
2885
+ 'stream.video.sfu.models.TrackType',
2886
+ TrackType,
2887
+ 'TRACK_TYPE_',
2888
+ ],
2889
+ },
2890
+ {
2891
+ no: 4,
2892
+ name: 'publish_option_id',
2893
+ kind: 'scalar',
2894
+ T: 5 /*ScalarType.INT32*/,
2895
+ },
2786
2896
  ]);
2787
2897
  }
2788
2898
  }
@@ -2835,6 +2945,22 @@ class VideoSender$Type extends MessageType {
2835
2945
  repeat: 1 /*RepeatType.PACKED*/,
2836
2946
  T: () => VideoLayerSetting,
2837
2947
  },
2948
+ {
2949
+ no: 4,
2950
+ name: 'track_type',
2951
+ kind: 'enum',
2952
+ T: () => [
2953
+ 'stream.video.sfu.models.TrackType',
2954
+ TrackType,
2955
+ 'TRACK_TYPE_',
2956
+ ],
2957
+ },
2958
+ {
2959
+ no: 5,
2960
+ name: 'publish_option_id',
2961
+ kind: 'scalar',
2962
+ T: 5 /*ScalarType.INT32*/,
2963
+ },
2838
2964
  ]);
2839
2965
  }
2840
2966
  }
@@ -2931,8 +3057,8 @@ var events = /*#__PURE__*/Object.freeze({
2931
3057
  CallEnded: CallEnded,
2932
3058
  CallGrantsUpdated: CallGrantsUpdated,
2933
3059
  ChangePublishOptions: ChangePublishOptions,
3060
+ ChangePublishOptionsComplete: ChangePublishOptionsComplete,
2934
3061
  ChangePublishQuality: ChangePublishQuality,
2935
- CodecNegotiationComplete: CodecNegotiationComplete,
2936
3062
  ConnectionQualityChanged: ConnectionQualityChanged,
2937
3063
  ConnectionQualityInfo: ConnectionQualityInfo,
2938
3064
  DominantSpeakerChanged: DominantSpeakerChanged,
@@ -3079,11 +3205,18 @@ const withHeaders = (headers) => {
3079
3205
  const withRequestLogger = (logger, level) => {
3080
3206
  return {
3081
3207
  interceptUnary: (next, method, input, options) => {
3082
- logger(level, `Calling SFU RPC method ${method.name}`, {
3083
- input,
3084
- options,
3085
- });
3086
- return next(method, input, options);
3208
+ let invocation;
3209
+ try {
3210
+ invocation = next(method, input, options);
3211
+ }
3212
+ finally {
3213
+ logger(level, `Invoked SFU RPC method ${method.name}`, {
3214
+ request: invocation?.request,
3215
+ headers: invocation?.requestHeaders,
3216
+ response: invocation?.response,
3217
+ });
3218
+ }
3219
+ return invocation;
3087
3220
  },
3088
3221
  };
3089
3222
  };
@@ -3300,374 +3433,98 @@ const retryable = async (rpc, signal) => {
3300
3433
  return result;
3301
3434
  };
3302
3435
 
3303
- const version = "1.13.1";
3304
- const [major, minor, patch] = version.split('.');
3305
- let sdkInfo = {
3306
- type: SdkType.PLAIN_JAVASCRIPT,
3307
- major,
3308
- minor,
3309
- patch,
3310
- };
3311
- let osInfo;
3312
- let deviceInfo;
3313
- let webRtcInfo;
3314
- let deviceState = { oneofKind: undefined };
3315
- const setSdkInfo = (info) => {
3316
- sdkInfo = info;
3317
- };
3318
- const getSdkInfo = () => {
3319
- return sdkInfo;
3320
- };
3321
- const setOSInfo = (info) => {
3322
- osInfo = info;
3323
- };
3324
- const getOSInfo = () => {
3325
- return osInfo;
3326
- };
3327
- const setDeviceInfo = (info) => {
3328
- deviceInfo = info;
3436
+ /**
3437
+ * Returns a generic SDP for the given direction.
3438
+ * We use this SDP to send it as part of our JoinRequest so that the SFU
3439
+ * can use it to determine the client's codec capabilities.
3440
+ *
3441
+ * @param direction the direction of the transceiver.
3442
+ */
3443
+ const getGenericSdp = async (direction) => {
3444
+ const tempPc = new RTCPeerConnection();
3445
+ tempPc.addTransceiver('video', { direction });
3446
+ tempPc.addTransceiver('audio', { direction });
3447
+ const offer = await tempPc.createOffer();
3448
+ const sdp = offer.sdp ?? '';
3449
+ tempPc.getTransceivers().forEach((t) => {
3450
+ t.stop?.();
3451
+ });
3452
+ tempPc.close();
3453
+ return sdp;
3329
3454
  };
3330
- const getDeviceInfo = () => {
3331
- return deviceInfo;
3455
+ /**
3456
+ * Returns whether the codec is an SVC codec.
3457
+ *
3458
+ * @param codecOrMimeType the codec to check.
3459
+ */
3460
+ const isSvcCodec = (codecOrMimeType) => {
3461
+ if (!codecOrMimeType)
3462
+ return false;
3463
+ codecOrMimeType = codecOrMimeType.toLowerCase();
3464
+ return (codecOrMimeType === 'vp9' ||
3465
+ codecOrMimeType === 'av1' ||
3466
+ codecOrMimeType === 'video/vp9' ||
3467
+ codecOrMimeType === 'video/av1');
3332
3468
  };
3333
- const getWebRTCInfo = () => {
3334
- return webRtcInfo;
3469
+
3470
+ const sfuEventKinds = {
3471
+ subscriberOffer: undefined,
3472
+ publisherAnswer: undefined,
3473
+ connectionQualityChanged: undefined,
3474
+ audioLevelChanged: undefined,
3475
+ iceTrickle: undefined,
3476
+ changePublishQuality: undefined,
3477
+ participantJoined: undefined,
3478
+ participantLeft: undefined,
3479
+ dominantSpeakerChanged: undefined,
3480
+ joinResponse: undefined,
3481
+ healthCheckResponse: undefined,
3482
+ trackPublished: undefined,
3483
+ trackUnpublished: undefined,
3484
+ error: undefined,
3485
+ callGrantsUpdated: undefined,
3486
+ goAway: undefined,
3487
+ iceRestart: undefined,
3488
+ pinsUpdated: undefined,
3489
+ callEnded: undefined,
3490
+ participantUpdated: undefined,
3491
+ participantMigrationComplete: undefined,
3492
+ changePublishOptions: undefined,
3335
3493
  };
3336
- const setWebRTCInfo = (info) => {
3337
- webRtcInfo = info;
3494
+ const isSfuEvent = (eventName) => {
3495
+ return Object.prototype.hasOwnProperty.call(sfuEventKinds, eventName);
3338
3496
  };
3339
- const setThermalState = (state) => {
3340
- if (!osInfo) {
3341
- deviceState = { oneofKind: undefined };
3342
- return;
3343
- }
3344
- if (osInfo.name === 'android') {
3345
- const thermalState = AndroidThermalState[state] ||
3346
- AndroidThermalState.UNSPECIFIED;
3347
- deviceState = {
3348
- oneofKind: 'android',
3349
- android: {
3350
- thermalState,
3351
- isPowerSaverMode: deviceState?.oneofKind === 'android' &&
3352
- deviceState.android.isPowerSaverMode,
3353
- },
3497
+ class Dispatcher {
3498
+ constructor() {
3499
+ this.logger = getLogger(['Dispatcher']);
3500
+ this.subscribers = {};
3501
+ this.dispatch = (message, logTag = '0') => {
3502
+ const eventKind = message.eventPayload.oneofKind;
3503
+ if (!eventKind)
3504
+ return;
3505
+ const payload = message.eventPayload[eventKind];
3506
+ this.logger('debug', `Dispatching ${eventKind}, tag=${logTag}`, payload);
3507
+ const listeners = this.subscribers[eventKind];
3508
+ if (!listeners)
3509
+ return;
3510
+ for (const fn of listeners) {
3511
+ try {
3512
+ fn(payload);
3513
+ }
3514
+ catch (e) {
3515
+ this.logger('warn', 'Listener failed with error', e);
3516
+ }
3517
+ }
3354
3518
  };
3355
- }
3356
- if (osInfo.name.toLowerCase() === 'ios') {
3357
- const thermalState = AppleThermalState[state] ||
3358
- AppleThermalState.UNSPECIFIED;
3359
- deviceState = {
3360
- oneofKind: 'apple',
3361
- apple: {
3362
- thermalState,
3363
- isLowPowerModeEnabled: deviceState?.oneofKind === 'apple' &&
3364
- deviceState.apple.isLowPowerModeEnabled,
3365
- },
3366
- };
3367
- }
3368
- };
3369
- const setPowerState = (powerMode) => {
3370
- if (!osInfo) {
3371
- deviceState = { oneofKind: undefined };
3372
- return;
3373
- }
3374
- if (osInfo.name === 'android') {
3375
- deviceState = {
3376
- oneofKind: 'android',
3377
- android: {
3378
- thermalState: deviceState?.oneofKind === 'android'
3379
- ? deviceState.android.thermalState
3380
- : AndroidThermalState.UNSPECIFIED,
3381
- isPowerSaverMode: powerMode,
3382
- },
3383
- };
3384
- }
3385
- if (osInfo.name.toLowerCase() === 'ios') {
3386
- deviceState = {
3387
- oneofKind: 'apple',
3388
- apple: {
3389
- thermalState: deviceState?.oneofKind === 'apple'
3390
- ? deviceState.apple.thermalState
3391
- : AppleThermalState.UNSPECIFIED,
3392
- isLowPowerModeEnabled: powerMode,
3393
- },
3394
- };
3395
- }
3396
- };
3397
- const getDeviceState = () => {
3398
- return deviceState;
3399
- };
3400
- const getClientDetails = () => {
3401
- if (isReactNative()) {
3402
- // Since RN doesn't support web, sharing browser info is not required
3403
- return {
3404
- sdk: getSdkInfo(),
3405
- os: getOSInfo(),
3406
- device: getDeviceInfo(),
3407
- };
3408
- }
3409
- const userAgent = new UAParser(navigator.userAgent);
3410
- const { browser, os, device, cpu } = userAgent.getResult();
3411
- return {
3412
- sdk: getSdkInfo(),
3413
- browser: {
3414
- name: browser.name || navigator.userAgent,
3415
- version: browser.version || '',
3416
- },
3417
- os: {
3418
- name: os.name || '',
3419
- version: os.version || '',
3420
- architecture: cpu.architecture || '',
3421
- },
3422
- device: {
3423
- name: [device.vendor, device.model, device.type]
3424
- .filter(Boolean)
3425
- .join(' '),
3426
- version: '',
3427
- },
3428
- };
3429
- };
3430
-
3431
- /**
3432
- * Checks whether the current browser is Safari.
3433
- */
3434
- const isSafari = () => {
3435
- if (typeof navigator === 'undefined')
3436
- return false;
3437
- return /^((?!chrome|android).)*safari/i.test(navigator.userAgent || '');
3438
- };
3439
- /**
3440
- * Checks whether the current browser is Firefox.
3441
- */
3442
- const isFirefox = () => {
3443
- if (typeof navigator === 'undefined')
3444
- return false;
3445
- return navigator.userAgent?.includes('Firefox');
3446
- };
3447
- /**
3448
- * Checks whether the current browser is Google Chrome.
3449
- */
3450
- const isChrome = () => {
3451
- if (typeof navigator === 'undefined')
3452
- return false;
3453
- return navigator.userAgent?.includes('Chrome');
3454
- };
3455
-
3456
- var browsers = /*#__PURE__*/Object.freeze({
3457
- __proto__: null,
3458
- isChrome: isChrome,
3459
- isFirefox: isFirefox,
3460
- isSafari: isSafari
3461
- });
3462
-
3463
- /**
3464
- * Returns back a list of sorted codecs, with the preferred codec first.
3465
- *
3466
- * @param kind the kind of codec to get.
3467
- * @param preferredCodec the codec to prioritize (vp8, h264, vp9, av1...).
3468
- * @param codecToRemove the codec to exclude from the list.
3469
- * @param codecPreferencesSource the source of the codec preferences.
3470
- */
3471
- const getPreferredCodecs = (kind, preferredCodec, codecToRemove, codecPreferencesSource) => {
3472
- const source = codecPreferencesSource === 'receiver' ? RTCRtpReceiver : RTCRtpSender;
3473
- if (!('getCapabilities' in source))
3474
- return;
3475
- const capabilities = source.getCapabilities(kind);
3476
- if (!capabilities)
3477
- return;
3478
- const preferred = [];
3479
- const partiallyPreferred = [];
3480
- const unpreferred = [];
3481
- const preferredCodecMimeType = `${kind}/${preferredCodec.toLowerCase()}`;
3482
- const codecToRemoveMimeType = codecToRemove && `${kind}/${codecToRemove.toLowerCase()}`;
3483
- for (const codec of capabilities.codecs) {
3484
- const codecMimeType = codec.mimeType.toLowerCase();
3485
- const shouldRemoveCodec = codecMimeType === codecToRemoveMimeType;
3486
- if (shouldRemoveCodec)
3487
- continue; // skip this codec
3488
- const isPreferredCodec = codecMimeType === preferredCodecMimeType;
3489
- if (!isPreferredCodec) {
3490
- unpreferred.push(codec);
3491
- continue;
3492
- }
3493
- // h264 is a special case, we want to prioritize the baseline codec with
3494
- // profile-level-id is 42e01f and packetization-mode=0 for maximum
3495
- // cross-browser compatibility.
3496
- // this branch covers the other cases, such as vp8.
3497
- if (codecMimeType !== 'video/h264') {
3498
- preferred.push(codec);
3499
- continue;
3500
- }
3501
- const sdpFmtpLine = codec.sdpFmtpLine;
3502
- if (!sdpFmtpLine || !sdpFmtpLine.includes('profile-level-id=42')) {
3503
- // this is not the baseline h264 codec, prioritize it lower
3504
- partiallyPreferred.push(codec);
3505
- continue;
3506
- }
3507
- if (sdpFmtpLine.includes('packetization-mode=1')) {
3508
- preferred.unshift(codec);
3509
- }
3510
- else {
3511
- preferred.push(codec);
3512
- }
3513
- }
3514
- // return a sorted list of codecs, with the preferred codecs first
3515
- return [...preferred, ...partiallyPreferred, ...unpreferred];
3516
- };
3517
- /**
3518
- * Returns a generic SDP for the given direction.
3519
- * We use this SDP to send it as part of our JoinRequest so that the SFU
3520
- * can use it to determine client's codec capabilities.
3521
- *
3522
- * @param direction the direction of the transceiver.
3523
- */
3524
- const getGenericSdp = async (direction) => {
3525
- const tempPc = new RTCPeerConnection();
3526
- tempPc.addTransceiver('video', { direction });
3527
- tempPc.addTransceiver('audio', { direction });
3528
- const offer = await tempPc.createOffer();
3529
- const sdp = offer.sdp ?? '';
3530
- tempPc.getTransceivers().forEach((t) => {
3531
- t.stop?.();
3532
- });
3533
- tempPc.close();
3534
- return sdp;
3535
- };
3536
- /**
3537
- * Returns the optimal video codec for the device.
3538
- */
3539
- const getOptimalVideoCodec = (preferredCodec) => {
3540
- if (isReactNative()) {
3541
- const os = getOSInfo()?.name.toLowerCase();
3542
- if (os === 'android')
3543
- return preferredOr(preferredCodec, 'vp8');
3544
- if (os === 'ios' || os === 'ipados') {
3545
- return supportsH264Baseline() ? 'h264' : 'vp8';
3546
- }
3547
- return preferredOr(preferredCodec, 'h264');
3548
- }
3549
- if (isSafari())
3550
- return 'h264';
3551
- if (isFirefox())
3552
- return 'vp8';
3553
- return preferredOr(preferredCodec, 'vp8');
3554
- };
3555
- /**
3556
- * Determines if the platform supports the preferred codec.
3557
- * If not, it returns the fallback codec.
3558
- */
3559
- const preferredOr = (codec, fallback) => {
3560
- if (!codec)
3561
- return fallback;
3562
- if (!('getCapabilities' in RTCRtpSender))
3563
- return fallback;
3564
- const capabilities = RTCRtpSender.getCapabilities('video');
3565
- if (!capabilities)
3566
- return fallback;
3567
- // Safari and Firefox do not have a good support encoding to SVC codecs,
3568
- // so we disable it for them.
3569
- if (isSvcCodec(codec) && (isSafari() || isFirefox()))
3570
- return fallback;
3571
- const { codecs } = capabilities;
3572
- const codecMimeType = `video/${codec}`.toLowerCase();
3573
- return codecs.some((c) => c.mimeType.toLowerCase() === codecMimeType)
3574
- ? codec
3575
- : fallback;
3576
- };
3577
- /**
3578
- * Returns whether the platform supports the H264 baseline codec.
3579
- */
3580
- const supportsH264Baseline = () => {
3581
- if (!('getCapabilities' in RTCRtpSender))
3582
- return false;
3583
- const capabilities = RTCRtpSender.getCapabilities('video');
3584
- if (!capabilities)
3585
- return false;
3586
- return capabilities.codecs.some((c) => c.mimeType.toLowerCase() === 'video/h264' &&
3587
- c.sdpFmtpLine?.includes('profile-level-id=42e01f'));
3588
- };
3589
- /**
3590
- * Returns whether the codec is an SVC codec.
3591
- *
3592
- * @param codecOrMimeType the codec to check.
3593
- */
3594
- const isSvcCodec = (codecOrMimeType) => {
3595
- if (!codecOrMimeType)
3596
- return false;
3597
- codecOrMimeType = codecOrMimeType.toLowerCase();
3598
- return (codecOrMimeType === 'vp9' ||
3599
- codecOrMimeType === 'av1' ||
3600
- codecOrMimeType === 'video/vp9' ||
3601
- codecOrMimeType === 'video/av1');
3602
- };
3603
-
3604
- const sfuEventKinds = {
3605
- subscriberOffer: undefined,
3606
- publisherAnswer: undefined,
3607
- connectionQualityChanged: undefined,
3608
- audioLevelChanged: undefined,
3609
- iceTrickle: undefined,
3610
- changePublishQuality: undefined,
3611
- participantJoined: undefined,
3612
- participantLeft: undefined,
3613
- dominantSpeakerChanged: undefined,
3614
- joinResponse: undefined,
3615
- healthCheckResponse: undefined,
3616
- trackPublished: undefined,
3617
- trackUnpublished: undefined,
3618
- error: undefined,
3619
- callGrantsUpdated: undefined,
3620
- goAway: undefined,
3621
- iceRestart: undefined,
3622
- pinsUpdated: undefined,
3623
- callEnded: undefined,
3624
- participantUpdated: undefined,
3625
- participantMigrationComplete: undefined,
3626
- codecNegotiationComplete: undefined,
3627
- changePublishOptions: undefined,
3628
- };
3629
- const isSfuEvent = (eventName) => {
3630
- return Object.prototype.hasOwnProperty.call(sfuEventKinds, eventName);
3631
- };
3632
- class Dispatcher {
3633
- constructor() {
3634
- this.logger = getLogger(['Dispatcher']);
3635
- this.subscribers = {};
3636
- this.dispatch = (message, logTag = '0') => {
3637
- const eventKind = message.eventPayload.oneofKind;
3638
- if (!eventKind)
3639
- return;
3640
- const payload = message.eventPayload[eventKind];
3641
- this.logger('debug', `Dispatching ${eventKind}, tag=${logTag}`, payload);
3642
- const listeners = this.subscribers[eventKind];
3643
- if (!listeners)
3644
- return;
3645
- for (const fn of listeners) {
3646
- try {
3647
- fn(payload);
3648
- }
3649
- catch (e) {
3650
- this.logger('warn', 'Listener failed with error', e);
3651
- }
3652
- }
3653
- };
3654
- this.on = (eventName, fn) => {
3655
- var _a;
3656
- ((_a = this.subscribers)[eventName] ?? (_a[eventName] = [])).push(fn);
3657
- return () => {
3658
- this.off(eventName, fn);
3659
- };
3660
- };
3661
- this.off = (eventName, fn) => {
3662
- this.subscribers[eventName] = (this.subscribers[eventName] || []).filter((f) => f !== fn);
3663
- };
3664
- this.offAll = (eventName) => {
3665
- if (eventName) {
3666
- this.subscribers[eventName] = [];
3667
- }
3668
- else {
3669
- this.subscribers = {};
3670
- }
3519
+ this.on = (eventName, fn) => {
3520
+ var _a;
3521
+ ((_a = this.subscribers)[eventName] ?? (_a[eventName] = [])).push(fn);
3522
+ return () => {
3523
+ this.off(eventName, fn);
3524
+ };
3525
+ };
3526
+ this.off = (eventName, fn) => {
3527
+ this.subscribers[eventName] = (this.subscribers[eventName] || []).filter((f) => f !== fn);
3671
3528
  };
3672
3529
  }
3673
3530
  }
@@ -3681,284 +3538,34 @@ class IceTrickleBuffer {
3681
3538
  this.subscriberCandidates = new ReplaySubject();
3682
3539
  this.publisherCandidates = new ReplaySubject();
3683
3540
  this.push = (iceTrickle) => {
3541
+ const iceCandidate = toIceCandidate(iceTrickle);
3542
+ if (!iceCandidate)
3543
+ return;
3684
3544
  if (iceTrickle.peerType === PeerType.SUBSCRIBER) {
3685
- this.subscriberCandidates.next(iceTrickle);
3545
+ this.subscriberCandidates.next(iceCandidate);
3686
3546
  }
3687
3547
  else if (iceTrickle.peerType === PeerType.PUBLISHER_UNSPECIFIED) {
3688
- this.publisherCandidates.next(iceTrickle);
3548
+ this.publisherCandidates.next(iceCandidate);
3689
3549
  }
3690
3550
  else {
3691
3551
  const logger = getLogger(['sfu-client']);
3692
3552
  logger('warn', `ICETrickle, Unknown peer type`, iceTrickle);
3693
3553
  }
3694
3554
  };
3695
- }
3696
- }
3697
-
3698
- function getIceCandidate(candidate) {
3699
- if (!candidate.usernameFragment) {
3700
- // react-native-webrtc doesn't include usernameFragment in the candidate
3701
- const splittedCandidate = candidate.candidate.split(' ');
3702
- const ufragIndex = splittedCandidate.findIndex((s) => s === 'ufrag') + 1;
3703
- const usernameFragment = splittedCandidate[ufragIndex];
3704
- return JSON.stringify({ ...candidate, usernameFragment });
3705
- }
3706
- else {
3707
- return JSON.stringify(candidate.toJSON());
3708
- }
3709
- }
3710
-
3711
- const bitrateLookupTable = {
3712
- h264: {
3713
- 2160: 5000000,
3714
- 1440: 3000000,
3715
- 1080: 2000000,
3716
- 720: 1250000,
3717
- 540: 750000,
3718
- 360: 400000,
3719
- default: 1250000,
3720
- },
3721
- vp8: {
3722
- 2160: 5000000,
3723
- 1440: 2750000,
3724
- 1080: 2000000,
3725
- 720: 1250000,
3726
- 540: 600000,
3727
- 360: 350000,
3728
- default: 1250000,
3729
- },
3730
- vp9: {
3731
- 2160: 3000000,
3732
- 1440: 2000000,
3733
- 1080: 1500000,
3734
- 720: 1250000,
3735
- 540: 500000,
3736
- 360: 275000,
3737
- default: 1250000,
3738
- },
3739
- av1: {
3740
- 2160: 2000000,
3741
- 1440: 1550000,
3742
- 1080: 1000000,
3743
- 720: 600000,
3744
- 540: 350000,
3745
- 360: 200000,
3746
- default: 600000,
3747
- },
3748
- };
3749
- const getOptimalBitrate = (codec, frameHeight) => {
3750
- const codecLookup = bitrateLookupTable[codec];
3751
- if (!codecLookup)
3752
- throw new Error(`Unknown codec: ${codec}`);
3753
- let bitrate = codecLookup[frameHeight];
3754
- if (!bitrate) {
3755
- const keys = Object.keys(codecLookup).map(Number);
3756
- const nearest = keys.reduce((a, b) => Math.abs(b - frameHeight) < Math.abs(a - frameHeight) ? b : a);
3757
- bitrate = codecLookup[nearest];
3758
- }
3759
- return bitrate ?? codecLookup.default;
3760
- };
3761
-
3762
- const DEFAULT_BITRATE = 1250000;
3763
- const defaultTargetResolution = {
3764
- bitrate: DEFAULT_BITRATE,
3765
- width: 1280,
3766
- height: 720,
3767
- };
3768
- const defaultBitratePerRid = {
3769
- q: 300000,
3770
- h: 750000,
3771
- f: DEFAULT_BITRATE,
3772
- };
3773
- /**
3774
- * In SVC, we need to send only one video encoding (layer).
3775
- * this layer will have the additional spatial and temporal layers
3776
- * defined via the scalabilityMode property.
3777
- *
3778
- * @param layers the layers to process.
3779
- */
3780
- const toSvcEncodings = (layers) => {
3781
- // we take the `f` layer, and we rename it to `q`.
3782
- return layers?.filter((l) => l.rid === 'f').map((l) => ({ ...l, rid: 'q' }));
3783
- };
3784
- /**
3785
- * Converts the rid to a video quality.
3786
- */
3787
- const ridToVideoQuality = (rid) => {
3788
- return rid === 'q'
3789
- ? VideoQuality.LOW_UNSPECIFIED
3790
- : rid === 'h'
3791
- ? VideoQuality.MID
3792
- : VideoQuality.HIGH; // default to HIGH
3793
- };
3794
- /**
3795
- * Determines the most optimal video layers for simulcasting
3796
- * for the given track.
3797
- *
3798
- * @param videoTrack the video track to find optimal layers for.
3799
- * @param targetResolution the expected target resolution.
3800
- * @param codecInUse the codec in use.
3801
- * @param publishOptions the publish options for the track.
3802
- */
3803
- const findOptimalVideoLayers = (videoTrack, targetResolution = defaultTargetResolution, codecInUse, publishOptions) => {
3804
- const optimalVideoLayers = [];
3805
- const settings = videoTrack.getSettings();
3806
- const { width = 0, height = 0 } = settings;
3807
- const { scalabilityMode, bitrateDownscaleFactor = 2, maxSimulcastLayers = 3, } = publishOptions || {};
3808
- const maxBitrate = getComputedMaxBitrate(targetResolution, width, height, codecInUse, publishOptions);
3809
- let downscaleFactor = 1;
3810
- let bitrateFactor = 1;
3811
- const svcCodec = isSvcCodec(codecInUse);
3812
- const totalLayers = svcCodec ? 3 : Math.min(3, maxSimulcastLayers);
3813
- for (const rid of ['f', 'h', 'q'].slice(0, totalLayers)) {
3814
- const layer = {
3815
- active: true,
3816
- rid,
3817
- width: Math.round(width / downscaleFactor),
3818
- height: Math.round(height / downscaleFactor),
3819
- maxBitrate: Math.round(maxBitrate / bitrateFactor) || defaultBitratePerRid[rid],
3820
- maxFramerate: 30,
3555
+ this.dispose = () => {
3556
+ this.subscriberCandidates.complete();
3557
+ this.publisherCandidates.complete();
3821
3558
  };
3822
- if (svcCodec) {
3823
- // for SVC codecs, we need to set the scalability mode, and the
3824
- // codec will handle the rest (layers, temporal layers, etc.)
3825
- layer.scalabilityMode = scalabilityMode || 'L3T2_KEY';
3826
- }
3827
- else {
3828
- // for non-SVC codecs, we need to downscale proportionally (simulcast)
3829
- layer.scaleResolutionDownBy = downscaleFactor;
3830
- }
3831
- downscaleFactor *= 2;
3832
- bitrateFactor *= bitrateDownscaleFactor;
3833
- // Reversing the order [f, h, q] to [q, h, f] as Chrome uses encoding index
3834
- // when deciding which layer to disable when CPU or bandwidth is constrained.
3835
- // Encodings should be ordered in increasing spatial resolution order.
3836
- optimalVideoLayers.unshift(layer);
3837
- }
3838
- // for simplicity, we start with all layers enabled, then this function
3839
- // will clear/reassign the layers that are not needed
3840
- return withSimulcastConstraints(settings, optimalVideoLayers);
3841
- };
3842
- /**
3843
- * Computes the maximum bitrate for a given resolution.
3844
- * If the current resolution is lower than the target resolution,
3845
- * we want to proportionally reduce the target bitrate.
3846
- * If the current resolution is higher than the target resolution,
3847
- * we want to use the target bitrate.
3848
- *
3849
- * @param targetResolution the target resolution.
3850
- * @param currentWidth the current width of the track.
3851
- * @param currentHeight the current height of the track.
3852
- * @param codecInUse the codec in use.
3853
- * @param publishOptions the publish options.
3854
- */
3855
- const getComputedMaxBitrate = (targetResolution, currentWidth, currentHeight, codecInUse, publishOptions) => {
3856
- // if the current resolution is lower than the target resolution,
3857
- // we want to proportionally reduce the target bitrate
3858
- const { width: targetWidth, height: targetHeight, bitrate: targetBitrate, } = targetResolution;
3859
- const { preferredBitrate } = publishOptions || {};
3860
- const frameHeight = currentWidth > currentHeight ? currentHeight : currentWidth;
3861
- const bitrate = preferredBitrate ||
3862
- (codecInUse ? getOptimalBitrate(codecInUse, frameHeight) : targetBitrate);
3863
- if (currentWidth < targetWidth || currentHeight < targetHeight) {
3864
- const currentPixels = currentWidth * currentHeight;
3865
- const targetPixels = targetWidth * targetHeight;
3866
- const reductionFactor = currentPixels / targetPixels;
3867
- return Math.round(bitrate * reductionFactor);
3868
- }
3869
- return bitrate;
3870
- };
3871
- /**
3872
- * Browsers have different simulcast constraints for different video resolutions.
3873
- *
3874
- * This function modifies the provided list of video layers according to the
3875
- * current implementation of simulcast constraints in the Chromium based browsers.
3876
- *
3877
- * https://chromium.googlesource.com/external/webrtc/+/refs/heads/main/media/engine/simulcast.cc#90
3878
- */
3879
- const withSimulcastConstraints = (settings, optimalVideoLayers) => {
3880
- let layers;
3881
- const size = Math.max(settings.width || 0, settings.height || 0);
3882
- if (size <= 320) {
3883
- // provide only one layer 320x240 (q), the one with the highest quality
3884
- layers = optimalVideoLayers.filter((layer) => layer.rid === 'f');
3885
- }
3886
- else if (size <= 640) {
3887
- // provide two layers, 160x120 (q) and 640x480 (h)
3888
- layers = optimalVideoLayers.filter((layer) => layer.rid !== 'h');
3889
- }
3890
- else {
3891
- // provide three layers for sizes > 640x480
3892
- layers = optimalVideoLayers;
3893
- }
3894
- const ridMapping = ['q', 'h', 'f'];
3895
- return layers.map((layer, index) => ({
3896
- ...layer,
3897
- rid: ridMapping[index], // reassign rid
3898
- }));
3899
- };
3900
- const findOptimalScreenSharingLayers = (videoTrack, publishOptions, defaultMaxBitrate = 3000000) => {
3901
- const { screenShareSettings: preferences } = publishOptions || {};
3902
- const settings = videoTrack.getSettings();
3903
- return [
3904
- {
3905
- active: true,
3906
- rid: 'q', // single track, start from 'q'
3907
- width: settings.width || 0,
3908
- height: settings.height || 0,
3909
- scaleResolutionDownBy: 1,
3910
- maxBitrate: preferences?.maxBitrate ?? defaultMaxBitrate,
3911
- maxFramerate: preferences?.maxFramerate ?? 30,
3912
- },
3913
- ];
3914
- };
3915
-
3916
- const ensureExhausted = (x, message) => {
3917
- getLogger(['helpers'])('warn', message, x);
3918
- };
3919
-
3920
- const trackTypeToParticipantStreamKey = (trackType) => {
3921
- switch (trackType) {
3922
- case TrackType.SCREEN_SHARE:
3923
- return 'screenShareStream';
3924
- case TrackType.SCREEN_SHARE_AUDIO:
3925
- return 'screenShareAudioStream';
3926
- case TrackType.VIDEO:
3927
- return 'videoStream';
3928
- case TrackType.AUDIO:
3929
- return 'audioStream';
3930
- case TrackType.UNSPECIFIED:
3931
- throw new Error('Track type is unspecified');
3932
- default:
3933
- ensureExhausted(trackType, 'Unknown track type');
3934
3559
  }
3935
- };
3936
- const muteTypeToTrackType = (muteType) => {
3937
- switch (muteType) {
3938
- case 'audio':
3939
- return TrackType.AUDIO;
3940
- case 'video':
3941
- return TrackType.VIDEO;
3942
- case 'screenshare':
3943
- return TrackType.SCREEN_SHARE;
3944
- case 'screenshare_audio':
3945
- return TrackType.SCREEN_SHARE_AUDIO;
3946
- default:
3947
- ensureExhausted(muteType, 'Unknown mute type');
3560
+ }
3561
+ const toIceCandidate = (iceTrickle) => {
3562
+ try {
3563
+ return JSON.parse(iceTrickle.iceCandidate);
3948
3564
  }
3949
- };
3950
- const toTrackType = (trackType) => {
3951
- switch (trackType) {
3952
- case 'TRACK_TYPE_AUDIO':
3953
- return TrackType.AUDIO;
3954
- case 'TRACK_TYPE_VIDEO':
3955
- return TrackType.VIDEO;
3956
- case 'TRACK_TYPE_SCREEN_SHARE':
3957
- return TrackType.SCREEN_SHARE;
3958
- case 'TRACK_TYPE_SCREEN_SHARE_AUDIO':
3959
- return TrackType.SCREEN_SHARE_AUDIO;
3960
- default:
3961
- return undefined;
3565
+ catch (e) {
3566
+ const logger = getLogger(['sfu-client']);
3567
+ logger('error', `Failed to parse ICE Trickle`, e, iceTrickle);
3568
+ return undefined;
3962
3569
  }
3963
3570
  };
3964
3571
 
@@ -4102,6 +3709,24 @@ const setCurrentValue = (subject, update) => {
4102
3709
  subject.next(next);
4103
3710
  return next;
4104
3711
  };
3712
+ /**
3713
+ * Updates the value of the provided Subject and returns the previous value
3714
+ * and a function to roll back the update.
3715
+ * This is useful when you want to optimistically update a value
3716
+ * and roll back the update if an error occurs.
3717
+ *
3718
+ * @param subject the subject to update.
3719
+ * @param update the update to apply to the subject.
3720
+ */
3721
+ const updateValue = (subject, update) => {
3722
+ const lastValue = subject.getValue();
3723
+ const value = setCurrentValue(subject, update);
3724
+ return {
3725
+ lastValue,
3726
+ value,
3727
+ rollback: () => setCurrentValue(subject, lastValue),
3728
+ };
3729
+ };
4105
3730
  /**
4106
3731
  * Creates a subscription and returns a function to unsubscribe.
4107
3732
  *
@@ -4135,7 +3760,8 @@ var rxUtils = /*#__PURE__*/Object.freeze({
4135
3760
  createSafeAsyncSubscription: createSafeAsyncSubscription,
4136
3761
  createSubscription: createSubscription,
4137
3762
  getCurrentValue: getCurrentValue,
4138
- setCurrentValue: setCurrentValue
3763
+ setCurrentValue: setCurrentValue,
3764
+ updateValue: updateValue
4139
3765
  });
4140
3766
 
4141
3767
  /**
@@ -4582,6 +4208,7 @@ class CallState {
4582
4208
  this.sessionSubject = new BehaviorSubject(undefined);
4583
4209
  this.settingsSubject = new BehaviorSubject(undefined);
4584
4210
  this.transcribingSubject = new BehaviorSubject(false);
4211
+ this.captioningSubject = new BehaviorSubject(false);
4585
4212
  this.endedBySubject = new BehaviorSubject(undefined);
4586
4213
  this.thumbnailsSubject = new BehaviorSubject(undefined);
4587
4214
  this.membersSubject = new BehaviorSubject([]);
@@ -4592,6 +4219,7 @@ class CallState {
4592
4219
  this.anonymousParticipantCountSubject = new BehaviorSubject(0);
4593
4220
  this.participantsSubject = new BehaviorSubject([]);
4594
4221
  this.callStatsReportSubject = new BehaviorSubject(undefined);
4222
+ this.closedCaptionsSubject = new BehaviorSubject([]);
4595
4223
  // These are tracks that were delivered to the Subscriber's onTrack event
4596
4224
  // that we couldn't associate with a participant yet.
4597
4225
  // This happens when the participantJoined event hasn't been received yet.
@@ -4600,10 +4228,18 @@ class CallState {
4600
4228
  this.logger = getLogger(['CallState']);
4601
4229
  /**
4602
4230
  * A list of comparators that are used to sort the participants.
4603
- *
4604
- * @private
4605
4231
  */
4606
4232
  this.sortParticipantsBy = defaultSortPreset;
4233
+ this.closedCaptionsTasks = new Map();
4234
+ /**
4235
+ * Runs the cleanup tasks.
4236
+ */
4237
+ this.dispose = () => {
4238
+ for (const [ccKey, taskId] of this.closedCaptionsTasks.entries()) {
4239
+ clearTimeout(taskId);
4240
+ this.closedCaptionsTasks.delete(ccKey);
4241
+ }
4242
+ };
4607
4243
  /**
4608
4244
  * Sets the list of criteria that are used to sort the participants.
4609
4245
  * To disable sorting, you can pass `noopComparator()`.
@@ -4652,6 +4288,15 @@ class CallState {
4652
4288
  this.setStartedAt = (startedAt) => {
4653
4289
  return this.setCurrentValue(this.startedAtSubject, startedAt);
4654
4290
  };
4291
+ /**
4292
+ * Sets the closed captioning state of the current call.
4293
+ *
4294
+ * @internal
4295
+ * @param captioning the closed captioning state.
4296
+ */
4297
+ this.setCaptioning = (captioning) => {
4298
+ return updateValue(this.captioningSubject, captioning);
4299
+ };
4655
4300
  /**
4656
4301
  * Sets the number of anonymous participants in the current call.
4657
4302
  *
@@ -4750,7 +4395,6 @@ class CallState {
4750
4395
  }
4751
4396
  const thePatch = typeof patch === 'function' ? patch(participant) : patch;
4752
4397
  const updatedParticipant = {
4753
- // FIXME OL: this is not a deep merge, we might want to revisit this
4754
4398
  ...participant,
4755
4399
  ...thePatch,
4756
4400
  };
@@ -4812,7 +4456,6 @@ class CallState {
4812
4456
  *
4813
4457
  * @param trackType the kind of subscription to update.
4814
4458
  * @param changes the list of subscription changes to do.
4815
- * @param type the debounce type to use for the update.
4816
4459
  */
4817
4460
  this.updateParticipantTracks = (trackType, changes) => {
4818
4461
  return this.updateParticipants(Object.entries(changes).reduce((acc, [sessionId, change]) => {
@@ -4915,6 +4558,14 @@ class CallState {
4915
4558
  }
4916
4559
  return orphans;
4917
4560
  };
4561
+ /**
4562
+ * Updates the closed captions settings.
4563
+ *
4564
+ * @param config the new closed captions settings.
4565
+ */
4566
+ this.updateClosedCaptionSettings = (config) => {
4567
+ this.closedCaptionsSettings = { ...this.closedCaptionsSettings, ...config };
4568
+ };
4918
4569
  /**
4919
4570
  * Updates the call state with the data received from the server.
4920
4571
  *
@@ -4938,6 +4589,7 @@ class CallState {
4938
4589
  this.updateParticipantCountFromSession(s);
4939
4590
  this.setCurrentValue(this.settingsSubject, call.settings);
4940
4591
  this.setCurrentValue(this.transcribingSubject, call.transcribing);
4592
+ this.setCurrentValue(this.captioningSubject, call.captioning);
4941
4593
  this.setCurrentValue(this.thumbnailsSubject, call.thumbnails);
4942
4594
  };
4943
4595
  /**
@@ -5129,6 +4781,35 @@ class CallState {
5129
4781
  this.setCurrentValue(this.ownCapabilitiesSubject, event.own_capabilities);
5130
4782
  }
5131
4783
  };
4784
+ this.updateFromClosedCaptions = (event) => {
4785
+ this.setCurrentValue(this.closedCaptionsSubject, (queue) => {
4786
+ const { closed_caption } = event;
4787
+ const keyOf = (c) => `${c.speaker_id}/${c.start_time}`;
4788
+ const currentKey = keyOf(closed_caption);
4789
+ const duplicate = queue.some((caption) => keyOf(caption) === currentKey);
4790
+ if (duplicate)
4791
+ return queue;
4792
+ const nextQueue = [...queue, closed_caption];
4793
+ const { visibilityDurationMs = 2700, maxVisibleCaptions = 2 } = this.closedCaptionsSettings || {};
4794
+ // schedule the removal of the closed caption after the retention time
4795
+ if (visibilityDurationMs > 0) {
4796
+ const taskId = setTimeout(() => {
4797
+ this.setCurrentValue(this.closedCaptionsSubject, (captions) => captions.filter((caption) => caption !== closed_caption));
4798
+ this.closedCaptionsTasks.delete(currentKey);
4799
+ }, visibilityDurationMs);
4800
+ this.closedCaptionsTasks.set(currentKey, taskId);
4801
+ // cancel the cleanup tasks for the closed captions that are no longer in the queue
4802
+ for (let i = 0; i < nextQueue.length - maxVisibleCaptions; i++) {
4803
+ const key = keyOf(nextQueue[i]);
4804
+ const task = this.closedCaptionsTasks.get(key);
4805
+ clearTimeout(task);
4806
+ this.closedCaptionsTasks.delete(key);
4807
+ }
4808
+ }
4809
+ // trim the queue
4810
+ return nextQueue.slice(-maxVisibleCaptions);
4811
+ });
4812
+ };
5132
4813
  this.participants$ = this.participantsSubject.asObservable().pipe(
5133
4814
  // maintain stable-sort by mutating the participants stored
5134
4815
  // in the original subject
@@ -5155,6 +4836,7 @@ class CallState {
5155
4836
  this.settings$ = this.settingsSubject.asObservable();
5156
4837
  this.endedBy$ = this.endedBySubject.asObservable();
5157
4838
  this.thumbnails$ = this.thumbnailsSubject.asObservable();
4839
+ this.closedCaptions$ = this.closedCaptionsSubject.asObservable();
5158
4840
  /**
5159
4841
  * Performs shallow comparison of two arrays.
5160
4842
  * Expects primitive values: [1, 2, 3] is equal to [2, 1, 3].
@@ -5184,9 +4866,9 @@ class CallState {
5184
4866
  this.participantCount$ = duc(this.participantCountSubject);
5185
4867
  this.recording$ = duc(this.recordingSubject);
5186
4868
  this.transcribing$ = duc(this.transcribingSubject);
4869
+ this.captioning$ = duc(this.captioningSubject);
5187
4870
  this.eventHandlers = {
5188
4871
  // these events are not updating the call state:
5189
- 'call.closed_caption': undefined,
5190
4872
  'call.deleted': undefined,
5191
4873
  'call.permission_request': undefined,
5192
4874
  'call.recording_ready': undefined,
@@ -5207,6 +4889,16 @@ class CallState {
5207
4889
  // events that update call state:
5208
4890
  'call.accepted': (e) => this.updateFromCallResponse(e.call),
5209
4891
  'call.blocked_user': this.blockUser,
4892
+ 'call.closed_caption': this.updateFromClosedCaptions,
4893
+ 'call.closed_captions_failed': () => {
4894
+ this.setCurrentValue(this.captioningSubject, false);
4895
+ },
4896
+ 'call.closed_captions_started': () => {
4897
+ this.setCurrentValue(this.captioningSubject, true);
4898
+ },
4899
+ 'call.closed_captions_stopped': () => {
4900
+ this.setCurrentValue(this.captioningSubject, false);
4901
+ },
5210
4902
  'call.created': (e) => this.updateFromCallResponse(e.call),
5211
4903
  'call.ended': (e) => {
5212
4904
  this.updateFromCallResponse(e.call);
@@ -5264,6 +4956,12 @@ class CallState {
5264
4956
  get startedAt() {
5265
4957
  return this.getCurrentValue(this.startedAt$);
5266
4958
  }
4959
+ /**
4960
+ * Returns whether closed captions are enabled in the current call.
4961
+ */
4962
+ get captioning() {
4963
+ return this.getCurrentValue(this.captioning$);
4964
+ }
5267
4965
  /**
5268
4966
  * The server-side counted number of anonymous participants connected to the current call.
5269
4967
  * This number includes the anonymous participants as well.
@@ -5427,200 +5125,454 @@ class CallState {
5427
5125
  get thumbnails() {
5428
5126
  return this.getCurrentValue(this.thumbnails$);
5429
5127
  }
5128
+ /**
5129
+ * Returns the current queue of closed captions.
5130
+ */
5131
+ get closedCaptions() {
5132
+ return this.getCurrentValue(this.closedCaptions$);
5133
+ }
5430
5134
  }
5431
5135
 
5432
- const getRtpMap = (line) => {
5433
- // Example: a=rtpmap:110 opus/48000/2
5434
- const rtpRegex = /^a=rtpmap:(\d*) ([\w\-.]*)(?:\s*\/(\d*)(?:\s*\/(\S*))?)?/;
5435
- // The first captured group is the payload type number, the second captured group is the encoding name, the third captured group is the clock rate, and the fourth captured group is any additional parameters.
5436
- const rtpMatch = rtpRegex.exec(line);
5437
- if (rtpMatch) {
5438
- return {
5439
- original: rtpMatch[0],
5440
- payload: rtpMatch[1],
5441
- codec: rtpMatch[2],
5136
+ /**
5137
+ * A base class for the `Publisher` and `Subscriber` classes.
5138
+ * @internal
5139
+ */
5140
+ class BasePeerConnection {
5141
+ /**
5142
+ * Constructs a new `BasePeerConnection` instance.
5143
+ */
5144
+ constructor(peerType, { sfuClient, connectionConfig, state, dispatcher, onUnrecoverableError, logTag, }) {
5145
+ this.isIceRestarting = false;
5146
+ this.subscriptions = [];
5147
+ /**
5148
+ * Disposes the `RTCPeerConnection` instance.
5149
+ */
5150
+ this.dispose = () => {
5151
+ this.detachEventHandlers();
5152
+ this.pc.close();
5153
+ };
5154
+ /**
5155
+ * Handles events synchronously.
5156
+ * Consecutive events are queued and executed one after the other.
5157
+ */
5158
+ this.on = (event, fn) => {
5159
+ this.subscriptions.push(this.dispatcher.on(event, (e) => {
5160
+ withoutConcurrency(`pc.${event}`, async () => fn(e)).catch((err) => {
5161
+ this.logger('warn', `Error handling ${event}`, err);
5162
+ });
5163
+ }));
5164
+ };
5165
+ /**
5166
+ * Appends the trickled ICE candidates to the `RTCPeerConnection`.
5167
+ */
5168
+ this.addTrickledIceCandidates = () => {
5169
+ const { iceTrickleBuffer } = this.sfuClient;
5170
+ const observable = this.peerType === PeerType.SUBSCRIBER
5171
+ ? iceTrickleBuffer.subscriberCandidates
5172
+ : iceTrickleBuffer.publisherCandidates;
5173
+ this.unsubscribeIceTrickle?.();
5174
+ this.unsubscribeIceTrickle = createSafeAsyncSubscription(observable, async (candidate) => {
5175
+ return this.pc.addIceCandidate(candidate).catch((e) => {
5176
+ this.logger('warn', `ICE candidate error`, e, candidate);
5177
+ });
5178
+ });
5179
+ };
5180
+ /**
5181
+ * Sets the SFU client to use.
5182
+ *
5183
+ * @param sfuClient the SFU client to use.
5184
+ */
5185
+ this.setSfuClient = (sfuClient) => {
5186
+ this.sfuClient = sfuClient;
5187
+ };
5188
+ /**
5189
+ * Returns the result of the `RTCPeerConnection.getStats()` method
5190
+ * @param selector an optional `MediaStreamTrack` to get the stats for.
5191
+ */
5192
+ this.getStats = (selector) => {
5193
+ return this.pc.getStats(selector);
5194
+ };
5195
+ /**
5196
+ * Handles the ICECandidate event and
5197
+ * Initiates an ICE Trickle process with the SFU.
5198
+ */
5199
+ this.onIceCandidate = (e) => {
5200
+ const { candidate } = e;
5201
+ if (!candidate) {
5202
+ this.logger('debug', 'null ice candidate');
5203
+ return;
5204
+ }
5205
+ const iceCandidate = this.toJSON(candidate);
5206
+ this.sfuClient
5207
+ .iceTrickle({ peerType: this.peerType, iceCandidate })
5208
+ .catch((err) => this.logger('warn', `ICETrickle failed`, err));
5209
+ };
5210
+ /**
5211
+ * Converts the ICE candidate to a JSON string.
5212
+ */
5213
+ this.toJSON = (candidate) => {
5214
+ if (!candidate.usernameFragment) {
5215
+ // react-native-webrtc doesn't include usernameFragment in the candidate
5216
+ const segments = candidate.candidate.split(' ');
5217
+ const ufragIndex = segments.findIndex((s) => s === 'ufrag') + 1;
5218
+ const usernameFragment = segments[ufragIndex];
5219
+ return JSON.stringify({ ...candidate, usernameFragment });
5220
+ }
5221
+ return JSON.stringify(candidate.toJSON());
5222
+ };
5223
+ /**
5224
+ * Handles the ICE connection state change event.
5225
+ */
5226
+ this.onIceConnectionStateChange = () => {
5227
+ const state = this.pc.iceConnectionState;
5228
+ this.logger('debug', `ICE connection state changed`, state);
5229
+ if (this.state.callingState === CallingState.RECONNECTING)
5230
+ return;
5231
+ // do nothing when ICE is restarting
5232
+ if (this.isIceRestarting)
5233
+ return;
5234
+ if (state === 'failed' || state === 'disconnected') {
5235
+ this.logger('debug', `Attempting to restart ICE`);
5236
+ this.restartIce().catch((e) => {
5237
+ this.logger('error', `ICE restart failed`, e);
5238
+ this.onUnrecoverableError?.();
5239
+ });
5240
+ }
5241
+ };
5242
+ /**
5243
+ * Handles the ICE candidate error event.
5244
+ */
5245
+ this.onIceCandidateError = (e) => {
5246
+ const errorMessage = e instanceof RTCPeerConnectionIceErrorEvent &&
5247
+ `${e.errorCode}: ${e.errorText}`;
5248
+ const iceState = this.pc.iceConnectionState;
5249
+ const logLevel = iceState === 'connected' || iceState === 'checking' ? 'debug' : 'warn';
5250
+ this.logger(logLevel, `ICE Candidate error`, errorMessage);
5251
+ };
5252
+ /**
5253
+ * Handles the ICE gathering state change event.
5254
+ */
5255
+ this.onIceGatherChange = () => {
5256
+ this.logger('debug', `ICE Gathering State`, this.pc.iceGatheringState);
5257
+ };
5258
+ /**
5259
+ * Handles the signaling state change event.
5260
+ */
5261
+ this.onSignalingChange = () => {
5262
+ this.logger('debug', `Signaling state changed`, this.pc.signalingState);
5442
5263
  };
5264
+ this.peerType = peerType;
5265
+ this.sfuClient = sfuClient;
5266
+ this.state = state;
5267
+ this.dispatcher = dispatcher;
5268
+ this.onUnrecoverableError = onUnrecoverableError;
5269
+ this.logger = getLogger([
5270
+ peerType === PeerType.SUBSCRIBER ? 'Subscriber' : 'Publisher',
5271
+ logTag,
5272
+ ]);
5273
+ this.pc = new RTCPeerConnection(connectionConfig);
5274
+ this.pc.addEventListener('icecandidate', this.onIceCandidate);
5275
+ this.pc.addEventListener('icecandidateerror', this.onIceCandidateError);
5276
+ this.pc.addEventListener('iceconnectionstatechange', this.onIceConnectionStateChange);
5277
+ this.pc.addEventListener('icegatheringstatechange', this.onIceGatherChange);
5278
+ this.pc.addEventListener('signalingstatechange', this.onSignalingChange);
5443
5279
  }
5444
- };
5445
- const getFmtp = (line) => {
5446
- // Example: a=fmtp:111 minptime=10; useinbandfec=1
5447
- const fmtpRegex = /^a=fmtp:(\d*) (.*)/;
5448
- const fmtpMatch = fmtpRegex.exec(line);
5449
- // The first captured group is the payload type number, the second captured group is any additional parameters.
5450
- if (fmtpMatch) {
5451
- return {
5452
- original: fmtpMatch[0],
5453
- payload: fmtpMatch[1],
5454
- config: fmtpMatch[2],
5280
+ /**
5281
+ * Detaches the event handlers from the `RTCPeerConnection`.
5282
+ */
5283
+ detachEventHandlers() {
5284
+ this.pc.removeEventListener('icecandidate', this.onIceCandidate);
5285
+ this.pc.removeEventListener('icecandidateerror', this.onIceCandidateError);
5286
+ this.pc.removeEventListener('signalingstatechange', this.onSignalingChange);
5287
+ this.pc.removeEventListener('iceconnectionstatechange', this.onIceConnectionStateChange);
5288
+ this.pc.removeEventListener('icegatheringstatechange', this.onIceGatherChange);
5289
+ this.unsubscribeIceTrickle?.();
5290
+ this.subscriptions.forEach((unsubscribe) => unsubscribe());
5291
+ }
5292
+ }
5293
+
5294
+ class TransceiverCache {
5295
+ constructor() {
5296
+ this.cache = [];
5297
+ this.layers = [];
5298
+ /**
5299
+ * An array maintaining the order how transceivers were added to the peer connection.
5300
+ * This is needed because some browsers (Firefox) don't reliably report
5301
+ * trackId and `mid` parameters.
5302
+ */
5303
+ this.transceiverOrder = [];
5304
+ /**
5305
+ * Adds a transceiver to the cache.
5306
+ */
5307
+ this.add = (publishOption, transceiver) => {
5308
+ this.cache.push({ publishOption, transceiver });
5309
+ this.transceiverOrder.push(transceiver);
5310
+ };
5311
+ /**
5312
+ * Gets the transceiver for the given publish option.
5313
+ */
5314
+ this.get = (publishOption) => {
5315
+ return this.findTransceiver(publishOption)?.transceiver;
5316
+ };
5317
+ /**
5318
+ * Gets the last transceiver for the given track type and publish option id.
5319
+ */
5320
+ this.getWith = (trackType, id) => {
5321
+ return this.findTransceiver({ trackType, id })?.transceiver;
5322
+ };
5323
+ /**
5324
+ * Checks if the cache has the given publish option.
5325
+ */
5326
+ this.has = (publishOption) => {
5327
+ return !!this.get(publishOption);
5328
+ };
5329
+ /**
5330
+ * Finds the first transceiver that satisfies the given predicate.
5331
+ */
5332
+ this.find = (predicate) => {
5333
+ return this.cache.find(predicate);
5334
+ };
5335
+ /**
5336
+ * Provides all the items in the cache.
5337
+ */
5338
+ this.items = () => {
5339
+ return this.cache;
5340
+ };
5341
+ /**
5342
+ * Init index of the transceiver in the cache.
5343
+ */
5344
+ this.indexOf = (transceiver) => {
5345
+ return this.transceiverOrder.indexOf(transceiver);
5455
5346
  };
5347
+ /**
5348
+ * Gets cached video layers for the given track.
5349
+ */
5350
+ this.getLayers = (publishOption) => {
5351
+ const entry = this.layers.find((item) => item.publishOption.id === publishOption.id &&
5352
+ item.publishOption.trackType === publishOption.trackType);
5353
+ return entry?.layers;
5354
+ };
5355
+ /**
5356
+ * Sets the video layers for the given track.
5357
+ */
5358
+ this.setLayers = (publishOption, layers = []) => {
5359
+ const entry = this.findLayer(publishOption);
5360
+ if (entry) {
5361
+ entry.layers = layers;
5362
+ }
5363
+ else {
5364
+ this.layers.push({ publishOption, layers });
5365
+ }
5366
+ };
5367
+ this.findTransceiver = (publishOption) => {
5368
+ return this.cache.find((item) => item.publishOption.id === publishOption.id &&
5369
+ item.publishOption.trackType === publishOption.trackType);
5370
+ };
5371
+ this.findLayer = (publishOption) => {
5372
+ return this.layers.find((item) => item.publishOption.id === publishOption.id &&
5373
+ item.publishOption.trackType === publishOption.trackType);
5374
+ };
5375
+ }
5376
+ }
5377
+
5378
+ const ensureExhausted = (x, message) => {
5379
+ getLogger(['helpers'])('warn', message, x);
5380
+ };
5381
+
5382
+ const trackTypeToParticipantStreamKey = (trackType) => {
5383
+ switch (trackType) {
5384
+ case TrackType.SCREEN_SHARE:
5385
+ return 'screenShareStream';
5386
+ case TrackType.SCREEN_SHARE_AUDIO:
5387
+ return 'screenShareAudioStream';
5388
+ case TrackType.VIDEO:
5389
+ return 'videoStream';
5390
+ case TrackType.AUDIO:
5391
+ return 'audioStream';
5392
+ case TrackType.UNSPECIFIED:
5393
+ throw new Error('Track type is unspecified');
5394
+ default:
5395
+ ensureExhausted(trackType, 'Unknown track type');
5456
5396
  }
5457
5397
  };
5398
+ const muteTypeToTrackType = (muteType) => {
5399
+ switch (muteType) {
5400
+ case 'audio':
5401
+ return TrackType.AUDIO;
5402
+ case 'video':
5403
+ return TrackType.VIDEO;
5404
+ case 'screenshare':
5405
+ return TrackType.SCREEN_SHARE;
5406
+ case 'screenshare_audio':
5407
+ return TrackType.SCREEN_SHARE_AUDIO;
5408
+ default:
5409
+ ensureExhausted(muteType, 'Unknown mute type');
5410
+ }
5411
+ };
5412
+ const toTrackType = (trackType) => {
5413
+ switch (trackType) {
5414
+ case 'TRACK_TYPE_AUDIO':
5415
+ return TrackType.AUDIO;
5416
+ case 'TRACK_TYPE_VIDEO':
5417
+ return TrackType.VIDEO;
5418
+ case 'TRACK_TYPE_SCREEN_SHARE':
5419
+ return TrackType.SCREEN_SHARE;
5420
+ case 'TRACK_TYPE_SCREEN_SHARE_AUDIO':
5421
+ return TrackType.SCREEN_SHARE_AUDIO;
5422
+ default:
5423
+ return undefined;
5424
+ }
5425
+ };
5426
+ const isAudioTrackType = (trackType) => trackType === TrackType.AUDIO || trackType === TrackType.SCREEN_SHARE_AUDIO;
5427
+
5428
+ const defaultBitratePerRid = {
5429
+ q: 300000,
5430
+ h: 750000,
5431
+ f: 1250000,
5432
+ };
5458
5433
  /**
5459
- * gets the media section for the specified media type.
5460
- * The media section contains the media type, port, codec, and payload type.
5461
- * Example: m=video 9 UDP/TLS/RTP/SAVPF 100 101 96 97 35 36 102 125 127
5434
+ * In SVC, we need to send only one video encoding (layer).
5435
+ * this layer will have the additional spatial and temporal layers
5436
+ * defined via the scalabilityMode property.
5437
+ *
5438
+ * @param layers the layers to process.
5462
5439
  */
5463
- const getMedia = (line, mediaType) => {
5464
- const regex = new RegExp(`(m=${mediaType} \\d+ [\\w/]+) ([\\d\\s]+)`);
5465
- const match = regex.exec(line);
5466
- if (match) {
5467
- return {
5468
- original: match[0],
5469
- mediaWithPorts: match[1],
5470
- codecOrder: match[2],
5471
- };
5472
- }
5440
+ const toSvcEncodings = (layers) => {
5441
+ if (!layers)
5442
+ return;
5443
+ // we take the highest quality layer, and we assign it to `q` encoder.
5444
+ const withRid = (rid) => (l) => l.rid === rid;
5445
+ const highestLayer = layers.find(withRid('f')) ||
5446
+ layers.find(withRid('h')) ||
5447
+ layers.find(withRid('q'));
5448
+ return [{ ...highestLayer, rid: 'q' }];
5473
5449
  };
5474
- const getMediaSection = (sdp, mediaType) => {
5475
- let media;
5476
- const rtpMap = [];
5477
- const fmtp = [];
5478
- let isTheRequiredMediaSection = false;
5479
- sdp.split(/(\r\n|\r|\n)/).forEach((line) => {
5480
- const isValidLine = /^([a-z])=(.*)/.test(line);
5481
- if (!isValidLine)
5482
- return;
5483
- /*
5484
- NOTE: according to https://www.rfc-editor.org/rfc/rfc8866.pdf
5485
- Each media description starts with an "m=" line and continues to the next media description or the end of the whole session description, whichever comes first
5486
- */
5487
- const type = line[0];
5488
- if (type === 'm') {
5489
- const _media = getMedia(line, mediaType);
5490
- isTheRequiredMediaSection = !!_media;
5491
- if (_media) {
5492
- media = _media;
5493
- }
5450
+ /**
5451
+ * Converts the rid to a video quality.
5452
+ */
5453
+ const ridToVideoQuality = (rid) => {
5454
+ return rid === 'q'
5455
+ ? VideoQuality.LOW_UNSPECIFIED
5456
+ : rid === 'h'
5457
+ ? VideoQuality.MID
5458
+ : VideoQuality.HIGH; // default to HIGH
5459
+ };
5460
+ /**
5461
+ * Converts the given video layers to SFU video layers.
5462
+ */
5463
+ const toVideoLayers = (layers = []) => {
5464
+ return layers.map((layer) => ({
5465
+ rid: layer.rid || '',
5466
+ bitrate: layer.maxBitrate || 0,
5467
+ fps: layer.maxFramerate || 0,
5468
+ quality: ridToVideoQuality(layer.rid || ''),
5469
+ videoDimension: { width: layer.width, height: layer.height },
5470
+ }));
5471
+ };
5472
+ /**
5473
+ * Converts the spatial and temporal layers to a scalability mode.
5474
+ */
5475
+ const toScalabilityMode = (spatialLayers, temporalLayers) => `L${spatialLayers}T${temporalLayers}${spatialLayers > 1 ? '_KEY' : ''}`;
5476
+ /**
5477
+ * Determines the most optimal video layers for the given track.
5478
+ *
5479
+ * @param videoTrack the video track to find optimal layers for.
5480
+ * @param publishOption the publish options for the track.
5481
+ */
5482
+ const computeVideoLayers = (videoTrack, publishOption) => {
5483
+ if (isAudioTrackType(publishOption.trackType))
5484
+ return;
5485
+ const optimalVideoLayers = [];
5486
+ const settings = videoTrack.getSettings();
5487
+ const { width = 0, height = 0 } = settings;
5488
+ const { bitrate, codec, fps, maxSpatialLayers = 3, maxTemporalLayers = 3, videoDimension = { width: 1280, height: 720 }, } = publishOption;
5489
+ const maxBitrate = getComputedMaxBitrate(videoDimension, width, height, bitrate);
5490
+ let downscaleFactor = 1;
5491
+ let bitrateFactor = 1;
5492
+ const svcCodec = isSvcCodec(codec?.name);
5493
+ for (const rid of ['f', 'h', 'q'].slice(0, maxSpatialLayers)) {
5494
+ const layer = {
5495
+ active: true,
5496
+ rid,
5497
+ width: Math.round(width / downscaleFactor),
5498
+ height: Math.round(height / downscaleFactor),
5499
+ maxBitrate: maxBitrate / bitrateFactor || defaultBitratePerRid[rid],
5500
+ maxFramerate: fps,
5501
+ };
5502
+ if (svcCodec) {
5503
+ // for SVC codecs, we need to set the scalability mode, and the
5504
+ // codec will handle the rest (layers, temporal layers, etc.)
5505
+ layer.scalabilityMode = toScalabilityMode(maxSpatialLayers, maxTemporalLayers);
5494
5506
  }
5495
- else if (isTheRequiredMediaSection && type === 'a') {
5496
- const rtpMapLine = getRtpMap(line);
5497
- const fmtpLine = getFmtp(line);
5498
- if (rtpMapLine) {
5499
- rtpMap.push(rtpMapLine);
5500
- }
5501
- else if (fmtpLine) {
5502
- fmtp.push(fmtpLine);
5503
- }
5507
+ else {
5508
+ // for non-SVC codecs, we need to downscale proportionally (simulcast)
5509
+ layer.scaleResolutionDownBy = downscaleFactor;
5504
5510
  }
5505
- });
5506
- if (media) {
5507
- return {
5508
- media,
5509
- rtpMap,
5510
- fmtp,
5511
- };
5511
+ downscaleFactor *= 2;
5512
+ bitrateFactor *= 2;
5513
+ // Reversing the order [f, h, q] to [q, h, f] as Chrome uses encoding index
5514
+ // when deciding which layer to disable when CPU or bandwidth is constrained.
5515
+ // Encodings should be ordered in increasing spatial resolution order.
5516
+ optimalVideoLayers.unshift(layer);
5512
5517
  }
5518
+ // for simplicity, we start with all layers enabled, then this function
5519
+ // will clear/reassign the layers that are not needed
5520
+ return withSimulcastConstraints(settings, optimalVideoLayers);
5513
5521
  };
5514
5522
  /**
5515
- * Gets the fmtp line corresponding to opus
5523
+ * Computes the maximum bitrate for a given resolution.
5524
+ * If the current resolution is lower than the target resolution,
5525
+ * we want to proportionally reduce the target bitrate.
5526
+ * If the current resolution is higher than the target resolution,
5527
+ * we want to use the target bitrate.
5528
+ *
5529
+ * @param targetResolution the target resolution.
5530
+ * @param currentWidth the current width of the track.
5531
+ * @param currentHeight the current height of the track.
5532
+ * @param bitrate the target bitrate.
5516
5533
  */
5517
- const getOpusFmtp = (sdp) => {
5518
- const section = getMediaSection(sdp, 'audio');
5519
- const rtpMap = section?.rtpMap.find((r) => r.codec.toLowerCase() === 'opus');
5520
- const codecId = rtpMap?.payload;
5521
- if (codecId) {
5522
- return section?.fmtp.find((f) => f.payload === codecId);
5534
+ const getComputedMaxBitrate = (targetResolution, currentWidth, currentHeight, bitrate) => {
5535
+ // if the current resolution is lower than the target resolution,
5536
+ // we want to proportionally reduce the target bitrate
5537
+ const { width: targetWidth, height: targetHeight } = targetResolution;
5538
+ if (currentWidth < targetWidth || currentHeight < targetHeight) {
5539
+ const currentPixels = currentWidth * currentHeight;
5540
+ const targetPixels = targetWidth * targetHeight;
5541
+ const reductionFactor = currentPixels / targetPixels;
5542
+ return Math.round(bitrate * reductionFactor);
5523
5543
  }
5544
+ return bitrate;
5524
5545
  };
5525
5546
  /**
5526
- * Returns an SDP with DTX enabled or disabled.
5527
- */
5528
- const toggleDtx = (sdp, enable) => {
5529
- const opusFmtp = getOpusFmtp(sdp);
5530
- if (!opusFmtp)
5531
- return sdp;
5532
- const matchDtx = /usedtx=(\d)/.exec(opusFmtp.config);
5533
- const requiredDtxConfig = `usedtx=${enable ? '1' : '0'}`;
5534
- const newFmtp = matchDtx
5535
- ? opusFmtp.original.replace(/usedtx=(\d)/, requiredDtxConfig)
5536
- : `${opusFmtp.original};${requiredDtxConfig}`;
5537
- return sdp.replace(opusFmtp.original, newFmtp);
5538
- };
5539
- /**
5540
- * Returns and SDP with all the codecs except the given codec removed.
5541
- */
5542
- const preserveCodec = (sdp, mid, codec) => {
5543
- const [kind, codecName] = codec.mimeType.toLowerCase().split('/');
5544
- const toSet = (fmtpLine) => new Set(fmtpLine.split(';').map((f) => f.trim().toLowerCase()));
5545
- const equal = (a, b) => {
5546
- if (a.size !== b.size)
5547
- return false;
5548
- for (const item of a)
5549
- if (!b.has(item))
5550
- return false;
5551
- return true;
5552
- };
5553
- const codecFmtp = toSet(codec.sdpFmtpLine || '');
5554
- const parsedSdp = SDP.parse(sdp);
5555
- for (const media of parsedSdp.media) {
5556
- if (media.type !== kind || String(media.mid) !== mid)
5557
- continue;
5558
- // find the payload id of the desired codec
5559
- const payloads = new Set();
5560
- for (const rtp of media.rtp) {
5561
- if (rtp.codec.toLowerCase() !== codecName)
5562
- continue;
5563
- const match =
5564
- // vp8 doesn't have any fmtp, we preserve it without any additional checks
5565
- codecName === 'vp8'
5566
- ? true
5567
- : media.fmtp.some((f) => f.payload === rtp.payload && equal(toSet(f.config), codecFmtp));
5568
- if (match) {
5569
- payloads.add(rtp.payload);
5570
- }
5571
- }
5572
- // find the corresponding rtx codec by matching apt=<preserved-codec-payload>
5573
- for (const fmtp of media.fmtp) {
5574
- const match = fmtp.config.match(/(apt)=(\d+)/);
5575
- if (!match)
5576
- continue;
5577
- const [, , preservedCodecPayload] = match;
5578
- if (payloads.has(Number(preservedCodecPayload))) {
5579
- payloads.add(fmtp.payload);
5580
- }
5581
- }
5582
- media.rtp = media.rtp.filter((r) => payloads.has(r.payload));
5583
- media.fmtp = media.fmtp.filter((f) => payloads.has(f.payload));
5584
- media.rtcpFb = media.rtcpFb?.filter((f) => payloads.has(f.payload));
5585
- media.payloads = Array.from(payloads).join(' ');
5586
- }
5587
- return SDP.write(parsedSdp);
5588
- };
5589
- /**
5590
- * Enables high-quality audio through SDP munging for the given trackMid.
5547
+ * Browsers have different simulcast constraints for different video resolutions.
5591
5548
  *
5592
- * @param sdp the SDP to munge.
5593
- * @param trackMid the trackMid.
5594
- * @param maxBitrate the max bitrate to set.
5595
- */
5596
- const enableHighQualityAudio = (sdp, trackMid, maxBitrate = 510000) => {
5597
- maxBitrate = Math.max(Math.min(maxBitrate, 510000), 96000);
5598
- const parsedSdp = SDP.parse(sdp);
5599
- const audioMedia = parsedSdp.media.find((m) => m.type === 'audio' && String(m.mid) === trackMid);
5600
- if (!audioMedia)
5601
- return sdp;
5602
- const opusRtp = audioMedia.rtp.find((r) => r.codec === 'opus');
5603
- if (!opusRtp)
5604
- return sdp;
5605
- const opusFmtp = audioMedia.fmtp.find((f) => f.payload === opusRtp.payload);
5606
- if (!opusFmtp)
5607
- return sdp;
5608
- // enable stereo, if not already enabled
5609
- if (opusFmtp.config.match(/stereo=(\d)/)) {
5610
- opusFmtp.config = opusFmtp.config.replace(/stereo=(\d)/, 'stereo=1');
5611
- }
5612
- else {
5613
- opusFmtp.config = `${opusFmtp.config};stereo=1`;
5549
+ * This function modifies the provided list of video layers according to the
5550
+ * current implementation of simulcast constraints in the Chromium based browsers.
5551
+ *
5552
+ * https://chromium.googlesource.com/external/webrtc/+/refs/heads/main/media/engine/simulcast.cc#90
5553
+ */
5554
+ const withSimulcastConstraints = (settings, optimalVideoLayers) => {
5555
+ let layers;
5556
+ const size = Math.max(settings.width || 0, settings.height || 0);
5557
+ if (size <= 320) {
5558
+ // provide only one layer 320x240 (q), the one with the highest quality
5559
+ layers = optimalVideoLayers.filter((layer) => layer.rid === 'f');
5614
5560
  }
5615
- // set maxaveragebitrate, to the given value
5616
- if (opusFmtp.config.match(/maxaveragebitrate=(\d*)/)) {
5617
- opusFmtp.config = opusFmtp.config.replace(/maxaveragebitrate=(\d*)/, `maxaveragebitrate=${maxBitrate}`);
5561
+ else if (size <= 640) {
5562
+ // provide two layers, 160x120 (q) and 640x480 (h)
5563
+ layers = optimalVideoLayers.filter((layer) => layer.rid !== 'h');
5618
5564
  }
5619
5565
  else {
5620
- opusFmtp.config = `${opusFmtp.config};maxaveragebitrate=${maxBitrate}`;
5566
+ // provide three layers for sizes > 640x480
5567
+ layers = optimalVideoLayers;
5621
5568
  }
5622
- return SDP.write(parsedSdp);
5569
+ const ridMapping = ['q', 'h', 'f'];
5570
+ return layers.map((layer, index) => ({
5571
+ ...layer,
5572
+ rid: ridMapping[index], // reassign rid
5573
+ }));
5623
5574
  };
5575
+
5624
5576
  /**
5625
5577
  * Extracts the mid from the transceiver or the SDP.
5626
5578
  *
@@ -5632,9 +5584,9 @@ const extractMid = (transceiver, transceiverInitIndex, sdp) => {
5632
5584
  if (transceiver.mid)
5633
5585
  return transceiver.mid;
5634
5586
  if (!sdp)
5635
- return '';
5587
+ return String(transceiverInitIndex);
5636
5588
  const track = transceiver.sender.track;
5637
- const parsedSdp = SDP.parse(sdp);
5589
+ const parsedSdp = parse(sdp);
5638
5590
  const media = parsedSdp.media.find((m) => {
5639
5591
  return (m.type === track.kind &&
5640
5592
  // if `msid` is not present, we assume that the track is the first one
@@ -5642,7 +5594,7 @@ const extractMid = (transceiver, transceiverInitIndex, sdp) => {
5642
5594
  });
5643
5595
  if (typeof media?.mid !== 'undefined')
5644
5596
  return String(media.mid);
5645
- if (transceiverInitIndex === -1)
5597
+ if (transceiverInitIndex < 0)
5646
5598
  return '';
5647
5599
  return String(transceiverInitIndex);
5648
5600
  };
@@ -5652,164 +5604,87 @@ const extractMid = (transceiver, transceiverInitIndex, sdp) => {
5652
5604
  *
5653
5605
  * @internal
5654
5606
  */
5655
- class Publisher {
5607
+ class Publisher extends BasePeerConnection {
5656
5608
  /**
5657
5609
  * Constructs a new `Publisher` instance.
5658
5610
  */
5659
- constructor({ connectionConfig, sfuClient, dispatcher, state, isDtxEnabled, isRedEnabled, onUnrecoverableError, logTag, }) {
5660
- this.transceiverCache = new Map();
5661
- this.trackLayersCache = new Map();
5662
- this.publishOptsForTrack = new Map();
5663
- /**
5664
- * An array maintaining the order how transceivers were added to the peer connection.
5665
- * This is needed because some browsers (Firefox) don't reliably report
5666
- * trackId and `mid` parameters.
5667
- *
5668
- * @internal
5669
- */
5670
- this.transceiverInitOrder = [];
5671
- this.isIceRestarting = false;
5672
- this.createPeerConnection = (connectionConfig) => {
5673
- const pc = new RTCPeerConnection(connectionConfig);
5674
- pc.addEventListener('icecandidate', this.onIceCandidate);
5675
- pc.addEventListener('negotiationneeded', this.onNegotiationNeeded);
5676
- pc.addEventListener('icecandidateerror', this.onIceCandidateError);
5677
- pc.addEventListener('iceconnectionstatechange', this.onIceConnectionStateChange);
5678
- pc.addEventListener('icegatheringstatechange', this.onIceGatheringStateChange);
5679
- pc.addEventListener('signalingstatechange', this.onSignalingStateChange);
5680
- return pc;
5681
- };
5682
- /**
5683
- * Closes the publisher PeerConnection and cleans up the resources.
5684
- */
5685
- this.close = ({ stopTracks }) => {
5686
- if (stopTracks) {
5687
- this.stopPublishing();
5688
- this.transceiverCache.clear();
5689
- this.trackLayersCache.clear();
5690
- }
5691
- this.detachEventHandlers();
5692
- this.pc.close();
5693
- };
5694
- /**
5695
- * Detaches the event handlers from the `RTCPeerConnection`.
5696
- * This is useful when we want to replace the `RTCPeerConnection`
5697
- * instance with a new one (in case of migration).
5698
- */
5699
- this.detachEventHandlers = () => {
5700
- this.unsubscribeOnIceRestart();
5701
- this.unsubscribeChangePublishQuality();
5702
- this.pc.removeEventListener('icecandidate', this.onIceCandidate);
5703
- this.pc.removeEventListener('negotiationneeded', this.onNegotiationNeeded);
5704
- this.pc.removeEventListener('icecandidateerror', this.onIceCandidateError);
5705
- this.pc.removeEventListener('iceconnectionstatechange', this.onIceConnectionStateChange);
5706
- this.pc.removeEventListener('icegatheringstatechange', this.onIceGatheringStateChange);
5707
- this.pc.removeEventListener('signalingstatechange', this.onSignalingStateChange);
5708
- };
5611
+ constructor({ publishOptions, ...baseOptions }) {
5612
+ super(PeerType.PUBLISHER_UNSPECIFIED, baseOptions);
5613
+ this.transceiverCache = new TransceiverCache();
5709
5614
  /**
5710
5615
  * Starts publishing the given track of the given media stream.
5711
5616
  *
5712
5617
  * Consecutive calls to this method will replace the stream.
5713
5618
  * The previous stream will be stopped.
5714
5619
  *
5715
- * @param mediaStream the media stream to publish.
5716
5620
  * @param track the track to publish.
5717
5621
  * @param trackType the track type to publish.
5718
- * @param opts the optional publish options to use.
5719
5622
  */
5720
- this.publishStream = async (mediaStream, track, trackType, opts = {}) => {
5721
- if (track.readyState === 'ended') {
5722
- throw new Error(`Can't publish a track that has ended already.`);
5723
- }
5724
- // enable the track if it is disabled
5725
- if (!track.enabled)
5726
- track.enabled = true;
5727
- const transceiver = this.transceiverCache.get(trackType);
5728
- if (!transceiver || !transceiver.sender.track) {
5729
- // listen for 'ended' event on the track as it might be ended abruptly
5730
- // by an external factors such as permission revokes, a disconnected device, etc.
5731
- // keep in mind that `track.stop()` doesn't trigger this event.
5732
- const handleTrackEnded = () => {
5733
- this.logger('info', `Track ${TrackType[trackType]} has ended abruptly`);
5734
- track.removeEventListener('ended', handleTrackEnded);
5735
- this.notifyTrackMuteStateChanged(mediaStream, trackType, true).catch((err) => this.logger('warn', `Couldn't notify track mute state`, err));
5736
- };
5737
- track.addEventListener('ended', handleTrackEnded);
5738
- this.addTransceiver(trackType, track, opts, mediaStream);
5623
+ this.publish = async (track, trackType) => {
5624
+ if (!this.publishOptions.some((o) => o.trackType === trackType)) {
5625
+ throw new Error(`No publish options found for ${TrackType[trackType]}`);
5739
5626
  }
5740
- else {
5741
- await this.updateTransceiver(transceiver, track);
5627
+ for (const publishOption of this.publishOptions) {
5628
+ if (publishOption.trackType !== trackType)
5629
+ continue;
5630
+ // create a clone of the track as otherwise the same trackId will
5631
+ // appear in the SDP in multiple transceivers
5632
+ const trackToPublish = track.clone();
5633
+ const transceiver = this.transceiverCache.get(publishOption);
5634
+ if (!transceiver) {
5635
+ this.addTransceiver(trackToPublish, publishOption);
5636
+ }
5637
+ else {
5638
+ await transceiver.sender.replaceTrack(trackToPublish);
5639
+ }
5742
5640
  }
5743
- await this.notifyTrackMuteStateChanged(mediaStream, trackType, false);
5744
5641
  };
5745
5642
  /**
5746
- * Adds a new transceiver to the peer connection.
5747
- * This needs to be called when a new track kind is added to the peer connection.
5748
- * In other cases, use `updateTransceiver` method.
5643
+ * Adds a new transceiver carrying the given track to the peer connection.
5749
5644
  */
5750
- this.addTransceiver = (trackType, track, opts, mediaStream) => {
5751
- const { forceCodec, preferredCodec } = opts;
5752
- const codecInUse = forceCodec || getOptimalVideoCodec(preferredCodec);
5753
- const videoEncodings = this.computeLayers(trackType, track, opts);
5645
+ this.addTransceiver = (track, publishOption) => {
5646
+ const videoEncodings = computeVideoLayers(track, publishOption);
5647
+ const sendEncodings = isSvcCodec(publishOption.codec?.name)
5648
+ ? toSvcEncodings(videoEncodings)
5649
+ : videoEncodings;
5754
5650
  const transceiver = this.pc.addTransceiver(track, {
5755
5651
  direction: 'sendonly',
5756
- streams: trackType === TrackType.VIDEO || trackType === TrackType.SCREEN_SHARE
5757
- ? [mediaStream]
5758
- : undefined,
5759
- sendEncodings: isSvcCodec(codecInUse)
5760
- ? toSvcEncodings(videoEncodings)
5761
- : videoEncodings,
5652
+ sendEncodings,
5762
5653
  });
5654
+ const trackType = publishOption.trackType;
5763
5655
  this.logger('debug', `Added ${TrackType[trackType]} transceiver`);
5764
- this.transceiverInitOrder.push(trackType);
5765
- this.transceiverCache.set(trackType, transceiver);
5766
- this.publishOptsForTrack.set(trackType, opts);
5767
- // handle codec preferences
5768
- if (!('setCodecPreferences' in transceiver))
5769
- return;
5770
- const codecPreferences = this.getCodecPreferences(trackType, trackType === TrackType.VIDEO ? codecInUse : undefined, 'receiver');
5771
- if (!codecPreferences)
5772
- return;
5773
- try {
5774
- this.logger('info', `Setting ${TrackType[trackType]} codec preferences`, codecPreferences);
5775
- transceiver.setCodecPreferences(codecPreferences);
5776
- }
5777
- catch (err) {
5778
- this.logger('warn', `Couldn't set codec preferences`, err);
5779
- }
5780
- };
5781
- /**
5782
- * Updates the given transceiver with the new track.
5783
- * Stops the previous track and replaces it with the new one.
5784
- */
5785
- this.updateTransceiver = async (transceiver, track) => {
5786
- const previousTrack = transceiver.sender.track;
5787
- // don't stop the track if we are re-publishing the same track
5788
- if (previousTrack && previousTrack !== track) {
5789
- previousTrack.stop();
5790
- }
5791
- await transceiver.sender.replaceTrack(track);
5656
+ this.transceiverCache.add(publishOption, transceiver);
5792
5657
  };
5793
5658
  /**
5794
- * Stops publishing the given track type to the SFU, if it is currently being published.
5795
- * Underlying track will be stopped and removed from the publisher.
5796
- * @param trackType the track type to unpublish.
5797
- * @param stopTrack specifies whether track should be stopped or just disabled
5659
+ * Synchronizes the current Publisher state with the provided publish options.
5798
5660
  */
5799
- this.unpublishStream = async (trackType, stopTrack) => {
5800
- const transceiver = this.transceiverCache.get(trackType);
5801
- if (transceiver &&
5802
- transceiver.sender.track &&
5803
- (stopTrack
5804
- ? transceiver.sender.track.readyState === 'live'
5805
- : transceiver.sender.track.enabled)) {
5806
- stopTrack
5807
- ? transceiver.sender.track.stop()
5808
- : (transceiver.sender.track.enabled = false);
5809
- // We don't need to notify SFU if unpublishing in response to remote soft mute
5810
- if (this.state.localParticipant?.publishedTracks.includes(trackType)) {
5811
- await this.notifyTrackMuteStateChanged(undefined, trackType, true);
5812
- }
5661
+ this.syncPublishOptions = async () => {
5662
+ // enable publishing with new options -> [av1, vp9]
5663
+ for (const publishOption of this.publishOptions) {
5664
+ const { trackType } = publishOption;
5665
+ if (!this.isPublishing(trackType))
5666
+ continue;
5667
+ if (this.transceiverCache.has(publishOption))
5668
+ continue;
5669
+ const item = this.transceiverCache.find((i) => !!i.transceiver.sender.track &&
5670
+ i.publishOption.trackType === trackType);
5671
+ if (!item || !item.transceiver)
5672
+ continue;
5673
+ // take the track from the existing transceiver for the same track type,
5674
+ // clone it and publish it with the new publish options
5675
+ const track = item.transceiver.sender.track.clone();
5676
+ this.addTransceiver(track, publishOption);
5677
+ }
5678
+ // stop publishing with options not required anymore -> [vp9]
5679
+ for (const item of this.transceiverCache.items()) {
5680
+ const { publishOption, transceiver } = item;
5681
+ const hasPublishOption = this.publishOptions.some((option) => option.id === publishOption.id &&
5682
+ option.trackType === publishOption.trackType);
5683
+ if (hasPublishOption)
5684
+ continue;
5685
+ // it is safe to stop the track here, it is a clone
5686
+ transceiver.sender.track?.stop();
5687
+ await transceiver.sender.replaceTrack(null);
5813
5688
  }
5814
5689
  };
5815
5690
  /**
@@ -5818,57 +5693,52 @@ class Publisher {
5818
5693
  * @param trackType the track type to check.
5819
5694
  */
5820
5695
  this.isPublishing = (trackType) => {
5821
- const transceiver = this.transceiverCache.get(trackType);
5822
- if (!transceiver || !transceiver.sender)
5823
- return false;
5824
- const track = transceiver.sender.track;
5825
- return !!track && track.readyState === 'live' && track.enabled;
5826
- };
5827
- this.notifyTrackMuteStateChanged = async (mediaStream, trackType, isMuted) => {
5828
- await this.sfuClient.updateMuteState(trackType, isMuted);
5829
- const audioOrVideoOrScreenShareStream = trackTypeToParticipantStreamKey(trackType);
5830
- if (!audioOrVideoOrScreenShareStream)
5831
- return;
5832
- if (isMuted) {
5833
- this.state.updateParticipant(this.sfuClient.sessionId, (p) => ({
5834
- publishedTracks: p.publishedTracks.filter((t) => t !== trackType),
5835
- [audioOrVideoOrScreenShareStream]: undefined,
5836
- }));
5837
- }
5838
- else {
5839
- this.state.updateParticipant(this.sfuClient.sessionId, (p) => {
5840
- return {
5841
- publishedTracks: p.publishedTracks.includes(trackType)
5842
- ? p.publishedTracks
5843
- : [...p.publishedTracks, trackType],
5844
- [audioOrVideoOrScreenShareStream]: mediaStream,
5845
- };
5846
- });
5696
+ for (const item of this.transceiverCache.items()) {
5697
+ if (item.publishOption.trackType !== trackType)
5698
+ continue;
5699
+ const track = item.transceiver.sender.track;
5700
+ if (!track)
5701
+ continue;
5702
+ if (track.readyState === 'live' && track.enabled)
5703
+ return true;
5847
5704
  }
5705
+ return false;
5848
5706
  };
5849
5707
  /**
5850
- * Stops publishing all tracks and stop all tracks.
5708
+ * Maps the given track ID to the corresponding track type.
5851
5709
  */
5852
- this.stopPublishing = () => {
5853
- this.logger('debug', 'Stopping publishing all tracks');
5854
- this.pc.getSenders().forEach((s) => {
5855
- s.track?.stop();
5856
- if (this.pc.signalingState !== 'closed') {
5857
- this.pc.removeTrack(s);
5710
+ this.getTrackType = (trackId) => {
5711
+ for (const transceiverId of this.transceiverCache.items()) {
5712
+ const { publishOption, transceiver } = transceiverId;
5713
+ if (transceiver.sender.track?.id === trackId) {
5714
+ return publishOption.trackType;
5858
5715
  }
5859
- });
5716
+ }
5717
+ return undefined;
5860
5718
  };
5861
- this.changePublishQuality = async (enabledLayers) => {
5862
- this.logger('info', 'Update publish quality, requested layers by SFU:', enabledLayers);
5863
- const videoSender = this.transceiverCache.get(TrackType.VIDEO)?.sender;
5864
- if (!videoSender) {
5865
- this.logger('warn', 'Update publish quality, no video sender found.');
5866
- return;
5719
+ /**
5720
+ * Stops the cloned track that is being published to the SFU.
5721
+ */
5722
+ this.stopTracks = (...trackTypes) => {
5723
+ for (const item of this.transceiverCache.items()) {
5724
+ const { publishOption, transceiver } = item;
5725
+ if (!trackTypes.includes(publishOption.trackType))
5726
+ continue;
5727
+ transceiver.sender.track?.stop();
5728
+ }
5729
+ };
5730
+ this.changePublishQuality = async (videoSender) => {
5731
+ const { trackType, layers, publishOptionId } = videoSender;
5732
+ const enabledLayers = layers.filter((l) => l.active);
5733
+ const tag = 'Update publish quality:';
5734
+ this.logger('info', `${tag} requested layers by SFU:`, enabledLayers);
5735
+ const sender = this.transceiverCache.getWith(trackType, publishOptionId)?.sender;
5736
+ if (!sender) {
5737
+ return this.logger('warn', `${tag} no video sender found.`);
5867
5738
  }
5868
- const params = videoSender.getParameters();
5739
+ const params = sender.getParameters();
5869
5740
  if (params.encodings.length === 0) {
5870
- this.logger('warn', 'Update publish quality, No suitable video encoding quality found');
5871
- return;
5741
+ return this.logger('warn', `${tag} there are no encodings set.`);
5872
5742
  }
5873
5743
  const [codecInUse] = params.codecs;
5874
5744
  const usesSvcCodec = codecInUse && isSvcCodec(codecInUse.mimeType);
@@ -5910,54 +5780,12 @@ class Publisher {
5910
5780
  changed = true;
5911
5781
  }
5912
5782
  }
5913
- const activeLayers = params.encodings.filter((e) => e.active);
5783
+ const activeEncoders = params.encodings.filter((e) => e.active);
5914
5784
  if (!changed) {
5915
- this.logger('info', `Update publish quality, no change:`, activeLayers);
5916
- return;
5917
- }
5918
- await videoSender.setParameters(params);
5919
- this.logger('info', `Update publish quality, enabled rids:`, activeLayers);
5920
- };
5921
- /**
5922
- * Returns the result of the `RTCPeerConnection.getStats()` method
5923
- * @param selector
5924
- * @returns
5925
- */
5926
- this.getStats = (selector) => {
5927
- return this.pc.getStats(selector);
5928
- };
5929
- this.getCodecPreferences = (trackType, preferredCodec, codecPreferencesSource) => {
5930
- if (trackType === TrackType.VIDEO) {
5931
- return getPreferredCodecs('video', preferredCodec || 'vp8', undefined, codecPreferencesSource);
5932
- }
5933
- if (trackType === TrackType.AUDIO) {
5934
- const defaultAudioCodec = this.isRedEnabled ? 'red' : 'opus';
5935
- const codecToRemove = !this.isRedEnabled ? 'red' : undefined;
5936
- return getPreferredCodecs('audio', preferredCodec ?? defaultAudioCodec, codecToRemove, codecPreferencesSource);
5937
- }
5938
- };
5939
- this.onIceCandidate = (e) => {
5940
- const { candidate } = e;
5941
- if (!candidate) {
5942
- this.logger('debug', 'null ice candidate');
5943
- return;
5785
+ return this.logger('info', `${tag} no change:`, activeEncoders);
5944
5786
  }
5945
- this.sfuClient
5946
- .iceTrickle({
5947
- iceCandidate: getIceCandidate(candidate),
5948
- peerType: PeerType.PUBLISHER_UNSPECIFIED,
5949
- })
5950
- .catch((err) => {
5951
- this.logger('warn', `ICETrickle failed`, err);
5952
- });
5953
- };
5954
- /**
5955
- * Sets the SFU client to use.
5956
- *
5957
- * @param sfuClient the SFU client to use.
5958
- */
5959
- this.setSfuClient = (sfuClient) => {
5960
- this.sfuClient = sfuClient;
5787
+ await sender.setParameters(params);
5788
+ this.logger('info', `${tag} enabled rids:`, activeEncoders);
5961
5789
  };
5962
5790
  /**
5963
5791
  * Restarts the ICE connection and renegotiates with the SFU.
@@ -5972,7 +5800,7 @@ class Publisher {
5972
5800
  await this.negotiate({ iceRestart: true });
5973
5801
  };
5974
5802
  this.onNegotiationNeeded = () => {
5975
- this.negotiate().catch((err) => {
5803
+ withoutConcurrency('publisher.negotiate', () => this.negotiate()).catch((err) => {
5976
5804
  this.logger('error', `Negotiation failed.`, err);
5977
5805
  this.onUnrecoverableError?.();
5978
5806
  });
@@ -5984,18 +5812,6 @@ class Publisher {
5984
5812
  */
5985
5813
  this.negotiate = async (options) => {
5986
5814
  const offer = await this.pc.createOffer(options);
5987
- if (offer.sdp) {
5988
- offer.sdp = toggleDtx(offer.sdp, this.isDtxEnabled);
5989
- if (this.isPublishing(TrackType.SCREEN_SHARE_AUDIO)) {
5990
- offer.sdp = this.enableHighQualityAudio(offer.sdp);
5991
- }
5992
- if (this.isPublishing(TrackType.VIDEO)) {
5993
- // Hotfix for platforms that don't respect the ordered codec list
5994
- // (Firefox, Android, Linux, etc...).
5995
- // We remove all the codecs from the SDP except the one we want to use.
5996
- offer.sdp = this.removeUnpreferredCodecs(offer.sdp, TrackType.VIDEO);
5997
- }
5998
- }
5999
5815
  const trackInfos = this.getAnnouncedTracks(offer.sdp);
6000
5816
  if (trackInfos.length === 0) {
6001
5817
  throw new Error(`Can't negotiate without announcing any tracks`);
@@ -6014,238 +5830,121 @@ class Publisher {
6014
5830
  finally {
6015
5831
  this.isIceRestarting = false;
6016
5832
  }
6017
- this.sfuClient.iceTrickleBuffer.publisherCandidates.subscribe(async (candidate) => {
6018
- try {
6019
- const iceCandidate = JSON.parse(candidate.iceCandidate);
6020
- await this.pc.addIceCandidate(iceCandidate);
6021
- }
6022
- catch (e) {
6023
- this.logger('warn', `ICE candidate error`, e, candidate);
6024
- }
6025
- });
6026
- };
6027
- this.enableHighQualityAudio = (sdp) => {
6028
- const transceiver = this.transceiverCache.get(TrackType.SCREEN_SHARE_AUDIO);
6029
- if (!transceiver)
6030
- return sdp;
6031
- const transceiverInitIndex = this.transceiverInitOrder.indexOf(TrackType.SCREEN_SHARE_AUDIO);
6032
- const mid = extractMid(transceiver, transceiverInitIndex, sdp);
6033
- return enableHighQualityAudio(sdp, mid);
5833
+ this.addTrickledIceCandidates();
6034
5834
  };
6035
5835
  /**
6036
5836
  * Returns a list of tracks that are currently being published.
6037
- *
6038
- * @internal
6039
- * @param sdp an optional SDP to extract the `mid` from.
6040
5837
  */
6041
- this.getAnnouncedTracks = (sdp) => {
6042
- sdp = sdp || this.pc.localDescription?.sdp;
6043
- return this.pc
6044
- .getTransceivers()
6045
- .filter((t) => t.direction === 'sendonly' && t.sender.track)
6046
- .map((transceiver) => {
6047
- let trackType;
6048
- this.transceiverCache.forEach((value, key) => {
6049
- if (value === transceiver)
6050
- trackType = key;
6051
- });
5838
+ this.getPublishedTracks = () => {
5839
+ const tracks = [];
5840
+ for (const { transceiver } of this.transceiverCache.items()) {
6052
5841
  const track = transceiver.sender.track;
6053
- let optimalLayers;
6054
- const isTrackLive = track.readyState === 'live';
6055
- if (isTrackLive) {
6056
- optimalLayers = this.computeLayers(trackType, track) || [];
6057
- this.trackLayersCache.set(trackType, optimalLayers);
6058
- }
6059
- else {
6060
- // we report the last known optimal layers for ended tracks
6061
- optimalLayers = this.trackLayersCache.get(trackType) || [];
6062
- this.logger('debug', `Track ${TrackType[trackType]} is ended. Announcing last known optimal layers`, optimalLayers);
6063
- }
6064
- const layers = optimalLayers.map((optimalLayer) => ({
6065
- rid: optimalLayer.rid || '',
6066
- bitrate: optimalLayer.maxBitrate || 0,
6067
- fps: optimalLayer.maxFramerate || 0,
6068
- quality: ridToVideoQuality(optimalLayer.rid || ''),
6069
- videoDimension: {
6070
- width: optimalLayer.width,
6071
- height: optimalLayer.height,
6072
- },
6073
- }));
6074
- const isAudioTrack = [
6075
- TrackType.AUDIO,
6076
- TrackType.SCREEN_SHARE_AUDIO,
6077
- ].includes(trackType);
6078
- const trackSettings = track.getSettings();
6079
- const isStereo = isAudioTrack && trackSettings.channelCount === 2;
6080
- const transceiverInitIndex = this.transceiverInitOrder.indexOf(trackType);
6081
- return {
6082
- trackId: track.id,
6083
- layers: layers,
6084
- trackType,
6085
- mid: extractMid(transceiver, transceiverInitIndex, sdp),
6086
- stereo: isStereo,
6087
- dtx: isAudioTrack && this.isDtxEnabled,
6088
- red: isAudioTrack && this.isRedEnabled,
6089
- muted: !isTrackLive,
6090
- };
6091
- });
6092
- };
6093
- this.computeLayers = (trackType, track, opts) => {
6094
- const { settings } = this.state;
6095
- const targetResolution = settings?.video
6096
- .target_resolution;
6097
- const screenShareBitrate = settings?.screensharing.target_resolution?.bitrate;
6098
- const publishOpts = opts || this.publishOptsForTrack.get(trackType);
6099
- const codecInUse = opts?.forceCodec || getOptimalVideoCodec(opts?.preferredCodec);
6100
- return trackType === TrackType.VIDEO
6101
- ? findOptimalVideoLayers(track, targetResolution, codecInUse, publishOpts)
6102
- : trackType === TrackType.SCREEN_SHARE
6103
- ? findOptimalScreenSharingLayers(track, publishOpts, screenShareBitrate)
6104
- : undefined;
6105
- };
6106
- this.onIceCandidateError = (e) => {
6107
- const errorMessage = e instanceof RTCPeerConnectionIceErrorEvent &&
6108
- `${e.errorCode}: ${e.errorText}`;
6109
- const iceState = this.pc.iceConnectionState;
6110
- const logLevel = iceState === 'connected' || iceState === 'checking' ? 'debug' : 'warn';
6111
- this.logger(logLevel, `ICE Candidate error`, errorMessage);
6112
- };
6113
- this.onIceConnectionStateChange = () => {
6114
- const state = this.pc.iceConnectionState;
6115
- this.logger('debug', `ICE Connection state changed to`, state);
6116
- if (this.state.callingState === CallingState.RECONNECTING)
6117
- return;
6118
- if (state === 'failed' || state === 'disconnected') {
6119
- this.logger('debug', `Attempting to restart ICE`);
6120
- this.restartIce().catch((e) => {
6121
- this.logger('error', `ICE restart error`, e);
6122
- this.onUnrecoverableError?.();
6123
- });
5842
+ if (track && track.readyState === 'live')
5843
+ tracks.push(track);
6124
5844
  }
6125
- };
6126
- this.onIceGatheringStateChange = () => {
6127
- this.logger('debug', `ICE Gathering State`, this.pc.iceGatheringState);
6128
- };
6129
- this.onSignalingStateChange = () => {
6130
- this.logger('debug', `Signaling state changed`, this.pc.signalingState);
6131
- };
6132
- this.logger = getLogger(['Publisher', logTag]);
6133
- this.pc = this.createPeerConnection(connectionConfig);
6134
- this.sfuClient = sfuClient;
6135
- this.state = state;
6136
- this.isDtxEnabled = isDtxEnabled;
6137
- this.isRedEnabled = isRedEnabled;
6138
- this.onUnrecoverableError = onUnrecoverableError;
6139
- this.unsubscribeOnIceRestart = dispatcher.on('iceRestart', (iceRestart) => {
6140
- if (iceRestart.peerType !== PeerType.PUBLISHER_UNSPECIFIED)
6141
- return;
6142
- this.restartIce().catch((err) => {
6143
- this.logger('warn', `ICERestart failed`, err);
6144
- this.onUnrecoverableError?.();
6145
- });
6146
- });
6147
- this.unsubscribeChangePublishQuality = dispatcher.on('changePublishQuality', ({ videoSenders }) => {
6148
- withoutConcurrency('publisher.changePublishQuality', async () => {
6149
- for (const videoSender of videoSenders) {
6150
- const { layers } = videoSender;
6151
- const enabledLayers = layers.filter((l) => l.active);
6152
- await this.changePublishQuality(enabledLayers);
6153
- }
6154
- }).catch((err) => {
6155
- this.logger('warn', 'Failed to change publish quality', err);
6156
- });
6157
- });
6158
- }
6159
- removeUnpreferredCodecs(sdp, trackType) {
6160
- const opts = this.publishOptsForTrack.get(trackType);
6161
- const forceSingleCodec = !!opts?.forceSingleCodec || isReactNative() || isFirefox();
6162
- if (!opts || !forceSingleCodec)
6163
- return sdp;
6164
- const codec = opts.forceCodec || getOptimalVideoCodec(opts.preferredCodec);
6165
- const orderedCodecs = this.getCodecPreferences(trackType, codec, 'sender');
6166
- if (!orderedCodecs || orderedCodecs.length === 0)
6167
- return sdp;
6168
- const transceiver = this.transceiverCache.get(trackType);
6169
- if (!transceiver)
6170
- return sdp;
6171
- const index = this.transceiverInitOrder.indexOf(trackType);
6172
- const mid = extractMid(transceiver, index, sdp);
6173
- const [codecToPreserve] = orderedCodecs;
6174
- return preserveCodec(sdp, mid, codecToPreserve);
6175
- }
6176
- }
6177
-
6178
- /**
6179
- * A wrapper around the `RTCPeerConnection` that handles the incoming
6180
- * media streams from the SFU.
6181
- *
6182
- * @internal
6183
- */
6184
- class Subscriber {
6185
- /**
6186
- * Constructs a new `Subscriber` instance.
6187
- *
6188
- * @param sfuClient the SFU client to use.
6189
- * @param dispatcher the dispatcher to use.
6190
- * @param state the state of the call.
6191
- * @param connectionConfig the connection configuration to use.
6192
- * @param iceRestartDelay the delay in milliseconds to wait before restarting ICE when connection goes to `disconnected` state.
6193
- * @param onUnrecoverableError a callback to call when an unrecoverable error occurs.
6194
- * @param logTag a tag to use for logging.
6195
- */
6196
- constructor({ sfuClient, dispatcher, state, connectionConfig, onUnrecoverableError, logTag, }) {
6197
- this.isIceRestarting = false;
6198
- /**
6199
- * Creates a new `RTCPeerConnection` instance with the given configuration.
6200
- *
6201
- * @param connectionConfig the connection configuration to use.
6202
- */
6203
- this.createPeerConnection = (connectionConfig) => {
6204
- const pc = new RTCPeerConnection(connectionConfig);
6205
- pc.addEventListener('icecandidate', this.onIceCandidate);
6206
- pc.addEventListener('track', this.handleOnTrack);
6207
- pc.addEventListener('icecandidateerror', this.onIceCandidateError);
6208
- pc.addEventListener('iceconnectionstatechange', this.onIceConnectionStateChange);
6209
- pc.addEventListener('icegatheringstatechange', this.onIceGatheringStateChange);
6210
- return pc;
6211
- };
6212
- /**
6213
- * Closes the `RTCPeerConnection` and unsubscribes from the dispatcher.
6214
- */
6215
- this.close = () => {
6216
- this.detachEventHandlers();
6217
- this.pc.close();
5845
+ return tracks;
6218
5846
  };
6219
5847
  /**
6220
- * Detaches the event handlers from the `RTCPeerConnection`.
6221
- * This is useful when we want to replace the `RTCPeerConnection`
6222
- * instance with a new one (in case of migration).
5848
+ * Returns a list of tracks that are currently being published.
5849
+ * @param sdp an optional SDP to extract the `mid` from.
6223
5850
  */
6224
- this.detachEventHandlers = () => {
6225
- this.unregisterOnSubscriberOffer();
6226
- this.unregisterOnIceRestart();
6227
- this.pc.removeEventListener('icecandidate', this.onIceCandidate);
6228
- this.pc.removeEventListener('track', this.handleOnTrack);
6229
- this.pc.removeEventListener('icecandidateerror', this.onIceCandidateError);
6230
- this.pc.removeEventListener('iceconnectionstatechange', this.onIceConnectionStateChange);
6231
- this.pc.removeEventListener('icegatheringstatechange', this.onIceGatheringStateChange);
5851
+ this.getAnnouncedTracks = (sdp) => {
5852
+ const trackInfos = [];
5853
+ for (const bundle of this.transceiverCache.items()) {
5854
+ const { transceiver, publishOption } = bundle;
5855
+ const track = transceiver.sender.track;
5856
+ if (!track)
5857
+ continue;
5858
+ trackInfos.push(this.toTrackInfo(transceiver, publishOption, sdp));
5859
+ }
5860
+ return trackInfos;
6232
5861
  };
6233
5862
  /**
6234
- * Returns the result of the `RTCPeerConnection.getStats()` method
6235
- * @param selector
6236
- * @returns
6237
- */
6238
- this.getStats = (selector) => {
6239
- return this.pc.getStats(selector);
5863
+ * Returns a list of tracks that are currently being published.
5864
+ * This method shall be used for the reconnection flow.
5865
+ * There we shouldn't announce the tracks that have been stopped due to a codec switch.
5866
+ */
5867
+ this.getAnnouncedTracksForReconnect = () => {
5868
+ const sdp = this.pc.localDescription?.sdp;
5869
+ const trackInfos = [];
5870
+ for (const publishOption of this.publishOptions) {
5871
+ const transceiver = this.transceiverCache.get(publishOption);
5872
+ if (!transceiver || !transceiver.sender.track)
5873
+ continue;
5874
+ trackInfos.push(this.toTrackInfo(transceiver, publishOption, sdp));
5875
+ }
5876
+ return trackInfos;
6240
5877
  };
6241
5878
  /**
6242
- * Sets the SFU client to use.
6243
- *
6244
- * @param sfuClient the SFU client to use.
5879
+ * Converts the given transceiver to a `TrackInfo` object.
6245
5880
  */
6246
- this.setSfuClient = (sfuClient) => {
6247
- this.sfuClient = sfuClient;
5881
+ this.toTrackInfo = (transceiver, publishOption, sdp) => {
5882
+ const track = transceiver.sender.track;
5883
+ const isTrackLive = track.readyState === 'live';
5884
+ const layers = isTrackLive
5885
+ ? computeVideoLayers(track, publishOption)
5886
+ : this.transceiverCache.getLayers(publishOption);
5887
+ this.transceiverCache.setLayers(publishOption, layers);
5888
+ const isAudioTrack = isAudioTrackType(publishOption.trackType);
5889
+ const isStereo = isAudioTrack && track.getSettings().channelCount === 2;
5890
+ const transceiverIndex = this.transceiverCache.indexOf(transceiver);
5891
+ const audioSettings = this.state.settings?.audio;
5892
+ return {
5893
+ trackId: track.id,
5894
+ layers: toVideoLayers(layers),
5895
+ trackType: publishOption.trackType,
5896
+ mid: extractMid(transceiver, transceiverIndex, sdp),
5897
+ stereo: isStereo,
5898
+ dtx: isAudioTrack && !!audioSettings?.opus_dtx_enabled,
5899
+ red: isAudioTrack && !!audioSettings?.redundant_coding_enabled,
5900
+ muted: !isTrackLive,
5901
+ codec: publishOption.codec,
5902
+ publishOptionId: publishOption.id,
5903
+ };
6248
5904
  };
5905
+ this.publishOptions = publishOptions;
5906
+ this.pc.addEventListener('negotiationneeded', this.onNegotiationNeeded);
5907
+ this.on('iceRestart', (iceRestart) => {
5908
+ if (iceRestart.peerType !== PeerType.PUBLISHER_UNSPECIFIED)
5909
+ return;
5910
+ this.restartIce().catch((err) => {
5911
+ this.logger('warn', `ICERestart failed`, err);
5912
+ this.onUnrecoverableError?.();
5913
+ });
5914
+ });
5915
+ this.on('changePublishQuality', async (event) => {
5916
+ for (const videoSender of event.videoSenders) {
5917
+ await this.changePublishQuality(videoSender);
5918
+ }
5919
+ });
5920
+ this.on('changePublishOptions', (event) => {
5921
+ this.publishOptions = event.publishOptions;
5922
+ return this.syncPublishOptions();
5923
+ });
5924
+ }
5925
+ /**
5926
+ * Detaches the event handlers from the `RTCPeerConnection`.
5927
+ * This is useful when we want to replace the `RTCPeerConnection`
5928
+ * instance with a new one (in case of migration).
5929
+ */
5930
+ detachEventHandlers() {
5931
+ super.detachEventHandlers();
5932
+ this.pc.removeEventListener('negotiationneeded', this.onNegotiationNeeded);
5933
+ }
5934
+ }
5935
+
5936
+ /**
5937
+ * A wrapper around the `RTCPeerConnection` that handles the incoming
5938
+ * media streams from the SFU.
5939
+ *
5940
+ * @internal
5941
+ */
5942
+ class Subscriber extends BasePeerConnection {
5943
+ /**
5944
+ * Constructs a new `Subscriber` instance.
5945
+ */
5946
+ constructor(opts) {
5947
+ super(PeerType.SUBSCRIBER, opts);
6249
5948
  /**
6250
5949
  * Restarts the ICE connection and renegotiates with the SFU.
6251
5950
  */
@@ -6308,7 +6007,15 @@ class Subscriber {
6308
6007
  this.logger('error', `Unknown track type: ${rawTrackType}`);
6309
6008
  return;
6310
6009
  }
6010
+ // get the previous stream to dispose it later
6011
+ // usually this happens during migration, when the stream is replaced
6012
+ // with a new one but the old one is still in the state
6311
6013
  const previousStream = participantToUpdate[streamKindProp];
6014
+ // replace the previous stream with the new one, prevents flickering
6015
+ this.state.updateParticipant(participantToUpdate.sessionId, {
6016
+ [streamKindProp]: primaryStream,
6017
+ });
6018
+ // now, dispose the previous stream if it exists
6312
6019
  if (previousStream) {
6313
6020
  this.logger('info', `[onTrack]: Cleaning up previous remote ${e.track.kind} tracks for userId: ${participantToUpdate.userId}`);
6314
6021
  previousStream.getTracks().forEach((t) => {
@@ -6316,24 +6023,6 @@ class Subscriber {
6316
6023
  previousStream.removeTrack(t);
6317
6024
  });
6318
6025
  }
6319
- this.state.updateParticipant(participantToUpdate.sessionId, {
6320
- [streamKindProp]: primaryStream,
6321
- });
6322
- };
6323
- this.onIceCandidate = (e) => {
6324
- const { candidate } = e;
6325
- if (!candidate) {
6326
- this.logger('debug', 'null ice candidate');
6327
- return;
6328
- }
6329
- this.sfuClient
6330
- .iceTrickle({
6331
- iceCandidate: getIceCandidate(candidate),
6332
- peerType: PeerType.SUBSCRIBER,
6333
- })
6334
- .catch((err) => {
6335
- this.logger('warn', `ICETrickle failed`, err);
6336
- });
6337
6026
  };
6338
6027
  this.negotiate = async (subscriberOffer) => {
6339
6028
  this.logger('info', `Received subscriberOffer`, subscriberOffer);
@@ -6341,15 +6030,7 @@ class Subscriber {
6341
6030
  type: 'offer',
6342
6031
  sdp: subscriberOffer.sdp,
6343
6032
  });
6344
- this.sfuClient.iceTrickleBuffer.subscriberCandidates.subscribe(async (candidate) => {
6345
- try {
6346
- const iceCandidate = JSON.parse(candidate.iceCandidate);
6347
- await this.pc.addIceCandidate(iceCandidate);
6348
- }
6349
- catch (e) {
6350
- this.logger('warn', `ICE candidate error`, [e, candidate]);
6351
- }
6352
- });
6033
+ this.addTrickledIceCandidates();
6353
6034
  const answer = await this.pc.createAnswer();
6354
6035
  await this.pc.setLocalDescription(answer);
6355
6036
  await this.sfuClient.sendAnswer({
@@ -6358,56 +6039,21 @@ class Subscriber {
6358
6039
  });
6359
6040
  this.isIceRestarting = false;
6360
6041
  };
6361
- this.onIceConnectionStateChange = () => {
6362
- const state = this.pc.iceConnectionState;
6363
- this.logger('debug', `ICE connection state changed`, state);
6364
- if (this.state.callingState === CallingState.RECONNECTING)
6365
- return;
6366
- // do nothing when ICE is restarting
6367
- if (this.isIceRestarting)
6368
- return;
6369
- if (state === 'failed' || state === 'disconnected') {
6370
- this.logger('debug', `Attempting to restart ICE`);
6371
- this.restartIce().catch((e) => {
6372
- this.logger('error', `ICE restart failed`, e);
6373
- this.onUnrecoverableError?.();
6374
- });
6375
- }
6376
- };
6377
- this.onIceGatheringStateChange = () => {
6378
- this.logger('debug', `ICE gathering state changed`, this.pc.iceGatheringState);
6379
- };
6380
- this.onIceCandidateError = (e) => {
6381
- const errorMessage = e instanceof RTCPeerConnectionIceErrorEvent &&
6382
- `${e.errorCode}: ${e.errorText}`;
6383
- const iceState = this.pc.iceConnectionState;
6384
- const logLevel = iceState === 'connected' || iceState === 'checking' ? 'debug' : 'warn';
6385
- this.logger(logLevel, `ICE Candidate error`, errorMessage);
6386
- };
6387
- this.logger = getLogger(['Subscriber', logTag]);
6388
- this.sfuClient = sfuClient;
6389
- this.state = state;
6390
- this.onUnrecoverableError = onUnrecoverableError;
6391
- this.pc = this.createPeerConnection(connectionConfig);
6392
- const subscriberOfferConcurrencyTag = Symbol('subscriberOffer');
6393
- this.unregisterOnSubscriberOffer = dispatcher.on('subscriberOffer', (subscriberOffer) => {
6394
- withoutConcurrency(subscriberOfferConcurrencyTag, () => {
6395
- return this.negotiate(subscriberOffer);
6396
- }).catch((err) => {
6042
+ this.pc.addEventListener('track', this.handleOnTrack);
6043
+ this.on('subscriberOffer', async (subscriberOffer) => {
6044
+ return this.negotiate(subscriberOffer).catch((err) => {
6397
6045
  this.logger('error', `Negotiation failed.`, err);
6398
6046
  });
6399
6047
  });
6400
- const iceRestartConcurrencyTag = Symbol('iceRestart');
6401
- this.unregisterOnIceRestart = dispatcher.on('iceRestart', (iceRestart) => {
6402
- withoutConcurrency(iceRestartConcurrencyTag, async () => {
6403
- if (iceRestart.peerType !== PeerType.SUBSCRIBER)
6404
- return;
6405
- await this.restartIce();
6406
- }).catch((err) => {
6407
- this.logger('error', `ICERestart failed`, err);
6408
- this.onUnrecoverableError?.();
6409
- });
6410
- });
6048
+ }
6049
+ /**
6050
+ * Detaches the event handlers from the `RTCPeerConnection`.
6051
+ * This is useful when we want to replace the `RTCPeerConnection`
6052
+ * instance with a new one (in case of migration).
6053
+ */
6054
+ detachEventHandlers() {
6055
+ super.detachEventHandlers();
6056
+ this.pc.removeEventListener('track', this.handleOnTrack);
6411
6057
  }
6412
6058
  }
6413
6059
 
@@ -6439,6 +6085,16 @@ const createWebSocketSignalChannel = (opts) => {
6439
6085
  return ws;
6440
6086
  };
6441
6087
 
6088
+ const toRtcConfiguration = (config) => {
6089
+ return {
6090
+ iceServers: config.map((ice) => ({
6091
+ urls: ice.urls,
6092
+ username: ice.username,
6093
+ credential: ice.password,
6094
+ })),
6095
+ };
6096
+ };
6097
+
6442
6098
  /**
6443
6099
  * Saving a long-lived reference to a promise that can reject can be unsafe,
6444
6100
  * since rejecting the promise causes an unhandled rejection error (even if the
@@ -6723,6 +6379,7 @@ class StreamSfuClient {
6723
6379
  clearTimeout(this.migrateAwayTimeout);
6724
6380
  this.abortController.abort();
6725
6381
  this.migrationTask?.resolve();
6382
+ this.iceTrickleBuffer.dispose();
6726
6383
  };
6727
6384
  this.leaveAndClose = async (reason) => {
6728
6385
  await this.joinTask;
@@ -6755,13 +6412,9 @@ class StreamSfuClient {
6755
6412
  await this.joinTask;
6756
6413
  return retryable(() => this.rpc.iceRestart({ ...data, sessionId: this.sessionId }), this.abortController.signal);
6757
6414
  };
6758
- this.updateMuteState = async (trackType, muted) => {
6415
+ this.updateMuteStates = async (muteStates) => {
6759
6416
  await this.joinTask;
6760
- return this.updateMuteStates({ muteStates: [{ trackType, muted }] });
6761
- };
6762
- this.updateMuteStates = async (data) => {
6763
- await this.joinTask;
6764
- return retryable(() => this.rpc.updateMuteStates({ ...data, sessionId: this.sessionId }), this.abortController.signal);
6417
+ return retryable(() => this.rpc.updateMuteStates({ muteStates, sessionId: this.sessionId }), this.abortController.signal);
6765
6418
  };
6766
6419
  this.sendStats = async (stats) => {
6767
6420
  await this.joinTask;
@@ -6941,16 +6594,6 @@ StreamSfuClient.ERROR_CONNECTION_UNHEALTHY = 4001;
6941
6594
  */
6942
6595
  StreamSfuClient.DISPOSE_OLD_SOCKET = 4002;
6943
6596
 
6944
- const toRtcConfiguration = (config) => {
6945
- return {
6946
- iceServers: config.map((ice) => ({
6947
- urls: ice.urls,
6948
- username: ice.username,
6949
- credential: ice.password,
6950
- })),
6951
- };
6952
- };
6953
-
6954
6597
  /**
6955
6598
  * Event handler that watched the delivery of `call.accepted`.
6956
6599
  * Once the event is received, the call is joined.
@@ -7169,6 +6812,21 @@ const handleRemoteSoftMute = (call) => {
7169
6812
  });
7170
6813
  };
7171
6814
 
6815
+ /**
6816
+ * Adds unique values to an array.
6817
+ *
6818
+ * @param arr the array to add to.
6819
+ * @param values the values to add.
6820
+ */
6821
+ const pushToIfMissing = (arr, ...values) => {
6822
+ for (const v of values) {
6823
+ if (!arr.includes(v)) {
6824
+ arr.push(v);
6825
+ }
6826
+ }
6827
+ return arr;
6828
+ };
6829
+
7172
6830
  /**
7173
6831
  * An event responder which handles the `participantJoined` event.
7174
6832
  */
@@ -7234,7 +6892,7 @@ const watchTrackPublished = (state) => {
7234
6892
  }
7235
6893
  else {
7236
6894
  state.updateParticipant(sessionId, (p) => ({
7237
- publishedTracks: [...p.publishedTracks, type].filter(unique),
6895
+ publishedTracks: pushToIfMissing([...p.publishedTracks], type),
7238
6896
  }));
7239
6897
  }
7240
6898
  };
@@ -7259,7 +6917,6 @@ const watchTrackUnpublished = (state) => {
7259
6917
  }
7260
6918
  };
7261
6919
  };
7262
- const unique = (v, i, arr) => arr.indexOf(v) === i;
7263
6920
  /**
7264
6921
  * Reconciles orphaned tracks (if any) for the given participant.
7265
6922
  *
@@ -7409,6 +7066,38 @@ const getSdkVersion = (sdk) => {
7409
7066
  return sdk ? `${sdk.major}.${sdk.minor}.${sdk.patch}` : '0.0.0-development';
7410
7067
  };
7411
7068
 
7069
+ /**
7070
+ * Checks whether the current browser is Safari.
7071
+ */
7072
+ const isSafari = () => {
7073
+ if (typeof navigator === 'undefined')
7074
+ return false;
7075
+ return /^((?!chrome|android).)*safari/i.test(navigator.userAgent || '');
7076
+ };
7077
+ /**
7078
+ * Checks whether the current browser is Firefox.
7079
+ */
7080
+ const isFirefox = () => {
7081
+ if (typeof navigator === 'undefined')
7082
+ return false;
7083
+ return navigator.userAgent?.includes('Firefox');
7084
+ };
7085
+ /**
7086
+ * Checks whether the current browser is Google Chrome.
7087
+ */
7088
+ const isChrome = () => {
7089
+ if (typeof navigator === 'undefined')
7090
+ return false;
7091
+ return navigator.userAgent?.includes('Chrome');
7092
+ };
7093
+
7094
+ var browsers = /*#__PURE__*/Object.freeze({
7095
+ __proto__: null,
7096
+ isChrome: isChrome,
7097
+ isFirefox: isFirefox,
7098
+ isSafari: isSafari
7099
+ });
7100
+
7412
7101
  /**
7413
7102
  * Creates a new StatsReporter instance that collects metrics about the ongoing call and reports them to the state store
7414
7103
  */
@@ -7425,12 +7114,12 @@ const createStatsReporter = ({ subscriber, publisher, state, datacenter, polling
7425
7114
  return undefined;
7426
7115
  }
7427
7116
  };
7428
- const getStatsForStream = async (kind, mediaStream) => {
7117
+ const getStatsForStream = async (kind, tracks) => {
7429
7118
  const pc = kind === 'subscriber' ? subscriber : publisher;
7430
7119
  if (!pc)
7431
7120
  return [];
7432
7121
  const statsForStream = [];
7433
- for (let track of mediaStream.getTracks()) {
7122
+ for (const track of tracks) {
7434
7123
  const report = await pc.getStats(track);
7435
7124
  const stats = transform(report, {
7436
7125
  // @ts-ignore
@@ -7455,26 +7144,24 @@ const createStatsReporter = ({ subscriber, publisher, state, datacenter, polling
7455
7144
  */
7456
7145
  const run = async () => {
7457
7146
  const participantStats = {};
7458
- const sessionIds = new Set(sessionIdsToTrack);
7459
- if (sessionIds.size > 0) {
7460
- for (let participant of state.participants) {
7147
+ if (sessionIdsToTrack.size > 0) {
7148
+ const sessionIds = new Set(sessionIdsToTrack);
7149
+ for (const participant of state.participants) {
7461
7150
  if (!sessionIds.has(participant.sessionId))
7462
7151
  continue;
7463
- const kind = participant.isLocalParticipant
7464
- ? 'publisher'
7465
- : 'subscriber';
7152
+ const { audioStream, isLocalParticipant, sessionId, userId, videoStream, } = participant;
7153
+ const kind = isLocalParticipant ? 'publisher' : 'subscriber';
7466
7154
  try {
7467
- const mergedStream = new MediaStream([
7468
- ...(participant.videoStream?.getVideoTracks() || []),
7469
- ...(participant.audioStream?.getAudioTracks() || []),
7470
- ]);
7471
- participantStats[participant.sessionId] = await getStatsForStream(kind, mergedStream);
7472
- mergedStream.getTracks().forEach((t) => {
7473
- mergedStream.removeTrack(t);
7474
- });
7155
+ const tracks = isLocalParticipant
7156
+ ? publisher?.getPublishedTracks() || []
7157
+ : [
7158
+ ...(videoStream?.getVideoTracks() || []),
7159
+ ...(audioStream?.getAudioTracks() || []),
7160
+ ];
7161
+ participantStats[sessionId] = await getStatsForStream(kind, tracks);
7475
7162
  }
7476
7163
  catch (e) {
7477
- logger('error', `Failed to collect stats for ${kind} of ${participant.userId}`, e);
7164
+ logger('warn', `Failed to collect ${kind} stats for ${userId}`, e);
7478
7165
  }
7479
7166
  }
7480
7167
  }
@@ -7484,6 +7171,7 @@ const createStatsReporter = ({ subscriber, publisher, state, datacenter, polling
7484
7171
  .then((report) => transform(report, {
7485
7172
  kind: 'subscriber',
7486
7173
  trackKind: 'video',
7174
+ publisher,
7487
7175
  }))
7488
7176
  .then(aggregate),
7489
7177
  publisher
@@ -7492,6 +7180,7 @@ const createStatsReporter = ({ subscriber, publisher, state, datacenter, polling
7492
7180
  .then((report) => transform(report, {
7493
7181
  kind: 'publisher',
7494
7182
  trackKind: 'video',
7183
+ publisher,
7495
7184
  }))
7496
7185
  .then(aggregate)
7497
7186
  : getEmptyStats(),
@@ -7540,7 +7229,7 @@ const createStatsReporter = ({ subscriber, publisher, state, datacenter, polling
7540
7229
  * @param opts the transform options.
7541
7230
  */
7542
7231
  const transform = (report, opts) => {
7543
- const { trackKind, kind } = opts;
7232
+ const { trackKind, kind, publisher } = opts;
7544
7233
  const direction = kind === 'subscriber' ? 'inbound-rtp' : 'outbound-rtp';
7545
7234
  const stats = flatten(report);
7546
7235
  const streams = stats
@@ -7556,6 +7245,16 @@ const transform = (report, opts) => {
7556
7245
  s.id === transport.selectedCandidatePairId);
7557
7246
  roundTripTime = candidatePair?.currentRoundTripTime;
7558
7247
  }
7248
+ let trackType;
7249
+ if (kind === 'publisher' && publisher) {
7250
+ const firefox = isFirefox();
7251
+ const mediaSource = stats.find((s) => s.type === 'media-source' &&
7252
+ // Firefox doesn't have mediaSourceId, so we need to guess the media source
7253
+ (firefox ? true : s.id === rtcStreamStats.mediaSourceId));
7254
+ if (mediaSource) {
7255
+ trackType = publisher.getTrackType(mediaSource.trackIdentifier);
7256
+ }
7257
+ }
7559
7258
  return {
7560
7259
  bytesSent: rtcStreamStats.bytesSent,
7561
7260
  bytesReceived: rtcStreamStats.bytesReceived,
@@ -7566,10 +7265,12 @@ const transform = (report, opts) => {
7566
7265
  framesPerSecond: rtcStreamStats.framesPerSecond,
7567
7266
  jitter: rtcStreamStats.jitter,
7568
7267
  kind: rtcStreamStats.kind,
7268
+ mediaSourceId: rtcStreamStats.mediaSourceId,
7569
7269
  // @ts-ignore: available in Chrome only, TS doesn't recognize this
7570
7270
  qualityLimitationReason: rtcStreamStats.qualityLimitationReason,
7571
7271
  rid: rtcStreamStats.rid,
7572
7272
  ssrc: rtcStreamStats.ssrc,
7273
+ trackType,
7573
7274
  };
7574
7275
  });
7575
7276
  return {
@@ -7590,6 +7291,7 @@ const getEmptyStats = (stats) => {
7590
7291
  highestFrameHeight: 0,
7591
7292
  highestFramesPerSecond: 0,
7592
7293
  codec: '',
7294
+ codecPerTrackType: {},
7593
7295
  timestamp: Date.now(),
7594
7296
  };
7595
7297
  };
@@ -7625,6 +7327,12 @@ const aggregate = (stats) => {
7625
7327
  report.averageRoundTripTimeInMs = Math.round((report.averageRoundTripTimeInMs / streams.length) * 1000);
7626
7328
  // we take the first codec we find, as it should be the same for all streams
7627
7329
  report.codec = streams[0].codec || '';
7330
+ report.codecPerTrackType = streams.reduce((acc, stream) => {
7331
+ if (stream.trackType) {
7332
+ acc[stream.trackType] = stream.codec || '';
7333
+ }
7334
+ return acc;
7335
+ }, {});
7628
7336
  }
7629
7337
  const qualityLimitationReason = [
7630
7338
  qualityLimitationReasons.has('cpu') && 'cpu',
@@ -7636,7 +7344,135 @@ const aggregate = (stats) => {
7636
7344
  if (qualityLimitationReason) {
7637
7345
  report.qualityLimitationReasons = qualityLimitationReason;
7638
7346
  }
7639
- return report;
7347
+ return report;
7348
+ };
7349
+
7350
+ const version = "1.15.0";
7351
+ const [major, minor, patch] = version.split('.');
7352
+ let sdkInfo = {
7353
+ type: SdkType.PLAIN_JAVASCRIPT,
7354
+ major,
7355
+ minor,
7356
+ patch,
7357
+ };
7358
+ let osInfo;
7359
+ let deviceInfo;
7360
+ let webRtcInfo;
7361
+ let deviceState = { oneofKind: undefined };
7362
+ const setSdkInfo = (info) => {
7363
+ sdkInfo = info;
7364
+ };
7365
+ const getSdkInfo = () => {
7366
+ return sdkInfo;
7367
+ };
7368
+ const setOSInfo = (info) => {
7369
+ osInfo = info;
7370
+ };
7371
+ const getOSInfo = () => {
7372
+ return osInfo;
7373
+ };
7374
+ const setDeviceInfo = (info) => {
7375
+ deviceInfo = info;
7376
+ };
7377
+ const getDeviceInfo = () => {
7378
+ return deviceInfo;
7379
+ };
7380
+ const getWebRTCInfo = () => {
7381
+ return webRtcInfo;
7382
+ };
7383
+ const setWebRTCInfo = (info) => {
7384
+ webRtcInfo = info;
7385
+ };
7386
+ const setThermalState = (state) => {
7387
+ if (!osInfo) {
7388
+ deviceState = { oneofKind: undefined };
7389
+ return;
7390
+ }
7391
+ if (osInfo.name === 'android') {
7392
+ const thermalState = AndroidThermalState[state] ||
7393
+ AndroidThermalState.UNSPECIFIED;
7394
+ deviceState = {
7395
+ oneofKind: 'android',
7396
+ android: {
7397
+ thermalState,
7398
+ isPowerSaverMode: deviceState?.oneofKind === 'android' &&
7399
+ deviceState.android.isPowerSaverMode,
7400
+ },
7401
+ };
7402
+ }
7403
+ if (osInfo.name.toLowerCase() === 'ios') {
7404
+ const thermalState = AppleThermalState[state] ||
7405
+ AppleThermalState.UNSPECIFIED;
7406
+ deviceState = {
7407
+ oneofKind: 'apple',
7408
+ apple: {
7409
+ thermalState,
7410
+ isLowPowerModeEnabled: deviceState?.oneofKind === 'apple' &&
7411
+ deviceState.apple.isLowPowerModeEnabled,
7412
+ },
7413
+ };
7414
+ }
7415
+ };
7416
+ const setPowerState = (powerMode) => {
7417
+ if (!osInfo) {
7418
+ deviceState = { oneofKind: undefined };
7419
+ return;
7420
+ }
7421
+ if (osInfo.name === 'android') {
7422
+ deviceState = {
7423
+ oneofKind: 'android',
7424
+ android: {
7425
+ thermalState: deviceState?.oneofKind === 'android'
7426
+ ? deviceState.android.thermalState
7427
+ : AndroidThermalState.UNSPECIFIED,
7428
+ isPowerSaverMode: powerMode,
7429
+ },
7430
+ };
7431
+ }
7432
+ if (osInfo.name.toLowerCase() === 'ios') {
7433
+ deviceState = {
7434
+ oneofKind: 'apple',
7435
+ apple: {
7436
+ thermalState: deviceState?.oneofKind === 'apple'
7437
+ ? deviceState.apple.thermalState
7438
+ : AppleThermalState.UNSPECIFIED,
7439
+ isLowPowerModeEnabled: powerMode,
7440
+ },
7441
+ };
7442
+ }
7443
+ };
7444
+ const getDeviceState = () => {
7445
+ return deviceState;
7446
+ };
7447
+ const getClientDetails = () => {
7448
+ if (isReactNative()) {
7449
+ // Since RN doesn't support web, sharing browser info is not required
7450
+ return {
7451
+ sdk: getSdkInfo(),
7452
+ os: getOSInfo(),
7453
+ device: getDeviceInfo(),
7454
+ };
7455
+ }
7456
+ const userAgent = new UAParser(navigator.userAgent);
7457
+ const { browser, os, device, cpu } = userAgent.getResult();
7458
+ return {
7459
+ sdk: getSdkInfo(),
7460
+ browser: {
7461
+ name: browser.name || navigator.userAgent,
7462
+ version: browser.version || '',
7463
+ },
7464
+ os: {
7465
+ name: os.name || '',
7466
+ version: os.version || '',
7467
+ architecture: cpu.architecture || '',
7468
+ },
7469
+ device: {
7470
+ name: [device.vendor, device.model, device.type]
7471
+ .filter(Boolean)
7472
+ .join(' '),
7473
+ version: '',
7474
+ },
7475
+ };
7640
7476
  };
7641
7477
 
7642
7478
  class SfuStatsReporter {
@@ -7672,8 +7508,28 @@ class SfuStatsReporter {
7672
7508
  });
7673
7509
  });
7674
7510
  };
7675
- this.sendTelemetryData = async (telemetryData) => {
7676
- return this.run(telemetryData);
7511
+ this.sendConnectionTime = (connectionTimeSeconds) => {
7512
+ this.sendTelemetryData({
7513
+ data: {
7514
+ oneofKind: 'connectionTimeSeconds',
7515
+ connectionTimeSeconds,
7516
+ },
7517
+ });
7518
+ };
7519
+ this.sendReconnectionTime = (strategy, timeSeconds) => {
7520
+ this.sendTelemetryData({
7521
+ data: {
7522
+ oneofKind: 'reconnection',
7523
+ reconnection: { strategy, timeSeconds },
7524
+ },
7525
+ });
7526
+ };
7527
+ this.sendTelemetryData = (telemetryData) => {
7528
+ // intentionally not awaiting the promise here
7529
+ // to avoid impeding with the ongoing actions.
7530
+ this.run(telemetryData).catch((err) => {
7531
+ this.logger('warn', 'Failed to send telemetry data', err);
7532
+ });
7677
7533
  };
7678
7534
  this.run = async (telemetryData) => {
7679
7535
  const [subscriberStats, publisherStats] = await Promise.all([
@@ -8241,6 +8097,25 @@ class PermissionsContext {
8241
8097
  this.hasPermission = (permission) => {
8242
8098
  return this.permissions.includes(permission);
8243
8099
  };
8100
+ /**
8101
+ * Helper method that checks whether the current user has the permission
8102
+ * to publish the given track type.
8103
+ */
8104
+ this.canPublish = (trackType) => {
8105
+ switch (trackType) {
8106
+ case TrackType.AUDIO:
8107
+ return this.hasPermission(OwnCapability.SEND_AUDIO);
8108
+ case TrackType.VIDEO:
8109
+ return this.hasPermission(OwnCapability.SEND_VIDEO);
8110
+ case TrackType.SCREEN_SHARE:
8111
+ case TrackType.SCREEN_SHARE_AUDIO:
8112
+ return this.hasPermission(OwnCapability.SCREENSHARE);
8113
+ case TrackType.UNSPECIFIED:
8114
+ return false;
8115
+ default:
8116
+ ensureExhausted(trackType, 'Unknown track type');
8117
+ }
8118
+ };
8244
8119
  /**
8245
8120
  * Checks if the current user can request a specific permission
8246
8121
  * within the call.
@@ -8879,36 +8754,42 @@ class InputMediaDeviceManager {
8879
8754
  }
8880
8755
  });
8881
8756
  }
8757
+ publishStream(stream) {
8758
+ return this.call.publish(stream, this.trackType);
8759
+ }
8760
+ stopPublishStream() {
8761
+ return this.call.stopPublish(this.trackType);
8762
+ }
8882
8763
  getTracks() {
8883
8764
  return this.state.mediaStream?.getTracks() ?? [];
8884
8765
  }
8885
8766
  async muteStream(stopTracks = true) {
8886
- if (!this.state.mediaStream)
8767
+ const mediaStream = this.state.mediaStream;
8768
+ if (!mediaStream)
8887
8769
  return;
8888
8770
  this.logger('debug', `${stopTracks ? 'Stopping' : 'Disabling'} stream`);
8889
8771
  if (this.call.state.callingState === CallingState.JOINED) {
8890
- await this.stopPublishStream(stopTracks);
8772
+ await this.stopPublishStream();
8891
8773
  }
8892
8774
  this.muteLocalStream(stopTracks);
8893
8775
  const allEnded = this.getTracks().every((t) => t.readyState === 'ended');
8894
8776
  if (allEnded) {
8895
- if (this.state.mediaStream &&
8896
- // @ts-expect-error release() is present in react-native-webrtc
8897
- typeof this.state.mediaStream.release === 'function') {
8777
+ // @ts-expect-error release() is present in react-native-webrtc
8778
+ if (typeof mediaStream.release === 'function') {
8898
8779
  // @ts-expect-error called to dispose the stream in RN
8899
- this.state.mediaStream.release();
8780
+ mediaStream.release();
8900
8781
  }
8901
8782
  this.state.setMediaStream(undefined, undefined);
8902
8783
  this.filters.forEach((entry) => entry.stop?.());
8903
8784
  }
8904
8785
  }
8905
- muteTracks() {
8786
+ disableTracks() {
8906
8787
  this.getTracks().forEach((track) => {
8907
8788
  if (track.enabled)
8908
8789
  track.enabled = false;
8909
8790
  });
8910
8791
  }
8911
- unmuteTracks() {
8792
+ enableTracks() {
8912
8793
  this.getTracks().forEach((track) => {
8913
8794
  if (!track.enabled)
8914
8795
  track.enabled = true;
@@ -8928,7 +8809,7 @@ class InputMediaDeviceManager {
8928
8809
  this.stopTracks();
8929
8810
  }
8930
8811
  else {
8931
- this.muteTracks();
8812
+ this.disableTracks();
8932
8813
  }
8933
8814
  }
8934
8815
  async unmuteStream() {
@@ -8938,7 +8819,7 @@ class InputMediaDeviceManager {
8938
8819
  if (this.state.mediaStream &&
8939
8820
  this.getTracks().every((t) => t.readyState === 'live')) {
8940
8821
  stream = this.state.mediaStream;
8941
- this.unmuteTracks();
8822
+ this.enableTracks();
8942
8823
  }
8943
8824
  else {
8944
8825
  const defaultConstraints = this.state.defaultConstraints;
@@ -9032,9 +8913,22 @@ class InputMediaDeviceManager {
9032
8913
  await this.disable();
9033
8914
  }
9034
8915
  };
9035
- this.getTracks().forEach((track) => {
8916
+ const createTrackMuteHandler = (muted) => () => {
8917
+ this.call.notifyTrackMuteState(muted, this.trackType).catch((err) => {
8918
+ this.logger('warn', 'Error while notifying track mute state', err);
8919
+ });
8920
+ };
8921
+ stream.getTracks().forEach((track) => {
8922
+ const muteHandler = createTrackMuteHandler(true);
8923
+ const unmuteHandler = createTrackMuteHandler(false);
8924
+ track.addEventListener('mute', muteHandler);
8925
+ track.addEventListener('unmute', unmuteHandler);
9036
8926
  track.addEventListener('ended', handleTrackEnded);
9037
- this.subscriptions.push(() => track.removeEventListener('ended', handleTrackEnded));
8927
+ this.subscriptions.push(() => {
8928
+ track.removeEventListener('mute', muteHandler);
8929
+ track.removeEventListener('unmute', unmuteHandler);
8930
+ track.removeEventListener('ended', handleTrackEnded);
8931
+ });
9038
8932
  });
9039
8933
  }
9040
8934
  }
@@ -9058,8 +8952,8 @@ class InputMediaDeviceManager {
9058
8952
  await this.statusChangeSettled();
9059
8953
  let isDeviceDisconnected = false;
9060
8954
  let isDeviceReplaced = false;
9061
- const currentDevice = this.findDeviceInList(currentDevices, deviceId);
9062
- const prevDevice = this.findDeviceInList(prevDevices, deviceId);
8955
+ const currentDevice = this.findDevice(currentDevices, deviceId);
8956
+ const prevDevice = this.findDevice(prevDevices, deviceId);
9063
8957
  if (!currentDevice && prevDevice) {
9064
8958
  isDeviceDisconnected = true;
9065
8959
  }
@@ -9089,8 +8983,9 @@ class InputMediaDeviceManager {
9089
8983
  }
9090
8984
  }));
9091
8985
  }
9092
- findDeviceInList(devices, deviceId) {
9093
- return devices.find((d) => d.deviceId === deviceId && d.kind === this.mediaDeviceKind);
8986
+ findDevice(devices, deviceId) {
8987
+ const kind = this.mediaDeviceKind;
8988
+ return devices.find((d) => d.deviceId === deviceId && d.kind === kind);
9094
8989
  }
9095
8990
  }
9096
8991
 
@@ -9354,14 +9249,35 @@ class CameraManager extends InputMediaDeviceManager {
9354
9249
  }
9355
9250
  }
9356
9251
  /**
9357
- * Sets the preferred codec for encoding the video.
9252
+ * Applies the video settings to the camera.
9358
9253
  *
9359
- * @internal internal use only, not part of the public API.
9360
- * @deprecated use {@link call.updatePublishOptions} instead.
9361
- * @param codec the codec to use for encoding the video.
9254
+ * @param settings the video settings to apply.
9255
+ * @param publish whether to publish the stream after applying the settings.
9362
9256
  */
9363
- setPreferredCodec(codec) {
9364
- this.call.updatePublishOptions({ preferredCodec: codec });
9257
+ async apply(settings, publish) {
9258
+ const hasPublishedVideo = !!this.call.state.localParticipant?.videoStream;
9259
+ const hasPermission = this.call.permissionsContext.hasPermission(OwnCapability.SEND_AUDIO);
9260
+ if (hasPublishedVideo || !hasPermission)
9261
+ return;
9262
+ // Wait for any in progress camera operation
9263
+ await this.statusChangeSettled();
9264
+ const { target_resolution, camera_facing, camera_default_on } = settings;
9265
+ await this.selectTargetResolution(target_resolution);
9266
+ // Set camera direction if it's not yet set
9267
+ if (!this.state.direction && !this.state.selectedDevice) {
9268
+ this.state.setDirection(camera_facing === 'front' ? 'front' : 'back');
9269
+ }
9270
+ if (!publish)
9271
+ return;
9272
+ const { mediaStream } = this.state;
9273
+ if (this.enabled && mediaStream) {
9274
+ // The camera is already enabled (e.g. lobby screen). Publish the stream
9275
+ await this.publishStream(mediaStream);
9276
+ }
9277
+ else if (this.state.status === undefined && camera_default_on) {
9278
+ // Start camera if backend config specifies, and there is no local setting
9279
+ await this.enable();
9280
+ }
9365
9281
  }
9366
9282
  getDevices() {
9367
9283
  return getVideoDevices();
@@ -9379,12 +9295,6 @@ class CameraManager extends InputMediaDeviceManager {
9379
9295
  }
9380
9296
  return getVideoStream(constraints);
9381
9297
  }
9382
- publishStream(stream) {
9383
- return this.call.publishVideoStream(stream);
9384
- }
9385
- stopPublishStream(stopTracks) {
9386
- return this.call.stopPublish(TrackType.VIDEO, stopTracks);
9387
- }
9388
9298
  }
9389
9299
 
9390
9300
  class MicrophoneManagerState extends InputMediaDeviceManagerState {
@@ -9712,18 +9622,37 @@ class MicrophoneManager extends InputMediaDeviceManager {
9712
9622
  this.speakingWhileMutedNotificationEnabled = false;
9713
9623
  await this.stopSpeakingWhileMutedDetection();
9714
9624
  }
9625
+ /**
9626
+ * Applies the audio settings to the microphone.
9627
+ * @param settings the audio settings to apply.
9628
+ * @param publish whether to publish the stream after applying the settings.
9629
+ */
9630
+ async apply(settings, publish) {
9631
+ if (!publish)
9632
+ return;
9633
+ const hasPublishedAudio = !!this.call.state.localParticipant?.audioStream;
9634
+ const hasPermission = this.call.permissionsContext.hasPermission(OwnCapability.SEND_AUDIO);
9635
+ if (hasPublishedAudio || !hasPermission)
9636
+ return;
9637
+ // Wait for any in progress mic operation
9638
+ await this.statusChangeSettled();
9639
+ // Publish media stream that was set before we joined
9640
+ const { mediaStream } = this.state;
9641
+ if (this.enabled && mediaStream) {
9642
+ // The mic is already enabled (e.g. lobby screen). Publish the stream
9643
+ await this.publishStream(mediaStream);
9644
+ }
9645
+ else if (this.state.status === undefined && settings.mic_default_on) {
9646
+ // Start mic if backend config specifies, and there is no local setting
9647
+ await this.enable();
9648
+ }
9649
+ }
9715
9650
  getDevices() {
9716
9651
  return getAudioDevices();
9717
9652
  }
9718
9653
  getStream(constraints) {
9719
9654
  return getAudioStream(constraints);
9720
9655
  }
9721
- publishStream(stream) {
9722
- return this.call.publishAudioStream(stream);
9723
- }
9724
- stopPublishStream(stopTracks) {
9725
- return this.call.stopPublish(TrackType.AUDIO, stopTracks);
9726
- }
9727
9656
  async startSpeakingWhileMutedDetection(deviceId) {
9728
9657
  await withoutConcurrency(this.soundDetectorConcurrencyTag, async () => {
9729
9658
  await this.stopSpeakingWhileMutedDetection();
@@ -9843,7 +9772,7 @@ class ScreenShareManager extends InputMediaDeviceManager {
9843
9772
  async disableScreenShareAudio() {
9844
9773
  this.state.setAudioEnabled(false);
9845
9774
  if (this.call.publisher?.isPublishing(TrackType.SCREEN_SHARE_AUDIO)) {
9846
- await this.call.stopPublish(TrackType.SCREEN_SHARE_AUDIO, true);
9775
+ await this.call.stopPublish(TrackType.SCREEN_SHARE_AUDIO);
9847
9776
  }
9848
9777
  }
9849
9778
  /**
@@ -9869,12 +9798,8 @@ class ScreenShareManager extends InputMediaDeviceManager {
9869
9798
  }
9870
9799
  return getScreenShareStream(constraints);
9871
9800
  }
9872
- publishStream(stream) {
9873
- return this.call.publishScreenShareStream(stream);
9874
- }
9875
- async stopPublishStream(stopTracks) {
9876
- await this.call.stopPublish(TrackType.SCREEN_SHARE, stopTracks);
9877
- await this.call.stopPublish(TrackType.SCREEN_SHARE_AUDIO, stopTracks);
9801
+ async stopPublishStream() {
9802
+ return this.call.stopPublish(TrackType.SCREEN_SHARE, TrackType.SCREEN_SHARE_AUDIO);
9878
9803
  }
9879
9804
  /**
9880
9805
  * Overrides the default `select` method to throw an error.
@@ -10084,6 +10009,112 @@ class Call {
10084
10009
  */
10085
10010
  this.leaveCallHooks = new Set();
10086
10011
  this.streamClientEventHandlers = new Map();
10012
+ this.setup = async () => {
10013
+ await withoutConcurrency(this.joinLeaveConcurrencyTag, async () => {
10014
+ if (this.initialized)
10015
+ return;
10016
+ this.leaveCallHooks.add(this.on('all', (event) => {
10017
+ // update state with the latest event data
10018
+ this.state.updateFromEvent(event);
10019
+ }));
10020
+ this.leaveCallHooks.add(this.on('changePublishOptions', (event) => {
10021
+ this.currentPublishOptions = event.publishOptions;
10022
+ }));
10023
+ this.leaveCallHooks.add(registerEventHandlers(this, this.dispatcher));
10024
+ this.registerEffects();
10025
+ this.registerReconnectHandlers();
10026
+ if (this.state.callingState === CallingState.LEFT) {
10027
+ this.state.setCallingState(CallingState.IDLE);
10028
+ }
10029
+ this.initialized = true;
10030
+ });
10031
+ };
10032
+ this.registerEffects = () => {
10033
+ this.leaveCallHooks.add(
10034
+ // handles updating the permissions context when the settings change.
10035
+ createSubscription(this.state.settings$, (settings) => {
10036
+ if (!settings)
10037
+ return;
10038
+ this.permissionsContext.setCallSettings(settings);
10039
+ }));
10040
+ this.leaveCallHooks.add(
10041
+ // handle the case when the user permissions are modified.
10042
+ createSafeAsyncSubscription(this.state.ownCapabilities$, this.handleOwnCapabilitiesUpdated));
10043
+ this.leaveCallHooks.add(
10044
+ // handles the case when the user is blocked by the call owner.
10045
+ createSubscription(this.state.blockedUserIds$, async (blockedUserIds) => {
10046
+ if (!blockedUserIds || blockedUserIds.length === 0)
10047
+ return;
10048
+ const currentUserId = this.currentUserId;
10049
+ if (currentUserId && blockedUserIds.includes(currentUserId)) {
10050
+ this.logger('info', 'Leaving call because of being blocked');
10051
+ await this.leave({ reason: 'user blocked' }).catch((err) => {
10052
+ this.logger('error', 'Error leaving call after being blocked', err);
10053
+ });
10054
+ }
10055
+ }));
10056
+ this.leaveCallHooks.add(
10057
+ // cancel auto-drop when call is
10058
+ createSubscription(this.state.session$, (session) => {
10059
+ if (!this.ringing)
10060
+ return;
10061
+ const receiverId = this.clientStore.connectedUser?.id;
10062
+ if (!receiverId)
10063
+ return;
10064
+ const isAcceptedByMe = Boolean(session?.accepted_by[receiverId]);
10065
+ const isRejectedByMe = Boolean(session?.rejected_by[receiverId]);
10066
+ if (isAcceptedByMe || isRejectedByMe) {
10067
+ this.cancelAutoDrop();
10068
+ }
10069
+ }));
10070
+ this.leaveCallHooks.add(
10071
+ // "ringing" mode effects and event handlers
10072
+ createSubscription(this.ringingSubject, (isRinging) => {
10073
+ if (!isRinging)
10074
+ return;
10075
+ const callSession = this.state.session;
10076
+ const receiver_id = this.clientStore.connectedUser?.id;
10077
+ const ended_at = callSession?.ended_at;
10078
+ const created_by_id = this.state.createdBy?.id;
10079
+ const rejected_by = callSession?.rejected_by;
10080
+ const accepted_by = callSession?.accepted_by;
10081
+ let leaveCallIdle = false;
10082
+ if (ended_at) {
10083
+ // call was ended before it was accepted or rejected so we should leave it to idle
10084
+ leaveCallIdle = true;
10085
+ }
10086
+ else if (created_by_id && rejected_by) {
10087
+ if (rejected_by[created_by_id]) {
10088
+ // call was cancelled by the caller
10089
+ leaveCallIdle = true;
10090
+ }
10091
+ }
10092
+ else if (receiver_id && rejected_by) {
10093
+ if (rejected_by[receiver_id]) {
10094
+ // call was rejected by the receiver in some other device
10095
+ leaveCallIdle = true;
10096
+ }
10097
+ }
10098
+ else if (receiver_id && accepted_by) {
10099
+ if (accepted_by[receiver_id]) {
10100
+ // call was accepted by the receiver in some other device
10101
+ leaveCallIdle = true;
10102
+ }
10103
+ }
10104
+ if (leaveCallIdle) {
10105
+ if (this.state.callingState !== CallingState.IDLE) {
10106
+ this.state.setCallingState(CallingState.IDLE);
10107
+ }
10108
+ }
10109
+ else {
10110
+ if (this.state.callingState === CallingState.IDLE) {
10111
+ this.state.setCallingState(CallingState.RINGING);
10112
+ }
10113
+ this.scheduleAutoDrop();
10114
+ this.leaveCallHooks.add(registerRingingCallEventHandlers(this));
10115
+ }
10116
+ }));
10117
+ };
10087
10118
  this.handleOwnCapabilitiesUpdated = async (ownCapabilities) => {
10088
10119
  // update the permission context.
10089
10120
  this.permissionsContext.setPermissions(ownCapabilities);
@@ -10196,14 +10227,15 @@ class Call {
10196
10227
  this.statsReporter = undefined;
10197
10228
  this.sfuStatsReporter?.stop();
10198
10229
  this.sfuStatsReporter = undefined;
10199
- this.subscriber?.close();
10230
+ this.subscriber?.dispose();
10200
10231
  this.subscriber = undefined;
10201
- this.publisher?.close({ stopTracks: true });
10232
+ this.publisher?.dispose();
10202
10233
  this.publisher = undefined;
10203
10234
  await this.sfuClient?.leaveAndClose(reason);
10204
10235
  this.sfuClient = undefined;
10205
10236
  this.dynascaleManager.setSfuClient(undefined);
10206
10237
  this.state.setCallingState(CallingState.LEFT);
10238
+ this.state.dispose();
10207
10239
  // Call all leave call hooks, e.g. to clean up global event handlers
10208
10240
  this.leaveCallHooks.forEach((hook) => hook());
10209
10241
  this.initialized = false;
@@ -10235,7 +10267,8 @@ class Call {
10235
10267
  // call.ring event excludes the call creator in the members list
10236
10268
  // as the creator does not get the ring event
10237
10269
  // so update the member list accordingly
10238
- const creator = this.state.members.find((m) => m.user.id === event.call.created_by.id);
10270
+ const { created_by, settings } = event.call;
10271
+ const creator = this.state.members.find((m) => m.user.id === created_by.id);
10239
10272
  if (!creator) {
10240
10273
  this.state.setMembers(event.members);
10241
10274
  }
@@ -10250,7 +10283,7 @@ class Call {
10250
10283
  // const calls = useCalls().filter((c) => c.ringing);
10251
10284
  const calls = this.clientStore.calls.filter((c) => c.cid !== this.cid);
10252
10285
  this.clientStore.setCalls([this, ...calls]);
10253
- await this.applyDeviceConfig(false);
10286
+ await this.applyDeviceConfig(settings, false);
10254
10287
  };
10255
10288
  /**
10256
10289
  * Loads the information about the call.
@@ -10273,7 +10306,7 @@ class Call {
10273
10306
  this.watching = true;
10274
10307
  this.clientStore.registerCall(this);
10275
10308
  }
10276
- await this.applyDeviceConfig(false);
10309
+ await this.applyDeviceConfig(response.call.settings, false);
10277
10310
  return response;
10278
10311
  };
10279
10312
  /**
@@ -10295,7 +10328,7 @@ class Call {
10295
10328
  this.watching = true;
10296
10329
  this.clientStore.registerCall(this);
10297
10330
  }
10298
- await this.applyDeviceConfig(false);
10331
+ await this.applyDeviceConfig(response.call.settings, false);
10299
10332
  return response;
10300
10333
  };
10301
10334
  /**
@@ -10397,19 +10430,32 @@ class Call {
10397
10430
  // we don't need to send JoinRequest if we are re-using an existing healthy SFU client
10398
10431
  if (previousSfuClient !== sfuClient) {
10399
10432
  // prepare a generic SDP and send it to the SFU.
10400
- // this is a throw-away SDP that the SFU will use to determine
10433
+ // these are throw-away SDPs that the SFU will use to determine
10401
10434
  // the capabilities of the client (codec support, etc.)
10402
- const receivingCapabilitiesSdp = await getGenericSdp('recvonly');
10403
- const reconnectDetails = this.reconnectStrategy !== WebsocketReconnectStrategy.UNSPECIFIED
10435
+ const [subscriberSdp, publisherSdp] = await Promise.all([
10436
+ getGenericSdp('recvonly'),
10437
+ getGenericSdp('sendonly'),
10438
+ ]);
10439
+ const isReconnecting = this.reconnectStrategy !== WebsocketReconnectStrategy.UNSPECIFIED;
10440
+ const reconnectDetails = isReconnecting
10404
10441
  ? this.getReconnectDetails(data?.migrating_from, previousSessionId)
10405
10442
  : undefined;
10406
- const { callState, fastReconnectDeadlineSeconds } = await sfuClient.join({
10407
- subscriberSdp: receivingCapabilitiesSdp,
10408
- publisherSdp: '',
10443
+ const preferredPublishOptions = !isReconnecting
10444
+ ? this.getPreferredPublishOptions()
10445
+ : this.currentPublishOptions || [];
10446
+ const preferredSubscribeOptions = !isReconnecting
10447
+ ? this.getPreferredSubscribeOptions()
10448
+ : [];
10449
+ const { callState, fastReconnectDeadlineSeconds, publishOptions } = await sfuClient.join({
10450
+ subscriberSdp,
10451
+ publisherSdp,
10409
10452
  clientDetails,
10410
10453
  fastReconnect: performingFastReconnect,
10411
10454
  reconnectDetails,
10455
+ preferredPublishOptions,
10456
+ preferredSubscribeOptions,
10412
10457
  });
10458
+ this.currentPublishOptions = publishOptions;
10413
10459
  this.fastReconnectDeadlineSeconds = fastReconnectDeadlineSeconds;
10414
10460
  if (callState) {
10415
10461
  this.state.updateFromSfuCallState(callState, sfuClient.sessionId, reconnectDetails);
@@ -10434,17 +10480,13 @@ class Call {
10434
10480
  connectionConfig,
10435
10481
  clientDetails,
10436
10482
  statsOptions,
10483
+ publishOptions: this.currentPublishOptions || [],
10437
10484
  closePreviousInstances: !performingMigration,
10438
10485
  });
10439
10486
  }
10440
10487
  // make sure we only track connection timing if we are not calling this method as part of a reconnection flow
10441
10488
  if (!performingRejoin && !performingFastReconnect && !performingMigration) {
10442
- this.sfuStatsReporter?.sendTelemetryData({
10443
- data: {
10444
- oneofKind: 'connectionTimeSeconds',
10445
- connectionTimeSeconds: (Date.now() - connectStartTime) / 1000,
10446
- },
10447
- });
10489
+ this.sfuStatsReporter?.sendConnectionTime((Date.now() - connectStartTime) / 1000);
10448
10490
  }
10449
10491
  if (performingRejoin) {
10450
10492
  const strategy = WebsocketReconnectStrategy[this.reconnectStrategy];
@@ -10455,8 +10497,8 @@ class Call {
10455
10497
  }
10456
10498
  // device settings should be applied only once, we don't have to
10457
10499
  // re-apply them on later reconnections or server-side data fetches
10458
- if (!this.deviceSettingsAppliedOnce) {
10459
- await this.applyDeviceConfig(true);
10500
+ if (!this.deviceSettingsAppliedOnce && this.state.settings) {
10501
+ await this.applyDeviceConfig(this.state.settings, true);
10460
10502
  this.deviceSettingsAppliedOnce = true;
10461
10503
  }
10462
10504
  // We shouldn't persist the `ring` and `notify` state after joining the call
@@ -10465,6 +10507,8 @@ class Call {
10465
10507
  // we will spam the other participants with push notifications and `call.ring` events.
10466
10508
  delete this.joinCallData?.ring;
10467
10509
  delete this.joinCallData?.notify;
10510
+ // reset the reconnect strategy to unspecified after a successful reconnection
10511
+ this.reconnectStrategy = WebsocketReconnectStrategy.UNSPECIFIED;
10468
10512
  this.logger('info', `Joined call ${this.cid}`);
10469
10513
  };
10470
10514
  /**
@@ -10474,7 +10518,7 @@ class Call {
10474
10518
  this.getReconnectDetails = (migratingFromSfuId, previousSessionId) => {
10475
10519
  const strategy = this.reconnectStrategy;
10476
10520
  const performingRejoin = strategy === WebsocketReconnectStrategy.REJOIN;
10477
- const announcedTracks = this.publisher?.getAnnouncedTracks() || [];
10521
+ const announcedTracks = this.publisher?.getAnnouncedTracksForReconnect() || [];
10478
10522
  return {
10479
10523
  strategy,
10480
10524
  announcedTracks,
@@ -10484,6 +10528,54 @@ class Call {
10484
10528
  previousSessionId: performingRejoin ? previousSessionId || '' : '',
10485
10529
  };
10486
10530
  };
10531
+ /**
10532
+ * Prepares the preferred codec for the call.
10533
+ * This is an experimental client feature and subject to change.
10534
+ * @internal
10535
+ */
10536
+ this.getPreferredPublishOptions = () => {
10537
+ const { preferredCodec, fmtpLine, preferredBitrate, maxSimulcastLayers } = this.clientPublishOptions || {};
10538
+ if (!preferredCodec && !preferredBitrate && !maxSimulcastLayers)
10539
+ return [];
10540
+ const codec = preferredCodec
10541
+ ? Codec.create({ name: preferredCodec.split('/').pop(), fmtp: fmtpLine })
10542
+ : undefined;
10543
+ const preferredPublishOptions = [
10544
+ PublishOption.create({
10545
+ trackType: TrackType.VIDEO,
10546
+ codec,
10547
+ bitrate: preferredBitrate,
10548
+ maxSpatialLayers: maxSimulcastLayers,
10549
+ }),
10550
+ ];
10551
+ const screenShareSettings = this.screenShare.getSettings();
10552
+ if (screenShareSettings) {
10553
+ preferredPublishOptions.push(PublishOption.create({
10554
+ trackType: TrackType.SCREEN_SHARE,
10555
+ fps: screenShareSettings.maxFramerate,
10556
+ bitrate: screenShareSettings.maxBitrate,
10557
+ }));
10558
+ }
10559
+ return preferredPublishOptions;
10560
+ };
10561
+ /**
10562
+ * Prepares the preferred options for subscribing to tracks.
10563
+ * This is an experimental client feature and subject to change.
10564
+ * @internal
10565
+ */
10566
+ this.getPreferredSubscribeOptions = () => {
10567
+ const { subscriberCodec, subscriberFmtpLine } = this.clientPublishOptions || {};
10568
+ if (!subscriberCodec || !subscriberFmtpLine)
10569
+ return [];
10570
+ return [
10571
+ SubscribeOption.create({
10572
+ trackType: TrackType.VIDEO,
10573
+ codecs: [
10574
+ { name: subscriberCodec.split('/').pop(), fmtp: subscriberFmtpLine },
10575
+ ],
10576
+ }),
10577
+ ];
10578
+ };
10487
10579
  /**
10488
10580
  * Performs an ICE restart on both the Publisher and Subscriber Peer Connections.
10489
10581
  * Uses the provided SFU client to restore the ICE connection.
@@ -10514,9 +10606,9 @@ class Call {
10514
10606
  * @internal
10515
10607
  */
10516
10608
  this.initPublisherAndSubscriber = (opts) => {
10517
- const { sfuClient, connectionConfig, clientDetails, statsOptions, closePreviousInstances, } = opts;
10609
+ const { sfuClient, connectionConfig, clientDetails, statsOptions, publishOptions, closePreviousInstances, } = opts;
10518
10610
  if (closePreviousInstances && this.subscriber) {
10519
- this.subscriber.close();
10611
+ this.subscriber.dispose();
10520
10612
  }
10521
10613
  this.subscriber = new Subscriber({
10522
10614
  sfuClient,
@@ -10535,18 +10627,14 @@ class Call {
10535
10627
  const isAnonymous = this.streamClient.user?.type === 'anonymous';
10536
10628
  if (!isAnonymous) {
10537
10629
  if (closePreviousInstances && this.publisher) {
10538
- this.publisher.close({ stopTracks: false });
10630
+ this.publisher.dispose();
10539
10631
  }
10540
- const audioSettings = this.state.settings?.audio;
10541
- const isDtxEnabled = !!audioSettings?.opus_dtx_enabled;
10542
- const isRedEnabled = !!audioSettings?.redundant_coding_enabled;
10543
10632
  this.publisher = new Publisher({
10544
10633
  sfuClient,
10545
10634
  dispatcher: this.dispatcher,
10546
10635
  state: this.state,
10547
10636
  connectionConfig,
10548
- isDtxEnabled,
10549
- isRedEnabled,
10637
+ publishOptions,
10550
10638
  logTag: String(this.sfuClientTag),
10551
10639
  onUnrecoverableError: () => {
10552
10640
  this.reconnect(WebsocketReconnectStrategy.REJOIN).catch((err) => {
@@ -10693,47 +10781,31 @@ class Call {
10693
10781
  * @internal
10694
10782
  */
10695
10783
  this.reconnectFast = async () => {
10696
- let reconnectStartTime = Date.now();
10784
+ const reconnectStartTime = Date.now();
10697
10785
  this.reconnectStrategy = WebsocketReconnectStrategy.FAST;
10698
10786
  this.state.setCallingState(CallingState.RECONNECTING);
10699
10787
  await this.join(this.joinCallData);
10700
- this.sfuStatsReporter?.sendTelemetryData({
10701
- data: {
10702
- oneofKind: 'reconnection',
10703
- reconnection: {
10704
- timeSeconds: (Date.now() - reconnectStartTime) / 1000,
10705
- strategy: WebsocketReconnectStrategy.FAST,
10706
- },
10707
- },
10708
- });
10788
+ this.sfuStatsReporter?.sendReconnectionTime(WebsocketReconnectStrategy.FAST, (Date.now() - reconnectStartTime) / 1000);
10709
10789
  };
10710
10790
  /**
10711
10791
  * Initiates the reconnection flow with the "rejoin" strategy.
10712
10792
  * @internal
10713
10793
  */
10714
10794
  this.reconnectRejoin = async () => {
10715
- let reconnectStartTime = Date.now();
10795
+ const reconnectStartTime = Date.now();
10716
10796
  this.reconnectStrategy = WebsocketReconnectStrategy.REJOIN;
10717
10797
  this.state.setCallingState(CallingState.RECONNECTING);
10718
10798
  await this.join(this.joinCallData);
10719
10799
  await this.restorePublishedTracks();
10720
10800
  this.restoreSubscribedTracks();
10721
- this.sfuStatsReporter?.sendTelemetryData({
10722
- data: {
10723
- oneofKind: 'reconnection',
10724
- reconnection: {
10725
- timeSeconds: (Date.now() - reconnectStartTime) / 1000,
10726
- strategy: WebsocketReconnectStrategy.REJOIN,
10727
- },
10728
- },
10729
- });
10801
+ this.sfuStatsReporter?.sendReconnectionTime(WebsocketReconnectStrategy.REJOIN, (Date.now() - reconnectStartTime) / 1000);
10730
10802
  };
10731
10803
  /**
10732
10804
  * Initiates the reconnection flow with the "migrate" strategy.
10733
10805
  * @internal
10734
10806
  */
10735
10807
  this.reconnectMigrate = async () => {
10736
- let reconnectStartTime = Date.now();
10808
+ const reconnectStartTime = Date.now();
10737
10809
  const currentSfuClient = this.sfuClient;
10738
10810
  if (!currentSfuClient) {
10739
10811
  throw new Error('Cannot migrate without an active SFU client');
@@ -10767,20 +10839,12 @@ class Call {
10767
10839
  this.state.setCallingState(CallingState.JOINED);
10768
10840
  }
10769
10841
  finally {
10770
- currentSubscriber?.close();
10771
- currentPublisher?.close({ stopTracks: false });
10842
+ currentSubscriber?.dispose();
10843
+ currentPublisher?.dispose();
10772
10844
  // and close the previous SFU client, without specifying close code
10773
10845
  currentSfuClient.close();
10774
10846
  }
10775
- this.sfuStatsReporter?.sendTelemetryData({
10776
- data: {
10777
- oneofKind: 'reconnection',
10778
- reconnection: {
10779
- timeSeconds: (Date.now() - reconnectStartTime) / 1000,
10780
- strategy: WebsocketReconnectStrategy.MIGRATE,
10781
- },
10782
- },
10783
- });
10847
+ this.sfuStatsReporter?.sendReconnectionTime(WebsocketReconnectStrategy.MIGRATE, (Date.now() - reconnectStartTime) / 1000);
10784
10848
  };
10785
10849
  /**
10786
10850
  * Registers the various event handlers for reconnection.
@@ -10857,23 +10921,16 @@ class Call {
10857
10921
  // the tracks need to be restored in their original order of publishing
10858
10922
  // otherwise, we might get `m-lines order mismatch` errors
10859
10923
  for (const trackType of this.trackPublishOrder) {
10924
+ let mediaStream;
10860
10925
  switch (trackType) {
10861
10926
  case TrackType.AUDIO:
10862
- const audioStream = this.microphone.state.mediaStream;
10863
- if (audioStream) {
10864
- await this.publishAudioStream(audioStream);
10865
- }
10927
+ mediaStream = this.microphone.state.mediaStream;
10866
10928
  break;
10867
10929
  case TrackType.VIDEO:
10868
- const videoStream = this.camera.state.mediaStream;
10869
- if (videoStream)
10870
- await this.publishVideoStream(videoStream);
10930
+ mediaStream = this.camera.state.mediaStream;
10871
10931
  break;
10872
10932
  case TrackType.SCREEN_SHARE:
10873
- const screenShareStream = this.screenShare.state.mediaStream;
10874
- if (screenShareStream) {
10875
- await this.publishScreenShareStream(screenShareStream);
10876
- }
10933
+ mediaStream = this.screenShare.state.mediaStream;
10877
10934
  break;
10878
10935
  // screen share audio can't exist without a screen share, so we handle it there
10879
10936
  case TrackType.SCREEN_SHARE_AUDIO:
@@ -10883,6 +10940,8 @@ class Call {
10883
10940
  ensureExhausted(trackType, 'Unknown track type');
10884
10941
  break;
10885
10942
  }
10943
+ if (mediaStream)
10944
+ await this.publish(mediaStream, trackType);
10886
10945
  }
10887
10946
  };
10888
10947
  /**
@@ -10897,105 +10956,111 @@ class Call {
10897
10956
  };
10898
10957
  /**
10899
10958
  * Starts publishing the given video stream to the call.
10900
- * The stream will be stopped if the user changes an input device, or if the user leaves the call.
10901
- *
10902
- * Consecutive calls to this method will replace the previously published stream.
10903
- * The previous video stream will be stopped.
10904
- *
10905
- * @param videoStream the video stream to publish.
10959
+ * @deprecated use `call.publish()`.
10906
10960
  */
10907
10961
  this.publishVideoStream = async (videoStream) => {
10908
- if (!this.sfuClient)
10909
- throw new Error(`Call not joined yet.`);
10910
- // joining is in progress, and we should wait until the client is ready
10911
- await this.sfuClient.joinTask;
10912
- if (!this.permissionsContext.hasPermission(OwnCapability.SEND_VIDEO)) {
10913
- throw new Error('No permission to publish video');
10914
- }
10915
- if (!this.publisher)
10916
- throw new Error('Publisher is not initialized');
10917
- const [videoTrack] = videoStream.getVideoTracks();
10918
- if (!videoTrack)
10919
- throw new Error('There is no video track in the stream');
10920
- if (!this.trackPublishOrder.includes(TrackType.VIDEO)) {
10921
- this.trackPublishOrder.push(TrackType.VIDEO);
10922
- }
10923
- await this.publisher.publishStream(videoStream, videoTrack, TrackType.VIDEO, this.publishOptions);
10962
+ await this.publish(videoStream, TrackType.VIDEO);
10924
10963
  };
10925
10964
  /**
10926
10965
  * Starts publishing the given audio stream to the call.
10927
- * The stream will be stopped if the user changes an input device, or if the user leaves the call.
10928
- *
10929
- * Consecutive calls to this method will replace the audio stream that is currently being published.
10930
- * The previous audio stream will be stopped.
10931
- *
10932
- * @param audioStream the audio stream to publish.
10966
+ * @deprecated use `call.publish()`
10933
10967
  */
10934
10968
  this.publishAudioStream = async (audioStream) => {
10935
- if (!this.sfuClient)
10936
- throw new Error(`Call not joined yet.`);
10937
- // joining is in progress, and we should wait until the client is ready
10938
- await this.sfuClient.joinTask;
10939
- if (!this.permissionsContext.hasPermission(OwnCapability.SEND_AUDIO)) {
10940
- throw new Error('No permission to publish audio');
10941
- }
10942
- if (!this.publisher)
10943
- throw new Error('Publisher is not initialized');
10944
- const [audioTrack] = audioStream.getAudioTracks();
10945
- if (!audioTrack)
10946
- throw new Error('There is no audio track in the stream');
10947
- if (!this.trackPublishOrder.includes(TrackType.AUDIO)) {
10948
- this.trackPublishOrder.push(TrackType.AUDIO);
10949
- }
10950
- await this.publisher.publishStream(audioStream, audioTrack, TrackType.AUDIO);
10969
+ await this.publish(audioStream, TrackType.AUDIO);
10951
10970
  };
10952
10971
  /**
10953
10972
  * Starts publishing the given screen-share stream to the call.
10954
- *
10955
- * Consecutive calls to this method will replace the previous screen-share stream.
10956
- * The previous screen-share stream will be stopped.
10957
- *
10958
- * @param screenShareStream the screen-share stream to publish.
10973
+ * @deprecated use `call.publish()`
10959
10974
  */
10960
10975
  this.publishScreenShareStream = async (screenShareStream) => {
10976
+ await this.publish(screenShareStream, TrackType.SCREEN_SHARE);
10977
+ };
10978
+ /**
10979
+ * Publishes the given media stream.
10980
+ *
10981
+ * @param mediaStream the media stream to publish.
10982
+ * @param trackType the type of the track to announce.
10983
+ */
10984
+ this.publish = async (mediaStream, trackType) => {
10961
10985
  if (!this.sfuClient)
10962
10986
  throw new Error(`Call not joined yet.`);
10963
10987
  // joining is in progress, and we should wait until the client is ready
10964
10988
  await this.sfuClient.joinTask;
10965
- if (!this.permissionsContext.hasPermission(OwnCapability.SCREENSHARE)) {
10966
- throw new Error('No permission to publish screen share');
10989
+ if (!this.permissionsContext.canPublish(trackType)) {
10990
+ throw new Error(`No permission to publish ${TrackType[trackType]}`);
10967
10991
  }
10968
10992
  if (!this.publisher)
10969
10993
  throw new Error('Publisher is not initialized');
10970
- const [screenShareTrack] = screenShareStream.getVideoTracks();
10971
- if (!screenShareTrack) {
10972
- throw new Error('There is no screen share track in the stream');
10994
+ const [track] = isAudioTrackType(trackType)
10995
+ ? mediaStream.getAudioTracks()
10996
+ : mediaStream.getVideoTracks();
10997
+ if (!track) {
10998
+ throw new Error(`There is no ${TrackType[trackType]} track in the stream`);
10973
10999
  }
10974
- if (!this.trackPublishOrder.includes(TrackType.SCREEN_SHARE)) {
10975
- this.trackPublishOrder.push(TrackType.SCREEN_SHARE);
10976
- }
10977
- const opts = {
10978
- screenShareSettings: this.screenShare.getSettings(),
10979
- };
10980
- await this.publisher.publishStream(screenShareStream, screenShareTrack, TrackType.SCREEN_SHARE, opts);
10981
- const [screenShareAudioTrack] = screenShareStream.getAudioTracks();
10982
- if (screenShareAudioTrack) {
10983
- if (!this.trackPublishOrder.includes(TrackType.SCREEN_SHARE_AUDIO)) {
10984
- this.trackPublishOrder.push(TrackType.SCREEN_SHARE_AUDIO);
11000
+ if (track.readyState === 'ended') {
11001
+ throw new Error(`Can't publish ended tracks.`);
11002
+ }
11003
+ pushToIfMissing(this.trackPublishOrder, trackType);
11004
+ await this.publisher.publish(track, trackType);
11005
+ const trackTypes = [trackType];
11006
+ if (trackType === TrackType.SCREEN_SHARE) {
11007
+ const [audioTrack] = mediaStream.getAudioTracks();
11008
+ if (audioTrack) {
11009
+ pushToIfMissing(this.trackPublishOrder, TrackType.SCREEN_SHARE_AUDIO);
11010
+ await this.publisher.publish(audioTrack, TrackType.SCREEN_SHARE_AUDIO);
11011
+ trackTypes.push(TrackType.SCREEN_SHARE_AUDIO);
10985
11012
  }
10986
- await this.publisher.publishStream(screenShareStream, screenShareAudioTrack, TrackType.SCREEN_SHARE_AUDIO, opts);
10987
11013
  }
11014
+ await this.updateLocalStreamState(mediaStream, ...trackTypes);
10988
11015
  };
10989
11016
  /**
10990
11017
  * Stops publishing the given track type to the call, if it is currently being published.
10991
- * Underlying track will be stopped and removed from the publisher.
10992
11018
  *
10993
- * @param trackType the track type to stop publishing.
10994
- * @param stopTrack if `true` the track will be stopped, else it will be just disabled
11019
+ * @param trackTypes the track types to stop publishing.
11020
+ */
11021
+ this.stopPublish = async (...trackTypes) => {
11022
+ if (!this.sfuClient || !this.publisher)
11023
+ return;
11024
+ this.publisher.stopTracks(...trackTypes);
11025
+ await this.updateLocalStreamState(undefined, ...trackTypes);
11026
+ };
11027
+ /**
11028
+ * Updates the call state with the new stream.
11029
+ *
11030
+ * @param mediaStream the new stream to update the call state with.
11031
+ * If undefined, the stream will be removed from the call state.
11032
+ * @param trackTypes the track types to update the call state with.
11033
+ */
11034
+ this.updateLocalStreamState = async (mediaStream, ...trackTypes) => {
11035
+ if (!this.sfuClient || !this.sfuClient.sessionId)
11036
+ return;
11037
+ await this.notifyTrackMuteState(!mediaStream, ...trackTypes);
11038
+ const { sessionId } = this.sfuClient;
11039
+ for (const trackType of trackTypes) {
11040
+ const streamStateProp = trackTypeToParticipantStreamKey(trackType);
11041
+ if (!streamStateProp)
11042
+ continue;
11043
+ this.state.updateParticipant(sessionId, (p) => ({
11044
+ publishedTracks: mediaStream
11045
+ ? pushToIfMissing([...p.publishedTracks], trackType)
11046
+ : p.publishedTracks.filter((t) => t !== trackType),
11047
+ [streamStateProp]: mediaStream,
11048
+ }));
11049
+ }
11050
+ };
11051
+ /**
11052
+ * Updates the preferred publishing options
11053
+ *
11054
+ * @internal
11055
+ * @param options the options to use.
10995
11056
  */
10996
- this.stopPublish = async (trackType, stopTrack = true) => {
10997
- this.logger('info', `stopPublish ${TrackType[trackType]}, stop tracks: ${stopTrack}`);
10998
- await this.publisher?.unpublishStream(trackType, stopTrack);
11057
+ this.updatePublishOptions = (options) => {
11058
+ this.logger('warn', '[call.updatePublishOptions]: You are manually overriding the publish options for this call. ' +
11059
+ 'This is not recommended, and it can cause call stability/compatibility issues. Use with caution.');
11060
+ if (this.state.callingState === CallingState.JOINED) {
11061
+ this.logger('warn', 'Updating publish options after joining the call does not have an effect');
11062
+ }
11063
+ this.clientPublishOptions = { ...this.clientPublishOptions, ...options };
10999
11064
  };
11000
11065
  /**
11001
11066
  * Notifies the SFU that a noise cancellation process has started.
@@ -11017,6 +11082,15 @@ class Call {
11017
11082
  this.logger('warn', 'Failed to notify stop of noise cancellation', err);
11018
11083
  });
11019
11084
  };
11085
+ /**
11086
+ * Notifies the SFU about the mute state of the given track types.
11087
+ * @internal
11088
+ */
11089
+ this.notifyTrackMuteState = async (muted, ...trackTypes) => {
11090
+ if (!this.sfuClient)
11091
+ return;
11092
+ await this.sfuClient.updateMuteStates(trackTypes.map((trackType) => ({ trackType, muted })));
11093
+ };
11020
11094
  /**
11021
11095
  * Will enhance the reported stats with additional participant-specific information (`callStatsReport$` state [store variable](./StreamVideoClient.md/#readonlystatestore)).
11022
11096
  * This is usually helpful when detailed stats for a specific participant are needed.
@@ -11161,7 +11235,43 @@ class Call {
11161
11235
  return this.streamClient.post(`${this.streamClientBasePath}/stop_transcription`);
11162
11236
  };
11163
11237
  /**
11164
- * Sends a `call.permission_request` event to all users connected to the call. The call settings object contains infomration about which permissions can be requested during a call (for example a user might be allowed to request permission to publish audio, but not video).
11238
+ * Starts the closed captions of the call.
11239
+ */
11240
+ this.startClosedCaptions = async (options) => {
11241
+ const trx = this.state.setCaptioning(true); // optimistic update
11242
+ try {
11243
+ return await this.streamClient.post(`${this.streamClientBasePath}/start_closed_captions`, options);
11244
+ }
11245
+ catch (err) {
11246
+ trx.rollback(); // revert the optimistic update
11247
+ throw err;
11248
+ }
11249
+ };
11250
+ /**
11251
+ * Stops the closed captions of the call.
11252
+ */
11253
+ this.stopClosedCaptions = async (options) => {
11254
+ const trx = this.state.setCaptioning(false); // optimistic update
11255
+ try {
11256
+ return await this.streamClient.post(`${this.streamClientBasePath}/stop_closed_captions`, options);
11257
+ }
11258
+ catch (err) {
11259
+ trx.rollback(); // revert the optimistic update
11260
+ throw err;
11261
+ }
11262
+ };
11263
+ /**
11264
+ * Updates the closed caption settings.
11265
+ *
11266
+ * @param config the closed caption settings to apply
11267
+ */
11268
+ this.updateClosedCaptionSettings = (config) => {
11269
+ this.state.updateClosedCaptionSettings(config);
11270
+ };
11271
+ /**
11272
+ * Sends a `call.permission_request` event to all users connected to the call.
11273
+ * The call settings object contains information about which permissions can be requested during a call
11274
+ * (for example, a user might be allowed to request permission to publish audio, but not video).
11165
11275
  */
11166
11276
  this.requestPermissions = async (data) => {
11167
11277
  const { permissions } = data;
@@ -11444,70 +11554,14 @@ class Call {
11444
11554
  *
11445
11555
  * @internal
11446
11556
  */
11447
- this.applyDeviceConfig = async (status) => {
11448
- await this.initCamera({ setStatus: status }).catch((err) => {
11557
+ this.applyDeviceConfig = async (settings, publish) => {
11558
+ await this.camera.apply(settings.video, publish).catch((err) => {
11449
11559
  this.logger('warn', 'Camera init failed', err);
11450
11560
  });
11451
- await this.initMic({ setStatus: status }).catch((err) => {
11561
+ await this.microphone.apply(settings.audio, publish).catch((err) => {
11452
11562
  this.logger('warn', 'Mic init failed', err);
11453
11563
  });
11454
11564
  };
11455
- this.initCamera = async (options) => {
11456
- // Wait for any in progress camera operation
11457
- await this.camera.statusChangeSettled();
11458
- if (this.state.localParticipant?.videoStream ||
11459
- !this.permissionsContext.hasPermission('send-video')) {
11460
- return;
11461
- }
11462
- // Set camera direction if it's not yet set
11463
- if (!this.camera.state.direction && !this.camera.state.selectedDevice) {
11464
- let defaultDirection = 'front';
11465
- const backendSetting = this.state.settings?.video.camera_facing;
11466
- if (backendSetting) {
11467
- defaultDirection = backendSetting === 'front' ? 'front' : 'back';
11468
- }
11469
- this.camera.state.setDirection(defaultDirection);
11470
- }
11471
- // Set target resolution
11472
- const targetResolution = this.state.settings?.video.target_resolution;
11473
- if (targetResolution) {
11474
- await this.camera.selectTargetResolution(targetResolution);
11475
- }
11476
- if (options.setStatus) {
11477
- // Publish already that was set before we joined
11478
- if (this.camera.enabled &&
11479
- this.camera.state.mediaStream &&
11480
- !this.publisher?.isPublishing(TrackType.VIDEO)) {
11481
- await this.publishVideoStream(this.camera.state.mediaStream);
11482
- }
11483
- // Start camera if backend config specifies, and there is no local setting
11484
- if (this.camera.state.status === undefined &&
11485
- this.state.settings?.video.camera_default_on) {
11486
- await this.camera.enable();
11487
- }
11488
- }
11489
- };
11490
- this.initMic = async (options) => {
11491
- // Wait for any in progress mic operation
11492
- await this.microphone.statusChangeSettled();
11493
- if (this.state.localParticipant?.audioStream ||
11494
- !this.permissionsContext.hasPermission('send-audio')) {
11495
- return;
11496
- }
11497
- if (options.setStatus) {
11498
- // Publish media stream that was set before we joined
11499
- if (this.microphone.enabled &&
11500
- this.microphone.state.mediaStream &&
11501
- !this.publisher?.isPublishing(TrackType.AUDIO)) {
11502
- await this.publishAudioStream(this.microphone.state.mediaStream);
11503
- }
11504
- // Start mic if backend config specifies, and there is no local setting
11505
- if (this.microphone.state.status === undefined &&
11506
- this.state.settings?.audio.mic_default_on) {
11507
- await this.microphone.enable();
11508
- }
11509
- }
11510
- };
11511
11565
  /**
11512
11566
  * Will begin tracking the given element for visibility changes within the
11513
11567
  * configured viewport element (`call.setViewport`).
@@ -11656,109 +11710,6 @@ class Call {
11656
11710
  this.screenShare = new ScreenShareManager(this);
11657
11711
  this.dynascaleManager = new DynascaleManager(this.state, this.speaker);
11658
11712
  }
11659
- async setup() {
11660
- await withoutConcurrency(this.joinLeaveConcurrencyTag, async () => {
11661
- if (this.initialized)
11662
- return;
11663
- this.leaveCallHooks.add(this.on('all', (event) => {
11664
- // update state with the latest event data
11665
- this.state.updateFromEvent(event);
11666
- }));
11667
- this.leaveCallHooks.add(registerEventHandlers(this, this.dispatcher));
11668
- this.registerEffects();
11669
- this.registerReconnectHandlers();
11670
- if (this.state.callingState === CallingState.LEFT) {
11671
- this.state.setCallingState(CallingState.IDLE);
11672
- }
11673
- this.initialized = true;
11674
- });
11675
- }
11676
- registerEffects() {
11677
- this.leaveCallHooks.add(
11678
- // handles updating the permissions context when the settings change.
11679
- createSubscription(this.state.settings$, (settings) => {
11680
- if (!settings)
11681
- return;
11682
- this.permissionsContext.setCallSettings(settings);
11683
- }));
11684
- this.leaveCallHooks.add(
11685
- // handle the case when the user permissions are modified.
11686
- createSafeAsyncSubscription(this.state.ownCapabilities$, this.handleOwnCapabilitiesUpdated));
11687
- this.leaveCallHooks.add(
11688
- // handles the case when the user is blocked by the call owner.
11689
- createSubscription(this.state.blockedUserIds$, async (blockedUserIds) => {
11690
- if (!blockedUserIds || blockedUserIds.length === 0)
11691
- return;
11692
- const currentUserId = this.currentUserId;
11693
- if (currentUserId && blockedUserIds.includes(currentUserId)) {
11694
- this.logger('info', 'Leaving call because of being blocked');
11695
- await this.leave({ reason: 'user blocked' }).catch((err) => {
11696
- this.logger('error', 'Error leaving call after being blocked', err);
11697
- });
11698
- }
11699
- }));
11700
- this.leaveCallHooks.add(
11701
- // cancel auto-drop when call is
11702
- createSubscription(this.state.session$, (session) => {
11703
- if (!this.ringing)
11704
- return;
11705
- const receiverId = this.clientStore.connectedUser?.id;
11706
- if (!receiverId)
11707
- return;
11708
- const isAcceptedByMe = Boolean(session?.accepted_by[receiverId]);
11709
- const isRejectedByMe = Boolean(session?.rejected_by[receiverId]);
11710
- if (isAcceptedByMe || isRejectedByMe) {
11711
- this.cancelAutoDrop();
11712
- }
11713
- }));
11714
- this.leaveCallHooks.add(
11715
- // "ringing" mode effects and event handlers
11716
- createSubscription(this.ringingSubject, (isRinging) => {
11717
- if (!isRinging)
11718
- return;
11719
- const callSession = this.state.session;
11720
- const receiver_id = this.clientStore.connectedUser?.id;
11721
- const ended_at = callSession?.ended_at;
11722
- const created_by_id = this.state.createdBy?.id;
11723
- const rejected_by = callSession?.rejected_by;
11724
- const accepted_by = callSession?.accepted_by;
11725
- let leaveCallIdle = false;
11726
- if (ended_at) {
11727
- // call was ended before it was accepted or rejected so we should leave it to idle
11728
- leaveCallIdle = true;
11729
- }
11730
- else if (created_by_id && rejected_by) {
11731
- if (rejected_by[created_by_id]) {
11732
- // call was cancelled by the caller
11733
- leaveCallIdle = true;
11734
- }
11735
- }
11736
- else if (receiver_id && rejected_by) {
11737
- if (rejected_by[receiver_id]) {
11738
- // call was rejected by the receiver in some other device
11739
- leaveCallIdle = true;
11740
- }
11741
- }
11742
- else if (receiver_id && accepted_by) {
11743
- if (accepted_by[receiver_id]) {
11744
- // call was accepted by the receiver in some other device
11745
- leaveCallIdle = true;
11746
- }
11747
- }
11748
- if (leaveCallIdle) {
11749
- if (this.state.callingState !== CallingState.IDLE) {
11750
- this.state.setCallingState(CallingState.IDLE);
11751
- }
11752
- }
11753
- else {
11754
- if (this.state.callingState === CallingState.IDLE) {
11755
- this.state.setCallingState(CallingState.RINGING);
11756
- }
11757
- this.scheduleAutoDrop();
11758
- this.leaveCallHooks.add(registerRingingCallEventHandlers(this));
11759
- }
11760
- }));
11761
- }
11762
11713
  /**
11763
11714
  * A flag indicating whether the call is "ringing" type of call.
11764
11715
  */
@@ -11777,15 +11728,6 @@ class Call {
11777
11728
  get isCreatedByMe() {
11778
11729
  return this.state.createdBy?.id === this.currentUserId;
11779
11730
  }
11780
- /**
11781
- * Updates the preferred publishing options
11782
- *
11783
- * @internal
11784
- * @param options the options to use.
11785
- */
11786
- updatePublishOptions(options) {
11787
- this.publishOptions = { ...this.publishOptions, ...options };
11788
- }
11789
11731
  }
11790
11732
 
11791
11733
  /**
@@ -12893,7 +12835,7 @@ class StreamClient {
12893
12835
  return await this.wsConnection.connect(this.defaultWSTimeout);
12894
12836
  };
12895
12837
  this.getUserAgent = () => {
12896
- const version = "1.13.1";
12838
+ const version = "1.15.0";
12897
12839
  return (this.userAgent ||
12898
12840
  `stream-video-javascript-client-${this.node ? 'node' : 'browser'}-${version}`);
12899
12841
  };
@@ -13193,7 +13135,7 @@ class StreamVideoClient {
13193
13135
  clientStore: this.writeableStateStore,
13194
13136
  });
13195
13137
  call.state.updateFromCallResponse(c.call);
13196
- await call.applyDeviceConfig(false);
13138
+ await call.applyDeviceConfig(c.call.settings, false);
13197
13139
  if (data.watch) {
13198
13140
  this.writeableStateStore.registerCall(call);
13199
13141
  }
@@ -13402,5 +13344,5 @@ class StreamVideoClient {
13402
13344
  }
13403
13345
  StreamVideoClient._instanceMap = new Map();
13404
13346
 
13405
- export { AudioSettingsRequestDefaultDeviceEnum, AudioSettingsResponseDefaultDeviceEnum, BlockListOptionsBehaviorEnum, browsers as Browsers, Call, CallState, CallType, CallTypes, CallingState, CameraManager, CameraManagerState, ChannelConfigWithInfoAutomodBehaviorEnum, ChannelConfigWithInfoAutomodEnum, ChannelConfigWithInfoBlocklistBehaviorEnum, CreateDeviceRequestPushProviderEnum, DebounceType, DynascaleManager, ErrorFromResponse, InputMediaDeviceManager, InputMediaDeviceManagerState, MicrophoneManager, MicrophoneManagerState, NoiseCancellationSettingsModeEnum, OwnCapability, RecordSettingsRequestModeEnum, RecordSettingsRequestQualityEnum, rxUtils as RxUtils, ScreenShareManager, ScreenShareState, events as SfuEvents, models as SfuModels, SpeakerManager, SpeakerState, StreamSfuClient, StreamVideoClient, StreamVideoReadOnlyStateStore, StreamVideoWriteableStateStore, TranscriptionSettingsRequestModeEnum, TranscriptionSettingsResponseModeEnum, VideoSettingsRequestCameraFacingEnum, VideoSettingsResponseCameraFacingEnum, ViewportTracker, VisibilityState, checkIfAudioOutputChangeSupported, combineComparators, conditional, createSoundDetector, defaultSortPreset, descending, deviceIds$, disposeOfMediaStream, dominantSpeaker, getAudioBrowserPermission, getAudioDevices, getAudioOutputDevices, getAudioStream, getClientDetails, getDeviceInfo, getDeviceState, getLogLevel, getLogger, getOSInfo, getScreenShareStream, getSdkInfo, getVideoBrowserPermission, getVideoDevices, getVideoStream, getWebRTCInfo, hasAudio, hasScreenShare, hasScreenShareAudio, hasVideo, isPinned, livestreamOrAudioRoomSortPreset, logLevels, logToConsole, name, noopComparator, paginatedLayoutSortPreset, pinned, publishingAudio, publishingVideo, reactionType, role, screenSharing, setDeviceInfo, setLogLevel, setLogger, setOSInfo, setPowerState, setSdkInfo, setThermalState, setWebRTCInfo, speakerLayoutSortPreset, speaking };
13347
+ export { AudioSettingsRequestDefaultDeviceEnum, AudioSettingsResponseDefaultDeviceEnum, BlockListOptionsBehaviorEnum, browsers as Browsers, Call, CallState, CallType, CallTypes, CallingState, CameraManager, CameraManagerState, ChannelConfigWithInfoAutomodBehaviorEnum, ChannelConfigWithInfoAutomodEnum, ChannelConfigWithInfoBlocklistBehaviorEnum, ChannelOwnCapability, CreateDeviceRequestPushProviderEnum, DebounceType, DynascaleManager, ErrorFromResponse, InputMediaDeviceManager, InputMediaDeviceManagerState, MicrophoneManager, MicrophoneManagerState, NoiseCancellationSettingsModeEnum, OwnCapability, RecordSettingsRequestModeEnum, RecordSettingsRequestQualityEnum, rxUtils as RxUtils, ScreenShareManager, ScreenShareState, events as SfuEvents, models as SfuModels, SpeakerManager, SpeakerState, StreamSfuClient, StreamVideoClient, StreamVideoReadOnlyStateStore, StreamVideoWriteableStateStore, TranscriptionSettingsRequestClosedCaptionModeEnum, TranscriptionSettingsRequestModeEnum, TranscriptionSettingsResponseClosedCaptionModeEnum, TranscriptionSettingsResponseModeEnum, VideoSettingsRequestCameraFacingEnum, VideoSettingsResponseCameraFacingEnum, ViewportTracker, VisibilityState, checkIfAudioOutputChangeSupported, combineComparators, conditional, createSoundDetector, defaultSortPreset, descending, deviceIds$, disposeOfMediaStream, dominantSpeaker, getAudioBrowserPermission, getAudioDevices, getAudioOutputDevices, getAudioStream, getClientDetails, getDeviceInfo, getDeviceState, getLogLevel, getLogger, getOSInfo, getScreenShareStream, getSdkInfo, getVideoBrowserPermission, getVideoDevices, getVideoStream, getWebRTCInfo, hasAudio, hasScreenShare, hasScreenShareAudio, hasVideo, isPinned, livestreamOrAudioRoomSortPreset, logLevels, logToConsole, name, noopComparator, paginatedLayoutSortPreset, pinned, publishingAudio, publishingVideo, reactionType, role, screenSharing, setDeviceInfo, setLogLevel, setLogger, setOSInfo, setPowerState, setSdkInfo, setThermalState, setWebRTCInfo, speakerLayoutSortPreset, speaking };
13406
13348
  //# sourceMappingURL=index.es.js.map