@stream-io/video-client 1.52.1-beta.0 → 1.53.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/CHANGELOG.md +17 -0
  2. package/dist/index.browser.es.js +819 -123
  3. package/dist/index.browser.es.js.map +1 -1
  4. package/dist/index.cjs.js +819 -122
  5. package/dist/index.cjs.js.map +1 -1
  6. package/dist/index.es.js +819 -123
  7. package/dist/index.es.js.map +1 -1
  8. package/dist/src/Call.d.ts +6 -14
  9. package/dist/src/StreamVideoClient.d.ts +2 -0
  10. package/dist/src/coordinator/connection/client.d.ts +1 -0
  11. package/dist/src/devices/MicrophoneManager.d.ts +6 -0
  12. package/dist/src/errors/SfuTimeoutError.d.ts +8 -0
  13. package/dist/src/errors/index.d.ts +1 -0
  14. package/dist/src/gen/google/protobuf/struct.d.ts +1 -3
  15. package/dist/src/gen/google/protobuf/timestamp.d.ts +1 -3
  16. package/dist/src/gen/video/sfu/event/events.d.ts +1 -22
  17. package/dist/src/gen/video/sfu/models/models.d.ts +0 -4
  18. package/dist/src/gen/video/sfu/signal_rpc/signal.client.d.ts +2 -23
  19. package/dist/src/helpers/firstVideoFrame.d.ts +7 -0
  20. package/dist/src/reporting/ClientEventReporter.d.ts +84 -0
  21. package/dist/src/reporting/index.d.ts +1 -0
  22. package/dist/src/rtc/BasePeerConnection.d.ts +5 -2
  23. package/dist/src/rtc/Publisher.d.ts +1 -4
  24. package/dist/src/rtc/Subscriber.d.ts +0 -7
  25. package/dist/src/rtc/types.d.ts +24 -1
  26. package/dist/src/types.d.ts +16 -0
  27. package/package.json +1 -1
  28. package/src/Call.ts +185 -106
  29. package/src/StreamSfuClient.ts +3 -3
  30. package/src/StreamVideoClient.ts +18 -3
  31. package/src/__tests__/Call.autodrop.test.ts +4 -1
  32. package/src/__tests__/Call.lifecycle.test.ts +4 -1
  33. package/src/__tests__/Call.publishing.test.ts +4 -1
  34. package/src/__tests__/Call.test.ts +23 -0
  35. package/src/coordinator/connection/client.ts +5 -0
  36. package/src/devices/MicrophoneManager.ts +16 -0
  37. package/src/devices/__tests__/CameraManager.test.ts +10 -1
  38. package/src/devices/__tests__/DeviceManager.test.ts +10 -1
  39. package/src/devices/__tests__/DeviceManagerFilters.test.ts +4 -1
  40. package/src/devices/__tests__/MicrophoneManager.test.ts +10 -1
  41. package/src/devices/__tests__/MicrophoneManagerRN.test.ts +78 -2
  42. package/src/devices/__tests__/ScreenShareManager.test.ts +4 -1
  43. package/src/devices/__tests__/SpeakerManager.test.ts +13 -4
  44. package/src/errors/SfuTimeoutError.ts +7 -0
  45. package/src/errors/index.ts +1 -0
  46. package/src/events/__tests__/call.test.ts +2 -0
  47. package/src/events/__tests__/mutes.test.ts +4 -1
  48. package/src/events/call.ts +8 -0
  49. package/src/gen/google/protobuf/struct.ts +12 -7
  50. package/src/gen/google/protobuf/timestamp.ts +7 -6
  51. package/src/gen/video/sfu/event/events.ts +25 -23
  52. package/src/gen/video/sfu/models/models.ts +1 -11
  53. package/src/gen/video/sfu/signal_rpc/signal.client.ts +29 -25
  54. package/src/gen/video/sfu/signal_rpc/signal.ts +0 -1
  55. package/src/helpers/__tests__/AudioBindingsWatchdog.test.ts +6 -3
  56. package/src/helpers/__tests__/DynascaleManager.test.ts +6 -3
  57. package/src/helpers/__tests__/ViewportTracker.test.ts +6 -3
  58. package/src/helpers/__tests__/firstVideoFrame.test.ts +91 -0
  59. package/src/helpers/client-details.ts +1 -1
  60. package/src/helpers/firstVideoFrame.ts +38 -0
  61. package/src/reporting/ClientEventReporter.ts +864 -0
  62. package/src/reporting/__tests__/ClientEventReporter.test.ts +620 -0
  63. package/src/reporting/index.ts +1 -0
  64. package/src/rtc/BasePeerConnection.ts +30 -0
  65. package/src/rtc/Publisher.ts +0 -4
  66. package/src/rtc/Subscriber.ts +2 -28
  67. package/src/rtc/__tests__/Call.reconnect.test.ts +3 -0
  68. package/src/rtc/types.ts +34 -0
  69. package/src/types.ts +18 -0
package/src/Call.ts CHANGED
@@ -130,6 +130,7 @@ import {
130
130
  ClientCapability,
131
131
  ClientDetails,
132
132
  Codec,
133
+ ErrorCode,
133
134
  ParticipantSource,
134
135
  PeerType,
135
136
  PublishOption,
@@ -145,10 +146,12 @@ import {
145
146
  StatsReporter,
146
147
  Tracer,
147
148
  } from './stats';
149
+ import type { ClientEventReporter, JoinReason } from './reporting';
148
150
  import { AudioBindingsWatchdog } from './helpers/AudioBindingsWatchdog';
149
151
  import { BlockedAudioTracker } from './helpers/BlockedAudioTracker';
150
152
  import { TrackSubscriptionManager } from './helpers/TrackSubscriptionManager';
151
153
  import { DynascaleManager } from './helpers/DynascaleManager';
154
+ import { createFirstVideoFrameDetector } from './helpers/firstVideoFrame';
152
155
  import { ViewportTracker } from './helpers/ViewportTracker';
153
156
  import { PermissionsContext } from './permissions';
154
157
  import { CallTypes } from './CallType';
@@ -292,6 +295,7 @@ export class Call {
292
295
 
293
296
  private readonly clientStore: StreamVideoWriteableStateStore;
294
297
  public readonly streamClient: StreamClient;
298
+ public readonly clientEventReporter: ClientEventReporter;
295
299
  private sfuClient?: StreamSfuClient;
296
300
  private sfuClientTag = 0;
297
301
  private unifiedSessionId?: string;
@@ -321,7 +325,6 @@ export class Call {
321
325
  private joinResponseTimeout?: number;
322
326
  private rpcRequestTimeout?: number;
323
327
  private joinCallData?: JoinCallData;
324
- private selfSubEnabled = false;
325
328
  private hasJoinedOnce = false;
326
329
  private deviceSettingsAppliedOnce = false;
327
330
  private credentials?: Credentials;
@@ -358,6 +361,7 @@ export class Call {
358
361
  type,
359
362
  id,
360
363
  streamClient,
364
+ clientEventReporter,
361
365
  members,
362
366
  ownCapabilities,
363
367
  sortParticipantsBy,
@@ -371,6 +375,7 @@ export class Call {
371
375
  this.ringingSubject = new BehaviorSubject(ringing);
372
376
  this.watching = watching;
373
377
  this.streamClient = streamClient;
378
+ this.clientEventReporter = clientEventReporter;
374
379
  this.clientStore = clientStore;
375
380
  this.streamClientBasePath = `/call/${this.type}/${this.id}`;
376
381
  this.logger = videoLoggerSystem.getLogger('Call');
@@ -741,11 +746,18 @@ export class Call {
741
746
  this.lastStatsOptions = undefined;
742
747
 
743
748
  await this.subscriber?.dispose();
749
+ this.clientEventReporter.abort(this.cid, {
750
+ code: 'CLIENT_ABORTED',
751
+ reason: leaveReason,
752
+ });
753
+
744
754
  this.subscriber = undefined;
745
755
 
746
756
  await this.publisher?.dispose();
747
757
  this.publisher = undefined;
748
758
 
759
+ this.clientEventReporter.unregisterCall(this.cid);
760
+
749
761
  await this.sfuClient?.leaveAndClose(leaveReason);
750
762
  this.sfuClient = undefined;
751
763
  this.trackSubscriptionManager.setSfuClient(undefined);
@@ -818,37 +830,6 @@ export class Call {
818
830
  return this.clientStore.connectedUser?.id;
819
831
  }
820
832
 
821
- /**
822
- * A flag indicating whether self-subscription is enabled for the call.
823
- */
824
- get isSelfSubEnabled() {
825
- return this.selfSubEnabled;
826
- }
827
-
828
- /**
829
- * The largest video publish dimension across the current publish options.
830
- *
831
- * @internal
832
- */
833
- getMaxVideoPublishDimension = (): VideoDimension | undefined => {
834
- if (!this.currentPublishOptions) return undefined;
835
- let maxDimension: VideoDimension | undefined;
836
- let maxArea = 0;
837
- for (const opt of this.currentPublishOptions) {
838
- if (opt.trackType !== TrackType.VIDEO) continue;
839
-
840
- const dim = opt.videoDimension;
841
- if (!dim || !dim.width || !dim.height) continue;
842
-
843
- const area = dim.width * dim.height;
844
- if (area > maxArea) {
845
- maxDimension = dim;
846
- maxArea = area;
847
- }
848
- }
849
- return maxDimension;
850
- };
851
-
852
833
  /**
853
834
  * A flag indicating whether the call was created by the current user.
854
835
  */
@@ -1061,13 +1042,11 @@ export class Call {
1061
1042
  maxJoinRetries = 3,
1062
1043
  joinResponseTimeout,
1063
1044
  rpcRequestTimeout,
1064
- selfSubEnabled = false,
1065
1045
  ...data
1066
1046
  }: JoinCallData & {
1067
1047
  maxJoinRetries?: number;
1068
1048
  joinResponseTimeout?: number;
1069
1049
  rpcRequestTimeout?: number;
1070
- selfSubEnabled?: boolean;
1071
1050
  } = {}): Promise<void> => {
1072
1051
  const callingState = this.state.callingState;
1073
1052
 
@@ -1078,11 +1057,6 @@ export class Call {
1078
1057
  if (data?.ring) {
1079
1058
  this.ringingSubject.next(true);
1080
1059
  }
1081
-
1082
- // we need this to be set before the callingx.joinCall() is
1083
- // called to avoid registering the test call in the CallKit/Telecom
1084
- this.selfSubEnabled = selfSubEnabled;
1085
-
1086
1060
  const callingX = globalThis.streamRNVideoSDK?.callingX;
1087
1061
  if (callingX) {
1088
1062
  // for Android/iOS, we need to start the call in the callingx library as soon as possible
@@ -1091,6 +1065,14 @@ export class Call {
1091
1065
 
1092
1066
  await this.setup();
1093
1067
 
1068
+ this.clientEventReporter.registerCall(this.cid, {
1069
+ callType: this.type,
1070
+ callId: this.id,
1071
+ getCallSessionId: () => this.state.session?.id ?? '',
1072
+ getSfuId: () => this.credentials?.server.edge_name ?? '',
1073
+ getUserSessionId: () => this.sfuClient?.sessionId ?? '',
1074
+ });
1075
+
1094
1076
  this.joinResponseTimeout = joinResponseTimeout;
1095
1077
  this.rpcRequestTimeout = rpcRequestTimeout;
1096
1078
  // we will count the number of join failures per SFU.
@@ -1100,44 +1082,56 @@ export class Call {
1100
1082
  const joinData: JoinCallData = data;
1101
1083
  maxJoinRetries = Math.max(maxJoinRetries, 1);
1102
1084
  try {
1103
- for (let attempt = 0; attempt < maxJoinRetries; attempt++) {
1104
- try {
1105
- this.logger.trace(`Joining call (${attempt})`, this.cid);
1106
- await this.doJoin(data);
1107
- delete joinData.migrating_from;
1108
- delete joinData.migrating_from_list;
1109
- break;
1110
- } catch (err) {
1111
- this.logger.warn(`Failed to join call (${attempt})`, this.cid);
1112
- if (
1113
- (err instanceof ErrorFromResponse && err.unrecoverable) ||
1114
- (err instanceof SfuJoinError && err.unrecoverable)
1115
- ) {
1116
- // if the error is unrecoverable, we should not retry as that signals
1117
- // that connectivity is good, but the coordinator doesn't allow the user
1118
- // to join the call due to some reason (e.g., ended call, expired token...)
1119
- throw err;
1120
- }
1085
+ await this.clientEventReporter.withJoinLifecycle(
1086
+ this.cid,
1087
+ 'first-attempt',
1088
+ async () => {
1089
+ for (let attempt = 0; attempt < maxJoinRetries; attempt++) {
1090
+ try {
1091
+ this.logger.trace(`Joining call (${attempt})`, this.cid);
1092
+ await this.doJoin(data);
1093
+ delete joinData.migrating_from;
1094
+ delete joinData.migrating_from_list;
1095
+ return;
1096
+ } catch (err) {
1097
+ this.logger.warn(`Failed to join call (${attempt})`, this.cid);
1098
+ if (
1099
+ (err instanceof ErrorFromResponse && err.unrecoverable) ||
1100
+ (err instanceof SfuJoinError && err.unrecoverable)
1101
+ ) {
1102
+ throw err;
1103
+ }
1121
1104
 
1122
- // immediately switch to a different SFU in case of recoverable join error
1123
- const switchSfu =
1124
- err instanceof SfuJoinError &&
1125
- SfuJoinError.isJoinErrorCode(err.errorEvent);
1126
-
1127
- const sfuId = this.credentials?.server.edge_name || '';
1128
- const failures = (sfuJoinFailures.get(sfuId) || 0) + 1;
1129
- sfuJoinFailures.set(sfuId, failures);
1130
- if (switchSfu || failures >= 2) {
1131
- joinData.migrating_from = sfuId;
1132
- joinData.migrating_from_list = Array.from(sfuJoinFailures.keys());
1133
- }
1105
+ const switchSfu =
1106
+ err instanceof SfuJoinError &&
1107
+ SfuJoinError.isJoinErrorCode(err.errorEvent);
1108
+
1109
+ const sfuId = this.credentials?.server.edge_name;
1110
+ if (sfuId) {
1111
+ const failures = (sfuJoinFailures.get(sfuId) || 0) + 1;
1112
+ sfuJoinFailures.set(sfuId, failures);
1113
+ if (switchSfu || failures >= 2) {
1114
+ joinData.migrating_from = sfuId;
1115
+ joinData.migrating_from_list = Array.from(
1116
+ sfuJoinFailures.keys(),
1117
+ );
1118
+ if (attempt < maxJoinRetries - 1) {
1119
+ this.clientEventReporter.startCorrelation(
1120
+ this.cid,
1121
+ 'first-attempt',
1122
+ );
1123
+ }
1124
+ }
1125
+ }
1134
1126
 
1135
- if (attempt === maxJoinRetries - 1) {
1136
- throw err;
1127
+ if (attempt === maxJoinRetries - 1) {
1128
+ throw err;
1129
+ }
1130
+ }
1131
+ await sleep(retryInterval(attempt));
1137
1132
  }
1138
- }
1139
- await sleep(retryInterval(attempt));
1140
- }
1133
+ },
1134
+ );
1141
1135
  } catch (error) {
1142
1136
  callingX?.endCall(this, 'error');
1143
1137
  throw error;
@@ -1175,7 +1169,11 @@ export class Call {
1175
1169
  data?.migrating_from
1176
1170
  ) {
1177
1171
  try {
1178
- const joinResponse = await this.doJoinRequest(data);
1172
+ const joinResponse = await this.clientEventReporter.track(
1173
+ this.cid,
1174
+ 'CoordinatorJoin',
1175
+ () => this.doJoinRequest(data),
1176
+ );
1179
1177
  this.credentials = joinResponse.credentials;
1180
1178
  statsOptions = joinResponse.stats_options;
1181
1179
  this.lastStatsOptions = statsOptions;
@@ -1241,20 +1239,24 @@ export class Call {
1241
1239
  ? this.getPreferredSubscribeOptions()
1242
1240
  : [];
1243
1241
 
1242
+ const unifiedSessionId = this.unifiedSessionId;
1243
+ const capabilities = Array.from(this.clientCapabilities);
1244
1244
  try {
1245
1245
  const { callState, fastReconnectDeadlineSeconds, publishOptions } =
1246
- await sfuClient.join({
1247
- unifiedSessionId: this.unifiedSessionId,
1248
- subscriberSdp,
1249
- publisherSdp,
1250
- clientDetails,
1251
- fastReconnect: performingFastReconnect,
1252
- reconnectDetails,
1253
- preferredPublishOptions,
1254
- preferredSubscribeOptions,
1255
- capabilities: Array.from(this.clientCapabilities),
1256
- source: ParticipantSource.WEBRTC_UNSPECIFIED,
1257
- });
1246
+ await this.clientEventReporter.track(this.cid, 'WSJoin', () =>
1247
+ sfuClient.join({
1248
+ unifiedSessionId,
1249
+ subscriberSdp,
1250
+ publisherSdp,
1251
+ clientDetails,
1252
+ fastReconnect: performingFastReconnect,
1253
+ reconnectDetails,
1254
+ preferredPublishOptions,
1255
+ preferredSubscribeOptions,
1256
+ capabilities,
1257
+ source: ParticipantSource.WEBRTC_UNSPECIFIED,
1258
+ }),
1259
+ );
1258
1260
 
1259
1261
  this.currentPublishOptions = publishOptions;
1260
1262
  this.fastReconnectDeadlineSeconds = fastReconnectDeadlineSeconds;
@@ -1525,6 +1527,18 @@ export class Call {
1525
1527
  // "ICE never connected" failure budget can be cleared.
1526
1528
  this.iceFailuresWithoutConnect = 0;
1527
1529
  },
1530
+ onPeerConnectionStateChange: (event) => {
1531
+ this.clientEventReporter.onPeerConnectionStateChange(this.cid, event);
1532
+ },
1533
+ onRemoteTrackUnmute: (trackType, trackId) => {
1534
+ const reportable =
1535
+ trackType === TrackType.AUDIO ||
1536
+ (isReactNative() && trackType === TrackType.VIDEO);
1537
+
1538
+ if (!reportable) return;
1539
+
1540
+ this.clientEventReporter.reportFirstFrame(this.cid, trackType, trackId);
1541
+ },
1528
1542
  };
1529
1543
 
1530
1544
  this.subscriber = new Subscriber(basePeerConnectionOptions);
@@ -1536,13 +1550,7 @@ export class Call {
1536
1550
  if (closePreviousInstances && this.publisher) {
1537
1551
  await this.publisher.dispose();
1538
1552
  }
1539
- this.publisher = new Publisher(
1540
- basePeerConnectionOptions,
1541
- publishOptions,
1542
- {
1543
- selfSubEnabled: this.selfSubEnabled,
1544
- },
1545
- );
1553
+ this.publisher = new Publisher(basePeerConnectionOptions, publishOptions);
1546
1554
  }
1547
1555
 
1548
1556
  this.statsReporter?.stop();
@@ -1589,6 +1597,7 @@ export class Call {
1589
1597
  JoinCallResponse,
1590
1598
  JoinCallRequest
1591
1599
  >(`${this.streamClientBasePath}/join`, request);
1600
+
1592
1601
  this.state.updateFromCallResponse(joinResponse.call);
1593
1602
  this.state.setMembers(joinResponse.members);
1594
1603
  this.state.setOwnCapabilities(joinResponse.own_capabilities);
@@ -1762,7 +1771,9 @@ export class Call {
1762
1771
  await this.networkAvailableTask?.promise;
1763
1772
 
1764
1773
  this.logger.info(
1765
- `[Reconnect] Reconnecting with strategy ${WebsocketReconnectStrategy[this.reconnectStrategy]}`,
1774
+ `[Reconnect] Reconnecting with strategy ${
1775
+ WebsocketReconnectStrategy[this.reconnectStrategy]
1776
+ }`,
1766
1777
  );
1767
1778
 
1768
1779
  switch (this.reconnectStrategy) {
@@ -1883,7 +1894,13 @@ export class Call {
1883
1894
  const reconnectStartTime = Date.now();
1884
1895
  this.reconnectStrategy = WebsocketReconnectStrategy.REJOIN;
1885
1896
  this.state.setCallingState(CallingState.RECONNECTING);
1886
- await this.doJoin(this.joinCallData);
1897
+ const joinReason: JoinReason =
1898
+ this.reconnectReason === ReconnectReason.NETWORK_BACK_ONLINE
1899
+ ? 'network-available'
1900
+ : 'full-rejoin';
1901
+ await this.clientEventReporter.withJoinLifecycle(this.cid, joinReason, () =>
1902
+ this.doJoin(this.joinCallData),
1903
+ );
1887
1904
  await this.restorePublishedTracks();
1888
1905
  this.restoreSubscribedTracks();
1889
1906
  this.sfuStatsReporter?.sendReconnectionTime(
@@ -1915,11 +1932,16 @@ export class Call {
1915
1932
 
1916
1933
  try {
1917
1934
  const currentSfu = currentSfuClient.edgeName;
1918
- await this.doJoin({
1919
- ...this.joinCallData,
1920
- migrating_from: currentSfu,
1921
- migrating_from_list: [currentSfu],
1922
- });
1935
+ await this.clientEventReporter.withJoinLifecycle(
1936
+ this.cid,
1937
+ 'migration',
1938
+ () =>
1939
+ this.doJoin({
1940
+ ...this.joinCallData,
1941
+ migrating_from: currentSfu,
1942
+ migrating_from_list: [currentSfu],
1943
+ }),
1944
+ );
1923
1945
  } finally {
1924
1946
  // cleanup the migration_from field after the migration is complete or failed
1925
1947
  // as we don't want to keep dirty data in the join call data
@@ -1961,6 +1983,10 @@ export class Call {
1961
1983
  private registerReconnectHandlers = () => {
1962
1984
  // handles the legacy "goAway" event
1963
1985
  const unregisterGoAway = this.on('goAway', () => {
1986
+ this.clientEventReporter.captureWsError(this.cid, {
1987
+ code: 'SFU_GO_AWAY',
1988
+ reason: 'SFU goAway received during WS join',
1989
+ });
1964
1990
  this.reconnect(
1965
1991
  WebsocketReconnectStrategy.MIGRATE,
1966
1992
  ReconnectReason.GO_AWAY,
@@ -1970,6 +1996,13 @@ export class Call {
1970
1996
  // handles the "error" event, through which the SFU can request a reconnect
1971
1997
  const unregisterOnError = this.on('error', (e) => {
1972
1998
  const { reconnectStrategy: strategy, error } = e;
1999
+ if (!SfuJoinError.isJoinErrorCode(e)) {
2000
+ const code = error?.code ? ErrorCode[error.code] : undefined;
2001
+ this.clientEventReporter.captureWsError(this.cid, {
2002
+ code: code ?? 'SFU_ERROR',
2003
+ reason: error?.message || 'SFU error during WS join',
2004
+ });
2005
+ }
1973
2006
  // SFU_FULL is a join error, and when emitted, although it specifies a
1974
2007
  // `migrate` strategy, we should actually perform a REJOIN to a new SFU.
1975
2008
  // This is now handled separately in the `call.join()` method.
@@ -2864,7 +2897,11 @@ export class Call {
2864
2897
  this.leave({
2865
2898
  reject: true,
2866
2899
  reason: 'timeout',
2867
- message: `ringing timeout - ${this.isCreatedByMe ? 'no one accepted' : `user didn't interact with incoming call screen`}`,
2900
+ message: `ringing timeout - ${
2901
+ this.isCreatedByMe
2902
+ ? 'no one accepted'
2903
+ : `user didn't interact with incoming call screen`
2904
+ }`,
2868
2905
  }).catch((err) => {
2869
2906
  this.logger.error('Failed to drop call', err);
2870
2907
  });
@@ -2925,7 +2962,9 @@ export class Call {
2925
2962
  filename: string,
2926
2963
  ): Promise<DeleteRecordingResponse> => {
2927
2964
  return this.streamClient.delete<DeleteRecordingResponse>(
2928
- `${this.streamClientBasePath}/${encodeURIComponent(callSessionId)}/recordings/${encodeURIComponent(filename)}`,
2965
+ `${this.streamClientBasePath}/${encodeURIComponent(
2966
+ callSessionId,
2967
+ )}/recordings/${encodeURIComponent(filename)}`,
2929
2968
  );
2930
2969
  };
2931
2970
 
@@ -2940,7 +2979,9 @@ export class Call {
2940
2979
  filename: string,
2941
2980
  ): Promise<DeleteTranscriptionResponse> => {
2942
2981
  return this.streamClient.delete<DeleteTranscriptionResponse>(
2943
- `${this.streamClientBasePath}/${encodeURIComponent(callSessionId)}/transcriptions/${encodeURIComponent(filename)}`,
2982
+ `${this.streamClientBasePath}/${encodeURIComponent(
2983
+ callSessionId,
2984
+ )}/transcriptions/${encodeURIComponent(filename)}`,
2944
2985
  );
2945
2986
  };
2946
2987
 
@@ -3166,13 +3207,25 @@ export class Call {
3166
3207
  sessionId: string,
3167
3208
  trackType: VideoTrackType,
3168
3209
  ) => {
3169
- const unbind = this.dynascaleManager?.bindVideoElement(
3210
+ const unbindDynascale = this.dynascaleManager?.bindVideoElement(
3170
3211
  videoElement,
3171
3212
  sessionId,
3172
3213
  trackType,
3173
3214
  );
3174
3215
 
3175
- if (!unbind) return;
3216
+ const stopFirstFrameDetector = this.bindFirstVideoFrameDetector(
3217
+ videoElement,
3218
+ sessionId,
3219
+ trackType,
3220
+ );
3221
+
3222
+ if (!unbindDynascale && !stopFirstFrameDetector) return;
3223
+
3224
+ const unbind = () => {
3225
+ stopFirstFrameDetector?.();
3226
+ unbindDynascale?.();
3227
+ };
3228
+
3176
3229
  this.leaveCallHooks.add(unbind);
3177
3230
  return () => {
3178
3231
  this.leaveCallHooks.delete(unbind);
@@ -3180,6 +3233,32 @@ export class Call {
3180
3233
  };
3181
3234
  };
3182
3235
 
3236
+ private bindFirstVideoFrameDetector = (
3237
+ videoElement: HTMLVideoElement,
3238
+ sessionId: string,
3239
+ trackType: VideoTrackType,
3240
+ ) => {
3241
+ if (trackType !== 'videoTrack') return;
3242
+
3243
+ return createFirstVideoFrameDetector(videoElement, () => {
3244
+ this.reportFirstRenderedVideoFrame(sessionId);
3245
+ });
3246
+ };
3247
+
3248
+ private reportFirstRenderedVideoFrame = (sessionId: string) => {
3249
+ const participant = this.state.findParticipantBySessionId(sessionId);
3250
+ if (participant?.isLocalParticipant) return;
3251
+
3252
+ const trackId = participant?.videoStream?.getVideoTracks()[0]?.id;
3253
+ if (!trackId) return;
3254
+
3255
+ this.clientEventReporter.reportFirstFrame(
3256
+ this.cid,
3257
+ TrackType.VIDEO,
3258
+ trackId,
3259
+ );
3260
+ };
3261
+
3183
3262
  /**
3184
3263
  * Binds a DOM <audio> element to the given session id.
3185
3264
  *
@@ -40,7 +40,7 @@ import {
40
40
  import { withoutConcurrency } from './helpers/concurrency';
41
41
  import { getTimers } from './timers';
42
42
  import { Tracer, TraceSlice } from './stats';
43
- import { SfuJoinError } from './errors';
43
+ import { SfuJoinError, SfuTimeoutError } from './errors';
44
44
 
45
45
  export type StreamSfuClientConstructor = {
46
46
  /**
@@ -353,7 +353,7 @@ export class StreamSfuClient {
353
353
  timeoutId = setTimeout(() => {
354
354
  const message = `SFU WS connection failed to open after ${this.joinResponseTimeout}ms`;
355
355
  this.tracer?.trace('signal.timeout', message);
356
- reject(new Error(message));
356
+ reject(new SfuTimeoutError(message));
357
357
  }, this.joinResponseTimeout);
358
358
  }),
359
359
  ]),
@@ -644,7 +644,7 @@ export class StreamSfuClient {
644
644
  cleanupJoinSubscriptions();
645
645
  const message = `Waiting for "joinResponse" has timed out after ${this.joinResponseTimeout}ms`;
646
646
  this.tracer?.trace('joinRequestTimeout', message);
647
- current.reject(new Error(message));
647
+ current.reject(new SfuTimeoutError(message));
648
648
  }, this.joinResponseTimeout);
649
649
 
650
650
  const joinRequest = SfuRequest.create({
@@ -42,6 +42,7 @@ import { logToConsole, ScopedLogger, videoLoggerSystem } from './logger';
42
42
  import { isReactNative } from './helpers/platforms';
43
43
  import { withoutConcurrency } from './helpers/concurrency';
44
44
  import { enableTimerWorker } from './timers';
45
+ import { ClientEventReporter } from './reporting';
45
46
 
46
47
  /**
47
48
  * A `StreamVideoClient` instance lets you communicate with our API, and authenticate users.
@@ -60,6 +61,7 @@ export class StreamVideoClient {
60
61
 
61
62
  protected readonly writeableStateStore: StreamVideoWriteableStateStore;
62
63
  streamClient: StreamClient;
64
+ readonly clientEventReporter: ClientEventReporter;
63
65
 
64
66
  private effectsRegistered = false;
65
67
  private eventHandlersToUnregister: Array<() => void> = [];
@@ -97,6 +99,9 @@ export class StreamVideoClient {
97
99
  this.rejectCallWhenBusy = clientOptions?.rejectCallWhenBusy ?? false;
98
100
 
99
101
  this.streamClient = createCoordinatorClient(apiKey, clientOptions);
102
+ this.clientEventReporter = new ClientEventReporter({
103
+ streamClient: this.streamClient,
104
+ });
100
105
 
101
106
  this.writeableStateStore = new StreamVideoWriteableStateStore();
102
107
  this.readOnlyStateStore = new StreamVideoReadOnlyStateStore(
@@ -214,6 +219,7 @@ export class StreamVideoClient {
214
219
 
215
220
  call = new Call({
216
221
  streamClient: this.streamClient,
222
+ clientEventReporter: this.clientEventReporter,
217
223
  type: e.call.type,
218
224
  id: e.call.id,
219
225
  members: e.members,
@@ -296,6 +302,9 @@ export class StreamVideoClient {
296
302
  return this.connectAnonymousUser(user as UserWithId, tokenOrProvider);
297
303
  }
298
304
 
305
+ const reporter = this.clientEventReporter;
306
+ reporter.startCoordinatorConnection(user.id);
307
+
299
308
  const connectUserResponse = await withoutConcurrency(
300
309
  this.connectionConcurrencyTag,
301
310
  async () => {
@@ -309,13 +318,16 @@ export class StreamVideoClient {
309
318
  for (let attempt = 0; attempt < maxConnectUserRetries; attempt++) {
310
319
  try {
311
320
  this.logger.trace(`Connecting user (${attempt})`, user);
312
- return user.type === 'guest'
313
- ? await client.connectGuestUser(user)
314
- : await client.connectUser(user, tokenOrProvider);
321
+ return await reporter.trackCoordinatorWs(() =>
322
+ user.type === 'guest'
323
+ ? client.connectGuestUser(user)
324
+ : client.connectUser(user, tokenOrProvider),
325
+ );
315
326
  } catch (err) {
316
327
  this.logger.warn(`Failed to connect a user (${attempt})`, err);
317
328
  errorQueue.push(err as Error);
318
329
  if (attempt === maxConnectUserRetries - 1) {
330
+ reporter.closeCoordinatorWs();
319
331
  onConnectUserError?.(err as Error, errorQueue);
320
332
  throw err;
321
333
  }
@@ -415,6 +427,7 @@ export class StreamVideoClient {
415
427
  call ??
416
428
  new Call({
417
429
  streamClient: this.streamClient,
430
+ clientEventReporter: this.clientEventReporter,
418
431
  id: id,
419
432
  type: type,
420
433
  clientStore: this.writeableStateStore,
@@ -448,6 +461,7 @@ export class StreamVideoClient {
448
461
  for (const c of response.calls) {
449
462
  const call = new Call({
450
463
  streamClient: this.streamClient,
464
+ clientEventReporter: this.clientEventReporter,
451
465
  id: c.call.id,
452
466
  type: c.call.type,
453
467
  members: c.members,
@@ -591,6 +605,7 @@ export class StreamVideoClient {
591
605
  const [callType, callId] = call_cid.split(':');
592
606
  call = new Call({
593
607
  streamClient: this.streamClient,
608
+ clientEventReporter: this.clientEventReporter,
594
609
  type: callType,
595
610
  id: callId,
596
611
  clientStore: this.writeableStateStore,
@@ -3,6 +3,7 @@ import '../rtc/__tests__/mocks/webrtc.mocks';
3
3
  import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
4
4
  import { Call } from '../Call';
5
5
  import { StreamClient } from '../coordinator/connection/client';
6
+ import { ClientEventReporter } from '../reporting';
6
7
  import { generateUUIDv4 } from '../coordinator/connection/utils';
7
8
  import { CallingState, StreamVideoWriteableStateStore } from '../store';
8
9
 
@@ -14,10 +15,12 @@ describe('Auto drop ringing calls', () => {
14
15
  vi.useFakeTimers();
15
16
 
16
17
  const clientStore = new StreamVideoWriteableStateStore();
18
+ const streamClient = new StreamClient('abc');
17
19
  call = new Call({
18
20
  type: 'test',
19
21
  id: generateUUIDv4(),
20
- streamClient: new StreamClient('abc'),
22
+ streamClient,
23
+ clientEventReporter: new ClientEventReporter({ streamClient }),
21
24
  clientStore: clientStore,
22
25
  });
23
26
 
@@ -7,6 +7,7 @@ import '../rtc/__tests__/mocks/webrtc.mocks';
7
7
  import { beforeEach, describe, expect, it, vi } from 'vitest';
8
8
  import { Call } from '../Call';
9
9
  import { StreamClient } from '../coordinator/connection/client';
10
+ import { ClientEventReporter } from '../reporting';
10
11
  import { generateUUIDv4 } from '../coordinator/connection/utils';
11
12
  import { StreamVideoWriteableStateStore } from '../store';
12
13
 
@@ -14,10 +15,12 @@ describe('Call lifecycle wiring', () => {
14
15
  let call: Call;
15
16
 
16
17
  beforeEach(() => {
18
+ const streamClient = new StreamClient('abc');
17
19
  call = new Call({
18
20
  type: 'test',
19
21
  id: generateUUIDv4(),
20
- streamClient: new StreamClient('abc'),
22
+ streamClient,
23
+ clientEventReporter: new ClientEventReporter({ streamClient }),
21
24
  clientStore: new StreamVideoWriteableStateStore(),
22
25
  });
23
26
  });
@@ -4,6 +4,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest';
4
4
  import { Call } from '../Call';
5
5
  import { Publisher } from '../rtc';
6
6
  import { StreamClient } from '../coordinator/connection/client';
7
+ import { ClientEventReporter } from '../reporting';
7
8
  import { generateUUIDv4 } from '../coordinator/connection/utils';
8
9
  import { PermissionsContext } from '../permissions';
9
10
  import { OwnCapability } from '../gen/coordinator';
@@ -15,10 +16,12 @@ describe('Publishing and Unpublishing tracks', () => {
15
16
  let call: Call;
16
17
 
17
18
  beforeEach(async () => {
19
+ const streamClient = new StreamClient('abc');
18
20
  call = new Call({
19
21
  type: 'test',
20
22
  id: generateUUIDv4(),
21
- streamClient: new StreamClient('abc'),
23
+ streamClient,
24
+ clientEventReporter: new ClientEventReporter({ streamClient }),
22
25
  clientStore: new StreamVideoWriteableStateStore(),
23
26
  });
24
27