@stream-io/video-client 1.61.0 → 1.61.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/CHANGELOG.md +15 -0
  2. package/dist/index.browser.es.js +292 -154
  3. package/dist/index.browser.es.js.map +1 -1
  4. package/dist/index.cjs.js +292 -154
  5. package/dist/index.cjs.js.map +1 -1
  6. package/dist/index.es.js +292 -154
  7. package/dist/index.es.js.map +1 -1
  8. package/dist/src/coordinator/connection/types.d.ts +1 -1
  9. package/dist/src/devices/DeviceManager.d.ts +6 -0
  10. package/dist/src/gen/video/sfu/signal_rpc/signal.d.ts +0 -27
  11. package/dist/src/helpers/AudioBindingsWatchdog.d.ts +7 -2
  12. package/dist/src/helpers/DynascaleManager.d.ts +15 -0
  13. package/dist/src/helpers/MediaPlaybackWatchdog.d.ts +18 -0
  14. package/dist/src/helpers/mediaInterruptions.d.ts +14 -0
  15. package/dist/src/reporting/ClientEventReporter.d.ts +1 -1
  16. package/dist/src/ringing/reconcileRingState.d.ts +8 -7
  17. package/dist/src/stats/SfuStatsReporter.d.ts +1 -13
  18. package/dist/src/stats/rtc/StatsTracer.d.ts +6 -6
  19. package/dist/src/stats/rtc/types.d.ts +0 -11
  20. package/package.json +1 -1
  21. package/src/Call.ts +0 -3
  22. package/src/StreamVideoClient.ts +8 -0
  23. package/src/__tests__/StreamVideoClient.rewatch.test.ts +136 -0
  24. package/src/coordinator/connection/types.ts +1 -1
  25. package/src/devices/DeviceManager.ts +20 -1
  26. package/src/devices/__tests__/CameraManager.test.ts +50 -1
  27. package/src/devices/__tests__/mediaStreamTestHelpers.ts +10 -6
  28. package/src/gen/video/sfu/signal_rpc/signal.ts +0 -53
  29. package/src/helpers/AudioBindingsWatchdog.ts +91 -35
  30. package/src/helpers/DynascaleManager.ts +84 -9
  31. package/src/helpers/MediaPlaybackWatchdog.ts +55 -6
  32. package/src/helpers/__tests__/DynascaleManager.test.ts +247 -9
  33. package/src/helpers/__tests__/MediaPlaybackWatchdog.test.ts +140 -0
  34. package/src/helpers/__tests__/mediaInterruptions.test.ts +131 -0
  35. package/src/helpers/mediaInterruptions.ts +37 -0
  36. package/src/reporting/ClientEventReporter.ts +8 -3
  37. package/src/reporting/__tests__/ClientEventReporter.test.ts +2 -2
  38. package/src/ringing/RingStatePoller.ts +1 -1
  39. package/src/ringing/__tests__/RingStatePoller.test.ts +1 -1
  40. package/src/ringing/__tests__/reconcileRingState.test.ts +33 -0
  41. package/src/ringing/reconcileRingState.ts +18 -10
  42. package/src/rtc/BasePeerConnection.ts +6 -1
  43. package/src/rtc/__tests__/Publisher.test.ts +50 -0
  44. package/src/stats/SfuStatsReporter.ts +19 -93
  45. package/src/stats/__tests__/SfuStatsReporter.test.ts +2 -18
  46. package/src/stats/rtc/StatsTracer.ts +6 -6
  47. package/src/stats/rtc/types.ts +0 -13
@@ -2665,18 +2665,6 @@ class SendStatsRequest$Type extends MessageType {
2665
2665
  constructor() {
2666
2666
  super('stream.video.sfu.signal.SendStatsRequest', [
2667
2667
  { no: 1, name: 'session_id', kind: 'scalar', T: 9 /*ScalarType.STRING*/ },
2668
- {
2669
- no: 2,
2670
- name: 'subscriber_stats',
2671
- kind: 'scalar',
2672
- T: 9 /*ScalarType.STRING*/,
2673
- },
2674
- {
2675
- no: 3,
2676
- name: 'publisher_stats',
2677
- kind: 'scalar',
2678
- T: 9 /*ScalarType.STRING*/,
2679
- },
2680
2668
  {
2681
2669
  no: 4,
2682
2670
  name: 'webrtc_version',
@@ -2690,8 +2678,6 @@ class SendStatsRequest$Type extends MessageType {
2690
2678
  kind: 'scalar',
2691
2679
  T: 9 /*ScalarType.STRING*/,
2692
2680
  },
2693
- { no: 7, name: 'audio_devices', kind: 'message', T: () => InputDevices },
2694
- { no: 8, name: 'video_devices', kind: 'message', T: () => InputDevices },
2695
2681
  {
2696
2682
  no: 9,
2697
2683
  name: 'android',
@@ -2708,18 +2694,6 @@ class SendStatsRequest$Type extends MessageType {
2708
2694
  },
2709
2695
  { no: 11, name: 'telemetry', kind: 'message', T: () => Telemetry },
2710
2696
  { no: 12, name: 'rtmp', kind: 'message', T: () => RTMPIngress },
2711
- {
2712
- no: 13,
2713
- name: 'subscriber_rtc_stats',
2714
- kind: 'scalar',
2715
- T: 9 /*ScalarType.STRING*/,
2716
- },
2717
- {
2718
- no: 14,
2719
- name: 'publisher_rtc_stats',
2720
- kind: 'scalar',
2721
- T: 9 /*ScalarType.STRING*/,
2722
- },
2723
2697
  { no: 15, name: 'rtc_stats', kind: 'scalar', T: 9 /*ScalarType.STRING*/ },
2724
2698
  {
2725
2699
  no: 16,
@@ -5081,7 +5055,7 @@ const toIceCandidate = (iceTrickle) => {
5081
5055
  }
5082
5056
  };
5083
5057
 
5084
- const version = "1.61.0";
5058
+ const version = "1.61.2";
5085
5059
  const [major, minor, patch] = version.split('.');
5086
5060
  let sdkInfo = {
5087
5061
  type: SdkType.PLAIN_JAVASCRIPT,
@@ -7809,30 +7783,21 @@ class SfuStatsReporter {
7809
7783
  sfuClient;
7810
7784
  subscriber;
7811
7785
  publisher;
7812
- microphone;
7813
- camera;
7814
- state;
7815
7786
  tracer;
7816
7787
  unifiedSessionId;
7817
7788
  intervalId;
7818
7789
  timeoutId;
7819
7790
  reportCount = 0;
7820
- unsubscribeDevicePermissionsSubscription;
7821
- unsubscribeListDevicesSubscription;
7822
7791
  sdkName;
7823
7792
  sdkVersion;
7824
7793
  webRTCVersion;
7825
- inputDevices = new Map();
7826
7794
  statsConcurrencyTag = Symbol('sfuStatsReporter');
7827
7795
  isStopped = false;
7828
- constructor(sfuClient, { options, clientDetails, subscriber, publisher, microphone, camera, state, tracer, unifiedSessionId, }) {
7796
+ constructor(sfuClient, { options, clientDetails, subscriber, publisher, tracer, unifiedSessionId, }) {
7829
7797
  this.sfuClient = sfuClient;
7830
7798
  this.options = options;
7831
7799
  this.subscriber = subscriber;
7832
7800
  this.publisher = publisher;
7833
- this.microphone = microphone;
7834
- this.camera = camera;
7835
- this.state = state;
7836
7801
  this.tracer = tracer;
7837
7802
  this.unifiedSessionId = unifiedSessionId;
7838
7803
  const { sdk, browser } = clientDetails;
@@ -7846,35 +7811,6 @@ class SfuStatsReporter {
7846
7811
  `${browser?.name || ''}-${browser?.version || ''}` ||
7847
7812
  'N/A';
7848
7813
  }
7849
- observeDevice = (device, kind) => {
7850
- const { browserPermissionState$ } = device.state;
7851
- this.unsubscribeDevicePermissionsSubscription?.();
7852
- this.unsubscribeDevicePermissionsSubscription = createSubscription(combineLatest([browserPermissionState$, this.state.ownCapabilities$]), ([browserPermissionState, ownCapabilities]) => {
7853
- // cleanup the previous listDevices() subscription in case
7854
- // permissions or capabilities have changed.
7855
- // we will subscribe again if everything is in order.
7856
- this.unsubscribeListDevicesSubscription?.();
7857
- const hasCapability = kind === 'mic'
7858
- ? ownCapabilities.includes(OwnCapability.SEND_AUDIO)
7859
- : ownCapabilities.includes(OwnCapability.SEND_VIDEO);
7860
- if (browserPermissionState !== 'granted' || !hasCapability) {
7861
- this.inputDevices.set(kind, {
7862
- currentDevice: '',
7863
- availableDevices: [],
7864
- isPermitted: false,
7865
- });
7866
- return;
7867
- }
7868
- this.unsubscribeListDevicesSubscription = createSubscription(combineLatest([device.listDevices(), device.state.selectedDevice$]), ([devices, deviceId]) => {
7869
- const selected = devices.find((d) => d.deviceId === deviceId);
7870
- this.inputDevices.set(kind, {
7871
- currentDevice: selected?.label || deviceId || '',
7872
- availableDevices: devices.map((d) => d.label),
7873
- isPermitted: true,
7874
- });
7875
- });
7876
- });
7877
- };
7878
7814
  sendConnectionTime = (connectionTimeSeconds) => {
7879
7815
  this.sendTelemetryData({
7880
7816
  data: {
@@ -7909,7 +7845,7 @@ class SfuStatsReporter {
7909
7845
  this.subscriber.stats.takeSample(),
7910
7846
  this.publisher?.stats.takeSample(),
7911
7847
  ]);
7912
- send = (subscriberStats, publisherStats, telemetry) => {
7848
+ send = (subscriberPerfStats, publisherPerfStats, telemetry) => {
7913
7849
  // serialize sends so overlapping ones can't race on the trace buffers or
7914
7850
  // deliver an older delta after a newer one already succeeded. Not gated by
7915
7851
  // `isStopped`: an explicit final flush must still deliver after stop().
@@ -7927,8 +7863,8 @@ class SfuStatsReporter {
7927
7863
  this.subscriber.stats.clearPendingDeltas();
7928
7864
  }
7929
7865
  let pubPending = [];
7930
- if (pubTracer && publisherStats) {
7931
- pubPending = this.publisher?.stats.getPendingDeltas() ?? [];
7866
+ if (pubTracer && this.publisher) {
7867
+ pubPending = this.publisher.stats.getPendingDeltas();
7932
7868
  }
7933
7869
  else {
7934
7870
  this.publisher?.stats.clearPendingDeltas();
@@ -7950,17 +7886,9 @@ class SfuStatsReporter {
7950
7886
  sdk: this.sdkName,
7951
7887
  sdkVersion: this.sdkVersion,
7952
7888
  webrtcVersion: this.webRTCVersion,
7953
- subscriberStats: JSON.stringify(flatten(subscriberStats.stats)),
7954
- publisherStats: publisherStats
7955
- ? JSON.stringify(flatten(publisherStats.stats))
7956
- : '[]',
7957
- subscriberRtcStats: '',
7958
- publisherRtcStats: '',
7959
7889
  rtcStats: JSON.stringify(traces),
7960
- encodeStats: publisherStats?.performanceStats ?? [],
7961
- decodeStats: subscriberStats.performanceStats,
7962
- audioDevices: this.inputDevices.get('mic'),
7963
- videoDevices: this.inputDevices.get('camera'),
7890
+ encodeStats: publisherPerfStats ?? [],
7891
+ decodeStats: subscriberPerfStats,
7964
7892
  unifiedSessionId: this.unifiedSessionId,
7965
7893
  deviceState: getDeviceState(),
7966
7894
  telemetry,
@@ -7974,8 +7902,8 @@ class SfuStatsReporter {
7974
7902
  // delivery confirmed: advance the delivery baseline for each chain.
7975
7903
  if (subTracer)
7976
7904
  this.subscriber.stats.commitDeltas(subPending);
7977
- if (pubTracer && publisherStats) {
7978
- this.publisher?.stats.commitDeltas(pubPending);
7905
+ if (pubTracer && this.publisher) {
7906
+ this.publisher.stats.commitDeltas(pubPending);
7979
7907
  }
7980
7908
  }
7981
7909
  catch (err) {
@@ -7997,8 +7925,8 @@ class SfuStatsReporter {
7997
7925
  run = async (telemetry) => {
7998
7926
  if (this.isStopped)
7999
7927
  return;
8000
- const [subscriberStats, publisherStats] = await this.sample();
8001
- await this.send(subscriberStats, publisherStats, telemetry);
7928
+ const [subscriberPerfStats, publisherPerfStats] = await this.sample();
7929
+ await this.send(subscriberPerfStats, publisherPerfStats, telemetry);
8002
7930
  };
8003
7931
  scheduleNextReport = () => {
8004
7932
  const intervals = [1500, 3000, 3000, 5000];
@@ -8019,8 +7947,6 @@ class SfuStatsReporter {
8019
7947
  start = () => {
8020
7948
  if (this.options.reporting_interval_ms <= 0)
8021
7949
  return;
8022
- this.observeDevice(this.microphone, 'mic');
8023
- this.observeDevice(this.camera, 'camera');
8024
7950
  this.isStopped = false;
8025
7951
  this.reportCount = 0;
8026
7952
  clearInterval(this.intervalId);
@@ -8029,11 +7955,6 @@ class SfuStatsReporter {
8029
7955
  };
8030
7956
  stop = () => {
8031
7957
  this.isStopped = true;
8032
- this.unsubscribeDevicePermissionsSubscription?.();
8033
- this.unsubscribeDevicePermissionsSubscription = undefined;
8034
- this.unsubscribeListDevicesSubscription?.();
8035
- this.unsubscribeListDevicesSubscription = undefined;
8036
- this.inputDevices.clear();
8037
7958
  clearInterval(this.intervalId);
8038
7959
  this.intervalId = undefined;
8039
7960
  clearTimeout(this.timeoutId);
@@ -8054,8 +7975,8 @@ class SfuStatsReporter {
8054
7975
  return;
8055
7976
  try {
8056
7977
  const [sample] = await timeboxed([this.sample()], 2000);
8057
- const [subscriberStats, publisherStats] = sample;
8058
- this.send(subscriberStats, publisherStats).catch((err) => {
7978
+ const [subscriberPerfStats, publisherPerfStats] = sample;
7979
+ this.send(subscriberPerfStats, publisherPerfStats).catch((err) => {
8059
7980
  this.logger.warn('Failed to flush report stats', err);
8060
7981
  });
8061
7982
  }
@@ -8182,9 +8103,9 @@ class StatsTracer {
8182
8103
  this.maxPendingDeltas = maxPendingDeltas;
8183
8104
  }
8184
8105
  /**
8185
- * Samples the RTCPeerConnection: returns the current stats report and the
8186
- * derived performance stats, and appends the delta-compressed sample to the
8187
- * un-acked delivery chain (retrieved via `getPendingDeltas`).
8106
+ * Samples the RTCPeerConnection: returns the derived performance stats and
8107
+ * appends the delta-compressed sample to the un-acked delivery chain
8108
+ * (retrieved via `getPendingDeltas`).
8188
8109
  *
8189
8110
  * @internal
8190
8111
  */
@@ -8209,7 +8130,7 @@ class StatsTracer {
8209
8130
  this.pendingDeltas.push({ delta, ts: now });
8210
8131
  this.frameTimeHistory = this.frameTimeHistory.slice(-2);
8211
8132
  this.fpsHistory = this.fpsHistory.slice(-2);
8212
- return { performanceStats, stats };
8133
+ return performanceStats;
8213
8134
  });
8214
8135
  };
8215
8136
  /**
@@ -8754,7 +8675,12 @@ class BasePeerConnection {
8754
8675
  }
8755
8676
  // we can't recover from a failed connection state (contrary to ICE)
8756
8677
  if (state === 'failed') {
8757
- this.onReconnectionNeeded?.(WebsocketReconnectStrategy.REJOIN, ReconnectReason.CONNECTION_FAILED, this.peerType);
8678
+ const reason = this.iceHasEverConnected
8679
+ ? ReconnectReason.CONNECTION_FAILED
8680
+ : ReconnectReason.ICE_NEVER_CONNECTED;
8681
+ clearTimeout(this.preConnectStuckTimeout);
8682
+ this.preConnectStuckTimeout = undefined;
8683
+ this.onReconnectionNeeded?.(WebsocketReconnectStrategy.REJOIN, reason, this.peerType);
8758
8684
  return;
8759
8685
  }
8760
8686
  this.handleConnectionStateUpdate(state);
@@ -10753,15 +10679,16 @@ class StreamSfuClient {
10753
10679
  /**
10754
10680
  * Decides what a ringing call should do next, based on the current call state.
10755
10681
  *
10756
- * The `call.accepted`, `call.rejected` and `call.missed` handlers and the ring
10757
- * state poller both run this. They differ only in how the state got there: the
10758
- * handlers rely on `CallState.updateFromEvent`, which runs before them, and the
10759
- * poller applies the polled ring state itself.
10682
+ * The `call.accepted` and `call.rejected` handlers, the ring state poller and
10683
+ * the rewatch after a WS reconnect all run this. They differ only in how the
10684
+ * state got there: the handlers rely on `CallState.updateFromEvent`, which runs
10685
+ * before them, while the poller applies the polled ring state itself and the
10686
+ * rewatch applies the queried call state.
10760
10687
  *
10761
10688
  * @param call the call to reconcile.
10762
- * @param joinSource which of the two triggered this run, reported on the
10763
- * caller's join: `ring-ws` for the event handlers, `ring-poll-api` for the
10764
- * poller.
10689
+ * @param joinSource what triggered this run, reported on the caller's join:
10690
+ * `ring-ws` for the event handlers, `ring-poll-api` for the poller and the
10691
+ * rewatch (both come from an API response).
10765
10692
  * @returns whether the ring reached a terminal state. A failed join is not
10766
10693
  * terminal: the caller should keep trying while the ring is open.
10767
10694
  */
@@ -10816,10 +10743,16 @@ const reconcileAsCaller = async (call, joinSource) => {
10816
10743
  return false;
10817
10744
  };
10818
10745
  // the current user's own accept or reject, on this or another device, is
10819
- // handled by `resolveOwnRingOutcome`, and `call.ended` by `watchCallEnded`.
10746
+ // handled by `resolveOwnRingOutcome`, and a live `call.ended` by
10747
+ // `watchCallEnded`. An end missed while offline is handled here.
10820
10748
  const reconcileAsCallee = async (call) => {
10821
- const createdById = call.state.createdBy?.id;
10822
- const rejectedBy = call.state.session?.rejected_by ?? {};
10749
+ const { session, endedAt, createdBy } = call.state;
10750
+ if (endedAt || session?.ended_at) {
10751
+ call.logger.info('ring: the call has ended, leaving');
10752
+ return leave(call, { reason: 'ended', message: 'ring: call ended' });
10753
+ }
10754
+ const createdById = createdBy?.id;
10755
+ const rejectedBy = session?.rejected_by ?? {};
10823
10756
  if (createdById && rejectedBy[createdById]) {
10824
10757
  call.logger.info('ring: the caller cancelled, leaving');
10825
10758
  return leave(call, {
@@ -10868,7 +10801,7 @@ class RingStatePoller {
10868
10801
  unsubscribe = [];
10869
10802
  constructor(call, options = {}) {
10870
10803
  this.call = call;
10871
- this.startAfterMs = options.startAfterMs ?? 15_000;
10804
+ this.startAfterMs = options.startAfterMs ?? 9_000;
10872
10805
  this.intervalMs = options.intervalMs ?? 5_000;
10873
10806
  }
10874
10807
  /**
@@ -11533,12 +11466,33 @@ const registerRingingCallEventHandlers = (call) => {
11533
11466
  };
11534
11467
 
11535
11468
  const toBindingKey = (sessionId, trackType = 'audioTrack') => `${sessionId}/${trackType}`;
11469
+ const toElementState = (element) => {
11470
+ const stream = element.srcObject;
11471
+ const [track] = stream?.getAudioTracks?.() ?? [];
11472
+ return {
11473
+ paused: element.paused,
11474
+ muted: element.muted,
11475
+ volume: Math.round(element.volume * 100) / 100,
11476
+ readyState: element.readyState,
11477
+ sinkId: element.sinkId,
11478
+ track: track
11479
+ ? {
11480
+ enabled: track.enabled,
11481
+ muted: track.muted,
11482
+ readyState: track.readyState,
11483
+ }
11484
+ : null,
11485
+ };
11486
+ };
11536
11487
  /**
11537
11488
  * Tracks audio element bindings and periodically warns about
11538
11489
  * remote participants whose audio streams have no bound element.
11490
+ * On the same tick, it traces the playback state of the bound elements,
11491
+ * limited to the bindings whose state changed since the previous trace.
11539
11492
  */
11540
11493
  class AudioBindingsWatchdog {
11541
11494
  bindings = new Map();
11495
+ tracedStates = new Map();
11542
11496
  enabled = true;
11543
11497
  watchdogInterval;
11544
11498
  unsubscribeCallingState;
@@ -11549,8 +11503,6 @@ class AudioBindingsWatchdog {
11549
11503
  this.tracer = tracer;
11550
11504
  this.state = state;
11551
11505
  this.unsubscribeCallingState = createSubscription(state.callingState$, (callingState) => {
11552
- if (!this.enabled)
11553
- return;
11554
11506
  if (callingState !== CallingState.JOINED) {
11555
11507
  this.stop();
11556
11508
  }
@@ -11579,15 +11531,15 @@ class AudioBindingsWatchdog {
11579
11531
  this.bindings.delete(toBindingKey(sessionId, trackType));
11580
11532
  };
11581
11533
  /**
11582
- * Enables or disables the watchdog.
11583
- * When disabled, the periodic check stops but bindings are still tracked.
11534
+ * Enables or disables the dangling binding warnings.
11535
+ * Bindings are still tracked and their playback state is still traced.
11584
11536
  */
11585
11537
  setEnabled = (enabled) => {
11586
11538
  this.enabled = enabled;
11587
11539
  if (enabled) {
11588
11540
  this.start();
11589
11541
  }
11590
- else {
11542
+ else if (this.state.callingState !== CallingState.JOINED) {
11591
11543
  this.stop();
11592
11544
  }
11593
11545
  };
@@ -11597,33 +11549,61 @@ class AudioBindingsWatchdog {
11597
11549
  dispose = () => {
11598
11550
  this.stop();
11599
11551
  this.bindings.clear();
11552
+ this.tracedStates.clear();
11600
11553
  this.unsubscribeCallingState();
11601
11554
  };
11602
11555
  start = () => {
11603
11556
  clearInterval(this.watchdogInterval);
11604
11557
  this.watchdogInterval = setInterval(() => {
11605
- const danglingUserIds = [];
11606
- for (const p of this.state.participants) {
11607
- if (p.isLocalParticipant)
11608
- continue;
11609
- const { audioStream, screenShareAudioStream, sessionId, userId, publishedTracks, } = p;
11610
- if (audioStream &&
11611
- publishedTracks.includes(TrackType.AUDIO) &&
11612
- !this.bindings.has(toBindingKey(sessionId))) {
11613
- danglingUserIds.push(userId);
11614
- }
11615
- if (screenShareAudioStream &&
11616
- publishedTracks.includes(TrackType.SCREEN_SHARE_AUDIO) &&
11617
- !this.bindings.has(toBindingKey(sessionId, 'screenShareAudioTrack'))) {
11618
- danglingUserIds.push(userId);
11619
- }
11558
+ if (this.enabled)
11559
+ this.warnAboutDanglingBindings();
11560
+ this.traceBindingStates();
11561
+ }, 3000);
11562
+ };
11563
+ warnAboutDanglingBindings = () => {
11564
+ const danglingUserIds = [];
11565
+ for (const p of this.state.participants) {
11566
+ if (p.isLocalParticipant)
11567
+ continue;
11568
+ const { audioStream, screenShareAudioStream, sessionId, userId, publishedTracks, } = p;
11569
+ if (audioStream &&
11570
+ publishedTracks.includes(TrackType.AUDIO) &&
11571
+ !this.bindings.has(toBindingKey(sessionId))) {
11572
+ danglingUserIds.push(userId);
11620
11573
  }
11621
- if (danglingUserIds.length > 0) {
11622
- const key = 'audioBinding.danglingWarning';
11623
- this.tracer.traceOnce(key, key, danglingUserIds);
11624
- this.logger.warn(`Dangling audio bindings detected. Did you forget to bind the audio element? user_ids: ${danglingUserIds}.`);
11574
+ if (screenShareAudioStream &&
11575
+ publishedTracks.includes(TrackType.SCREEN_SHARE_AUDIO) &&
11576
+ !this.bindings.has(toBindingKey(sessionId, 'screenShareAudioTrack'))) {
11577
+ danglingUserIds.push(userId);
11625
11578
  }
11626
- }, 3000);
11579
+ }
11580
+ if (danglingUserIds.length > 0) {
11581
+ const key = 'audioBinding.danglingWarning';
11582
+ this.tracer.traceOnce(key, key, danglingUserIds);
11583
+ this.logger.warn(`Dangling audio bindings detected. Did you forget to bind the audio element? user_ids: ${danglingUserIds}.`);
11584
+ }
11585
+ };
11586
+ traceBindingStates = () => {
11587
+ const changes = {};
11588
+ let hasChanges = false;
11589
+ for (const [key, element] of this.bindings) {
11590
+ const state = toElementState(element);
11591
+ const serialized = JSON.stringify(state);
11592
+ if (this.tracedStates.get(key) === serialized)
11593
+ continue;
11594
+ this.tracedStates.set(key, serialized);
11595
+ changes[key] = state;
11596
+ hasChanges = true;
11597
+ }
11598
+ for (const key of this.tracedStates.keys()) {
11599
+ if (this.bindings.has(key))
11600
+ continue;
11601
+ this.tracedStates.delete(key);
11602
+ changes[key] = null;
11603
+ hasChanges = true;
11604
+ }
11605
+ if (hasChanges)
11606
+ this.tracer.trace('audioBinding.state', changes);
11627
11607
  };
11628
11608
  stop = () => {
11629
11609
  clearInterval(this.watchdogInterval);
@@ -11902,7 +11882,9 @@ class TrackSubscriptionManager {
11902
11882
  class MediaPlaybackWatchdog {
11903
11883
  logger = videoLoggerSystem.getLogger('MediaPlaybackWatchdog');
11904
11884
  kind;
11885
+ traceInfo;
11905
11886
  isBlocked;
11887
+ onGiveUp;
11906
11888
  element;
11907
11889
  tracer;
11908
11890
  controller = new AbortController();
@@ -11914,8 +11896,14 @@ class MediaPlaybackWatchdog {
11914
11896
  constructor(opts) {
11915
11897
  this.element = opts.element;
11916
11898
  this.kind = opts.kind;
11899
+ this.traceInfo = {
11900
+ kind: opts.kind,
11901
+ sessionId: opts.sessionId,
11902
+ trackType: opts.trackType,
11903
+ };
11917
11904
  this.tracer = opts.tracer;
11918
11905
  this.isBlocked = opts.isBlocked ?? (() => false);
11906
+ this.onGiveUp = opts.onGiveUp ?? (() => { });
11919
11907
  this.attach();
11920
11908
  }
11921
11909
  attach = () => {
@@ -11933,6 +11921,23 @@ class MediaPlaybackWatchdog {
11933
11921
  this.controller.abort();
11934
11922
  this.clearTimers();
11935
11923
  };
11924
+ /**
11925
+ * Starts a fresh recovery cycle, e.g. after a media interruption ended.
11926
+ * Unlike a regular pause-triggered recovery, it also tries elements that
11927
+ * were skipped as blocked or not ready, since those conditions may have
11928
+ * cleared together with the interruption.
11929
+ */
11930
+ rearm = () => {
11931
+ if (this.disposed)
11932
+ return;
11933
+ this.clearTimers();
11934
+ this.attempt = 0;
11935
+ this.stopped = false;
11936
+ const { paused, srcObject, ended } = this.element;
11937
+ if (!paused || !srcObject || ended)
11938
+ return;
11939
+ this.pendingTimer = setTimeout(this.attemptPlay, 0);
11940
+ };
11936
11941
  clearTimers = () => {
11937
11942
  if (this.pendingTimer)
11938
11943
  clearTimeout(this.pendingTimer);
@@ -11954,17 +11959,34 @@ class MediaPlaybackWatchdog {
11954
11959
  if (this.element.paused)
11955
11960
  return;
11956
11961
  this.tracer.trace('mediaPlayback.recover.success', {
11957
- kind: this.kind,
11962
+ ...this.traceInfo,
11958
11963
  attempts: this.attempt,
11959
11964
  });
11960
11965
  this.attempt = 0;
11961
11966
  this.stopped = false;
11962
11967
  };
11968
+ /**
11969
+ * Media elements backed by a MediaStream fire `suspend` routinely (after
11970
+ * the initial load, on source swaps, on layer switches), so a `suspend`
11971
+ * that needs no recovery is ignored without a trace. An explicit `pause`
11972
+ * is rare and always traced, along with the reason recovery was skipped.
11973
+ */
11963
11974
  onPauseOrSuspend = (event) => {
11964
- if (this.disposed || this.stopped)
11975
+ if (this.disposed || this.stopped || this.pendingTimer)
11976
+ return;
11977
+ const skipReason = this.computeSkipReason();
11978
+ if (skipReason) {
11979
+ if (event.type === 'pause') {
11980
+ this.tracer.trace('mediaPlayback.paused', {
11981
+ ...this.traceInfo,
11982
+ reason: event.type,
11983
+ skipped: skipReason,
11984
+ });
11985
+ }
11965
11986
  return;
11987
+ }
11966
11988
  this.tracer.trace('mediaPlayback.paused', {
11967
- kind: this.kind,
11989
+ ...this.traceInfo,
11968
11990
  reason: event.type,
11969
11991
  });
11970
11992
  if (this.settleTimer)
@@ -11978,7 +12000,7 @@ class MediaPlaybackWatchdog {
11978
12000
  const skipReason = this.computeSkipReason();
11979
12001
  if (skipReason) {
11980
12002
  this.tracer.trace('mediaPlayback.recover.skipped', {
11981
- kind: this.kind,
12003
+ ...this.traceInfo,
11982
12004
  reason: skipReason,
11983
12005
  });
11984
12006
  return;
@@ -11987,9 +12009,10 @@ class MediaPlaybackWatchdog {
11987
12009
  this.stopped = true;
11988
12010
  this.clearTimers();
11989
12011
  this.tracer.trace('mediaPlayback.recover.giveUp', {
11990
- kind: this.kind,
12012
+ ...this.traceInfo,
11991
12013
  attempts: this.attempt,
11992
12014
  });
12015
+ this.onGiveUp();
11993
12016
  return;
11994
12017
  }
11995
12018
  const delay = this.attempt === 0 ? 0 : retryInterval(this.attempt);
@@ -12017,7 +12040,7 @@ class MediaPlaybackWatchdog {
12017
12040
  return;
12018
12041
  this.attempt += 1;
12019
12042
  this.tracer.trace('mediaPlayback.recover.attempt', {
12020
- kind: this.kind,
12043
+ ...this.traceInfo,
12021
12044
  attempt: this.attempt,
12022
12045
  });
12023
12046
  try {
@@ -12032,6 +12055,41 @@ class MediaPlaybackWatchdog {
12032
12055
  };
12033
12056
  }
12034
12057
 
12058
+ /**
12059
+ * Checks whether media playback and capture are likely interrupted: the page
12060
+ * is hidden, or WebKit reports the audio session as interrupted (a phone
12061
+ * call, Siri, or another app taking over audio).
12062
+ */
12063
+ const isMediaInterrupted = () => {
12064
+ if (typeof document === 'undefined')
12065
+ return false;
12066
+ return (document.visibilityState === 'hidden' ||
12067
+ navigator.audioSession?.state === 'interrupted');
12068
+ };
12069
+ /**
12070
+ * Invokes `onResumed` every time the page leaves a media interruption,
12071
+ * as defined by {@link isMediaInterrupted}.
12072
+ *
12073
+ * @param onResumed the callback to invoke.
12074
+ * @returns a function that stops watching.
12075
+ */
12076
+ const watchMediaInterruptions = (onResumed) => {
12077
+ if (typeof document === 'undefined')
12078
+ return () => { };
12079
+ let interrupted = isMediaInterrupted();
12080
+ const check = () => {
12081
+ const wasInterrupted = interrupted;
12082
+ interrupted = isMediaInterrupted();
12083
+ if (wasInterrupted && !interrupted)
12084
+ onResumed();
12085
+ };
12086
+ const controller = new AbortController();
12087
+ const { signal } = controller;
12088
+ document.addEventListener('visibilitychange', check, { signal });
12089
+ navigator.audioSession?.addEventListener('statechange', check, { signal });
12090
+ return () => controller.abort();
12091
+ };
12092
+
12035
12093
  /**
12036
12094
  * A manager class that handles dynascale related tasks like:
12037
12095
  *
@@ -12045,6 +12103,9 @@ class DynascaleManager {
12045
12103
  tracer;
12046
12104
  useWebAudio = false;
12047
12105
  audioContext;
12106
+ playbackWatchdogs = new Set();
12107
+ stopWatchingInterruptions;
12108
+ stopTracingAudioSession;
12048
12109
  trackSubscriptionManager;
12049
12110
  blockedAudioTracker;
12050
12111
  /**
@@ -12061,6 +12122,10 @@ class DynascaleManager {
12061
12122
  * Closes the audio context if it was created.
12062
12123
  */
12063
12124
  dispose = async () => {
12125
+ this.stopWatchingInterruptions?.();
12126
+ this.stopWatchingInterruptions = undefined;
12127
+ this.stopTracingAudioSession?.();
12128
+ this.stopTracingAudioSession = undefined;
12064
12129
  const context = this.audioContext;
12065
12130
  if (context && context.state !== 'closed') {
12066
12131
  document.removeEventListener('click', this.resumeAudioContext);
@@ -12200,10 +12265,12 @@ class DynascaleManager {
12200
12265
  // without prior user interaction:
12201
12266
  // https://developer.mozilla.org/en-US/docs/Web/Media/Autoplay_guide
12202
12267
  videoElement.muted = true;
12203
- const playbackWatchdog = new MediaPlaybackWatchdog({
12268
+ const playbackWatchdog = this.createPlaybackWatchdog({
12204
12269
  element: videoElement,
12205
12270
  kind: 'video',
12206
12271
  tracer: this.tracer,
12272
+ sessionId,
12273
+ trackType,
12207
12274
  });
12208
12275
  const trackKey = isVideoTrack ? 'videoStream' : 'screenShareStream';
12209
12276
  const streamSubscription = participant$
@@ -12231,7 +12298,7 @@ class DynascaleManager {
12231
12298
  publishedTracksSubscription?.unsubscribe();
12232
12299
  streamSubscription.unsubscribe();
12233
12300
  resizeObserver?.disconnect();
12234
- playbackWatchdog.dispose();
12301
+ this.disposePlaybackWatchdog(playbackWatchdog);
12235
12302
  };
12236
12303
  };
12237
12304
  /**
@@ -12249,6 +12316,7 @@ class DynascaleManager {
12249
12316
  const participant = this.callState.findParticipantBySessionId(sessionId);
12250
12317
  if (!participant || participant.isLocalParticipant)
12251
12318
  return;
12319
+ this.traceAudioSessionState();
12252
12320
  const participant$ = this.callState.participants$.pipe(map((ps) => ps.find((p) => p.sessionId === sessionId)), takeWhile((p) => !!p), distinctUntilChanged(), shareReplay({ bufferSize: 1, refCount: true }));
12253
12321
  const updateSinkId = (deviceId, audioContext) => {
12254
12322
  if (!deviceId)
@@ -12268,7 +12336,9 @@ class DynascaleManager {
12268
12336
  let sourceNode = undefined;
12269
12337
  let gainNode = undefined;
12270
12338
  let audioWatchdog = undefined;
12339
+ let blockedByGiveUp = false;
12271
12340
  const clearBlockedAudio = () => {
12341
+ blockedByGiveUp = false;
12272
12342
  if (!this.blockedAudioTracker.isBlocked(audioElement))
12273
12343
  return;
12274
12344
  this.blockedAudioTracker.markBlocked(audioElement, false);
@@ -12276,15 +12346,18 @@ class DynascaleManager {
12276
12346
  audioElement.addEventListener('playing', clearBlockedAudio);
12277
12347
  const isAudioTrack = trackType === 'audioTrack';
12278
12348
  const trackKey = isAudioTrack ? 'audioStream' : 'screenShareAudioStream';
12349
+ let sourceChangeTimer;
12279
12350
  const updateMediaStreamSubscription = participant$
12280
12351
  .pipe(distinctUntilKeyChanged(trackKey))
12281
12352
  .subscribe((p) => {
12282
12353
  const source = isAudioTrack ? p.audioStream : p.screenShareAudioStream;
12283
12354
  if (audioElement.srcObject === source)
12284
12355
  return;
12285
- setTimeout(() => {
12356
+ clearTimeout(sourceChangeTimer);
12357
+ sourceChangeTimer = setTimeout(() => {
12358
+ sourceChangeTimer = undefined;
12286
12359
  audioElement.srcObject = source ?? null;
12287
- audioWatchdog?.dispose();
12360
+ this.disposePlaybackWatchdog(audioWatchdog);
12288
12361
  audioWatchdog = undefined;
12289
12362
  if (!source) {
12290
12363
  clearBlockedAudio();
@@ -12315,15 +12388,25 @@ class DynascaleManager {
12315
12388
  this.tracer.trace('audioPlaybackError', e.message);
12316
12389
  if (e.name === 'NotAllowedError') {
12317
12390
  this.tracer.trace('audioPlaybackBlocked', null);
12391
+ blockedByGiveUp = false;
12318
12392
  this.blockedAudioTracker.markBlocked(audioElement, true, sessionId);
12319
12393
  }
12320
12394
  this.logger.warn(`Failed to play audio stream`, e);
12321
12395
  });
12322
- audioWatchdog = new MediaPlaybackWatchdog({
12396
+ audioWatchdog = this.createPlaybackWatchdog({
12323
12397
  element: audioElement,
12324
12398
  kind: 'audio',
12325
12399
  tracer: this.tracer,
12326
- isBlocked: () => this.blockedAudioTracker.isBlocked(audioElement),
12400
+ sessionId,
12401
+ trackType,
12402
+ // only an autoplay-policy block needs a user gesture; a give-up
12403
+ // block is for the UI and must not stop a re-armed cycle
12404
+ isBlocked: () => !blockedByGiveUp &&
12405
+ this.blockedAudioTracker.isBlocked(audioElement),
12406
+ onGiveUp: () => {
12407
+ blockedByGiveUp = true;
12408
+ this.blockedAudioTracker.markBlocked(audioElement, true, sessionId);
12409
+ },
12327
12410
  });
12328
12411
  }
12329
12412
  const { selectedDevice } = this.speaker.state;
@@ -12353,13 +12436,51 @@ class DynascaleManager {
12353
12436
  sinkIdSubscription?.unsubscribe();
12354
12437
  volumeSubscription.unsubscribe();
12355
12438
  updateMediaStreamSubscription.unsubscribe();
12439
+ clearTimeout(sourceChangeTimer);
12356
12440
  audioElement.srcObject = null;
12357
12441
  sourceNode?.disconnect();
12358
12442
  gainNode?.disconnect();
12359
- audioWatchdog?.dispose();
12443
+ this.disposePlaybackWatchdog(audioWatchdog);
12360
12444
  audioWatchdog = undefined;
12361
12445
  };
12362
12446
  };
12447
+ /**
12448
+ * Creates a playback watchdog that is re-armed whenever a media
12449
+ * interruption (backgrounding, phone call, Siri) ends.
12450
+ */
12451
+ createPlaybackWatchdog = (opts) => {
12452
+ const watchdog = new MediaPlaybackWatchdog(opts);
12453
+ this.playbackWatchdogs.add(watchdog);
12454
+ this.stopWatchingInterruptions ??= watchMediaInterruptions(this.rearmPlayback);
12455
+ return watchdog;
12456
+ };
12457
+ disposePlaybackWatchdog = (watchdog) => {
12458
+ if (!watchdog)
12459
+ return;
12460
+ watchdog.dispose();
12461
+ this.playbackWatchdogs.delete(watchdog);
12462
+ };
12463
+ rearmPlayback = () => {
12464
+ this.tracer.trace('mediaPlayback.rearm', this.playbackWatchdogs.size);
12465
+ this.playbackWatchdogs.forEach((watchdog) => watchdog.rearm());
12466
+ };
12467
+ /**
12468
+ * Traces the WebKit audio session state (initial value and changes) in
12469
+ * both playback modes, as it explains most iOS playback interruptions.
12470
+ */
12471
+ traceAudioSessionState = () => {
12472
+ const audioSession = navigator.audioSession;
12473
+ if (!audioSession || this.stopTracingAudioSession)
12474
+ return;
12475
+ const trace = () => {
12476
+ this.tracer.trace('audioSession.state', audioSession.state);
12477
+ };
12478
+ trace();
12479
+ audioSession.addEventListener('statechange', trace);
12480
+ this.stopTracingAudioSession = () => {
12481
+ audioSession.removeEventListener('statechange', trace);
12482
+ };
12483
+ };
12363
12484
  getOrCreateAudioContext = () => {
12364
12485
  if (!this.useWebAudio)
12365
12486
  return;
@@ -12382,7 +12503,6 @@ class DynascaleManager {
12382
12503
  audioSession.type = 'play-and-record';
12383
12504
  let isSessionInterrupted = false;
12384
12505
  audioSession.addEventListener('statechange', () => {
12385
- this.tracer.trace('audioSession.state', audioSession.state);
12386
12506
  if (audioSession.state === 'interrupted') {
12387
12507
  isSessionInterrupted = true;
12388
12508
  }
@@ -13941,9 +14061,22 @@ class DeviceManager {
13941
14061
  }
13942
14062
  const devices = getCurrentValue(this.listDevices()) || [];
13943
14063
  const currentDevice = this.findDevice(devices, selectedDevice) ??
13944
- createSyntheticDevice(selectedDevice, deviceKind);
14064
+ createSyntheticDevice(selectedDevice, deviceKind, this.getActiveTrackLabel(selectedDevice));
13945
14065
  writePreferences(currentDevice, deviceKey, muted, storageKey);
13946
14066
  }
14067
+ /**
14068
+ * Returns the label of the active track if it was captured from the given
14069
+ * device. Used as a fallback when the device list hasn't been enumerated yet,
14070
+ * so the persisted preference still carries the device label.
14071
+ */
14072
+ getActiveTrackLabel = (deviceId) => {
14073
+ const stream = this.state.rootMediaStream;
14074
+ const tracks = this.trackType === TrackType.AUDIO
14075
+ ? stream?.getAudioTracks()
14076
+ : stream?.getVideoTracks();
14077
+ const track = tracks?.find((t) => t.getSettings().deviceId === deviceId);
14078
+ return track?.label;
14079
+ };
13947
14080
  async applyPersistedPreferences(enabledInCallType, forceDisabled = false) {
13948
14081
  const deviceKey = this.trackType === TrackType.AUDIO ? 'microphone' : 'camera';
13949
14082
  const preferences = readPreferences(this.devicePersistence.storageKey);
@@ -16779,9 +16912,6 @@ class Call {
16779
16912
  options: statsOptions,
16780
16913
  subscriber: this.subscriber,
16781
16914
  publisher: this.publisher,
16782
- microphone: this.microphone,
16783
- camera: this.camera,
16784
- state: this.state,
16785
16915
  tracer: this.tracer,
16786
16916
  unifiedSessionId,
16787
16917
  });
@@ -19633,11 +19763,11 @@ class StreamClient {
19633
19763
  return await this.wsConnection.connect(this.defaultWSTimeout);
19634
19764
  };
19635
19765
  getSdkVersion = () => this.options.clientAppIdentifier?.sdkVersion ||
19636
- "1.61.0";
19766
+ "1.61.2";
19637
19767
  getUserAgent = () => {
19638
19768
  if (!this.cachedUserAgent) {
19639
19769
  const { clientAppIdentifier = {} } = this.options;
19640
- const { sdkName = 'js', sdkVersion = "1.61.0", ...extras } = clientAppIdentifier;
19770
+ const { sdkName = 'js', sdkVersion = "1.61.2", ...extras } = clientAppIdentifier;
19641
19771
  this.cachedUserAgent = [
19642
19772
  `stream-video-${sdkName}-v${sdkVersion}`,
19643
19773
  ...Object.entries(extras).map(([key, value]) => `${key}=${value}`),
@@ -20172,7 +20302,11 @@ class ClientEventReporter {
20172
20302
  return;
20173
20303
  }
20174
20304
  if (event.state === 'failed') {
20175
- this.emitPeerConnectionFailure(cid, role, 'DTLS_CONNECTIVITY_FAILED', 'DTLS connectivity checks failed');
20305
+ const iceConnected = event.iceConnectionState === 'connected' ||
20306
+ event.iceConnectionState === 'completed';
20307
+ this.emitPeerConnectionFailure(cid, role, iceConnected ? 'DTLS_CONNECTIVITY_FAILED' : 'ICE_CONNECTIVITY_FAILED', iceConnected
20308
+ ? 'DTLS connectivity checks failed'
20309
+ : 'ICE connectivity checks failed');
20176
20310
  return;
20177
20311
  }
20178
20312
  switch (event.state) {
@@ -20573,6 +20707,10 @@ class StreamVideoClient {
20573
20707
  const call = this.writeableStateStore.findCall(c.call.type, c.call.id);
20574
20708
  if (call) {
20575
20709
  call.updateFromCallStateResponse(c);
20710
+ // ring events missed while the WS was down aren't replayed
20711
+ reconcileRingState(call, 'ring-poll-api').catch((err) => {
20712
+ call.logger.error('Failed to reconcile the ring state after rewatch', err);
20713
+ });
20576
20714
  }
20577
20715
  }
20578
20716
  return;