@stream-io/video-client 1.44.6-beta.0 → 1.46.0

package/dist/src/Call.d.ts

@@ -804,6 +804,10 @@ export declare class Call {
      * @param trackType the kind of audio.
      */
     bindAudioElement: (audioElement: HTMLAudioElement, sessionId: string, trackType?: AudioTrackType) => (() => void) | undefined;
+    /**
+     * Plays all audio elements blocked by the browser's autoplay policy.
+     */
+    resumeAudio: () => Promise<void>;
     /**
      * Binds a DOM <img> element to this call's thumbnail (if enabled in settings).
      *

package/dist/src/coordinator/connection/types.d.ts

@@ -3,6 +3,7 @@ import { ConnectedEvent, UserRequest, VideoEvent } from '../../gen/coordinator';
 import { AllSfuEvents } from '../../rtc';
 import type { ConfigureLoggersOptions, LogLevel } from '@stream-io/logger';
 import type { DevicePersistenceOptions } from '../../devices/devicePersistence';
+import { InputDeviceStatus } from '../../devices';
 export type UR = Record<string, unknown>;
 export type User = (Omit<UserRequest, 'role'> & {
     type?: 'authenticated';
@@ -79,7 +80,27 @@ export type MicCaptureReportEvent = {
      */
     label?: string;
 };
-export type StreamVideoEvent = (VideoEvent | NetworkChangedEvent | ConnectionChangedEvent | TransportChangedEvent | ConnectionRecoveredEvent | MicCaptureReportEvent) & {
+export type DeviceDisconnectedEvent = {
+    type: 'device.disconnected';
+    call_cid: string;
+    /**
+     * The device status at the time it was disconnected.
+     */
+    status: InputDeviceStatus;
+    /**
+     * The disconnected device ID.
+     */
+    deviceId: string;
+    /**
+     * The human-readable label of the disconnected device.
+     */
+    label?: string;
+    /**
+     * The disconnected device kind.
+     */
+    kind: MediaDeviceKind;
+};
+export type StreamVideoEvent = (VideoEvent | NetworkChangedEvent | ConnectionChangedEvent | TransportChangedEvent | ConnectionRecoveredEvent | MicCaptureReportEvent | DeviceDisconnectedEvent) & {
     received_at?: string | Date;
 };
 export type StreamCallEvent = Extract<StreamVideoEvent, {

package/dist/src/devices/DeviceManager.d.ts

@@ -104,6 +104,7 @@ export declare abstract class DeviceManager<S extends DeviceManagerState<C>, C =
     private get mediaDeviceKind();
     private handleDisconnectedOrReplacedDevices;
     protected findDevice(devices: MediaDeviceInfo[], deviceId: string): MediaDeviceInfo | undefined;
+    private dispatchDeviceDisconnectedEvent;
     private persistPreference;
     protected applyPersistedPreferences(enabledInCallType: boolean): Promise<boolean>;
     private applyMutedState;

package/dist/src/gen/video/sfu/event/events.d.ts

@@ -574,6 +574,10 @@ export interface ParticipantJoined {
      * @generated from protobuf field: stream.video.sfu.models.Participant participant = 2;
      */
     participant?: Participant;
+    /**
+     * @generated from protobuf field: bool is_pinned = 3;
+     */
+    isPinned: boolean;
 }
 /**
  * ParticipantJoined is fired when a user leaves a call

package/dist/src/helpers/DynascaleManager.d.ts

@@ -42,6 +42,19 @@ export declare class DynascaleManager {
     private sfuClient;
     private pendingSubscriptionsUpdate;
     readonly audioBindingsWatchdog: AudioBindingsWatchdog | undefined;
+    /**
+     * Audio elements that were blocked by the browser's autoplay policy.
+     * These can be retried by calling `resumeAudio()` from a user gesture.
+     */
+    private blockedAudioElementsSubject;
+    /**
+     * Whether the browser's autoplay policy is blocking audio playback.
+     * Will be `true` when the browser blocks autoplay (e.g., no prior user interaction).
+     * Use `resumeAudio()` within a user gesture to unblock.
+     */
+    autoplayBlocked$: import("rxjs").Observable<boolean>;
+    private addBlockedAudioElement;
+    private removeBlockedAudioElement;
     private videoTrackSubscriptionOverridesSubject;
     videoTrackSubscriptionOverrides$: import("rxjs").Observable<VideoTrackSubscriptionOverrides>;
     incomingVideoSettings$: import("rxjs").Observable<{
@@ -121,6 +134,13 @@ export declare class DynascaleManager {
      * @returns a cleanup function that will unbind the audio element.
      */
     bindAudioElement: (audioElement: HTMLAudioElement, sessionId: string, trackType: AudioTrackType) => (() => void) | undefined;
+    /**
+     * Plays all audio elements blocked by the browser's autoplay policy.
+     * Must be called from within a user gesture (e.g., click handler).
+     *
+     * @returns a promise that resolves when all blocked elements have been retried.
+     */
+    resumeAudio: () => Promise<void>;
     private getOrCreateAudioContext;
     private resumeAudioContext;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stream-io/video-client",
-  "version": "1.44.6-beta.0",
+  "version": "1.46.0",
   "main": "dist/index.cjs.js",
   "module": "dist/index.es.js",
   "browser": "dist/index.browser.es.js",

package/src/Call.ts

@@ -656,15 +656,15 @@ export class Call {
             reasonToEndCallReason[
               rejectReason as keyof typeof reasonToEndCallReason
             ] ?? 'rejected';
-          globalThis.streamRNVideoSDK?.callingX?.endCall(this, endCallReason);
           await this.reject(rejectReason);
+          globalThis.streamRNVideoSDK?.callingX?.endCall(this, endCallReason);
         } else {
           // if reject was undefined, we still have to cancel the call automatically
           // when I am the creator and everyone else left the call
           const hasOtherParticipants = this.state.remoteParticipants.length > 0;
           if (this.isCreatedByMe && !hasOtherParticipants) {
-            globalThis.streamRNVideoSDK?.callingX?.endCall(this, 'canceled');
             await this.reject('cancel');
+            globalThis.streamRNVideoSDK?.callingX?.endCall(this, 'canceled');
           }
         }
       }
@@ -2967,6 +2967,13 @@ export class Call {
     };
   };
 
+  /**
+   * Plays all audio elements blocked by the browser's autoplay policy.
+   */
+  resumeAudio = () => {
+    return this.dynascaleManager.resumeAudio();
+  };
+
   /**
    * Binds a DOM <img> element to this call's thumbnail (if enabled in settings).
    *

package/src/coordinator/connection/types.ts

@@ -3,6 +3,7 @@ import { ConnectedEvent, UserRequest, VideoEvent } from '../../gen/coordinator';
 import { AllSfuEvents } from '../../rtc';
 import type { ConfigureLoggersOptions, LogLevel } from '@stream-io/logger';
 import type { DevicePersistenceOptions } from '../../devices/devicePersistence';
+import { InputDeviceStatus } from '../../devices';
 
 export type UR = Record<string, unknown>;
 
@@ -126,6 +127,27 @@ export type MicCaptureReportEvent = {
   label?: string;
 };
 
+export type DeviceDisconnectedEvent = {
+  type: 'device.disconnected';
+  call_cid: string;
+  /**
+   * The device status at the time it was disconnected.
+   */
+  status: InputDeviceStatus;
+  /**
+   * The disconnected device ID.
+   */
+  deviceId: string;
+  /**
+   * The human-readable label of the disconnected device.
+   */
+  label?: string;
+  /**
+   * The disconnected device kind.
+   */
+  kind: MediaDeviceKind;
+};
+
 export type StreamVideoEvent = (
   | VideoEvent
   | NetworkChangedEvent
@@ -133,6 +155,7 @@ export type StreamVideoEvent = (
   | TransportChangedEvent
   | ConnectionRecoveredEvent
   | MicCaptureReportEvent
+  | DeviceDisconnectedEvent
 ) & { received_at?: string | Date };
 
 // TODO: we should use WSCallEvent here but that needs fixing

package/src/devices/DeviceManager.ts

@@ -1,5 +1,6 @@
 import { combineLatest, firstValueFrom, Observable, pairwise } from 'rxjs';
 import { Call } from '../Call';
+import type { DeviceDisconnectedEvent } from '../coordinator/connection/types';
 import { TrackPublishOptions } from '../rtc';
 import { CallingState } from '../store';
 import { createSubscription, getCurrentValue } from '../store/rxUtils';
@@ -13,6 +14,7 @@ import { ScopedLogger, videoLoggerSystem } from '../logger';
 import { TrackType } from '../gen/video/sfu/models/models';
 import { deviceIds$ } from './devices';
 import {
+  hasPending,
   settled,
   withCancellation,
   withoutConcurrency,
@@ -543,6 +545,7 @@ export abstract class DeviceManager<
             }
 
             if (isDeviceDisconnected) {
+              this.dispatchDeviceDisconnectedEvent(prevDevice!);
               await this.disable();
               await this.select(undefined);
             }
@@ -553,7 +556,7 @@ export abstract class DeviceManager<
               ) {
                 await this.enable();
                 this.isTrackStoppedDueToTrackEnd = false;
-              } else {
+              } else if (!hasPending(this.statusChangeConcurrencyTag)) {
                 await this.applySettingsToStream();
               }
             }
@@ -573,6 +576,22 @@ export abstract class DeviceManager<
     return devices.find((d) => d.deviceId === deviceId && d.kind === kind);
   }
 
+  private dispatchDeviceDisconnectedEvent(device: MediaDeviceInfo) {
+    const event: DeviceDisconnectedEvent = {
+      type: 'device.disconnected',
+      call_cid: this.call.cid,
+      status: this.isTrackStoppedDueToTrackEnd
+        ? this.state.prevStatus
+        : this.state.status,
+      deviceId: device.deviceId,
+      label: device.label,
+      kind: device.kind,
+    };
+
+    this.call.tracer.trace('device.disconnected', event);
+    this.call.streamClient.dispatchEvent(event);
+  }
+
   private persistPreference(
     selectedDevice: string | undefined,
     status: InputDeviceStatus,

package/src/devices/MicrophoneManager.ts

@@ -158,6 +158,7 @@ export class MicrophoneManager extends AudioDeviceManager<MicrophoneManagerState
           const devices = await firstValueFrom(this.listDevices());
           const label = devices.find((d) => d.deviceId === deviceId)?.label;
 
+          let lastCapturesAudio: boolean | undefined;
           this.noAudioDetectorCleanup = createNoAudioDetector(mediaStream, {
             noAudioThresholdMs: this.silenceThresholdMs,
             emitIntervalMs: this.silenceThresholdMs,
@@ -169,7 +170,12 @@ export class MicrophoneManager extends AudioDeviceManager<MicrophoneManagerState
                 deviceId,
                 label,
               };
-              this.call.tracer.trace('mic.capture_report', event);
+
+              if (capturesAudio !== lastCapturesAudio) {
+                lastCapturesAudio = capturesAudio;
+                this.call.tracer.trace('mic.capture_report', event);
+              }
+
               this.call.streamClient.dispatchEvent(event);
             },
           });

package/src/devices/__tests__/DeviceManager.test.ts

@@ -380,6 +380,14 @@ describe('Device Manager', () => {
 
     expect(manager.state.selectedDevice).toBe(undefined);
     expect(manager.state.status).toBe('disabled');
+    expect(manager['call'].streamClient.dispatchEvent).toHaveBeenCalledWith({
+      type: 'device.disconnected',
+      call_cid: manager['call'].cid,
+      status: 'enabled',
+      deviceId: device.deviceId,
+      label: device.label,
+      kind: device.kind,
+    });
 
     vi.useRealTimers();
   });

package/src/devices/__tests__/mocks.ts

@@ -95,9 +95,13 @@ export const mockCall = (): Partial<Call> => {
     }),
   );
   return {
+    cid: 'default:test-call',
     state: callState,
     publish: vi.fn(),
     stopPublish: vi.fn(),
+    streamClient: fromPartial({
+      dispatchEvent: vi.fn(),
+    }),
     notifyNoiseCancellationStarting: vi.fn().mockResolvedValue(undefined),
     notifyNoiseCancellationStopped: vi.fn().mockResolvedValue(undefined),
     tracer: new Tracer('tests'),

package/src/events/__tests__/participant.test.ts

@@ -74,6 +74,47 @@ describe('Participant events', () => {
 
       expect(state.participants).toEqual([]);
     });
+
+    it('sets a server-side pin when isPinned is true', () => {
+      const state = new CallState();
+      state.setSortParticipantsBy(noopComparator());
+
+      const onParticipantJoined = watchParticipantJoined(state);
+      const now = Date.now();
+
+      onParticipantJoined({
+        // @ts-expect-error incomplete data
+        participant: {
+          userId: 'user-id',
+          sessionId: 'session-id',
+        },
+        isPinned: true,
+      });
+
+      const participant = state.findParticipantBySessionId('session-id');
+      expect(participant?.pin).toBeDefined();
+      expect(participant?.pin?.isLocalPin).toBe(false);
+      expect(participant?.pin?.pinnedAt).toBeGreaterThanOrEqual(now);
+    });
+
+    it('does not set a pin when isPinned is false', () => {
+      const state = new CallState();
+      state.setSortParticipantsBy(noopComparator());
+
+      const onParticipantJoined = watchParticipantJoined(state);
+
+      onParticipantJoined({
+        // @ts-expect-error incomplete data
+        participant: {
+          userId: 'user-id',
+          sessionId: 'session-id',
+        },
+        isPinned: false,
+      });
+
+      const participant = state.findParticipantBySessionId('session-id');
+      expect(participant?.pin).toBeUndefined();
+    });
   });
 
   describe('orphaned tracks reconciliation', () => {

package/src/events/participant.ts

@@ -38,6 +38,7 @@ export const watchParticipantJoined = (state: CallState) => {
         StreamVideoParticipantPatch | undefined,
         Partial<StreamVideoParticipant>
       >(participant, orphanedTracks, {
+        ...(e.isPinned && { pin: { isLocalPin: false, pinnedAt: Date.now() } }),
         viewportVisibilityState: {
           videoTrack: VisibilityState.UNKNOWN,
           screenShareTrack: VisibilityState.UNKNOWN,

package/src/gen/video/sfu/event/events.ts

@@ -625,6 +625,10 @@ export interface ParticipantJoined {
    * @generated from protobuf field: stream.video.sfu.models.Participant participant = 2;
    */
   participant?: Participant;
+  /**
+   * @generated from protobuf field: bool is_pinned = 3;
+   */
+  isPinned: boolean;
 }
 /**
  * ParticipantJoined is fired when a user leaves a call
@@ -1557,6 +1561,7 @@ class ParticipantJoined$Type extends MessageType<ParticipantJoined> {
     super('stream.video.sfu.event.ParticipantJoined', [
       { no: 1, name: 'call_cid', kind: 'scalar', T: 9 /*ScalarType.STRING*/ },
       { no: 2, name: 'participant', kind: 'message', T: () => Participant },
+      { no: 3, name: 'is_pinned', kind: 'scalar', T: 8 /*ScalarType.BOOL*/ },
     ]);
   }
 }

package/src/helpers/AudioBindingsWatchdog.ts

@@ -3,6 +3,7 @@ import { CallingState, CallState } from '../store';
 import { createSubscription } from '../store/rxUtils';
 import { videoLoggerSystem } from '../logger';
 import { Tracer } from '../stats';
+import { TrackType } from '../gen/video/sfu/models/models';
 
 const toBindingKey = (
   sessionId: string,
@@ -91,12 +92,23 @@ export class AudioBindingsWatchdog {
       const danglingUserIds: string[] = [];
       for (const p of this.state.participants) {
         if (p.isLocalParticipant) continue;
-        const { audioStream, screenShareAudioStream, sessionId, userId } = p;
-        if (audioStream && !this.bindings.has(toBindingKey(sessionId))) {
+        const {
+          audioStream,
+          screenShareAudioStream,
+          sessionId,
+          userId,
+          publishedTracks,
+        } = p;
+        if (
+          audioStream &&
+          publishedTracks.includes(TrackType.AUDIO) &&
+          !this.bindings.has(toBindingKey(sessionId))
+        ) {
           danglingUserIds.push(userId);
         }
         if (
           screenShareAudioStream &&
+          publishedTracks.includes(TrackType.SCREEN_SHARE_AUDIO) &&
           !this.bindings.has(toBindingKey(sessionId, 'screenShareAudioTrack'))
         ) {
           danglingUserIds.push(userId);

package/src/helpers/DynascaleManager.ts

@@ -79,6 +79,40 @@ export class DynascaleManager {
   private pendingSubscriptionsUpdate: NodeJS.Timeout | null = null;
   readonly audioBindingsWatchdog: AudioBindingsWatchdog | undefined;
 
+  /**
+   * Audio elements that were blocked by the browser's autoplay policy.
+   * These can be retried by calling `resumeAudio()` from a user gesture.
+   */
+  private blockedAudioElementsSubject = new BehaviorSubject<
+    Set<HTMLAudioElement>
+  >(new Set());
+
+  /**
+   * Whether the browser's autoplay policy is blocking audio playback.
+   * Will be `true` when the browser blocks autoplay (e.g., no prior user interaction).
+   * Use `resumeAudio()` within a user gesture to unblock.
+   */
+  autoplayBlocked$ = this.blockedAudioElementsSubject.pipe(
+    map((elements) => elements.size > 0),
+    distinctUntilChanged(),
+  );
+
+  private addBlockedAudioElement = (audioElement: HTMLAudioElement) => {
+    setCurrentValue(this.blockedAudioElementsSubject, (elements) => {
+      const next = new Set(elements);
+      next.add(audioElement);
+      return next;
+    });
+  };
+
+  private removeBlockedAudioElement = (audioElement: HTMLAudioElement) => {
+    setCurrentValue(this.blockedAudioElementsSubject, (elements) => {
+      const nextElements = new Set(elements);
+      nextElements.delete(audioElement);
+      return nextElements;
+    });
+  };
+
   private videoTrackSubscriptionOverridesSubject =
     new BehaviorSubject<VideoTrackSubscriptionOverrides>({});
 
@@ -136,6 +170,7 @@ export class DynascaleManager {
       clearTimeout(this.pendingSubscriptionsUpdate);
     }
     this.audioBindingsWatchdog?.dispose();
+    setCurrentValue(this.blockedAudioElementsSubject, new Set());
     const context = this.audioContext;
     if (context && context.state !== 'closed') {
       document.removeEventListener('click', this.resumeAudioContext);
@@ -575,7 +610,10 @@ export class DynascaleManager {
 
         setTimeout(() => {
           audioElement.srcObject = source ?? null;
-          if (!source) return;
+          if (!source) {
+            this.removeBlockedAudioElement(audioElement);
+            return;
+          }
 
           // Safari has a special quirk that prevents playing audio until the user
           // interacts with the page or focuses on the tab where the call happens.
@@ -599,6 +637,10 @@ export class DynascaleManager {
             audioElement.muted = false;
             audioElement.play().catch((e) => {
               this.tracer.trace('audioPlaybackError', e.message);
+              if (e.name === 'NotAllowedError') {
+                this.tracer.trace('audioPlaybackBlocked', null);
+                this.addBlockedAudioElement(audioElement);
+              }
               this.logger.warn(`Failed to play audio stream`, e);
             });
           }
@@ -628,6 +670,7 @@ export class DynascaleManager {
 
     return () => {
       this.audioBindingsWatchdog?.unregister(sessionId, trackType);
+      this.removeBlockedAudioElement(audioElement);
       sinkIdSubscription?.unsubscribe();
       volumeSubscription.unsubscribe();
       updateMediaStreamSubscription.unsubscribe();
@@ -637,6 +680,34 @@ export class DynascaleManager {
     };
   };
 
+  /**
+   * Plays all audio elements blocked by the browser's autoplay policy.
+   * Must be called from within a user gesture (e.g., click handler).
+   *
+   * @returns a promise that resolves when all blocked elements have been retried.
+   */
+  resumeAudio = async () => {
+    this.tracer.trace('resumeAudio', null);
+    const blocked = new Set<HTMLAudioElement>();
+    await Promise.all(
+      Array.from(
+        getCurrentValue(this.blockedAudioElementsSubject),
+        async (el) => {
+          try {
+            if (el.srcObject) {
+              await el.play();
+            }
+          } catch {
+            this.logger.warn(`Can't resume audio for element: `, el);
+            blocked.add(el);
+          }
+        },
+      ),
+    );
+
+    setCurrentValue(this.blockedAudioElementsSubject, blocked);
+  };
+
   private getOrCreateAudioContext = (): AudioContext | undefined => {
     if (!this.useWebAudio) return;
     if (this.audioContext) return this.audioContext;

package/src/helpers/RNSpeechDetector.ts

@@ -23,11 +23,15 @@ export class RNSpeechDetector {
           : await navigator.mediaDevices.getUserMedia({ audio: true });
       this.audioStream = audioStream;
 
-      this.pc1.addEventListener('icecandidate', async (e) => {
-        await this.pc2.addIceCandidate(e.candidate);
+      this.pc1.addEventListener('icecandidate', (e) => {
+        this.pc2.addIceCandidate(e.candidate).catch(() => {
+          // do nothing
+        });
       });
       this.pc2.addEventListener('icecandidate', async (e) => {
-        await this.pc1.addIceCandidate(e.candidate);
+        this.pc1.addIceCandidate(e.candidate).catch(() => {
+          // do nothing
+        });
       });
       this.pc2.addEventListener('track', (e) => {
         e.streams[0].getTracks().forEach((track) => {

package/src/helpers/__tests__/AudioBindingsWatchdog.test.ts

@@ -11,6 +11,7 @@ import { StreamClient } from '../../coordinator/connection/client';
 import { CallingState, StreamVideoWriteableStateStore } from '../../store';
 import { noopComparator } from '../../sorting';
 import { fromPartial } from '@total-typescript/shoehorn';
+import { TrackType } from '../../gen/video/sfu/models/models';
 
 describe('AudioBindingsWatchdog', () => {
   let watchdog: AudioBindingsWatchdog;
@@ -44,12 +45,17 @@ describe('AudioBindingsWatchdog', () => {
       screenShareAudioStream?: MediaStream;
     },
   ) => {
+    const publishedTracks = [];
+    if (streams?.audioStream) publishedTracks.push(TrackType.AUDIO);
+    if (streams?.screenShareAudioStream) {
+      publishedTracks.push(TrackType.SCREEN_SHARE_AUDIO);
+    }
     call.state.updateOrAddParticipant(
       sessionId,
       fromPartial({
         userId,
         sessionId,
-        publishedTracks: [],
+        publishedTracks,
         ...streams,
       }),
     );
@@ -233,6 +239,26 @@ describe('AudioBindingsWatchdog', () => {
     expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('user-1'));
   });
 
+  it('should not warn when audioStream exists but audio is not published', () => {
+    // @ts-expect-error private property
+    const warnSpy = vi.spyOn(watchdog.logger, 'warn');
+
+    call.state.updateOrAddParticipant(
+      'session-1',
+      fromPartial({
+        userId: 'user-1',
+        sessionId: 'session-1',
+        publishedTracks: [],
+        audioStream: new MediaStream(),
+      }),
+    );
+
+    call.state.setCallingState(CallingState.JOINED);
+    vi.advanceTimersByTime(3000);
+
+    expect(warnSpy).not.toHaveBeenCalled();
+  });
+
   it('should not warn when screenShareAudio element is bound', () => {
     // @ts-expect-error private property
     const warnSpy = vi.spyOn(watchdog.logger, 'warn');