@stream-io/video-client 0.3.13 → 0.3.15

package/dist/src/devices/CameraManager.d.ts

@@ -28,6 +28,5 @@ export declare class CameraManager extends InputMediaDeviceManager<CameraManager
     protected getStream(constraints: MediaTrackConstraints): Promise<MediaStream>;
     protected publishStream(stream: MediaStream): Promise<void>;
     protected stopPublishStream(stopTracks: boolean): Promise<void>;
-    protected muteTracks(): void;
-    protected unmuteTracks(): void;
+    protected getTrack(): MediaStreamTrack | undefined;
 }

package/dist/src/devices/InputMediaDeviceManager.d.ts

@@ -1,9 +1,12 @@
 import { Observable } from 'rxjs';
 import { Call } from '../Call';
 import { InputMediaDeviceManagerState } from './InputMediaDeviceManagerState';
+import { Logger } from '../coordinator/connection/types';
+import { TrackType } from '../gen/video/sfu/models/models';
 export declare abstract class InputMediaDeviceManager<T extends InputMediaDeviceManagerState> {
     protected readonly call: Call;
     readonly state: T;
+    protected readonly trackType: TrackType;
     /**
      * @internal
      */
@@ -12,7 +15,8 @@ export declare abstract class InputMediaDeviceManager<T extends InputMediaDevice
      * @internal
      */
     disablePromise?: Promise<void>;
-    constructor(call: Call, state: T);
+    logger: Logger;
+    constructor(call: Call, state: T, trackType: TrackType);
     /**
      * Lists the available audio/video devices
      *
@@ -54,8 +58,11 @@ export declare abstract class InputMediaDeviceManager<T extends InputMediaDevice
     protected abstract getStream(constraints: MediaTrackConstraints): Promise<MediaStream>;
     protected abstract publishStream(stream: MediaStream): Promise<void>;
     protected abstract stopPublishStream(stopTracks: boolean): Promise<void>;
-    protected abstract muteTracks(): void;
-    protected abstract unmuteTracks(): void;
+    protected abstract getTrack(): undefined | MediaStreamTrack;
     private muteStream;
+    private muteTrack;
+    private unmuteTrack;
+    private stopTrack;
+    private muteLocalStream;
     private unmuteStream;
 }

package/dist/src/devices/MicrophoneManager.d.ts

@@ -8,6 +8,5 @@ export declare class MicrophoneManager extends InputMediaDeviceManager<Microphon
     protected getStream(constraints: MediaTrackConstraints): Promise<MediaStream>;
     protected publishStream(stream: MediaStream): Promise<void>;
     protected stopPublishStream(stopTracks: boolean): Promise<void>;
-    protected muteTracks(): void;
-    protected unmuteTracks(): void;
+    protected getTrack(): MediaStreamTrack | undefined;
 }

package/dist/src/rtc/Publisher.d.ts

@@ -31,6 +31,14 @@ export declare class Publisher {
     private transceiverInitOrder;
     private readonly trackKindMapping;
     private readonly trackLayersCache;
+    /**
+     * A map keeping track of track types that were published to the SFU.
+     * This map shouldn't be cleared when unpublishing a track, as it is used
+     * to determine whether a track was published before.
+     *
+     * @private
+     */
+    private readonly trackTypePublishHistory;
     private readonly isDtxEnabled;
     private readonly isRedEnabled;
     private readonly preferredVideoCodec?;
@@ -85,6 +93,20 @@ export declare class Publisher {
      * @param trackType the track type to check.
      */
     isPublishing: (trackType: TrackType) => boolean;
+    /**
+     * Returns true if the given track type was ever published to the SFU.
+     * Contrary to `isPublishing`, this method returns true if a certain
+     * track type was published before, even if it is currently unpublished.
+     *
+     * @param trackType the track type to check.
+     */
+    hasEverPublished: (trackType: TrackType) => boolean;
+    /**
+     * Returns true if the given track type is currently live
+     *
+     * @param trackType the track type to check.
+     */
+    isLive: (trackType: TrackType) => boolean;
     private notifyTrackMuteStateChanged;
     /**
      * Stops publishing all tracks and stop all tracks.

package/dist/version.d.ts

@@ -1 +1 @@
-export declare const version = "0.3.13";
+export declare const version = "0.3.15";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stream-io/video-client",
-  "version": "0.3.13",
+  "version": "0.3.15",
   "packageManager": "yarn@3.2.4",
   "main": "dist/index.cjs.js",
   "module": "dist/index.es.js",

package/src/Call.ts

@@ -277,6 +277,41 @@ export class Call {
 
     this.camera = new CameraManager(this);
     this.microphone = new MicrophoneManager(this);
+
+    // FIXME OL: disable soft-mutes as they are not working properly
+    // this.state.localParticipant$.subscribe(async (p) => {
+    //   if (!this.publisher) return;
+    //   // Mute via device manager
+    //   // If integrator doesn't use device manager, we mute using stopPublish
+    //   if (
+    //     this.publisher.hasEverPublished(TrackType.VIDEO) &&
+    //     this.publisher.isPublishing(TrackType.VIDEO) &&
+    //     !p?.publishedTracks.includes(TrackType.VIDEO)
+    //   ) {
+    //     this.logger(
+    //       'info',
+    //       `Local participant's video track is muted remotely`,
+    //     );
+    //     await this.camera.disable();
+    //     if (this.publisher.isPublishing(TrackType.VIDEO)) {
+    //       await this.stopPublish(TrackType.VIDEO);
+    //     }
+    //   }
+    //   if (
+    //     this.publisher.hasEverPublished(TrackType.AUDIO) &&
+    //     this.publisher.isPublishing(TrackType.AUDIO) &&
+    //     !p?.publishedTracks.includes(TrackType.AUDIO)
+    //   ) {
+    //     this.logger(
+    //       'info',
+    //       `Local participant's audio track is muted remotely`,
+    //     );
+    //     await this.microphone.disable();
+    //     if (this.publisher.isPublishing(TrackType.AUDIO)) {
+    //       await this.stopPublish(TrackType.AUDIO);
+    //     }
+    //   }
+    // });
     this.speaker = new SpeakerManager();
   }
 
@@ -309,10 +344,50 @@ export class Call {
           const hasPermission = this.permissionsContext.hasPermission(
             permission as OwnCapability,
           );
-          if (!hasPermission && this.publisher.isPublishing(trackType)) {
-            this.stopPublish(trackType).catch((err) => {
-              this.logger('error', `Error stopping publish ${trackType}`, err);
-            });
+          if (
+            !hasPermission &&
+            (this.publisher.isPublishing(trackType) ||
+              this.publisher.isLive(trackType))
+          ) {
+            // Stop tracks, then notify device manager
+            this.stopPublish(trackType)
+              .catch((err) => {
+                this.logger(
+                  'error',
+                  `Error stopping publish ${trackType}`,
+                  err,
+                );
+              })
+              .then(() => {
+                if (
+                  trackType === TrackType.VIDEO &&
+                  this.camera.state.status === 'enabled'
+                ) {
+                  this.camera
+                    .disable()
+                    .catch((err) =>
+                      this.logger(
+                        'error',
+                        `Error disabling camera after pemission revoked`,
+                        err,
+                      ),
+                    );
+                }
+                if (
+                  trackType === TrackType.AUDIO &&
+                  this.microphone.state.status === 'enabled'
+                ) {
+                  this.microphone
+                    .disable()
+                    .catch((err) =>
+                      this.logger(
+                        'error',
+                        `Error disabling microphone after pemission revoked`,
+                        err,
+                      ),
+                    );
+                }
+              });
           }
         }
       }),
@@ -1112,7 +1187,10 @@ export class Call {
    * @param stopTrack if `true` the track will be stopped, else it will be just disabled
    */
   stopPublish = async (trackType: TrackType, stopTrack: boolean = true) => {
-    this.logger('info', `stopPublish ${TrackType[trackType]}`);
+    this.logger(
+      'info',
+      `stopPublish ${TrackType[trackType]}, stop tracks: ${stopTrack}`,
+    );
     await this.publisher?.unpublishStream(trackType, stopTrack);
   };
 

package/src/StreamSfuClient.ts

@@ -396,5 +396,9 @@ const retryable = async <I extends object, O extends SfuResponseWithError>(
     retryAttempt < MAX_RETRIES
   );
 
+  if (rpcCallResult.response.error) {
+    throw rpcCallResult.response.error;
+  }
+
   return rpcCallResult;
 };

package/src/coordinator/connection/location.ts

@@ -5,7 +5,7 @@ const HINT_URL = `https://hint.stream-io-video.com/`;
 
 export const getLocationHint = async (
   hintUrl: string = HINT_URL,
-  timeout: number = 1500,
+  timeout: number = 2000,
 ) => {
   const abortController = new AbortController();
   const timeoutId = setTimeout(() => abortController.abort(), timeout);
@@ -18,7 +18,7 @@ export const getLocationHint = async (
     logger('debug', `Location header: ${awsPop}`);
     return awsPop.substring(0, 3); // AMS1-P2 -> AMS
   } catch (e) {
-    logger('error', `Failed to get location hint from ${HINT_URL}`, e);
+    logger('warn', `Failed to get location hint from ${HINT_URL}`, e);
     return 'ERR';
   } finally {
     clearTimeout(timeoutId);

package/src/devices/CameraManager.ts

@@ -12,7 +12,7 @@ export class CameraManager extends InputMediaDeviceManager<CameraManagerState> {
   };
 
   constructor(call: Call) {
-    super(call, new CameraManagerState());
+    super(call, new CameraManagerState(), TrackType.VIDEO);
   }
 
   /**
@@ -59,6 +59,10 @@ export class CameraManager extends InputMediaDeviceManager<CameraManagerState> {
         height !== this.targetResolution.height
       )
         await this.applySettingsToStream();
+      this.logger(
+        'debug',
+        `${width}x${height} target resolution applied to media stream`,
+      );
     }
   }
 
@@ -85,12 +89,7 @@ export class CameraManager extends InputMediaDeviceManager<CameraManagerState> {
     return this.call.stopPublish(TrackType.VIDEO, stopTracks);
   }
 
-  protected muteTracks(): void {
-    this.state.mediaStream
-      ?.getVideoTracks()
-      .forEach((t) => (t.enabled = false));
-  }
-  protected unmuteTracks(): void {
-    this.state.mediaStream?.getVideoTracks().forEach((t) => (t.enabled = true));
+  protected getTrack() {
+    return this.state.mediaStream?.getVideoTracks()[0];
   }
 }

package/src/devices/InputMediaDeviceManager.ts

@@ -2,8 +2,10 @@ import { Observable } from 'rxjs';
 import { Call } from '../Call';
 import { CallingState } from '../store';
 import { InputMediaDeviceManagerState } from './InputMediaDeviceManagerState';
-import { disposeOfMediaStream } from './devices';
 import { isReactNative } from '../helpers/platforms';
+import { Logger } from '../coordinator/connection/types';
+import { getLogger } from '../logger';
+import { TrackType } from '../gen/video/sfu/models/models';
 
 export abstract class InputMediaDeviceManager<
   T extends InputMediaDeviceManagerState,
@@ -16,7 +18,15 @@ export abstract class InputMediaDeviceManager<
    * @internal
    */
   disablePromise?: Promise<void>;
-  constructor(protected readonly call: Call, public readonly state: T) {}
+  logger: Logger;
+
+  constructor(
+    protected readonly call: Call,
+    public readonly state: T,
+    protected readonly trackType: TrackType,
+  ) {
+    this.logger = getLogger([`${TrackType[trackType].toLowerCase()} manager`]);
+  }
 
   /**
    * Lists the available audio/video devices
@@ -129,32 +139,68 @@ export abstract class InputMediaDeviceManager<
 
   protected abstract stopPublishStream(stopTracks: boolean): Promise<void>;
 
-  protected abstract muteTracks(): void;
-
-  protected abstract unmuteTracks(): void;
+  protected abstract getTrack(): undefined | MediaStreamTrack;
 
   private async muteStream(stopTracks: boolean = true) {
     if (!this.state.mediaStream) {
       return;
     }
+    this.logger('debug', `${stopTracks ? 'Stopping' : 'Disabling'} stream`);
     if (this.call.state.callingState === CallingState.JOINED) {
       await this.stopPublishStream(stopTracks);
-    } else if (this.state.mediaStream) {
-      stopTracks
-        ? disposeOfMediaStream(this.state.mediaStream)
-        : this.muteTracks();
     }
-    if (stopTracks) {
+    this.muteLocalStream(stopTracks);
+    if (this.getTrack()?.readyState === 'ended') {
+      // @ts-expect-error release() is present in react-native-webrtc and must be called to dispose the stream
+      if (typeof this.state.mediaStream.release === 'function') {
+        // @ts-expect-error
+        this.state.mediaStream.release();
+      }
       this.state.setMediaStream(undefined);
     }
   }
 
+  private muteTrack() {
+    const track = this.getTrack();
+    if (!track || !track.enabled) {
+      return;
+    }
+    track.enabled = false;
+  }
+
+  private unmuteTrack() {
+    const track = this.getTrack();
+    if (!track || track.enabled) {
+      return;
+    }
+    track.enabled = true;
+  }
+
+  private stopTrack() {
+    const track = this.getTrack();
+    if (!track || track.readyState === 'ended') {
+      return;
+    }
+    track.stop();
+  }
+
+  private muteLocalStream(stopTracks: boolean) {
+    if (!this.state.mediaStream) {
+      return;
+    }
+    stopTracks ? this.stopTrack() : this.muteTrack();
+  }
+
   private async unmuteStream() {
+    this.logger('debug', 'Starting stream');
     let stream: MediaStream;
-    if (this.state.mediaStream) {
+    if (this.state.mediaStream && this.getTrack()?.readyState === 'live') {
       stream = this.state.mediaStream;
-      this.unmuteTracks();
+      this.unmuteTrack();
     } else {
+      if (this.state.mediaStream) {
+        this.stopTrack();
+      }
       const constraints = { deviceId: this.state.selectedDevice };
       stream = await this.getStream(constraints);
     }

package/src/devices/MicrophoneManager.ts

@@ -7,7 +7,7 @@ import { TrackType } from '../gen/video/sfu/models/models';
 
 export class MicrophoneManager extends InputMediaDeviceManager<MicrophoneManagerState> {
   constructor(call: Call) {
-    super(call, new MicrophoneManagerState());
+    super(call, new MicrophoneManagerState(), TrackType.AUDIO);
   }
 
   protected getDevices(): Observable<MediaDeviceInfo[]> {
@@ -25,12 +25,7 @@ export class MicrophoneManager extends InputMediaDeviceManager<MicrophoneManager
     return this.call.stopPublish(TrackType.AUDIO, stopTracks);
   }
 
-  protected muteTracks(): void {
-    this.state.mediaStream
-      ?.getAudioTracks()
-      .forEach((t) => (t.enabled = false));
-  }
-  protected unmuteTracks(): void {
-    this.state.mediaStream?.getAudioTracks().forEach((t) => (t.enabled = true));
+  protected getTrack() {
+    return this.state.mediaStream?.getAudioTracks()[0];
   }
 }

package/src/devices/__tests__/InputMediaDeviceManager.test.ts

@@ -7,14 +7,7 @@ import { mockCall, mockVideoDevices, mockVideoStream } from './mocks';
 import { InputMediaDeviceManager } from '../InputMediaDeviceManager';
 import { InputMediaDeviceManagerState } from '../InputMediaDeviceManagerState';
 import { of } from 'rxjs';
-import { disposeOfMediaStream } from '../devices';
-
-vi.mock('../devices.ts', () => {
-  console.log('MOCKING devices');
-  return {
-    disposeOfMediaStream: vi.fn(),
-  };
-});
+import { TrackType } from '../../gen/video/sfu/models/models';
 
 vi.mock('../../Call.ts', () => {
   console.log('MOCKING Call');
@@ -32,11 +25,10 @@ class TestInputMediaDeviceManager extends InputMediaDeviceManager<TestInputMedia
   public getStream = vi.fn(() => Promise.resolve(mockVideoStream()));
   public publishStream = vi.fn();
   public stopPublishStream = vi.fn();
-  public muteTracks = vi.fn();
-  public unmuteTracks = vi.fn();
+  public getTrack = () => this.state.mediaStream!.getVideoTracks()[0];
 
   constructor(call: Call) {
-    super(call, new TestInputMediaDeviceManagerState());
+    super(call, new TestInputMediaDeviceManagerState(), TrackType.VIDEO);
   }
 }
 
@@ -135,11 +127,12 @@ describe('InputMediaDeviceManager.test', () => {
   it('select device when status is enabled', async () => {
     await manager.enable();
     const prevStream = manager.state.mediaStream;
+    vi.spyOn(prevStream!.getVideoTracks()[0], 'stop');
 
     const deviceId = mockVideoDevices[1].deviceId;
     await manager.select(deviceId);
 
-    expect(disposeOfMediaStream).toHaveBeenCalledWith(prevStream);
+    expect(prevStream!.getVideoTracks()[0].stop).toHaveBeenCalledWith();
   });
 
   it('select device when status is enabled and in call', async () => {

package/src/devices/__tests__/mocks.ts

@@ -93,6 +93,10 @@ export const mockVideoStream = () => {
       height: 720,
     }),
     enabled: true,
+    readyState: 'live',
+    stop: () => {
+      track.readyState = 'eneded';
+    },
   };
   return {
     getVideoTracks: () => [track],

package/src/helpers/__tests__/DynascaleManager.test.ts

@@ -53,6 +53,7 @@ describe('DynascaleManager', () => {
       call.state.updateOrAddParticipant('session-id', {
         userId: 'user-id',
         sessionId: 'session-id',
+        publishedTracks: [],
       });
 
       const element = document.createElement('div');
@@ -113,6 +114,7 @@ describe('DynascaleManager', () => {
       call.state.updateOrAddParticipant('session-id', {
         userId: 'user-id',
         sessionId: 'session-id',
+        publishedTracks: [],
       });
 
       // @ts-ignore
@@ -120,6 +122,7 @@ describe('DynascaleManager', () => {
         userId: 'user-id-local',
         sessionId: 'session-id-local',
         isLocalParticipant: true,
+        publishedTracks: [],
       });
 
       const cleanup = dynascaleManager.bindAudioElement(

package/src/rtc/Publisher.ts

@@ -86,6 +86,15 @@ export class Publisher {
     [TrackType.UNSPECIFIED]: undefined,
   };
 
+  /**
+   * A map keeping track of track types that were published to the SFU.
+   * This map shouldn't be cleared when unpublishing a track, as it is used
+   * to determine whether a track was published before.
+   *
+   * @private
+   */
+  private readonly trackTypePublishHistory = new Map<TrackType, boolean>();
+
   private readonly isDtxEnabled: boolean;
   private readonly isRedEnabled: boolean;
   private readonly preferredVideoCodec?: string;
@@ -253,6 +262,9 @@ export class Publisher {
       // by an external factor as permission revokes, device disconnected, etc.
       // keep in mind that `track.stop()` doesn't trigger this event.
       track.addEventListener('ended', handleTrackEnded);
+      if (!track.enabled) {
+        track.enabled = true;
+      }
 
       transceiver = this.pc.addTransceiver(track, {
         direction: 'sendonly',
@@ -266,6 +278,7 @@ export class Publisher {
       logger('debug', `Added ${TrackType[trackType]} transceiver`);
       this.transceiverInitOrder.push(trackType);
       this.transceiverRegistry[trackType] = transceiver;
+      this.trackTypePublishHistory.set(trackType, true);
 
       if ('setCodecPreferences' in transceiver && codecPreferences) {
         logger(
@@ -310,17 +323,24 @@ export class Publisher {
     if (
       transceiver &&
       transceiver.sender.track &&
-      transceiver.sender.track.readyState === 'live'
+      (stopTrack
+        ? transceiver.sender.track.readyState === 'live'
+        : transceiver.sender.track.enabled)
     ) {
       stopTrack
         ? transceiver.sender.track.stop()
         : (transceiver.sender.track.enabled = false);
-      return this.notifyTrackMuteStateChanged(
-        undefined,
-        transceiver.sender.track,
-        trackType,
-        true,
-      );
+      // We don't need to notify SFU if unpublishing in response to remote soft mute
+      if (!this.state.localParticipant?.publishedTracks.includes(trackType)) {
+        return;
+      } else {
+        return this.notifyTrackMuteStateChanged(
+          undefined,
+          transceiver.sender.track,
+          trackType,
+          true,
+        );
+      }
     }
   };
 
@@ -330,6 +350,35 @@ export class Publisher {
    * @param trackType the track type to check.
    */
   isPublishing = (trackType: TrackType): boolean => {
+    const transceiverForTrackType = this.transceiverRegistry[trackType];
+    if (transceiverForTrackType && transceiverForTrackType.sender) {
+      const sender = transceiverForTrackType.sender;
+      return (
+        !!sender.track &&
+        sender.track.readyState === 'live' &&
+        sender.track.enabled
+      );
+    }
+    return false;
+  };
+
+  /**
+   * Returns true if the given track type was ever published to the SFU.
+   * Contrary to `isPublishing`, this method returns true if a certain
+   * track type was published before, even if it is currently unpublished.
+   *
+   * @param trackType the track type to check.
+   */
+  hasEverPublished = (trackType: TrackType): boolean => {
+    return this.trackTypePublishHistory.get(trackType) ?? false;
+  };
+
+  /**
+   * Returns true if the given track type is currently live
+   *
+   * @param trackType the track type to check.
+   */
+  isLive = (trackType: TrackType): boolean => {
     const transceiverForTrackType = this.transceiverRegistry[trackType];
     if (transceiverForTrackType && transceiverForTrackType.sender) {
       const sender = transceiverForTrackType.sender;

package/src/rtc/__tests__/Publisher.test.ts

@@ -180,6 +180,7 @@ describe('Publisher', () => {
 
     expect(state.localParticipant?.videoDeviceId).toEqual('test-device-id');
     expect(state.localParticipant?.publishedTracks).toContain(TrackType.VIDEO);
+    expect(track.enabled).toBe(true);
     expect(state.localParticipant?.videoStream).toEqual(mediaStream);
     expect(transceiver.setCodecPreferences).toHaveBeenCalled();
     expect(sfuClient.updateMuteState).toHaveBeenCalledWith(

package/src/rtc/codecs.ts

@@ -1,5 +1,3 @@
-import { isReactNative } from '../helpers/platforms';
-import { removeCodec, setPreferredCodec } from '../helpers/sdp-munging';
 import { getLogger } from '../logger';
 
 export const getPreferredCodecs = (