@stream-io/video-client 1.13.1 → 1.15.0

package/src/store/CallState.ts

@@ -9,6 +9,7 @@ import type { Patch } from './rxUtils';
 import * as RxUtils from './rxUtils';
 import { CallingState } from './CallingState';
 import {
+  type ClosedCaptionsSettings,
   type StreamVideoParticipant,
   type StreamVideoParticipantPatch,
   type StreamVideoParticipantPatches,
@@ -19,6 +20,7 @@ import {
 import { CallStatsReport } from '../stats';
 import {
   BlockedUserEvent,
+  CallClosedCaption,
   CallHLSBroadcastingStartedEvent,
   CallIngressResponse,
   CallMemberAddedEvent,
@@ -32,6 +34,7 @@ import {
   CallSessionParticipantLeftEvent,
   CallSessionResponse,
   CallSettingsResponse,
+  ClosedCaptionEvent,
   EgressResponse,
   MemberResponse,
   OwnCapability,
@@ -97,6 +100,7 @@ export class CallState {
     CallSettingsResponse | undefined
   >(undefined);
   private transcribingSubject = new BehaviorSubject<boolean>(false);
+  private captioningSubject = new BehaviorSubject<boolean>(false);
   private endedBySubject = new BehaviorSubject<UserResponse | undefined>(
     undefined,
   );
@@ -117,6 +121,7 @@ export class CallState {
   private callStatsReportSubject = new BehaviorSubject<
     CallStatsReport | undefined
   >(undefined);
+  private closedCaptionsSubject = new BehaviorSubject<CallClosedCaption[]>([]);
 
   // These are tracks that were delivered to the Subscriber's onTrack event
   // that we couldn't associate with a participant yet.
@@ -275,6 +280,11 @@ export class CallState {
    */
   transcribing$: Observable<boolean>;
 
+  /**
+   * Will provide the closed captioning state of this call.
+   */
+  captioning$: Observable<boolean>;
+
   /**
    * Will provide the user who ended this call.
    */
@@ -285,15 +295,24 @@ export class CallState {
    */
   thumbnails$: Observable<ThumbnailResponse | undefined>;
 
+  /**
+   * The queue of closed captions.
+   */
+  closedCaptions$: Observable<CallClosedCaption[]>;
+
   readonly logger = getLogger(['CallState']);
 
   /**
    * A list of comparators that are used to sort the participants.
-   *
-   * @private
    */
   private sortParticipantsBy = defaultSortPreset;
 
+  /**
+   * The closed captions configuration.
+   */
+  private closedCaptionsSettings: ClosedCaptionsSettings | undefined;
+  private closedCaptionsTasks = new Map<string, NodeJS.Timeout>();
+
   private readonly eventHandlers: {
     [EventType in WSEvent['type']]:
       | ((event: Extract<WSEvent, { type: EventType }>) => void)
@@ -357,6 +376,7 @@ export class CallState {
     this.settings$ = this.settingsSubject.asObservable();
     this.endedBy$ = this.endedBySubject.asObservable();
     this.thumbnails$ = this.thumbnailsSubject.asObservable();
+    this.closedCaptions$ = this.closedCaptionsSubject.asObservable();
 
     /**
      * Performs shallow comparison of two arrays.
@@ -390,10 +410,10 @@ export class CallState {
     this.participantCount$ = duc(this.participantCountSubject);
     this.recording$ = duc(this.recordingSubject);
     this.transcribing$ = duc(this.transcribingSubject);
+    this.captioning$ = duc(this.captioningSubject);
 
     this.eventHandlers = {
       // these events are not updating the call state:
-      'call.closed_caption': undefined,
       'call.deleted': undefined,
       'call.permission_request': undefined,
       'call.recording_ready': undefined,
@@ -415,6 +435,16 @@ export class CallState {
       // events that update call state:
       'call.accepted': (e) => this.updateFromCallResponse(e.call),
       'call.blocked_user': this.blockUser,
+      'call.closed_caption': this.updateFromClosedCaptions,
+      'call.closed_captions_failed': () => {
+        this.setCurrentValue(this.captioningSubject, false);
+      },
+      'call.closed_captions_started': () => {
+        this.setCurrentValue(this.captioningSubject, true);
+      },
+      'call.closed_captions_stopped': () => {
+        this.setCurrentValue(this.captioningSubject, false);
+      },
       'call.created': (e) => this.updateFromCallResponse(e.call),
       'call.ended': (e) => {
         this.updateFromCallResponse(e.call);
@@ -464,6 +494,16 @@ export class CallState {
     };
   }
 
+  /**
+   * Runs the cleanup tasks.
+   */
+  dispose = () => {
+    for (const [ccKey, taskId] of this.closedCaptionsTasks.entries()) {
+      clearTimeout(taskId);
+      this.closedCaptionsTasks.delete(ccKey);
+    }
+  };
+
   /**
    * Sets the list of criteria that are used to sort the participants.
    * To disable sorting, you can pass `noopComparator()`.
@@ -533,6 +573,23 @@ export class CallState {
     return this.setCurrentValue(this.startedAtSubject, startedAt);
   };
 
+  /**
+   * Returns whether closed captions are enabled in the current call.
+   */
+  get captioning() {
+    return this.getCurrentValue(this.captioning$);
+  }
+
+  /**
+   * Sets the closed captioning state of the current call.
+   *
+   * @internal
+   * @param captioning the closed captioning state.
+   */
+  setCaptioning = (captioning: boolean) => {
+    return RxUtils.updateValue(this.captioningSubject, captioning);
+  };
+
   /**
    * The server-side counted number of anonymous participants connected to the current call.
    * This number includes the anonymous participants as well.
@@ -792,6 +849,13 @@ export class CallState {
     return this.getCurrentValue(this.thumbnails$);
   }
 
+  /**
+   * Returns the current queue of closed captions.
+   */
+  get closedCaptions() {
+    return this.getCurrentValue(this.closedCaptions$);
+  }
+
   /**
    * Will try to find the participant with the given sessionId in the current call.
    *
@@ -840,7 +904,6 @@ export class CallState {
 
     const thePatch = typeof patch === 'function' ? patch(participant) : patch;
     const updatedParticipant: StreamVideoParticipant = {
-      // FIXME OL: this is not a deep merge, we might want to revisit this
       ...participant,
       ...thePatch,
     };
@@ -912,7 +975,6 @@ export class CallState {
    *
    * @param trackType the kind of subscription to update.
    * @param changes the list of subscription changes to do.
-   * @param type the debounce type to use for the update.
    */
   updateParticipantTracks = (
     trackType: VideoTrackType,
@@ -1040,6 +1102,15 @@ export class CallState {
     return orphans;
   };
 
+  /**
+   * Updates the closed captions settings.
+   *
+   * @param config the new closed captions settings.
+   */
+  updateClosedCaptionSettings = (config: Partial<ClosedCaptionsSettings>) => {
+    this.closedCaptionsSettings = { ...this.closedCaptionsSettings, ...config };
+  };
+
   /**
    * Updates the call state with the data received from the server.
    *
@@ -1066,6 +1137,7 @@ export class CallState {
     this.updateParticipantCountFromSession(s);
     this.setCurrentValue(this.settingsSubject, call.settings);
     this.setCurrentValue(this.transcribingSubject, call.transcribing);
+    this.setCurrentValue(this.captioningSubject, call.captioning);
     this.setCurrentValue(this.thumbnailsSubject, call.thumbnails);
   };
 
@@ -1299,4 +1371,42 @@ export class CallState {
       this.setCurrentValue(this.ownCapabilitiesSubject, event.own_capabilities);
     }
   };
+
+  private updateFromClosedCaptions = (event: ClosedCaptionEvent) => {
+    this.setCurrentValue(this.closedCaptionsSubject, (queue) => {
+      const { closed_caption } = event;
+
+      const keyOf = (c: CallClosedCaption) => `${c.speaker_id}/${c.start_time}`;
+      const currentKey = keyOf(closed_caption);
+
+      const duplicate = queue.some((caption) => keyOf(caption) === currentKey);
+      if (duplicate) return queue;
+
+      const nextQueue = [...queue, closed_caption];
+
+      const { visibilityDurationMs = 2700, maxVisibleCaptions = 2 } =
+        this.closedCaptionsSettings || {};
+      // schedule the removal of the closed caption after the retention time
+      if (visibilityDurationMs > 0) {
+        const taskId = setTimeout(() => {
+          this.setCurrentValue(this.closedCaptionsSubject, (captions) =>
+            captions.filter((caption) => caption !== closed_caption),
+          );
+          this.closedCaptionsTasks.delete(currentKey);
+        }, visibilityDurationMs);
+        this.closedCaptionsTasks.set(currentKey, taskId);
+
+        // cancel the cleanup tasks for the closed captions that are no longer in the queue
+        for (let i = 0; i < nextQueue.length - maxVisibleCaptions; i++) {
+          const key = keyOf(nextQueue[i]);
+          const task = this.closedCaptionsTasks.get(key);
+          clearTimeout(task);
+          this.closedCaptionsTasks.delete(key);
+        }
+      }
+
+      // trim the queue
+      return nextQueue.slice(-maxVisibleCaptions);
+    });
+  };
 }

package/src/store/__tests__/CallState.test.ts

@@ -967,4 +967,105 @@ describe('CallState', () => {
       expect(state['orphanedTracks'].length).toBe(0);
     });
   });
+
+  describe('closed captions', () => {
+    it('should add closed captions to the queue', () => {
+      const state = new CallState();
+      state.updateFromEvent({
+        type: 'call.closed_caption',
+        // @ts-expect-error incomplete data
+        closed_caption: {
+          speaker_id: '123',
+          text: 'Hello world',
+          start_time: '2021-01-01T00:00:00.000Z',
+          end_time: '2021-01-01T00:02:00.000Z',
+        },
+      });
+      expect(state.closedCaptions.length).toBe(1);
+    });
+
+    it('should maintain predefined queue size', () => {
+      const state = new CallState();
+      state.updateClosedCaptionSettings({ maxVisibleCaptions: 2 });
+      for (let i = 0; i < 5; i++) {
+        state.updateFromEvent({
+          type: 'call.closed_caption',
+          // @ts-expect-error incomplete data
+          closed_caption: {
+            speaker_id: `123-${i}`,
+            text: `Hello world ${i}`,
+            start_time: '2021-01-01T00:00:00.000Z',
+            end_time: '2021-01-01T00:02:00.000Z',
+          },
+        });
+      }
+      expect(state.closedCaptions.length).toBe(2);
+      expect(state['closedCaptionsTasks'].size).toBe(2);
+      expect(state.closedCaptions.map((cc) => cc.text)).toEqual([
+        'Hello world 3',
+        'Hello world 4',
+      ]);
+    });
+
+    it('should remove stale captions from the queue', () => {
+      const state = new CallState();
+      vi.useFakeTimers();
+      state.updateFromEvent({
+        type: 'call.closed_caption',
+        // @ts-expect-error incomplete data
+        closed_caption: {
+          speaker_id: `123`,
+          text: `Hello world`,
+          start_time: '2021-01-01T00:00:00.000Z',
+          end_time: '2021-01-01T00:02:00.000Z',
+        },
+      });
+      expect(state.closedCaptions.length).toBe(1);
+      expect(state['closedCaptionsTasks'].size).toBe(1);
+
+      vi.runAllTimers();
+      expect(state.closedCaptions.length).toBe(0);
+      expect(state['closedCaptionsTasks'].size).toBe(0);
+    });
+
+    it('should remove stale captions from the queue after timer runs', () => {
+      const state = new CallState();
+      state.updateClosedCaptionSettings({ visibilityDurationMs: 100 });
+      vi.useFakeTimers();
+      state.updateFromEvent({
+        type: 'call.closed_caption',
+        // @ts-expect-error incomplete data
+        closed_caption: {
+          speaker_id: `123`,
+          text: `Hello world`,
+          start_time: '2021-01-01T00:00:00.000Z',
+          end_time: '2021-01-01T00:02:00.000Z',
+        },
+      });
+      expect(state.closedCaptions.length).toBe(1);
+      expect(state['closedCaptionsTasks'].size).toBe(1);
+
+      vi.advanceTimersByTime(101);
+      expect(state.closedCaptions.length).toBe(0);
+      expect(state['closedCaptionsTasks'].size).toBe(0);
+    });
+
+    it('dispose cancels all cleanup tasks', () => {
+      const state = new CallState();
+      state.updateFromEvent({
+        type: 'call.closed_caption',
+        // @ts-expect-error incomplete data
+        closed_caption: {
+          speaker_id: `123`,
+          text: `Hello world`,
+          start_time: '2021-01-01T00:00:00.000Z',
+        },
+      });
+      expect(state.closedCaptions.length).toBe(1);
+      expect(state['closedCaptionsTasks'].size).toBe(1);
+
+      state.dispose();
+      expect(state['closedCaptionsTasks'].size).toBe(0);
+    });
+  });
 });

package/src/store/rxUtils.ts

@@ -1,4 +1,4 @@
-import { combineLatest, Observable, Subject } from 'rxjs';
+import { BehaviorSubject, combineLatest, Observable, Subject } from 'rxjs';
 import { withoutConcurrency } from '../helpers/concurrency';
 import { getLogger } from '../logger';
 
@@ -59,6 +59,28 @@ export const setCurrentValue = <T>(subject: Subject<T>, update: Patch<T>) => {
   return next;
 };
 
+/**
+ * Updates the value of the provided Subject and returns the previous value
+ * and a function to roll back the update.
+ * This is useful when you want to optimistically update a value
+ * and roll back the update if an error occurs.
+ *
+ * @param subject the subject to update.
+ * @param update the update to apply to the subject.
+ */
+export const updateValue = <T>(
+  subject: BehaviorSubject<T>,
+  update: Patch<T>,
+) => {
+  const lastValue = subject.getValue();
+  const value = setCurrentValue(subject, update);
+  return {
+    lastValue,
+    value,
+    rollback: () => setCurrentValue(subject, lastValue),
+  };
+};
+
 /**
  * Creates a subscription and returns a function to unsubscribe.
  *

package/src/types.ts

@@ -122,6 +122,21 @@ export type ParticipantPin = {
   pinnedAt: number;
 };
 
+export type ClosedCaptionsSettings = {
+  /**
+   * The time in milliseconds to keep a closed caption in the state (visible).
+   * Default is 2700 ms.
+   */
+  visibilityDurationMs?: number;
+  /**
+   * The maximum number of closed captions to keep in the state (visible).
+   * When the maximum number is reached, the oldest closed caption is removed.
+   *
+   * Default is 2.
+   */
+  maxVisibleCaptions?: number;
+};
+
 /**
  * A partial representation of the StreamVideoParticipant.
  */
@@ -147,7 +162,7 @@ export type SubscriptionChanges = {
 };
 
 /**
- * A preferred codec to use when publishing a video track.
+ * A preferred codec to use when publishing a video or audio track.
  * @internal
  */
 export type PreferredCodec = 'vp8' | 'h264' | 'vp9' | 'av1';
@@ -156,41 +171,31 @@ export type PreferredCodec = 'vp8' | 'h264' | 'vp9' | 'av1';
  * A collection of track publication options.
  * @internal
  */
-export type PublishOptions = {
+export type ClientPublishOptions = {
   /**
    * The preferred codec to use when publishing the video stream.
    */
   preferredCodec?: PreferredCodec;
   /**
-   * Force the codec to use when publishing the video stream.
-   * This will override the preferred codec and the internal codec selection logic.
-   * Use with caution.
-   */
-  forceCodec?: PreferredCodec;
-  /**
-   * When using a preferred codec, force the use of a single codec.
-   * Enabling this, it will remove all other supported codecs from the SDP.
-   * Defaults to false.
+   * The fmtp line for the video codec.
    */
-  forceSingleCodec?: boolean;
-  /**
-   * The preferred scalability to use when publishing the video stream.
-   * Applicable only for SVC codecs.
-   */
-  scalabilityMode?: string;
+  fmtpLine?: string;
   /**
    * The preferred bitrate to use when publishing the video stream.
    */
   preferredBitrate?: number;
-  /**
-   * The preferred downscale factor to use when publishing the video stream
-   * in simulcast mode (non-SVC).
-   */
-  bitrateDownscaleFactor?: number;
   /**
    * The maximum number of simulcast layers to use when publishing the video stream.
    */
   maxSimulcastLayers?: number;
+  /**
+   * The preferred subscription (incoming video stream) codec.
+   */
+  subscriberCodec?: PreferredCodec;
+  /**
+   * The fmtp line for the subscriber codec.
+   */
+  subscriberFmtpLine?: string;
   /**
    * Screen share settings.
    */

package/dist/src/helpers/sdp-munging.d.ts

@@ -1,24 +0,0 @@
-/**
- * Returns an SDP with DTX enabled or disabled.
- */
-export declare const toggleDtx: (sdp: string, enable: boolean) => string;
-/**
- * Returns and SDP with all the codecs except the given codec removed.
- */
-export declare const preserveCodec: (sdp: string, mid: string, codec: RTCRtpCodec) => string;
-/**
- * Enables high-quality audio through SDP munging for the given trackMid.
- *
- * @param sdp the SDP to munge.
- * @param trackMid the trackMid.
- * @param maxBitrate the max bitrate to set.
- */
-export declare const enableHighQualityAudio: (sdp: string, trackMid: string, maxBitrate?: number) => string;
-/**
- * Extracts the mid from the transceiver or the SDP.
- *
- * @param transceiver the transceiver.
- * @param transceiverInitIndex the index of the transceiver in the transceiver's init array.
- * @param sdp the SDP.
- */
-export declare const extractMid: (transceiver: RTCRtpTransceiver, transceiverInitIndex: number, sdp: string | undefined) => string;

package/dist/src/rtc/bitrateLookup.d.ts

@@ -1,2 +0,0 @@
-import { PreferredCodec } from '../types';
-export declare const getOptimalBitrate: (codec: PreferredCodec, frameHeight: number) => number;

package/dist/src/rtc/helpers/iceCandidate.d.ts

@@ -1,2 +0,0 @@
-import { ICETrickle } from '../../gen/video/sfu/models/models';
-export declare function getIceCandidate(candidate: RTCIceCandidate): ICETrickle['iceCandidate'];