@microsoft/live-share-media 1.3.1 → 1.4.0

package/src/MediaPlayerSynchronizer.ts

@@ -20,7 +20,7 @@ import {
 import { VolumeManager } from "./VolumeManager";
 import { LiveMediaSession } from "./LiveMediaSession";
 import { IMediaPlayer } from "./IMediaPlayer";
-import { TelemetryEvents } from "./internals";
+import { ITriggerActionEvent, TelemetryEvents } from "./internals";
 import { waitUntilConnected } from "@microsoft/live-share/bin/internals";
 
 /**
@@ -64,6 +64,7 @@ export class MediaPlayerSynchronizer extends EventEmitter {
         "pause",
         "seekto",
         "settrack",
+        "ratechange",
         "datachange",
         "catchup",
         "wait",
@@ -89,6 +90,7 @@ export class MediaPlayerSynchronizer extends EventEmitter {
     private _viewOnly = false;
     private _blockUnexpectedPlayerEvents = true;
     private _expectedPlaybackState: ExtendedMediaSessionPlaybackState = "none";
+    private _expectedPlaybackRate: number = 1.0;
     private _metadata: ExtendedMediaMetadata | null = null;
     private _trackData: object | null = null;
 
@@ -142,7 +144,7 @@ export class MediaPlayerSynchronizer extends EventEmitter {
                 playbackState: playbackState,
                 positionState: {
                     position: this._player.currentTime,
-                    playbackRate: this._player.playbackRate,
+                    playbackRate: this._player.playbackRate ?? 1,
                     duration: this._player.duration,
                 },
                 trackData: this._trackData,
@@ -227,22 +229,19 @@ export class MediaPlayerSynchronizer extends EventEmitter {
                     }
                     break;
                 case "ratechange":
-                    // Block rate changes unless suspended.
                     if (
-                        this._player.playbackRate != 1.0 &&
+                        this._blockUnexpectedPlayerEvents &&
+                        this._expectedPlaybackRate !==
+                            this._player.playbackRate &&
                         !this._mediaSession.coordinator.isSuspended
                     ) {
-                        this._logger.sendTelemetryEvent(
-                            TelemetryEvents.MediaPlayerSynchronizer
-                                .PlaybackRateChangeBlocked
-                        );
-                        this._player.playbackRate = 1.0;
+                        this._player.playbackRate = this._expectedPlaybackRate;
                     }
                     break;
                 case "blocked":
                     this._logger.sendTelemetryEvent(
                         TelemetryEvents.MediaPlayerSynchronizer
-                            .PlaybackRateChangeBlocked
+                            .PlayerBlockedOperation
                     );
                     break;
             }
@@ -254,6 +253,13 @@ export class MediaPlayerSynchronizer extends EventEmitter {
             (evt) => this.emit(evt.type, evt)
         );
 
+        this._mediaSession.coordinator.on(
+            MediaSessionCoordinatorEvents.triggeractionignored,
+            (evt: ITriggerActionEvent) => {
+                this.dispatchGroupAction(evt.details, undefined);
+            }
+        );
+
         // Register media session actions
         for (const action of MediaPlayerSynchronizer.SESSION_ACTIONS) {
             this._mediaSession.setActionHandler(
@@ -344,6 +350,22 @@ export class MediaPlayerSynchronizer extends EventEmitter {
                                         .DataChangeAction
                                 );
                                 break;
+                            case "ratechange":
+                                if (!details.playbackRate) {
+                                    break;
+                                }
+
+                                this._logger.sendTelemetryEvent(
+                                    TelemetryEvents.MediaPlayerSynchronizer
+                                        .RateChangeAction,
+                                    null,
+                                    { playbackRate: details.playbackRate }
+                                );
+                                this._expectedPlaybackRate =
+                                    details.playbackRate;
+                                this._player.playbackRate =
+                                    details.playbackRate;
+                                break;
                             case "catchup":
                                 if (typeof details.seekTime == "number") {
                                     this._logger.sendTelemetryEvent(
@@ -401,10 +423,10 @@ export class MediaPlayerSynchronizer extends EventEmitter {
      *
      * @remarks
      * Toggling this value to true results in `mediaSession.coordinator.canPlayPause`,
-     * `mediaSession.coordinator.canSeek`, `mediaSession.coordinator.canSetTrack`, and
-     * `mediaSession.coordinator.canSetTrackData` all being set to false.  For more fine
-     * grained control over the local clients policies, call the `mediaSession.coordinator`
-     * directly.
+     * `mediaSession.coordinator.canSeek`, `mediaSession.coordinator.canSetTrack`,
+     * `mediaSession.coordinator.canSetTrackData`, and `mediaSession.coordinator.canSetPlaybackRate`
+     *  all being set to false.  For more fine grained control over the local clients policies,
+     *  call the `mediaSession.coordinator` directly.
      */
     public get viewOnly(): boolean {
         return this._viewOnly;
@@ -416,6 +438,7 @@ export class MediaPlayerSynchronizer extends EventEmitter {
         this.mediaSession.coordinator.canSeek = !value;
         this.mediaSession.coordinator.canSetTrack = !value;
         this.mediaSession.coordinator.canSetTrackData = !value;
+        this.mediaSession.coordinator.canSetPlaybackRate = !value;
     }
 
     /**
@@ -635,6 +658,24 @@ export class MediaPlayerSynchronizer extends EventEmitter {
         });
     }
 
+    public async setPlaybackRate(playbackRate: number): Promise<void> {
+        this._logger.sendTelemetryEvent(
+            TelemetryEvents.MediaPlayerSynchronizer.RateChangeCalled,
+            null,
+            { playbackRate }
+        );
+
+        await this._mediaSession.coordinator.setPlaybackRate(playbackRate);
+
+        this.dispatchUserAction({
+            action: "ratechange",
+            source: "user",
+            clientId: await waitUntilConnected(this._runtime),
+            local: true,
+            playbackRate,
+        });
+    }
+
     /**
      * Tells the group to change to a new track.
      *
@@ -687,7 +728,6 @@ export class MediaPlayerSynchronizer extends EventEmitter {
         });
     }
 
-    // TODO: what is delay useful for? we do not use, delete?
     private dispatchGroupAction(
         details: ExtendedMediaSessionActionDetails,
         error: Error | undefined

package/src/MediaSessionExtensions.ts

@@ -59,6 +59,10 @@ export enum MediaSessionCoordinatorEvents {
      * Event emitted when an action was triggered.
      */
     triggeraction = "triggeraction",
+    /**
+     * Event emitted when an action would have been triggered, but a suspension is active for the local client.
+     */
+    triggeractionignored = "triggeractionignored",
 }
 
 /**
@@ -71,6 +75,7 @@ export enum MediaSessionCoordinatorEvents {
 export type ExtendedMediaSessionAction =
     | MediaSessionAction
     | "settrack"
+    | "ratechange"
     | "wait"
     | "catchup"
     | "datachange"
@@ -116,6 +121,15 @@ export interface ExtendedMediaMetadata extends MediaMetadata {
     liveStream?: boolean;
 }
 
+/**
+ * The reasons why an action is ignored.
+ *
+ * @remarks
+ * Is `localusersuspended` when the action updated the group state, but ignored locally due to the local client being suspended.
+
+ */
+export type ExtendedMediaSessionActionIgnore = "localusersuspended";
+
 /**
  * Details for emitted actions from `LiveMediaSession`.
  *
@@ -169,9 +183,11 @@ export interface ExtendedMediaSessionActionDetails {
      */
     data?: object | null;
     /**
-     * Action type that was blocked.
+     * Reason an action is ignored by the local client, undefined if not ignored.
      */
-    blocked?: ExtendedMediaSessionAction | null;
+    ignoreReason?: ExtendedMediaSessionActionIgnore | null;
+
+    playbackRate?: number | null;
 }
 
 /**

package/src/internals/GroupCoordinatorState.ts

@@ -23,8 +23,13 @@ import {
     GroupPlaybackTrack,
     GroupPlaybackTrackEvents,
     IPlaybackTrack,
+    IPlaybackTrackChangeEvent,
 } from "./GroupPlaybackTrack";
-import { GroupTransportState, ITransportState } from "./GroupTransportState";
+import {
+    GroupTransportState,
+    ITransportState,
+    ITransportStateChangeEvent,
+} from "./GroupTransportState";
 import {
     GroupPlaybackPosition,
     ICurrentPlaybackPosition,
@@ -35,9 +40,16 @@ import {
     GroupPlaybackTrackData,
     PlaybackTrackDataEvents,
     IPlaybackTrackData,
+    IPlaybackTrackDataChangeEvent,
 } from "./GroupPlaybackTrackData";
 import { TelemetryEvents } from "./consts";
 import { waitUntilConnected } from "@microsoft/live-share/bin/internals";
+import {
+    GroupPlaybackRate,
+    IPlaybackRate,
+    IPlaybackRateChangeEvent,
+    PlaybackRateEvents,
+} from "./GroupPlaybackRate";
 
 /**
  * @hidden
@@ -46,6 +58,7 @@ export interface IPositionUpdateEvent {
     track: IPlaybackTrack;
     trackData: IPlaybackTrackData;
     transport: ITransportState;
+    playbackRate: IPlaybackRate;
     playbackState: ExtendedMediaSessionPlaybackState;
     position: number;
     waitPoint?: CoordinationWaitPoint;
@@ -59,6 +72,13 @@ export interface ITransportCommandEvent {
     position: number;
 }
 
+/**
+ * @hidden
+ */
+export interface IRateChangeCommandEvent {
+    playbackRate: number;
+}
+
 /**
  * @hidden
  */
@@ -87,6 +107,7 @@ export interface ITriggerActionEvent extends IEvent {
 export enum GroupCoordinatorStateEvents {
     newwaitpoint = "newwaitpoint",
     triggeraction = "triggeraction",
+    triggeractionignored = "triggeractionignored",
 }
 
 /**
@@ -104,6 +125,7 @@ export class GroupCoordinatorState extends EventEmitter {
     private _playbackTrackData: GroupPlaybackTrackData;
     private _transportState: GroupTransportState;
     private _playbackPosition: GroupPlaybackPosition;
+    private _playbackRate: GroupPlaybackRate;
 
     // Suspension tracking
     private _suspensionCnt: number = 0;
@@ -132,13 +154,16 @@ export class GroupCoordinatorState extends EventEmitter {
         this._playbackTrackData = new GroupPlaybackTrackData(
             this._playbackTrack
         );
+        this._playbackRate = new GroupPlaybackRate();
         this._transportState = new GroupTransportState(
             this._playbackTrack,
+            this._playbackRate,
             getMediaPlayerState,
             this._liveRuntime
         );
         this._playbackPosition = new GroupPlaybackPosition(
             this._transportState,
+            this._playbackRate,
             this._runtime,
             this._liveRuntime,
             positionUpdateInterval
@@ -146,104 +171,122 @@ export class GroupCoordinatorState extends EventEmitter {
         this._getMediaPlayerState = getMediaPlayerState;
 
         // Listen track related events
-        this._playbackTrack.on(GroupPlaybackTrackEvents.trackChange, (evt) => {
-            if (!this.isSuspended) {
-                this._logger.sendTelemetryEvent(
-                    TelemetryEvents.GroupCoordinator.TrackChanged
-                );
-                this.emitSetTrack(evt.clientId, evt.metadata!, evt.source);
-            } else {
-                this._logger.sendTelemetryEvent(
-                    TelemetryEvents.GroupCoordinator.TrackChangeDelayed
-                );
+        this._playbackTrack.on(
+            GroupPlaybackTrackEvents.trackChange,
+            (evt: IPlaybackTrackChangeEvent) => {
+                if (!this.isSuspended) {
+                    this._logger.sendTelemetryEvent(
+                        TelemetryEvents.GroupCoordinator.TrackChanged
+                    );
+                    this.emitSetTrack(evt.clientId, evt.metadata!, evt.source);
+                } else {
+                    this._logger.sendTelemetryEvent(
+                        TelemetryEvents.GroupCoordinator.TrackChangeDelayed
+                    );
+                }
             }
-        });
+        );
 
         // Listen for track data changes
         this._playbackTrackData.on(
             PlaybackTrackDataEvents.dataChange,
-            async (evt) => {
+            async (evt: IPlaybackTrackDataChangeEvent) => {
                 if (!this.isSuspended && !this.isWaiting) {
                     this._logger.sendTelemetryEvent(
                         TelemetryEvents.GroupCoordinator.TrackDataChanged
                     );
-                    const clientId = await waitUntilConnected(this._runtime);
-                    this.emitTriggerAction({
-                        action: "datachange",
-                        source: evt.source,
-                        data: evt.data,
-                        clientId: evt.clientId,
-                        local: evt.clientId === clientId,
-                    });
                 } else {
                     this._logger.sendTelemetryEvent(
                         TelemetryEvents.GroupCoordinator.TrackDataChangeDelayed
                     );
                 }
+                const clientId = await waitUntilConnected(this._runtime);
+                this.emitTriggerActionOrIgnored({
+                    action: "datachange",
+                    source: evt.source,
+                    data: evt.data,
+                    clientId: evt.clientId,
+                    local: evt.clientId === clientId,
+                });
             }
         );
 
-        // Listen to transport related events
-        this._transportState.on(
-            GroupTransportStateEvents.transportStateChange,
-            async (evt) => {
+        this._playbackRate.on(
+            PlaybackRateEvents.rateChange,
+            async (evt: IPlaybackRateChangeEvent) => {
                 if (!this.isSuspended && !this.isWaiting) {
                     this._logger.sendTelemetryEvent(
-                        TelemetryEvents.GroupCoordinator.TransportStateChanged,
-                        null,
-                        {
-                            action: evt.action,
-                            seekTime: evt.seekTime,
-                        }
+                        TelemetryEvents.GroupCoordinator.PlaybackRateChanged
                     );
-
-                    const localClientId = await waitUntilConnected(
-                        this._runtime
-                    );
-                    const local = evt.clientId === localClientId;
-                    // Trigger action
-                    switch (evt.action) {
-                        case "play":
-                            this.emitTriggerAction({
-                                action: "play",
-                                source: evt.source,
-                                clientId: evt.clientId,
-                                local,
-                                seekTime: evt.seekTime,
-                            });
-                            break;
-
-                        case "pause":
-                            this.emitTriggerAction({
-                                action: "pause",
-                                source: evt.source,
-                                clientId: evt.clientId,
-                                local,
-                                seekTime: evt.seekTime,
-                            });
-                            break;
-
-                        case "seekto":
-                            this.emitTriggerAction({
-                                action: "seekto",
-                                source: evt.source,
-                                clientId: evt.clientId,
-                                local,
-                                seekTime: evt.seekTime,
-                            });
-                            break;
-                    }
                 } else {
                     this._logger.sendTelemetryEvent(
                         TelemetryEvents.GroupCoordinator
-                            .TransportStateChangeDelayed,
-                        null,
-                        {
-                            action: evt.action,
-                            seekTime: evt.seekTime,
-                        }
+                            .PlaybackRateChangedDelayed
                     );
                 }
+
+                const localClientId = await waitUntilConnected(this._runtime);
+                const local = evt.clientId === localClientId;
+                this.emitTriggerActionOrIgnored({
+                    action: "ratechange",
+                    source: evt.source,
+                    clientId: evt.clientId,
+                    local,
+                    playbackRate: evt.playbackRate,
+                });
+            }
+        );
+
+        // Listen to transport related events
+        this._transportState.on(
+            GroupTransportStateEvents.transportStateChange,
+            async (evt: ITransportStateChangeEvent) => {
+                const tag =
+                    this.isSuspended || this.isWaiting
+                        ? TelemetryEvents.GroupCoordinator
+                              .TransportStateChangeDelayed
+                        : TelemetryEvents.GroupCoordinator
+                              .TransportStateChanged;
+
+                this._logger.sendTelemetryEvent(tag, null, {
+                    action: evt.action,
+                    seekTime: evt.seekTime,
+                });
+
+                const localClientId = await waitUntilConnected(this._runtime);
+                const local = evt.clientId === localClientId;
+                // Trigger action
+                switch (evt.action) {
+                    case "play":
+                        this.emitTriggerActionOrIgnored({
+                            action: "play",
+                            source: evt.source,
+                            clientId: evt.clientId,
+                            local,
+                            seekTime: evt.seekTime,
+                        });
+                        break;
+
+                    case "pause":
+                        this.emitTriggerActionOrIgnored({
+                            action: "pause",
+                            source: evt.source,
+                            clientId: evt.clientId,
+                            local,
+                            seekTime: evt.seekTime,
+                        });
+                        break;
+
+                    case "seekto":
+                        this.emitTriggerActionOrIgnored({
+                            action: "seekto",
+                            source: evt.source,
+                            clientId: evt.clientId,
+                            local,
+                            seekTime: evt.seekTime,
+                        });
+                        break;
+                }
             }
         );
     }
@@ -264,6 +307,10 @@ export class GroupCoordinatorState extends EventEmitter {
         return this._playbackPosition;
     }
 
+    public get playbackRate(): GroupPlaybackRate {
+        return this._playbackRate;
+    }
+
     public get waitingAt(): CoordinationWaitPoint | undefined {
         return this._waitPoint;
     }
@@ -312,6 +359,7 @@ export class GroupCoordinatorState extends EventEmitter {
                 track: this.playbackTrack.current,
                 trackData: this.playbackTrackData.current,
                 transport: this.transportState.current,
+                playbackRate: this.playbackRate.current,
                 playbackState: "suspended",
                 position: positionState?.position || 0.0,
                 waitPoint: this._waitPoint,
@@ -322,6 +370,7 @@ export class GroupCoordinatorState extends EventEmitter {
                 track: this.playbackTrack.current,
                 trackData: this.playbackTrackData.current,
                 transport: this.transportState.current,
+                playbackRate: this.playbackRate.current,
                 playbackState: this._waitPoint ? "waiting" : playbackState,
                 position: positionState?.position || 0.0,
                 waitPoint: this._waitPoint,
@@ -388,6 +437,32 @@ export class GroupCoordinatorState extends EventEmitter {
         }
     }
 
+    public handleRateChangeCommand(
+        event: ILiveEvent<IRateChangeCommandEvent>,
+        local: boolean
+    ): void {
+        const newState: IPlaybackRate = {
+            playbackRate: event.data.playbackRate,
+            clientId: event.clientId,
+            timestamp: event.timestamp,
+        };
+        const updated = this.playbackRate.updatePlaybackRate(newState, "user");
+
+        if (updated) {
+            const correlationId = LiveTelemetryLogger.formatCorrelationId(
+                event.clientId,
+                event.timestamp
+            );
+            this._logger.sendTelemetryEvent(
+                TelemetryEvents.SessionCoordinator.RemoteRateChangeReceived,
+                null,
+                {
+                    correlationId: correlationId,
+                }
+            );
+        }
+    }
+
     public handleTransportCommand(
         event: ILiveEvent<ITransportCommandEvent>,
         local: boolean
@@ -579,7 +654,11 @@ export class GroupCoordinatorState extends EventEmitter {
                 if (
                     target >= 0.0 &&
                     Math.abs(target - position) >=
-                        this._maxPlaybackDrift.seconds
+                        Math.max(
+                            this._maxPlaybackDrift.seconds,
+                            this._maxPlaybackDrift.seconds *
+                                this._playbackRate.rate
+                        )
                 ) {
                     this._logger.sendTelemetryEvent(
                         TelemetryEvents.GroupCoordinator.PositionOutOfSync,
@@ -589,7 +668,7 @@ export class GroupCoordinatorState extends EventEmitter {
                             target: target,
                         }
                     );
-                    await this.emitTriggerAction({
+                    this.emitTriggerAction({
                         action: "catchup",
                         source: "system",
                         clientId: await waitUntilConnected(this._runtime),
@@ -613,7 +692,7 @@ export class GroupCoordinatorState extends EventEmitter {
                         localClientId === this.transportState.current.clientId;
                     switch (this.transportState.playbackState) {
                         case "playing":
-                            await this.emitTriggerAction({
+                            this.emitTriggerAction({
                                 action: "play",
                                 source: "system",
                                 clientId: this.transportState.current.clientId,
@@ -621,7 +700,7 @@ export class GroupCoordinatorState extends EventEmitter {
                             });
                             break;
                         case "paused":
-                            await this.emitTriggerAction({
+                            this.emitTriggerAction({
                                 action: "pause",
                                 source: "system",
                                 clientId: this.transportState.current.clientId,
@@ -642,7 +721,7 @@ export class GroupCoordinatorState extends EventEmitter {
                     const local =
                         localClientId ===
                         this.playbackTrackData.current.clientId;
-                    await this.emitTriggerAction({
+                    this.emitTriggerAction({
                         action: "datachange",
                         source: "system",
                         clientId: this.playbackTrackData.current.clientId,
@@ -650,6 +729,23 @@ export class GroupCoordinatorState extends EventEmitter {
                         data: this.playbackTrackData.data,
                     });
                 }
+
+                // Sync playback rate
+                if (this.playbackRate.rate != positionState?.playbackRate) {
+                    this._logger.sendTelemetryEvent(
+                        TelemetryEvents.GroupCoordinator.PlaybackRateOutOfSync
+                    );
+                    const local =
+                        localClientId ===
+                        this.playbackTrackData.current.clientId;
+                    this.emitTriggerAction({
+                        action: "ratechange",
+                        source: "system",
+                        clientId: this.playbackTrackData.current.clientId,
+                        local,
+                        playbackRate: this.playbackRate.rate,
+                    });
+                }
             }
         }
     }
@@ -673,6 +769,16 @@ export class GroupCoordinatorState extends EventEmitter {
         });
     }
 
+    private emitTriggerActionOrIgnored(
+        details: ExtendedMediaSessionActionDetails
+    ): void {
+        if (this.isSuspended || this.isWaiting) {
+            return this.emitTriggerActionIgnored(details);
+        } else {
+            return this.emitTriggerAction(details);
+        }
+    }
+
     private emitTriggerAction(
         details: ExtendedMediaSessionActionDetails
     ): void {
@@ -682,4 +788,15 @@ export class GroupCoordinatorState extends EventEmitter {
         };
         this.emit(GroupCoordinatorStateEvents.triggeraction, evt);
     }
+
+    private emitTriggerActionIgnored(
+        details: ExtendedMediaSessionActionDetails
+    ): void {
+        details.ignoreReason = "localusersuspended";
+        const evt: ITriggerActionEvent = {
+            name: GroupCoordinatorStateEvents.triggeractionignored,
+            details: details,
+        };
+        this.emit(GroupCoordinatorStateEvents.triggeractionignored, evt);
+    }
 }

package/src/internals/GroupPlaybackPosition.ts

@@ -14,6 +14,7 @@ import {
     CoordinationWaitPoint,
     ExtendedMediaSessionPlaybackState,
 } from "../MediaSessionExtensions";
+import { GroupPlaybackRate } from "./GroupPlaybackRate";
 
 /**
  * Per client position
@@ -33,6 +34,7 @@ export interface ICurrentPlaybackPosition {
  */
 export class GroupPlaybackPosition {
     private _transportState: GroupTransportState;
+    private _playbackRate: GroupPlaybackRate;
     private _runtime: IRuntimeSignaler;
     private _liveRuntime: LiveShareRuntime;
     private _updateInterval: TimeInterval;
@@ -40,11 +42,13 @@ export class GroupPlaybackPosition {
 
     constructor(
         transportState: GroupTransportState,
+        playbackRate: GroupPlaybackRate,
         runtime: IRuntimeSignaler,
         liveRuntime: LiveShareRuntime,
         updateInterval: TimeInterval
     ) {
         this._transportState = transportState;
+        this._playbackRate = playbackRate;
         this._runtime = runtime;
         this._liveRuntime = liveRuntime;
         this._updateInterval = updateInterval;
@@ -135,7 +139,8 @@ export class GroupPlaybackPosition {
             const now = this._liveRuntime.getTimestamp();
             const projected =
                 this._transportState.startPosition +
-                (now - this._transportState.timestamp) / 1000;
+                ((now - this._transportState.timestamp) / 1000) *
+                    this._playbackRate.rate;
             return this.limitProjectedPosition(projected);
         } else {
             return this._transportState.startPosition;
@@ -175,7 +180,9 @@ export class GroupPlaybackPosition {
                 // - This computation does not take into account future wait points.
                 const projected =
                     position.playbackState == "playing" && shouldProject
-                        ? position.position + (now - position.timestamp) / 1000
+                        ? position.position +
+                          ((now - position.timestamp) / 1000) *
+                              this._playbackRate.rate
                         : position.position;
                 callbackFn(position, this.limitProjectedPosition(projected));
             }