@unwanted/matrix-sdk-mini 34.12.0-1

package/src/webrtc/mediaHandler.ts

@@ -0,0 +1,484 @@
+/*
+Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2017 New Vector Ltd
+Copyright 2019, 2020 The Matrix.org Foundation C.I.C.
+Copyright 2021 - 2022 Šimon Brandner <simon.bra.ag@gmail.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { TypedEventEmitter } from "../models/typed-event-emitter.ts";
+import { GroupCallType, GroupCallState } from "../webrtc/groupCall.ts";
+import { logger } from "../logger.ts";
+import { MatrixClient } from "../client.ts";
+
+export enum MediaHandlerEvent {
+    LocalStreamsChanged = "local_streams_changed",
+}
+
+export type MediaHandlerEventHandlerMap = {
+    [MediaHandlerEvent.LocalStreamsChanged]: () => void;
+};
+
+export interface IScreensharingOpts {
+    /**
+     * sourceId for Electron DesktopCapturer
+     */
+    desktopCapturerSourceId?: string;
+    audio?: boolean;
+    // For electron screen capture, there are very few options for detecting electron
+    // apart from inspecting the user agent or just trying getDisplayMedia() and
+    // catching the failure, so we do the latter - this flag tells the function to just
+    // throw an error so we can catch it in this case, rather than logging and emitting.
+    throwOnFail?: boolean;
+}
+
+export interface AudioSettings {
+    autoGainControl: boolean;
+    echoCancellation: boolean;
+    noiseSuppression: boolean;
+}
+
+export class MediaHandler extends TypedEventEmitter<
+    MediaHandlerEvent.LocalStreamsChanged,
+    MediaHandlerEventHandlerMap
+> {
+    private audioInput?: string;
+    private audioSettings?: AudioSettings;
+    private videoInput?: string;
+    private localUserMediaStream?: MediaStream;
+    public userMediaStreams: MediaStream[] = [];
+    public screensharingStreams: MediaStream[] = [];
+
+    // Promise chain to serialise calls to getMediaStream
+    private getMediaStreamPromise?: Promise<MediaStream>;
+
+    public constructor(private client: MatrixClient) {
+        super();
+    }
+
+    public restoreMediaSettings(audioInput: string, videoInput: string): void {
+        this.audioInput = audioInput;
+        this.videoInput = videoInput;
+    }
+
+    /**
+     * Set an audio input device to use for MatrixCalls
+     * @param deviceId - the identifier for the device
+     * undefined treated as unset
+     */
+    public async setAudioInput(deviceId: string): Promise<void> {
+        logger.info(`MediaHandler setAudioInput() running (deviceId=${deviceId})`);
+
+        if (this.audioInput === deviceId) return;
+
+        this.audioInput = deviceId;
+        await this.updateLocalUsermediaStreams();
+    }
+
+    /**
+     * Set audio settings for MatrixCalls
+     * @param opts - audio options to set
+     */
+    public async setAudioSettings(opts: AudioSettings): Promise<void> {
+        logger.info(`MediaHandler setAudioSettings() running (opts=${JSON.stringify(opts)})`);
+
+        this.audioSettings = Object.assign({}, opts) as AudioSettings;
+        await this.updateLocalUsermediaStreams();
+    }
+
+    /**
+     * Set a video input device to use for MatrixCalls
+     * @param deviceId - the identifier for the device
+     * undefined treated as unset
+     */
+    public async setVideoInput(deviceId: string): Promise<void> {
+        logger.info(`MediaHandler setVideoInput() running (deviceId=${deviceId})`);
+
+        if (this.videoInput === deviceId) return;
+
+        this.videoInput = deviceId;
+        await this.updateLocalUsermediaStreams();
+    }
+
+    /**
+     * Set media input devices to use for MatrixCalls
+     * @param audioInput - the identifier for the audio device
+     * @param videoInput - the identifier for the video device
+     * undefined treated as unset
+     */
+    public async setMediaInputs(audioInput: string, videoInput: string): Promise<void> {
+        logger.log(`MediaHandler setMediaInputs() running (audioInput: ${audioInput} videoInput: ${videoInput})`);
+        this.audioInput = audioInput;
+        this.videoInput = videoInput;
+        await this.updateLocalUsermediaStreams();
+    }
+
+    /*
+     * Requests new usermedia streams and replace the old ones
+     */
+    public async updateLocalUsermediaStreams(): Promise<void> {
+        if (this.userMediaStreams.length === 0) return;
+
+        const callMediaStreamParams: Map<string, { audio: boolean; video: boolean }> = new Map();
+        for (const call of this.client.callEventHandler!.calls.values()) {
+            callMediaStreamParams.set(call.callId, {
+                audio: call.hasLocalUserMediaAudioTrack,
+                video: call.hasLocalUserMediaVideoTrack,
+            });
+        }
+
+        for (const stream of this.userMediaStreams) {
+            logger.log(`MediaHandler updateLocalUsermediaStreams() stopping all tracks (streamId=${stream.id})`);
+            for (const track of stream.getTracks()) {
+                track.stop();
+            }
+        }
+
+        this.userMediaStreams = [];
+        this.localUserMediaStream = undefined;
+
+        for (const call of this.client.callEventHandler!.calls.values()) {
+            if (call.callHasEnded() || !callMediaStreamParams.has(call.callId)) {
+                continue;
+            }
+
+            const { audio, video } = callMediaStreamParams.get(call.callId)!;
+
+            logger.log(
+                `MediaHandler updateLocalUsermediaStreams() calling getUserMediaStream() (callId=${call.callId})`,
+            );
+            const stream = await this.getUserMediaStream(audio, video);
+
+            if (call.callHasEnded()) {
+                continue;
+            }
+
+            await call.updateLocalUsermediaStream(stream);
+        }
+
+        for (const groupCall of this.client.groupCallEventHandler!.groupCalls.values()) {
+            if (!groupCall.localCallFeed) {
+                continue;
+            }
+
+            logger.log(
+                `MediaHandler updateLocalUsermediaStreams() calling getUserMediaStream() (groupCallId=${groupCall.groupCallId})`,
+            );
+            const stream = await this.getUserMediaStream(true, groupCall.type === GroupCallType.Video);
+
+            if (groupCall.state === GroupCallState.Ended) {
+                continue;
+            }
+
+            await groupCall.updateLocalUsermediaStream(stream);
+        }
+
+        this.emit(MediaHandlerEvent.LocalStreamsChanged);
+    }
+
+    public async hasAudioDevice(): Promise<boolean> {
+        try {
+            const devices = await navigator.mediaDevices.enumerateDevices();
+            return devices.filter((device) => device.kind === "audioinput").length > 0;
+        } catch (err) {
+            logger.log(`MediaHandler hasAudioDevice() calling navigator.mediaDevices.enumerateDevices with error`, err);
+            return false;
+        }
+    }
+
+    public async hasVideoDevice(): Promise<boolean> {
+        try {
+            const devices = await navigator.mediaDevices.enumerateDevices();
+            return devices.filter((device) => device.kind === "videoinput").length > 0;
+        } catch (err) {
+            logger.log(`MediaHandler hasVideoDevice() calling navigator.mediaDevices.enumerateDevices with error`, err);
+            return false;
+        }
+    }
+
+    /**
+     * @param audio - should have an audio track
+     * @param video - should have a video track
+     * @param reusable - is allowed to be reused by the MediaHandler
+     * @returns based on passed parameters
+     */
+    public async getUserMediaStream(audio: boolean, video: boolean, reusable = true): Promise<MediaStream> {
+        // Serialise calls, othertwise we can't sensibly re-use the stream
+        if (this.getMediaStreamPromise) {
+            this.getMediaStreamPromise = this.getMediaStreamPromise.then(() => {
+                return this.getUserMediaStreamInternal(audio, video, reusable);
+            });
+        } else {
+            this.getMediaStreamPromise = this.getUserMediaStreamInternal(audio, video, reusable);
+        }
+
+        return this.getMediaStreamPromise;
+    }
+
+    private async getUserMediaStreamInternal(audio: boolean, video: boolean, reusable: boolean): Promise<MediaStream> {
+        const shouldRequestAudio = audio && (await this.hasAudioDevice());
+        const shouldRequestVideo = video && (await this.hasVideoDevice());
+
+        let stream: MediaStream;
+
+        let canReuseStream = true;
+        if (this.localUserMediaStream) {
+            // This figures out if we can reuse the current localUsermediaStream
+            // based on whether or not the "mute state" (presence of tracks of a
+            // given kind) matches what is being requested
+            if (shouldRequestAudio !== this.localUserMediaStream.getAudioTracks().length > 0) {
+                canReuseStream = false;
+            }
+            if (shouldRequestVideo !== this.localUserMediaStream.getVideoTracks().length > 0) {
+                canReuseStream = false;
+            }
+
+            // This code checks that the device ID is the same as the localUserMediaStream stream, but we update
+            // the localUserMediaStream whenever the device ID changes (apart from when restoring) so it's not
+            // clear why this would ever be different, unless there's a race.
+            if (
+                shouldRequestAudio &&
+                this.localUserMediaStream.getAudioTracks()[0]?.getSettings()?.deviceId !== this.audioInput
+            ) {
+                canReuseStream = false;
+            }
+            if (
+                shouldRequestVideo &&
+                this.localUserMediaStream.getVideoTracks()[0]?.getSettings()?.deviceId !== this.videoInput
+            ) {
+                canReuseStream = false;
+            }
+        } else {
+            canReuseStream = false;
+        }
+
+        if (!canReuseStream) {
+            const constraints = this.getUserMediaContraints(shouldRequestAudio, shouldRequestVideo);
+            stream = await navigator.mediaDevices.getUserMedia(constraints);
+            logger.log(
+                `MediaHandler getUserMediaStreamInternal() calling getUserMediaStream (streamId=${
+                    stream.id
+                }, shouldRequestAudio=${shouldRequestAudio}, shouldRequestVideo=${shouldRequestVideo}, constraints=${JSON.stringify(
+                    constraints,
+                )})`,
+            );
+
+            for (const track of stream.getTracks()) {
+                const settings = track.getSettings();
+
+                if (track.kind === "audio") {
+                    this.audioInput = settings.deviceId!;
+                } else if (track.kind === "video") {
+                    this.videoInput = settings.deviceId!;
+                }
+            }
+
+            if (reusable) {
+                this.localUserMediaStream = stream;
+            }
+        } else {
+            stream = this.localUserMediaStream!.clone();
+            logger.log(
+                `MediaHandler getUserMediaStreamInternal() cloning (oldStreamId=${this.localUserMediaStream?.id} newStreamId=${stream.id} shouldRequestAudio=${shouldRequestAudio} shouldRequestVideo=${shouldRequestVideo})`,
+            );
+
+            if (!shouldRequestAudio) {
+                for (const track of stream.getAudioTracks()) {
+                    stream.removeTrack(track);
+                }
+            }
+
+            if (!shouldRequestVideo) {
+                for (const track of stream.getVideoTracks()) {
+                    stream.removeTrack(track);
+                }
+            }
+        }
+
+        if (reusable) {
+            this.userMediaStreams.push(stream);
+        }
+
+        this.emit(MediaHandlerEvent.LocalStreamsChanged);
+
+        return stream;
+    }
+
+    /**
+     * Stops all tracks on the provided usermedia stream
+     */
+    public stopUserMediaStream(mediaStream: MediaStream): void {
+        logger.log(`MediaHandler stopUserMediaStream() stopping (streamId=${mediaStream.id})`);
+        for (const track of mediaStream.getTracks()) {
+            track.stop();
+        }
+
+        const index = this.userMediaStreams.indexOf(mediaStream);
+
+        if (index !== -1) {
+            logger.debug(
+                `MediaHandler stopUserMediaStream() splicing usermedia stream out stream array (streamId=${mediaStream.id})`,
+                mediaStream.id,
+            );
+            this.userMediaStreams.splice(index, 1);
+        }
+
+        this.emit(MediaHandlerEvent.LocalStreamsChanged);
+
+        if (this.localUserMediaStream === mediaStream) {
+            // if we have this stream cahced, remove it, because we've stopped it
+            this.localUserMediaStream = undefined;
+        } else {
+            // If it's not the same stream. remove any tracks from the cached stream that
+            // we have just stopped, and if we do stop any, call the same method on the
+            // cached stream too in order to stop all its tracks (in case they are different)
+            // and un-cache it.
+            for (const track of mediaStream.getTracks()) {
+                if (this.localUserMediaStream?.getTrackById(track.id)) {
+                    this.stopUserMediaStream(this.localUserMediaStream);
+                    break;
+                }
+            }
+        }
+    }
+
+    /**
+     * @param opts - screensharing stream options
+     * @param reusable - is allowed to be reused by the MediaHandler
+     * @returns based on passed parameters
+     */
+    public async getScreensharingStream(opts: IScreensharingOpts = {}, reusable = true): Promise<MediaStream> {
+        let stream: MediaStream;
+
+        if (this.screensharingStreams.length === 0) {
+            const screenshareConstraints = this.getScreenshareContraints(opts);
+
+            if (opts.desktopCapturerSourceId) {
+                // We are using Electron
+                logger.debug(
+                    `MediaHandler getScreensharingStream() calling getUserMedia() (opts=${JSON.stringify(opts)})`,
+                );
+                stream = await navigator.mediaDevices.getUserMedia(screenshareConstraints);
+            } else {
+                // We are not using Electron
+                logger.debug(
+                    `MediaHandler getScreensharingStream() calling getDisplayMedia() (opts=${JSON.stringify(opts)})`,
+                );
+                stream = await navigator.mediaDevices.getDisplayMedia(screenshareConstraints);
+            }
+        } else {
+            const matchingStream = this.screensharingStreams[this.screensharingStreams.length - 1];
+            logger.log(`MediaHandler getScreensharingStream() cloning (streamId=${matchingStream.id})`);
+            stream = matchingStream.clone();
+        }
+
+        if (reusable) {
+            this.screensharingStreams.push(stream);
+        }
+
+        this.emit(MediaHandlerEvent.LocalStreamsChanged);
+
+        return stream;
+    }
+
+    /**
+     * Stops all tracks on the provided screensharing stream
+     */
+    public stopScreensharingStream(mediaStream: MediaStream): void {
+        logger.debug(`MediaHandler stopScreensharingStream() stopping stream (streamId=${mediaStream.id})`);
+        for (const track of mediaStream.getTracks()) {
+            track.stop();
+        }
+
+        const index = this.screensharingStreams.indexOf(mediaStream);
+
+        if (index !== -1) {
+            logger.debug(`MediaHandler stopScreensharingStream() splicing stream out (streamId=${mediaStream.id})`);
+            this.screensharingStreams.splice(index, 1);
+        }
+
+        this.emit(MediaHandlerEvent.LocalStreamsChanged);
+    }
+
+    /**
+     * Stops all local media tracks
+     */
+    public stopAllStreams(): void {
+        for (const stream of this.userMediaStreams) {
+            logger.log(`MediaHandler stopAllStreams() stopping (streamId=${stream.id})`);
+            for (const track of stream.getTracks()) {
+                track.stop();
+            }
+        }
+
+        for (const stream of this.screensharingStreams) {
+            for (const track of stream.getTracks()) {
+                track.stop();
+            }
+        }
+
+        this.userMediaStreams = [];
+        this.screensharingStreams = [];
+        this.localUserMediaStream = undefined;
+
+        this.emit(MediaHandlerEvent.LocalStreamsChanged);
+    }
+
+    private getUserMediaContraints(audio: boolean, video: boolean): MediaStreamConstraints {
+        const isWebkit = !!navigator.webkitGetUserMedia;
+
+        return {
+            audio: audio
+                ? {
+                      deviceId: this.audioInput ? { ideal: this.audioInput } : undefined,
+                      autoGainControl: this.audioSettings ? { ideal: this.audioSettings.autoGainControl } : undefined,
+                      echoCancellation: this.audioSettings ? { ideal: this.audioSettings.echoCancellation } : undefined,
+                      noiseSuppression: this.audioSettings ? { ideal: this.audioSettings.noiseSuppression } : undefined,
+                  }
+                : false,
+            video: video
+                ? {
+                      deviceId: this.videoInput ? { ideal: this.videoInput } : undefined,
+                      /* We want 640x360.  Chrome will give it only if we ask exactly,
+                   FF refuses entirely if we ask exactly, so have to ask for ideal
+                   instead
+                   XXX: Is this still true?
+                 */
+                      width: isWebkit ? { exact: 640 } : { ideal: 640 },
+                      height: isWebkit ? { exact: 360 } : { ideal: 360 },
+                  }
+                : false,
+        };
+    }
+
+    private getScreenshareContraints(opts: IScreensharingOpts): DesktopCapturerConstraints {
+        const { desktopCapturerSourceId, audio } = opts;
+        if (desktopCapturerSourceId) {
+            return {
+                audio: audio ?? false,
+                video: {
+                    mandatory: {
+                        chromeMediaSource: "desktop",
+                        chromeMediaSourceId: desktopCapturerSourceId,
+                    },
+                },
+            };
+        } else {
+            return {
+                audio: audio ?? false,
+                video: true,
+            };
+        }
+    }
+}

package/src/webrtc/stats/callFeedStatsReporter.ts

@@ -0,0 +1,94 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+import { CallFeedReport, CallFeedStats, TrackStats, TransceiverStats } from "./statsReport.ts";
+import { CallFeed } from "../callFeed.ts";
+
+export class CallFeedStatsReporter {
+    public static buildCallFeedReport(callId: string, opponentMemberId: string, pc: RTCPeerConnection): CallFeedReport {
+        const rtpTransceivers = pc.getTransceivers();
+        const transceiver: TransceiverStats[] = [];
+        const callFeeds: CallFeedStats[] = [];
+
+        rtpTransceivers.forEach((t) => {
+            const sender = t.sender?.track ? CallFeedStatsReporter.buildTrackStats(t.sender.track, "sender") : null;
+            const receiver = CallFeedStatsReporter.buildTrackStats(t.receiver.track, "receiver");
+            transceiver.push({
+                mid: t.mid == null ? "null" : t.mid,
+                direction: t.direction,
+                currentDirection: t.currentDirection == null ? "null" : t.currentDirection,
+                sender,
+                receiver,
+            });
+        });
+
+        return {
+            callId,
+            opponentMemberId,
+            transceiver,
+            callFeeds,
+        };
+    }
+
+    private static buildTrackStats(track: MediaStreamTrack, label = "--"): TrackStats {
+        const settingDeviceId = track.getSettings()?.deviceId;
+        const constrainDeviceId = track.getConstraints()?.deviceId;
+
+        return {
+            id: track.id,
+            kind: track.kind,
+            settingDeviceId: settingDeviceId ? settingDeviceId : "unknown",
+            constrainDeviceId: constrainDeviceId ? constrainDeviceId : "unknown",
+            muted: track.muted,
+            enabled: track.enabled,
+            readyState: track.readyState,
+            label,
+        } as TrackStats;
+    }
+
+    public static expandCallFeedReport(
+        report: CallFeedReport,
+        callFeeds: CallFeed[],
+        prefix = "unknown",
+    ): CallFeedReport {
+        if (!report.callFeeds) {
+            report.callFeeds = [];
+        }
+        callFeeds.forEach((feed) => {
+            const audioTracks = feed.stream.getAudioTracks();
+            const videoTracks = feed.stream.getVideoTracks();
+            const audio =
+                audioTracks.length > 0
+                    ? CallFeedStatsReporter.buildTrackStats(feed.stream.getAudioTracks()[0], feed.purpose)
+                    : null;
+            const video =
+                videoTracks.length > 0
+                    ? CallFeedStatsReporter.buildTrackStats(feed.stream.getVideoTracks()[0], feed.purpose)
+                    : null;
+            const feedStats = {
+                stream: feed.stream.id,
+                type: feed.isLocal() ? "local" : "remote",
+                audio,
+                video,
+                purpose: feed.purpose,
+                prefix,
+                isVideoMuted: feed.isVideoMuted(),
+                isAudioMuted: feed.isAudioMuted(),
+            } as CallFeedStats;
+            report.callFeeds.push(feedStats);
+        });
+        return report;
+    }
+}