@unwanted/matrix-sdk-mini 34.12.0-1

package/src/webrtc/stats/media/mediaTrackHandler.ts

@@ -0,0 +1,76 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+export type TrackId = string;
+
+export class MediaTrackHandler {
+    public constructor(private readonly pc: RTCPeerConnection) {}
+
+    public getLocalTracks(kind: "audio" | "video"): MediaStreamTrack[] {
+        const isNotNullAndKind = (track: MediaStreamTrack | null): boolean => {
+            return track !== null && track.kind === kind;
+        };
+        return this.pc
+            .getTransceivers()
+            .filter((t) => t.currentDirection === "sendonly" || t.currentDirection === "sendrecv")
+            .filter((t) => t.sender !== null)
+            .map((t) => t.sender)
+            .map((s) => s.track)
+            .filter(isNotNullAndKind) as MediaStreamTrack[];
+    }
+
+    public getTackById(trackId: string): MediaStreamTrack | undefined {
+        return this.pc
+            .getTransceivers()
+            .map((t) => {
+                if (t?.sender.track !== null && t.sender.track.id === trackId) {
+                    return t.sender.track;
+                }
+                if (t?.receiver.track !== null && t.receiver.track.id === trackId) {
+                    return t.receiver.track;
+                }
+                return undefined;
+            })
+            .find((t) => t !== undefined);
+    }
+
+    public getLocalTrackIdByMid(mid: string): string | undefined {
+        const transceiver = this.pc.getTransceivers().find((t) => t.mid === mid);
+        if (transceiver !== undefined && !!transceiver.sender && !!transceiver.sender.track) {
+            return transceiver.sender.track.id;
+        }
+        return undefined;
+    }
+
+    public getRemoteTrackIdByMid(mid: string): string | undefined {
+        const transceiver = this.pc.getTransceivers().find((t) => t.mid === mid);
+        if (transceiver !== undefined && !!transceiver.receiver && !!transceiver.receiver.track) {
+            return transceiver.receiver.track.id;
+        }
+        return undefined;
+    }
+
+    public getActiveSimulcastStreams(): number {
+        //@TODO implement this right.. Check how many layer configured
+        return 3;
+    }
+
+    public getTransceiverByTrackId(trackId: TrackId): RTCRtpTransceiver | undefined {
+        return this.pc.getTransceivers().find((t) => {
+            return t.receiver.track.id === trackId || (t.sender.track !== null && t.sender.track.id === trackId);
+        });
+    }
+}

package/src/webrtc/stats/media/mediaTrackStats.ts

@@ -0,0 +1,176 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { AudioConcealment } from "../statsReport.ts";
+import { TrackId } from "./mediaTrackHandler.ts";
+
+export interface PacketLoss {
+    packetsTotal: number;
+    packetsLost: number;
+    isDownloadStream: boolean;
+}
+
+export interface Bitrate {
+    /**
+     * bytes per second
+     */
+    download: number;
+    /**
+     * bytes per second
+     */
+    upload: number;
+}
+export interface ConcealedAudio {
+    /**
+     * duration in ms
+     */
+    duration: number;
+
+    ratio: number;
+}
+export interface Resolution {
+    width: number;
+    height: number;
+}
+
+export type TrackStatsType = "local" | "remote";
+
+export class MediaTrackStats {
+    private loss: PacketLoss = { packetsTotal: 0, packetsLost: 0, isDownloadStream: false };
+    private bitrate: Bitrate = { download: 0, upload: 0 };
+    private resolution: Resolution = { width: -1, height: -1 };
+    private audioConcealment: AudioConcealment = { concealedAudio: 0, totalAudioDuration: 0 };
+    private framerate = 0;
+    private jitter = 0;
+    private codec = "";
+    private isAlive = true;
+    private isMuted = false;
+    private isEnabled = true;
+
+    public constructor(
+        public readonly trackId: TrackId,
+        public readonly type: TrackStatsType,
+        public readonly kind: "audio" | "video",
+    ) {}
+
+    public getType(): TrackStatsType {
+        return this.type;
+    }
+
+    public setLoss(loss: PacketLoss): void {
+        this.loss = loss;
+    }
+
+    public getLoss(): PacketLoss {
+        return this.loss;
+    }
+
+    public setResolution(resolution: Resolution): void {
+        this.resolution = resolution;
+    }
+
+    public getResolution(): Resolution {
+        return this.resolution;
+    }
+
+    public setFramerate(framerate: number): void {
+        this.framerate = framerate;
+    }
+
+    public getFramerate(): number {
+        return this.framerate;
+    }
+
+    public setBitrate(bitrate: Bitrate): void {
+        this.bitrate = bitrate;
+    }
+
+    public getBitrate(): Bitrate {
+        return this.bitrate;
+    }
+
+    public setCodec(codecShortType: string): boolean {
+        this.codec = codecShortType;
+        return true;
+    }
+
+    public getCodec(): string {
+        return this.codec;
+    }
+
+    public resetBitrate(): void {
+        this.bitrate = { download: 0, upload: 0 };
+    }
+
+    public set alive(isAlive: boolean) {
+        this.isAlive = isAlive;
+    }
+
+    /**
+     * A MediaTrackState is alive if the corresponding MediaStreamTrack track bound to a transceiver and the
+     * MediaStreamTrack is in state MediaStreamTrack.readyState === live
+     */
+    public get alive(): boolean {
+        return this.isAlive;
+    }
+
+    public set muted(isMuted: boolean) {
+        this.isMuted = isMuted;
+    }
+
+    /**
+     * A MediaTrackState.isMuted corresponding to MediaStreamTrack.muted.
+     * But these values only match if MediaTrackState.isAlive.
+     */
+    public get muted(): boolean {
+        return this.isMuted;
+    }
+
+    public set enabled(isEnabled: boolean) {
+        this.isEnabled = isEnabled;
+    }
+
+    /**
+     * A MediaTrackState.isEnabled corresponding to MediaStreamTrack.enabled.
+     * But these values only match if MediaTrackState.isAlive.
+     */
+    public get enabled(): boolean {
+        return this.isEnabled;
+    }
+
+    public setJitter(jitter: number): void {
+        this.jitter = jitter;
+    }
+
+    /**
+     * Jitter in milliseconds
+     */
+    public getJitter(): number {
+        return this.jitter;
+    }
+
+    /**
+     * Audio concealment ration (conceled duration / total duration)
+     */
+    public setAudioConcealment(concealedAudioDuration: number, totalAudioDuration: number): void {
+        this.audioConcealment.concealedAudio = concealedAudioDuration;
+        this.audioConcealment.totalAudioDuration = totalAudioDuration;
+    }
+
+    public getAudioConcealment(): AudioConcealment {
+        return this.audioConcealment;
+    }
+}

package/src/webrtc/stats/media/mediaTrackStatsHandler.ts

@@ -0,0 +1,90 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+import { TrackID } from "../statsReport.ts";
+import { MediaTrackStats } from "./mediaTrackStats.ts";
+import { MediaTrackHandler, TrackId } from "./mediaTrackHandler.ts";
+import { MediaSsrcHandler } from "./mediaSsrcHandler.ts";
+
+export class MediaTrackStatsHandler {
+    private readonly track2stats = new Map<TrackID, MediaTrackStats>();
+
+    public constructor(
+        public readonly mediaSsrcHandler: MediaSsrcHandler,
+        public readonly mediaTrackHandler: MediaTrackHandler,
+    ) {}
+
+    /**
+     * Find tracks by rtc stats
+     * Argument report is any because the stats api is not consistent:
+     * For example `trackIdentifier`, `mid` not existing in every implementations
+     * https://www.w3.org/TR/webrtc-stats/#dom-rtcinboundrtpstreamstats
+     * https://developer.mozilla.org/en-US/docs/Web/API/RTCInboundRtpStreamStats
+     */
+    public findTrack2Stats(report: any, type: "remote" | "local"): MediaTrackStats | undefined {
+        let trackID;
+        if (report.trackIdentifier) {
+            trackID = report.trackIdentifier;
+        } else if (report.mid) {
+            trackID =
+                type === "remote"
+                    ? this.mediaTrackHandler.getRemoteTrackIdByMid(report.mid)
+                    : this.mediaTrackHandler.getLocalTrackIdByMid(report.mid);
+        } else if (report.ssrc) {
+            const mid = this.mediaSsrcHandler.findMidBySsrc(report.ssrc, type);
+            if (!mid) {
+                return undefined;
+            }
+            trackID =
+                type === "remote"
+                    ? this.mediaTrackHandler.getRemoteTrackIdByMid(report.mid)
+                    : this.mediaTrackHandler.getLocalTrackIdByMid(report.mid);
+        }
+
+        if (!trackID) {
+            return undefined;
+        }
+
+        let trackStats = this.track2stats.get(trackID);
+
+        if (!trackStats) {
+            const track = this.mediaTrackHandler.getTackById(trackID);
+            if (track !== undefined) {
+                const kind: "audio" | "video" = track.kind === "audio" ? track.kind : "video";
+                trackStats = new MediaTrackStats(trackID, type, kind);
+                this.track2stats.set(trackID, trackStats);
+            } else {
+                return undefined;
+            }
+        }
+        return trackStats;
+    }
+
+    public findLocalVideoTrackStats(report: any): MediaTrackStats | undefined {
+        const localVideoTracks = this.mediaTrackHandler.getLocalTracks("video");
+        if (localVideoTracks.length === 0) {
+            return undefined;
+        }
+        return this.findTrack2Stats(report, "local");
+    }
+
+    public getTrack2stats(): Map<TrackID, MediaTrackStats> {
+        return this.track2stats;
+    }
+
+    public findTransceiverByTrackId(trackID: TrackId): undefined | RTCRtpTransceiver {
+        return this.mediaTrackHandler.getTransceiverByTrackId(trackID);
+    }
+}

package/src/webrtc/stats/statsReport.ts

@@ -0,0 +1,133 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { ConnectionStatsBandwidth, ConnectionStatsBitrate, PacketLoss } from "./connectionStats.ts";
+import { TransportStats } from "./transportStats.ts";
+import { Resolution } from "./media/mediaTrackStats.ts";
+
+export enum StatsReport {
+    CONNECTION_STATS = "StatsReport.connection_stats",
+    CALL_FEED_REPORT = "StatsReport.call_feed_report",
+    BYTE_SENT_STATS = "StatsReport.byte_sent_stats",
+    SUMMARY_STATS = "StatsReport.summary_stats",
+}
+
+/// ByteSentStatsReport ################################################################################################
+export interface ByteSentStatsReport extends Map<TrackID, ByteSend> {
+    callId?: string;
+    opponentMemberId?: string;
+    // is a map: `local trackID` => byte send
+}
+
+export type TrackID = string;
+export type ByteSend = number;
+
+/// ConnectionStatsReport ##############################################################################################
+export interface ConnectionStatsReport {
+    callId?: string;
+    opponentMemberId?: string;
+    bandwidth: ConnectionStatsBandwidth;
+    bitrate: ConnectionStatsBitrate;
+    packetLoss: PacketLoss;
+    audioConcealment: Map<TrackID, AudioConcealment>;
+    totalAudioConcealment: AudioConcealment;
+    resolution: ResolutionMap;
+    framerate: FramerateMap;
+    codec: CodecMap;
+    jitter: Map<TrackID, number>;
+    transport: TransportStats[];
+}
+
+export interface AudioConcealment {
+    concealedAudio: number;
+    totalAudioDuration: number;
+}
+
+export interface ResolutionMap {
+    local: Map<TrackID, Resolution>;
+    remote: Map<TrackID, Resolution>;
+}
+
+export interface FramerateMap {
+    local: Map<TrackID, number>;
+    remote: Map<TrackID, number>;
+}
+
+export interface CodecMap {
+    local: Map<TrackID, string>;
+    remote: Map<TrackID, string>;
+}
+
+/// SummaryStatsReport #################################################################################################
+export interface SummaryStatsReport {
+    /**
+     * Aggregated the information for percentage of received media
+     *
+     * This measure whether the current user receive data from a call participants.
+     * As soon as a participant sends at least a byte media to this user, this counts as one measurement unit.
+     * The units of measure divided by the total number of participants is a value between 0 and 1.
+     */
+    percentageReceivedMedia: number;
+    percentageReceivedAudioMedia: number;
+    percentageReceivedVideoMedia: number;
+    maxJitter: number;
+    maxPacketLoss: number;
+    percentageConcealedAudio: number;
+    peerConnections: number;
+    opponentUsersInCall?: number;
+    opponentDevicesInCall?: number;
+    diffDevicesToPeerConnections?: number;
+    ratioPeerConnectionToDevices?: number;
+    // Todo: Decide if we want an index (or a timestamp) of this report in relation to the group call, to help differenciate when issues occur and ignore/track initial connection delays.
+}
+
+/// CallFeedReport #####################################################################################################
+export interface CallFeedReport {
+    callId: string;
+    opponentMemberId: string;
+    transceiver: TransceiverStats[];
+    callFeeds: CallFeedStats[];
+}
+
+export interface CallFeedStats {
+    stream: string;
+    type: "remote" | "local";
+    audio: TrackStats | null;
+    video: TrackStats | null;
+    purpose: string;
+    prefix: string;
+    isVideoMuted: boolean;
+    isAudioMuted: boolean;
+}
+
+export interface TransceiverStats {
+    readonly mid: string;
+    readonly sender: TrackStats | null;
+    readonly receiver: TrackStats | null;
+    readonly direction: string;
+    readonly currentDirection: string;
+}
+
+export interface TrackStats {
+    readonly id: string;
+    readonly kind: "audio" | "video";
+    readonly settingDeviceId: string;
+    readonly constrainDeviceId: string;
+    readonly muted: boolean;
+    readonly enabled: boolean;
+    readonly readyState: "ended" | "live";
+    readonly label: string;
+}

package/src/webrtc/stats/statsReportEmitter.ts

@@ -0,0 +1,49 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { TypedEventEmitter } from "../../models/typed-event-emitter.ts";
+import {
+    ByteSentStatsReport,
+    CallFeedReport,
+    ConnectionStatsReport,
+    StatsReport,
+    SummaryStatsReport,
+} from "./statsReport.ts";
+
+export type StatsReportHandlerMap = {
+    [StatsReport.BYTE_SENT_STATS]: (report: ByteSentStatsReport) => void;
+    [StatsReport.CONNECTION_STATS]: (report: ConnectionStatsReport) => void;
+    [StatsReport.CALL_FEED_REPORT]: (report: CallFeedReport) => void;
+    [StatsReport.SUMMARY_STATS]: (report: SummaryStatsReport) => void;
+};
+
+export class StatsReportEmitter extends TypedEventEmitter<StatsReport, StatsReportHandlerMap> {
+    public emitByteSendReport(byteSentStats: ByteSentStatsReport): void {
+        this.emit(StatsReport.BYTE_SENT_STATS, byteSentStats);
+    }
+
+    public emitConnectionStatsReport(report: ConnectionStatsReport): void {
+        this.emit(StatsReport.CONNECTION_STATS, report);
+    }
+
+    public emitCallFeedReport(report: CallFeedReport): void {
+        this.emit(StatsReport.CALL_FEED_REPORT, report);
+    }
+
+    public emitSummaryStatsReport(report: SummaryStatsReport): void {
+        this.emit(StatsReport.SUMMARY_STATS, report);
+    }
+}

package/src/webrtc/stats/summaryStatsReportGatherer.ts

@@ -0,0 +1,148 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+import { StatsReportEmitter } from "./statsReportEmitter.ts";
+import { CallStatsReportSummary } from "./callStatsReportSummary.ts";
+import { SummaryStatsReport } from "./statsReport.ts";
+import { ParticipantState } from "../groupCall.ts";
+import { RoomMember } from "../../matrix.ts";
+
+interface CallStatsReportSummaryCounter {
+    receivedAudio: number;
+    receivedVideo: number;
+    receivedMedia: number;
+    concealedAudio: number;
+    totalAudio: number;
+}
+
+export class SummaryStatsReportGatherer {
+    public constructor(private emitter: StatsReportEmitter) {}
+
+    public build(allSummary: CallStatsReportSummary[]): void {
+        // Filter all stats which collect the first time webrtc stats.
+        // Because stats based on time interval and the first collection of a summery stats has no previous
+        // webrtcStats as basement all the calculation are 0. We don't want track the 0 stats.
+        const summary = allSummary.filter((s) => !s.isFirstCollection);
+        const summaryTotalCount = summary.length;
+        // For counting the peer connections we also want to consider the ignored summaries
+        const peerConnectionsCount = allSummary.length;
+        if (summaryTotalCount === 0) {
+            return;
+        }
+
+        const summaryCounter: CallStatsReportSummaryCounter = {
+            receivedAudio: 0,
+            receivedVideo: 0,
+            receivedMedia: 0,
+            concealedAudio: 0,
+            totalAudio: 0,
+        };
+        let maxJitter = 0;
+        let maxPacketLoss = 0;
+        summary.forEach((stats) => {
+            this.countTrackListReceivedMedia(summaryCounter, stats);
+            this.countConcealedAudio(summaryCounter, stats);
+            maxJitter = this.buildMaxJitter(maxJitter, stats);
+            maxPacketLoss = this.buildMaxPacketLoss(maxPacketLoss, stats);
+        });
+        const decimalPlaces = 5;
+        const report = {
+            percentageReceivedMedia: Number((summaryCounter.receivedMedia / summaryTotalCount).toFixed(decimalPlaces)),
+            percentageReceivedVideoMedia: Number(
+                (summaryCounter.receivedVideo / summaryTotalCount).toFixed(decimalPlaces),
+            ),
+            percentageReceivedAudioMedia: Number(
+                (summaryCounter.receivedAudio / summaryTotalCount).toFixed(decimalPlaces),
+            ),
+            maxJitter,
+            maxPacketLoss,
+            percentageConcealedAudio: Number(
+                summaryCounter.totalAudio > 0
+                    ? (summaryCounter.concealedAudio / summaryCounter.totalAudio).toFixed(decimalPlaces)
+                    : 0,
+            ),
+            peerConnections: peerConnectionsCount,
+        } as SummaryStatsReport;
+        this.emitter.emitSummaryStatsReport(report);
+    }
+
+    public static extendSummaryReport(
+        report: SummaryStatsReport,
+        callParticipants: Map<RoomMember, Map<string, ParticipantState>>,
+    ): void {
+        // Calculate the actual number of devices based on the participants state event
+        // (this is used, to compare the expected participant count from the room state with the acutal peer connections)
+        // const devices = callParticipants.()
+        const devices: [string, ParticipantState][] = [];
+        const users: [RoomMember, Map<string, ParticipantState>][] = [];
+        for (const userEntry of callParticipants) {
+            users.push(userEntry);
+            for (const device of userEntry[1]) {
+                devices.push(device);
+            }
+        }
+        report.opponentDevicesInCall = Math.max(0, devices.length - 1);
+        report.opponentUsersInCall = Math.max(0, users.length - 1);
+        report.diffDevicesToPeerConnections = Math.max(0, devices.length - 1) - report.peerConnections;
+        report.ratioPeerConnectionToDevices =
+            Math.max(0, devices.length - 1) == 0 ? 0 : report.peerConnections / (devices.length - 1);
+    }
+
+    private countTrackListReceivedMedia(counter: CallStatsReportSummaryCounter, stats: CallStatsReportSummary): void {
+        let hasReceivedAudio = false;
+        let hasReceivedVideo = false;
+        if (stats.receivedAudioMedia > 0 || stats.audioTrackSummary.count === 0) {
+            counter.receivedAudio++;
+            hasReceivedAudio = true;
+        }
+        if (stats.receivedVideoMedia > 0 || stats.videoTrackSummary.count === 0) {
+            counter.receivedVideo++;
+            hasReceivedVideo = true;
+        } else {
+            if (stats.videoTrackSummary.muted > 0 && stats.videoTrackSummary.muted === stats.videoTrackSummary.count) {
+                counter.receivedVideo++;
+                hasReceivedVideo = true;
+            }
+        }
+
+        if (hasReceivedVideo && hasReceivedAudio) {
+            counter.receivedMedia++;
+        }
+    }
+
+    private buildMaxJitter(maxJitter: number, stats: CallStatsReportSummary): number {
+        if (maxJitter < stats.videoTrackSummary.maxJitter) {
+            maxJitter = stats.videoTrackSummary.maxJitter;
+        }
+
+        if (maxJitter < stats.audioTrackSummary.maxJitter) {
+            maxJitter = stats.audioTrackSummary.maxJitter;
+        }
+        return maxJitter;
+    }
+
+    private buildMaxPacketLoss(maxPacketLoss: number, stats: CallStatsReportSummary): number {
+        if (maxPacketLoss < stats.videoTrackSummary.maxPacketLoss) {
+            maxPacketLoss = stats.videoTrackSummary.maxPacketLoss;
+        }
+
+        if (maxPacketLoss < stats.audioTrackSummary.maxPacketLoss) {
+            maxPacketLoss = stats.audioTrackSummary.maxPacketLoss;
+        }
+        return maxPacketLoss;
+    }
+
+    private countConcealedAudio(summaryCounter: CallStatsReportSummaryCounter, stats: CallStatsReportSummary): void {
+        summaryCounter.concealedAudio += stats.audioTrackSummary.concealedAudio;
+        summaryCounter.totalAudio += stats.audioTrackSummary.totalAudio;
+    }
+}