@lakutata/webrtc 0.0.0-stage → 3.1.4

package/dist/esm/lib/peer/media/rtpReceiver.js

@@ -0,0 +1,403 @@
+import { randomUUID } from 'node:crypto';
+import { setTimeout } from 'node:timers/promises';
+import { Event, int } from '../imports/common.js';
+import { PictureLossIndication, RTP_EXTENSION_URI, Red, RedHandler, RtcpPayloadSpecificFeedback, RtcpReceiverInfo, RtcpRrPacket, RtcpSrPacket, debug, unwrapRtx } from '../imports/rtp.js';
+import { compactNtp, ntpTimeToEpochMs, timestampSeconds } from '../utils.js';
+import { deliveredTrackCodec } from './codecCompatibility.js';
+import { NackHandler } from './receiver/nack.js';
+import { StreamStatistics } from './receiver/statistics.js';
+import { codecParametersFromString } from '../sdp.js';
+import { usePLI, useTWCC } from './extension/rtcpFeedback.js';
+import { buildStatsReport, generateCodecStatsId, generateStatsId, getStatsTimestamp } from './stats.js';
+import { MediaStreamTrack } from './track.js';
+const log = debug('webrtc/peer/media/rtpReceiver');
+export class RTCRtpReceiver {
+    get codecArray() {
+        this.sortedCodecs ??= Object.values(this.codecs).sort((a, b) => a.payloadType - b.payloadType);
+        return this.sortedCodecs;
+    }
+    constructor(config, kind, rtcpSsrc) {
+        this.config = config;
+        this.kind = kind;
+        this.rtcpSsrc = rtcpSsrc;
+        this.codecs = {};
+        this.ssrcByRtx = {};
+        this.nack = new NackHandler(this);
+        this.audioRedHandler = new RedHandler();
+        this.type = 'receiver';
+        this.uuid = randomUUID().toString();
+        this.tracks = [];
+        this.trackBySSRC = {};
+        this.trackByRID = {};
+        /**last sender Report Timestamp
+       * compactNtp
+       */
+        this.lastSRtimestamp = {};
+        /**seconds */
+        this.receiveLastSRTimestamp = {};
+        this.onPacketLost = this.nack.onPacketLost;
+        this.onRtcp = new Event();
+        this.stopped = false;
+        this.remoteStreamIds = [];
+        this.rtcpRunning = false;
+        this.rtcpCancel = new AbortController();
+        this.remoteStreams = {};
+        this.senderReportsReceivedBySsrc = {};
+        this.remoteTimestampsBySsrc = {};
+        this.remotePacketCountBySsrc = {};
+        this.remoteOctetCountBySsrc = {};
+        this.nackCountBySsrc = {};
+        this.pliCountBySsrc = {};
+        this.handleRtpBySsrc = (packet, extensions) => {
+            const track = this.trackBySSRC[packet.header.ssrc];
+            this.handleRTP(packet, extensions, track);
+        };
+        this.handleRtpByRid = (packet, rid, extensions) => {
+            const track = this.trackByRID[rid];
+            if (!this.trackBySSRC[packet.header.ssrc]) {
+                this.trackBySSRC[packet.header.ssrc] = track;
+            }
+            this.handleRTP(packet, extensions, track);
+        };
+        this.defaultTrack = new MediaStreamTrack({ kind, remote: true });
+        this.onPacketLost.subscribe((nack) => {
+            this.nackCountBySsrc[nack.mediaSourceSsrc] =
+                (this.nackCountBySsrc[nack.mediaSourceSsrc] ?? 0) + 1;
+        });
+    }
+    get transport() {
+        return this.dtlsTransport ?? null;
+    }
+    setDtlsTransport(dtls) {
+        this.dtlsTransport = dtls;
+    }
+    // todo fix
+    get track() {
+        return this.tracks[0] ?? this.defaultTrack;
+    }
+    get nackEnabled() {
+        return this.codecArray[0]?.rtcpFeedback.find((f) => f.type === 'nack');
+    }
+    get twccEnabled() {
+        return this.codecArray[0]?.rtcpFeedback.find((f) => f.type === useTWCC().type);
+    }
+    get pliEnabled() {
+        return this.codecArray[0]?.rtcpFeedback.find((f) => f.type === usePLI().type);
+    }
+    prepareReceive(params) {
+        params.codecs.forEach((c) => {
+            this.codecs[c.payloadType] = c;
+        });
+        this.sortedCodecs = undefined;
+        params.encodings.forEach((e) => {
+            if (e.rtx) {
+                this.ssrcByRtx[e.rtx.ssrc] = e.ssrc;
+            }
+        });
+    }
+    /**
+   * negotiated codec と receiver の実処理状態を再同期する。
+   * codec map と RTX mapping を置換し、既存 remote track の codec
+   * metadata も新しい先頭 codec に更新する。
+   * transport-cc の有無は twccEnabled が新しい codec から判断する。
+   * setCodecPreferences() 後の createAnswer() 再解決や remote からの
+   * 再交渉では、除外された payload type の RTP を受け付けないようにする。
+   */
+    resyncCodecs(params) {
+        for (const key of Object.keys(this.codecs)) {
+            delete this.codecs[Number(key)];
+        }
+        this.sortedCodecs = undefined;
+        for (const key of Object.keys(this.ssrcByRtx)) {
+            delete this.ssrcByRtx[Number(key)];
+        }
+        this.prepareReceive(params);
+        const codec = deliveredTrackCodec(this.kind, params.codecs[0], params.codecs);
+        if (codec) {
+            for (const track of this.tracks) {
+                track.codec = codec;
+            }
+        }
+    }
+    addTrack(track) {
+        const exist = this.tracks.find((t) => {
+            if (t.rid) {
+                return t.rid === track.rid;
+            }
+            if (t.ssrc) {
+                return t.ssrc === track.ssrc;
+            }
+        });
+        if (exist) {
+            return false;
+        }
+        this.tracks.push(track);
+        if (track.ssrc) {
+            this.trackBySSRC[track.ssrc] = track;
+        }
+        if (track.rid) {
+            this.trackByRID[track.rid] = track;
+        }
+        return true;
+    }
+    stop() {
+        this.stopped = true;
+        this.rtcpRunning = false;
+        this.rtcpCancel.abort();
+        this.nack.close();
+    }
+    /**transceiver の停止確定時に remote track を ended にする */
+    endTracks() {
+        for (const track of [...this.tracks, this.defaultTrack]) {
+            track.stop();
+        }
+    }
+    async runRtcp() {
+        if (this.rtcpRunning || this.stopped)
+            return;
+        this.rtcpRunning = true;
+        try {
+            while (this.rtcpRunning) {
+                await setTimeout(500 + Math.random() * 1000, undefined, {
+                    signal: this.rtcpCancel.signal
+                });
+                const reports = Object.entries(this.remoteStreams).map(([ssrc, stream]) => {
+                    let lastSRtimestamp = 0, delaySinceLastSR = 0;
+                    if (this.lastSRtimestamp[ssrc]) {
+                        lastSRtimestamp = this.lastSRtimestamp[ssrc];
+                        const delaySeconds = timestampSeconds() - this.receiveLastSRTimestamp[ssrc];
+                        if (delaySeconds > 0 && delaySeconds < 65536) {
+                            delaySinceLastSR = int(delaySeconds * 65536);
+                        }
+                    }
+                    return new RtcpReceiverInfo({
+                        ssrc: Number(ssrc),
+                        fractionLost: stream.fraction_lost,
+                        packetsLost: stream.packets_lost,
+                        highestSequence: stream.max_seq,
+                        jitter: stream.jitter,
+                        lsr: lastSRtimestamp,
+                        dlsr: delaySinceLastSR
+                    });
+                });
+                const packet = new RtcpRrPacket({ ssrc: this.rtcpSsrc, reports });
+                try {
+                    if (this.config.debug.receiverReportDelay) {
+                        await setTimeout(this.config.debug.receiverReportDelay);
+                    }
+                    await this.dtlsTransport.sendRtcp([packet]);
+                }
+                catch (error) {
+                    log('sendRtcp failed', error);
+                    await setTimeout(500 + Math.random() * 1000);
+                }
+            }
+        }
+        catch (error) {
+            //The loop of the RTCP reports ends with the transport
+            log('rtcp loop ended', error);
+        }
+    }
+    getInboundRtpStatsId(track) {
+        return generateStatsId('inbound-rtp', track.id ?? track.uuid);
+    }
+    getRemoteOutboundRtpStatsId(track) {
+        return generateStatsId('remote-outbound-rtp', track.id ?? track.uuid);
+    }
+    getStatsRootIds(selector) {
+        return this.tracks
+            .filter((track) => (!selector ? true : track === selector))
+            .filter((track) => track.ssrc)
+            .map((track) => this.getInboundRtpStatsId(track));
+    }
+    collectStats(timestamp) {
+        const stats = [];
+        const transportId = this.dtlsTransport
+            ? generateStatsId('transport', this.dtlsTransport.id)
+            : undefined;
+        const activeCodec = this.codecArray[0];
+        const emittedCodecIds = new Set();
+        // Collect stats for each track
+        for (const track of this.tracks) {
+            if (!track.ssrc)
+                continue;
+            const streamStats = this.remoteStreams[track.ssrc];
+            // Inbound RTP stats
+            const hasRemoteTimestamp = this.remoteTimestampsBySsrc[track.ssrc] !== undefined;
+            const remoteId = this.lastSRtimestamp[track.ssrc] !== undefined || hasRemoteTimestamp
+                ? this.getRemoteOutboundRtpStatsId(track)
+                : undefined;
+            const codecId = activeCodec && transportId
+                ? generateCodecStatsId(transportId, activeCodec.payloadType, track.id ?? track.uuid)
+                : undefined;
+            if (activeCodec &&
+                transportId &&
+                codecId &&
+                !emittedCodecIds.has(codecId)) {
+                emittedCodecIds.add(codecId);
+                const codecStats = {
+                    type: 'codec',
+                    id: codecId,
+                    timestamp,
+                    payloadType: activeCodec.payloadType,
+                    transportId,
+                    mimeType: activeCodec.mimeType,
+                    clockRate: activeCodec.clockRate,
+                    channels: activeCodec.channels,
+                    sdpFmtpLine: activeCodec.parameters
+                };
+                stats.push(codecStats);
+            }
+            const inboundRtpStats = {
+                type: 'inbound-rtp',
+                id: this.getInboundRtpStatsId(track),
+                timestamp,
+                ssrc: track.ssrc,
+                kind: this.kind,
+                transportId,
+                codecId,
+                mid: this.sdesMid,
+                trackIdentifier: track.id ?? track.uuid,
+                packetsReceived: streamStats?.packets_received ?? 0,
+                bytesReceived: streamStats?.bytesReceived ?? 0,
+                headerBytesReceived: streamStats?.headerBytesReceived ?? 0,
+                packetsLost: streamStats?.packets_lost ?? 0,
+                jitter: streamStats?.clockRate
+                    ? streamStats.jitter / streamStats.clockRate
+                    : undefined,
+                lastPacketReceivedTimestamp: streamStats?.lastPacketReceivedTimestamp,
+                remoteId,
+                nackCount: this.nackCountBySsrc[track.ssrc] || undefined,
+                pliCount: this.pliCountBySsrc[track.ssrc] || undefined
+            };
+            stats.push(inboundRtpStats);
+            // Remote outbound RTP stats (if we have SR info)
+            if (remoteId) {
+                const remoteOutboundStats = {
+                    type: 'remote-outbound-rtp',
+                    id: this.getRemoteOutboundRtpStatsId(track),
+                    timestamp,
+                    ssrc: track.ssrc,
+                    kind: this.kind,
+                    transportId,
+                    codecId: inboundRtpStats.codecId,
+                    localId: inboundRtpStats.id,
+                    remoteTimestamp: this.remoteTimestampsBySsrc[track.ssrc],
+                    reportsSent: this.senderReportsReceivedBySsrc[track.ssrc] ?? 0,
+                    packetsSent: this.remotePacketCountBySsrc[track.ssrc],
+                    bytesSent: this.remoteOctetCountBySsrc[track.ssrc]
+                };
+                stats.push(remoteOutboundStats);
+            }
+        }
+        return stats;
+    }
+    async getStats() {
+        const timestamp = getStatsTimestamp();
+        const stats = this.collectStats(timestamp);
+        if (this.dtlsTransport) {
+            stats.push(...(await this.dtlsTransport.getStats(timestamp)));
+        }
+        return buildStatsReport(stats, this.getStatsRootIds());
+    }
+    async sendRtcpPLI(mediaSsrc) {
+        if (!this.pliEnabled) {
+            log('pli not supported', { mediaSsrc });
+            return;
+        }
+        if (this.stopped) {
+            return;
+        }
+        log('sendRtcpPLI', { mediaSsrc });
+        const packet = new RtcpPayloadSpecificFeedback({
+            feedback: new PictureLossIndication({
+                senderSsrc: this.rtcpSsrc,
+                mediaSsrc
+            })
+        });
+        try {
+            this.pliCountBySsrc[mediaSsrc] =
+                (this.pliCountBySsrc[mediaSsrc] ?? 0) + 1;
+            await this.dtlsTransport.sendRtcp([packet]);
+        }
+        catch (error) {
+            log(error);
+        }
+    }
+    handleRtcpPacket(packet) {
+        switch (packet.type) {
+            case RtcpSrPacket.type:
+                {
+                    const sr = packet;
+                    this.lastSRtimestamp[sr.ssrc] = compactNtp(sr.senderInfo.ntpTimestamp);
+                    this.receiveLastSRTimestamp[sr.ssrc] = timestampSeconds();
+                    this.senderReportsReceivedBySsrc[sr.ssrc] =
+                        (this.senderReportsReceivedBySsrc[sr.ssrc] ?? 0) + 1;
+                    this.remoteTimestampsBySsrc[sr.ssrc] = ntpTimeToEpochMs(sr.senderInfo.ntpTimestamp);
+                    this.remotePacketCountBySsrc[sr.ssrc] = sr.senderInfo.packetCount;
+                    this.remoteOctetCountBySsrc[sr.ssrc] = sr.senderInfo.octetCount;
+                    const track = this.trackBySSRC[packet.ssrc];
+                    if (track) {
+                        track.onReceiveRtcp.execute(packet);
+                    }
+                }
+                break;
+        }
+        this.onRtcp.execute(packet);
+    }
+    handleRTP(packet, extensions, track) {
+        if (this.stopped) {
+            return;
+        }
+        const codec = this.codecs[packet.header.payloadType];
+        if (!codec) {
+            // log("unknown codec " + packet.header.payloadType);
+            return;
+        }
+        this.remoteStreams[packet.header.ssrc] =
+            this.remoteStreams[packet.header.ssrc] ??
+                new StreamStatistics(codec.clockRate);
+        this.remoteStreams[packet.header.ssrc].add(packet);
+        //The transport reports the arrival of the packets of the streams negotiating transport-cc; a packet without the
+        //transport-wide sequence number (the extension not negotiated) is not counted
+        if (this.twccEnabled) {
+            const transportSequenceNumber = extensions[RTP_EXTENSION_URI.transportWideCC];
+            if (transportSequenceNumber !== undefined) {
+                this.dtlsTransport.receiverTWCC.handleTWCC(transportSequenceNumber, packet.header.ssrc);
+            }
+        }
+        if (codec.name.toLowerCase() === 'rtx') {
+            const originalSsrc = this.ssrcByRtx[packet.header.ssrc];
+            const codecParams = codecParametersFromString(codec.parameters ?? '');
+            const rtxCodec = this.codecs[codecParams['apt']];
+            if (packet.payload.length < 2)
+                return;
+            packet = unwrapRtx(packet, rtxCodec.payloadType, originalSsrc);
+            track = this.trackBySSRC[originalSsrc];
+        }
+        let red;
+        if (codec.name.toLowerCase() === 'red') {
+            red = Red.deSerialize(packet.payload);
+            if (!Object.keys(this.codecs).includes(red.header.fields[0].blockPT.toString())) {
+                return;
+            }
+        }
+        if (track?.kind === 'video' && this.nackEnabled) {
+            this.nack.addPacket(packet);
+        }
+        if (track) {
+            if (red) {
+                if (track.kind === 'audio') {
+                    const payloads = this.audioRedHandler.push(red, packet);
+                    for (const packet of payloads) {
+                        track.onReceiveRtp.execute(packet.clone(), extensions);
+                    }
+                }
+            }
+            else {
+                track.onReceiveRtp.execute(packet.clone(), extensions);
+            }
+        }
+        this.runRtcp().catch((error) => log('rtcp failed', error));
+    }
+}