@m7mdxzx1/discord-video-strem 0.0.1

package/src/client/packet/VideoPacketizerAnnexB.ts

@@ -0,0 +1,263 @@
+import type { MediaUdp } from "../voice/MediaUdp.js";
+import { BaseMediaPacketizer } from "./BaseMediaPacketizer.js";
+import {
+    H264Helpers,
+    H265Helpers,
+    type AnnexBHelpers
+} from "../processing/AnnexBHelper.js";
+import { extensions } from "../../utils.js";
+import { splitNalu } from "../processing/AnnexBHelper.js";
+import { CodecPayloadType } from "../voice/BaseMediaConnection.js";
+
+/**
+ * Annex B format
+ * 
+ * Packetizer for Annex B NAL. This method does NOT support aggregation packets
+ * where multiple NALs are sent as a single RTP payload. The supported payload
+ * type is Single NAL Unit Packet and Fragmentation Unit A (FU-A). The headers
+ * produced correspond to packetization-mode=1.
+
+         RTP Payload Format for H.264 Video:
+         https://tools.ietf.org/html/rfc6184
+
+         RTP Payload Format for HEVC Video:
+         https://tools.ietf.org/html/rfc7798
+         
+         FFmpeg H264/HEVC RTP packetisation code:
+         https://github.com/FFmpeg/FFmpeg/blob/master/libavformat/rtpenc_h264_hevc.c
+         
+         When the payload size is less than or equal to max RTP payload, send as 
+         Single NAL Unit Packet:
+         https://tools.ietf.org/html/rfc6184#section-5.6
+         https://tools.ietf.org/html/rfc7798#section-4.4.1
+         
+         0                   1                   2                   3
+         0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+         |F|NRI|  Type   |                                               |
+         +-+-+-+-+-+-+-+-+                                               |
+         |                                                               |
+         |               Bytes 2..n of a single NAL unit                 |
+         |                                                               |
+         |                               +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+         |                               :...OPTIONAL RTP padding        |
+         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+         
+         Type = 24 for STAP-A (NOTE: this is the type of the RTP header 
+         and NOT the NAL type).
+         
+         When the payload size is greater than max RTP payload, send as 
+         Fragmentation Unit A (FU-A):
+         https://tools.ietf.org/html/rfc6184#section-5.8
+              0                   1                   2                   3
+         0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+         | FU indicator  |   FU header   |                               |
+         +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 
+         |   Fragmentation Unit (FU) Payload
+         |
+         ...
+ */
+class VideoPacketizerAnnexB extends BaseMediaPacketizer {
+    private _nalFunctions: AnnexBHelpers;
+
+    constructor(connection: MediaUdp, ssrc: number, payloadType: number, nalFunctions: AnnexBHelpers) {
+        super(connection, ssrc, payloadType, true);
+        this._nalFunctions = nalFunctions;
+    }
+
+    /**
+     * Sends packets after partitioning the video frame into
+     * MTU-sized chunks
+     * @param frame Annex B video frame
+     */
+    public override async sendFrame(frame: Buffer, frametime: number): Promise<void> {
+        super.sendFrame(frame, frametime);
+
+        const nalus = splitNalu(frame);
+
+        let packetsSent = 0;
+        let bytesSent = 0;
+        let index = 0;
+        for (const nalu of nalus) {
+            const isLastNal = index === nalus.length - 1;
+            if (nalu.length <= this.mtu) {
+                // Send as Single NAL Unit Packet.
+                const packetHeader = Buffer.concat([this.makeRtpHeader(isLastNal), this.createExtensionHeader(extensions)]);
+
+                const [ciphertext, nonceBuffer] = await this.encryptData(
+                    Buffer.concat([this.createExtensionPayload(extensions), nalu]), packetHeader
+                );
+                const packet = Buffer.concat([
+                    packetHeader, ciphertext,
+                    nonceBuffer.subarray(0, 4),
+                ]);
+                this.mediaUdp.sendPacket(packet);
+                packetsSent++;
+                bytesSent += packet.length;
+            } else {
+                const [naluHeader, naluData] = this._nalFunctions.splitHeader(nalu);
+                const data = this.partitionDataMTUSizedChunks(naluData);
+                const encryptedPackets: Promise<Buffer>[] = [];
+
+                // Send as Fragmentation Unit A (FU-A):
+                for (let i = 0; i < data.length; i++) {
+                    const isFirstPacket = i === 0;
+                    const isFinalPacket = i === data.length - 1;
+
+                    const markerBit = isLastNal && isFinalPacket;
+
+                    const packetHeader = Buffer.concat([this.makeRtpHeader(markerBit), this.createExtensionHeader(extensions)]);
+
+                    const packetData = Buffer.concat([
+                        this.createExtensionPayload(extensions),
+                        this.makeFragmentationUnitHeader(
+                            isFirstPacket,
+                            isFinalPacket,
+                            naluHeader
+                        ),
+                        data[i]
+                    ]);
+
+                    encryptedPackets.push(this.encryptData(packetData, packetHeader)
+                        .then(([ciphertext, nonceBuffer]) => Buffer.concat([
+                            packetHeader,
+                            ciphertext,
+                            nonceBuffer.subarray(0, 4),
+                        ]))
+                    );
+                }
+
+                for (const packet of await Promise.all(encryptedPackets))
+                {
+                    this.mediaUdp.sendPacket(packet);
+                    packetsSent++;
+                    bytesSent += packet.length;
+                }
+            }
+            index++;
+        }
+
+        await this.onFrameSent(packetsSent, bytesSent, frametime);
+    }
+
+    protected makeFragmentationUnitHeader(isFirstPacket: boolean, isLastPacket: boolean, naluHeader: Buffer): Buffer {
+        throw new Error("Not implemented");
+    }
+
+    public override async onFrameSent(packetsSent: number, bytesSent: number, frametime: number): Promise<void> {
+        await super.onFrameSent(packetsSent, bytesSent, frametime);
+        // video RTP packet timestamp incremental value = 90,000Hz / fps
+        this.incrementTimestamp(90000 / 1000 * frametime);
+    }
+}
+
+export class VideoPacketizerH264 extends VideoPacketizerAnnexB {
+    constructor(connection: MediaUdp, ssrc: number) {
+        super(connection, ssrc, CodecPayloadType.H264.payload_type, H264Helpers);
+    }
+    /**
+     * The FU indicator octet has the following format:
+        
+            +---------------+
+            |0|1|2|3|4|5|6|7|
+            +-+-+-+-+-+-+-+-+
+            |F|NRI|  Type   |
+            +---------------+
+            
+            F and NRI bits come from the NAL being transmitted.
+            Type = 28 for FU-A (NOTE: this is the type of the H264 RTP header 
+            and NOT the NAL type).
+            
+            The FU header has the following format:
+            
+            +---------------+
+            |0|1|2|3|4|5|6|7|
+            +-+-+-+-+-+-+-+-+
+            |S|E|R|  Type   |
+            +---------------+
+            
+            S: Set to 1 for the start of the NAL FU (i.e. first packet in frame).
+            E: Set to 1 for the end of the NAL FU (i.e. the last packet in the frame).
+            R: Reserved bit must be 0.
+            Type: The NAL unit payload type, comes from NAL packet (NOTE: this IS the type of the NAL message).
+ * @param isFirstPacket 
+ * @param isLastPacket 
+ * @param naluHeader
+ * @returns FU-A packets
+ */
+    protected override makeFragmentationUnitHeader(isFirstPacket: boolean, isLastPacket: boolean, naluHeader: Buffer): Buffer {
+        const nal0 = naluHeader[0];
+        const fuPayloadHeader = Buffer.alloc(2);
+        const nalType = H264Helpers.getUnitType(naluHeader);
+        const fnri = nal0 & 0xe0;
+
+        // set fu indicator
+        fuPayloadHeader[0] = 0x1c | fnri; // type 28 with fnri from original frame
+
+        // set fu header
+        if (isFirstPacket) {
+            fuPayloadHeader[1] = 0x80 | nalType; // set start bit
+        } else if (isLastPacket) {
+            fuPayloadHeader[1] = 0x40 | nalType; // set last bit
+        } else {
+            fuPayloadHeader[1] = nalType; // no start or end bit
+        }
+
+        return fuPayloadHeader;
+    }
+}
+
+export class VideoPacketizerH265 extends VideoPacketizerAnnexB {
+    constructor(connection: MediaUdp, ssrc: number) {
+        super(connection, ssrc, CodecPayloadType.H265.payload_type, H265Helpers);
+    }
+    /**
+     * The FU indicator octet has the following format:
+
+            +---------------+---------------+
+            |0|1|2|3|4|5|6|7|0|1|2|3|4|5|6|7|
+            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+            |F|   Type    |  LayerId  | TID |
+            +-------------+-----------------+
+            
+            All other fields except Type come from the NAL being transmitted.
+            Type = 49 for FU-A (NOTE: this is the type of the H265 RTP header 
+            and NOT the NAL type).
+            
+            The FU header has the following format:
+            
+            +---------------+
+            |0|1|2|3|4|5|6|7|
+            +-+-+-+-+-+-+-+-+
+            |S|E|    Type   |
+            +---------------+
+            
+            S: Set to 1 for the start of the NAL FU (i.e. first packet in frame).
+            E: Set to 1 for the end of the NAL FU (i.e. the last packet in the frame).
+            Type: The NAL unit payload type, comes from NAL packet (NOTE: this IS the type of the NAL message).
+ * @param isFirstPacket 
+ * @param isLastPacket 
+ * @param naluHeader
+ * @returns FU-A packets
+ */
+    protected override makeFragmentationUnitHeader(isFirstPacket: boolean, isLastPacket: boolean, naluHeader: Buffer): Buffer {
+        const fuIndicatorHeader = Buffer.allocUnsafe(3);
+        naluHeader.copy(fuIndicatorHeader);
+        const nalType = H265Helpers.getUnitType(naluHeader);
+
+        // clear NAL type and set it to 49
+        fuIndicatorHeader[0] = (fuIndicatorHeader[0] & 0b10000001) | (49 << 1);
+
+        // set fu header
+        if (isFirstPacket) {
+            fuIndicatorHeader[2] = 0x80 | nalType; // set start bit
+        } else if (isLastPacket) {
+            fuIndicatorHeader[2] = 0x40 | nalType; // set last bit
+        } else {
+            fuIndicatorHeader[2] = nalType; // no start or end bit
+        }
+
+        return fuIndicatorHeader;
+    }
+}

package/src/client/packet/VideoPacketizerVP8.ts

@@ -0,0 +1,73 @@
+import type { MediaUdp } from "../voice/MediaUdp.js";
+import { extensions, max_int16bit } from "../../utils.js";
+import { BaseMediaPacketizer } from "./BaseMediaPacketizer.js";
+import { CodecPayloadType } from "../voice/BaseMediaConnection.js";
+
+/**
+ * VP8 payload format
+ * 
+ */
+export class VideoPacketizerVP8 extends BaseMediaPacketizer {
+    private _pictureId: number;
+
+    constructor(connection: MediaUdp, ssrc: number) {
+        super(connection, ssrc, CodecPayloadType.VP8.payload_type, true);
+        this._pictureId = 0;
+    }
+
+    private incrementPictureId(): void {
+        this._pictureId = (this._pictureId + 1) % max_int16bit;
+    }
+
+    public override async sendFrame(frame: Buffer, frametime: number): Promise<void> {
+        super.sendFrame(frame, frametime);
+        const data = this.partitionDataMTUSizedChunks(frame);
+
+        let bytesSent = 0;
+        const encryptedPackets = data.map((chunk, i) => this.createPacket(chunk, i === (data.length - 1), i === 0))
+        for (const packet of await Promise.all(encryptedPackets)) {
+            this.mediaUdp.sendPacket(packet);
+            bytesSent += packet.length;
+        }
+
+        await this.onFrameSent(data.length, bytesSent, frametime);
+    }
+
+    public async createPacket(chunk: Buffer, isLastPacket = true, isFirstPacket = true): Promise<Buffer> {
+        if(chunk.length > this.mtu) throw Error('error packetizing video frame: frame is larger than mtu');
+
+        const packetHeader = Buffer.concat([this.makeRtpHeader(isLastPacket), this.createExtensionHeader(extensions)]);
+
+        const packetData = Buffer.concat([this.createExtensionPayload(extensions), this.makeChunk(chunk, isFirstPacket)]);
+    
+        // nonce buffer used for encryption. 4 bytes are appended to end of packet
+        const [ciphertext, nonceBuffer] = await this.encryptData(packetData, packetHeader);
+        return Buffer.concat([packetHeader, ciphertext, nonceBuffer.subarray(0, 4)]);
+    }
+
+    public override async onFrameSent(packetsSent: number, bytesSent: number, frametime: number): Promise<void> {
+        await super.onFrameSent(packetsSent, bytesSent, frametime);
+        // video RTP packet timestamp incremental value = 90,000Hz / fps
+        this.incrementTimestamp(90000 / 1000 * frametime);
+        this.incrementPictureId();
+    }
+
+    private makeChunk(frameData: Buffer, isFirstPacket: boolean): Buffer {
+        // vp8 payload descriptor
+        const payloadDescriptorBuf = Buffer.alloc(2);
+    
+        payloadDescriptorBuf[0] = 0x80;
+        payloadDescriptorBuf[1] = 0x80;
+        if (isFirstPacket) {
+            payloadDescriptorBuf[0] |= 0b00010000; // mark S bit, indicates start of frame
+        }
+    
+        // vp8 pictureid payload extension
+        const pictureIdBuf = Buffer.alloc(2);
+    
+        pictureIdBuf.writeUIntBE(this._pictureId, 0, 2);
+        pictureIdBuf[0] |= 0b10000000;
+    
+        return Buffer.concat([payloadDescriptorBuf, pictureIdBuf, frameData]);
+    }
+}

package/src/client/packet/index.ts

@@ -0,0 +1,4 @@
+export * from './AudioPacketizer.js';
+export * from './BaseMediaPacketizer.js';
+export * from './VideoPacketizerVP8.js';
+export * from './VideoPacketizerAnnexB.js';

package/src/client/processing/AnnexBHelper.ts

@@ -0,0 +1,142 @@
+export enum H264NalUnitTypes {
+    Unspecified = 0,
+    CodedSliceNonIDR = 1,
+    CodedSlicePartitionA = 2,
+    CodedSlicePartitionB = 3,
+    CodedSlicePartitionC = 4,
+    CodedSliceIdr = 5,
+    SEI = 6,
+    SPS = 7,
+    PPS = 8,
+    AccessUnitDelimiter = 9,
+    EndOfSequence = 10,
+    EndOfStream = 11,
+    FillerData = 12,
+    SEIExtenstion = 13,
+    PrefixNalUnit = 14,
+    SubsetSPS = 15,
+}
+
+export enum H265NalUnitTypes {
+    TRAIL_N = 0,
+    TRAIL_R = 1,
+    TSA_N = 2,
+    TSA_R = 3,
+    STSA_N = 4,
+    STSA_R = 5,
+    RADL_N = 6,
+    RADL_R = 7,
+    RASL_N = 8,
+    RASL_R = 9,
+    RSV_VCL_N10 = 10,
+    RSV_VCL_R11 = 11,
+    RSV_VCL_N12 = 12,
+    RSV_VCL_R13 = 13,
+    RSV_VCL_N14 = 14,
+    RSV_VCL_R15 = 15,
+    BLA_W_LP = 16,
+    BLA_W_RADL = 17,
+    BLA_N_LP = 18,
+    IDR_W_RADL = 19,
+    IDR_N_LP = 20,
+    CRA_NUT = 21,
+    RSV_IRAP_VCL22 = 22,
+    RSV_IRAP_VCL23 = 23,
+    RSV_VCL24 = 24,
+    RSV_VCL25 = 25,
+    RSV_VCL26 = 26,
+    RSV_VCL27 = 27,
+    RSV_VCL28 = 28,
+    RSV_VCL29 = 29,
+    RSV_VCL30 = 30,
+    RSV_VCL31 = 31,
+    VPS_NUT = 32,
+    SPS_NUT = 33,
+    PPS_NUT = 34,
+    AUD_NUT = 35,
+    EOS_NUT = 36,
+    EOB_NUT = 37,
+    FD_NUT = 38,
+    PREFIX_SEI_NUT = 39,
+    SUFFIX_SEI_NUT = 40,
+    RSV_NVCL41 = 41,
+    RSV_NVCL42 = 42,
+    RSV_NVCL43 = 43,
+    RSV_NVCL44 = 44,
+    RSV_NVCL45 = 45,
+    RSV_NVCL46 = 46,
+    RSV_NVCL47 = 47,
+    UNSPEC48 = 48,
+    UNSPEC49 = 49,
+    UNSPEC50 = 50,
+    UNSPEC51 = 51,
+    UNSPEC52 = 52,
+    UNSPEC53 = 53,
+    UNSPEC54 = 54,
+    UNSPEC55 = 55,
+    UNSPEC56 = 56,
+    UNSPEC57 = 57,
+    UNSPEC58 = 58,
+    UNSPEC59 = 59,
+    UNSPEC60 = 60,
+    UNSPEC61 = 61,
+    UNSPEC62 = 62,
+    UNSPEC63 = 63,
+};
+
+export interface AnnexBHelpers {
+    getUnitType(frame: Buffer): number;
+    splitHeader(frame: Buffer): [Buffer, Buffer];
+    isAUD(unitType: number): boolean;
+}
+
+export const H264Helpers: AnnexBHelpers = {
+    getUnitType(frame) {
+        return frame[0] & 0x1f;
+    },
+    splitHeader(frame) {
+        return [frame.subarray(0, 1), frame.subarray(1)];
+    },
+    isAUD(unitType) {
+        return unitType === H264NalUnitTypes.AccessUnitDelimiter;
+    }
+}
+
+export const H265Helpers: AnnexBHelpers = {
+    getUnitType(frame) {
+        return (frame[0] >> 1) & 0x3f;
+    },
+    splitHeader(frame) {
+        return [frame.subarray(0, 2), frame.subarray(2)];
+    },
+    isAUD(unitType) {
+        return unitType === H265NalUnitTypes.AUD_NUT;
+    }
+}
+
+// Get individual NAL units from an AVPacket frame
+export function splitNalu(frame: Buffer) {
+    const nalus = [];
+    let offset = 0;
+    while (offset < frame.length) {
+        const naluSize = frame.readUInt32BE(offset);
+        offset += 4;
+        const nalu = frame.subarray(offset, offset + naluSize);
+        nalus.push(nalu);
+        offset += nalu.length;
+    }
+    return nalus;
+}
+
+// Merge NAL units into an AVPacket frame
+export function mergeNalu(nalus: Buffer[])
+{
+    const chunks = [];
+    for (const nalu of nalus)
+    {
+        const size = Buffer.allocUnsafe(4);
+        size.writeUInt32BE(nalu.length);
+        chunks.push(size, nalu);
+    }
+    return Buffer.concat(chunks);
+}