@m7mdxzx1/discord-video-strem 0.0.1

package/dist/client/voice/BaseMediaConnection.js

@@ -0,0 +1,319 @@
+import { VoiceOpCodes } from "./VoiceOpCodes.js";
+import { MediaUdp } from "./MediaUdp.js";
+import { AES256TransportEncryptor, Chacha20TransportEncryptor } from "../encryptor/TransportEncryptor.js";
+import { STREAMS_SIMULCAST, SupportedEncryptionModes } from "../../utils.js";
+import WebSocket from 'ws';
+import EventEmitter from "node:events";
+export const CodecPayloadType = {
+    "opus": {
+        name: "opus", type: "audio", priority: 1000, payload_type: 120
+    },
+    "H264": {
+        name: "H264", type: "video", priority: 1000, payload_type: 101, rtx_payload_type: 102, encode: true, decode: true
+    },
+    "H265": {
+        name: "H265", type: "video", priority: 1000, payload_type: 103, rtx_payload_type: 104, encode: true, decode: true
+    },
+    "VP8": {
+        name: "VP8", type: "video", priority: 1000, payload_type: 105, rtx_payload_type: 106, encode: true, decode: true
+    },
+    "VP9": {
+        name: "VP9", type: "video", priority: 1000, payload_type: 107, rtx_payload_type: 108, encode: true, decode: true
+    },
+    "AV1": {
+        name: "AV1", type: "video", priority: 1000, payload_type: 109, rtx_payload_type: 110, encode: true, decode: true
+    }
+};
+export class BaseMediaConnection extends EventEmitter {
+    constructor(streamer, guildId, botId, channelId, callback) {
+        super();
+        this.interval = null;
+        this.guildId = null;
+        this.ws = null;
+        this.server = null; //websocket url
+        this.token = null;
+        this.session_id = null;
+        this.webRtcParams = null;
+        this._sequenceNumber = -1;
+        this._streamer = streamer;
+        this.status = {
+            hasSession: false,
+            hasToken: false,
+            started: false,
+            resuming: false
+        };
+        // make udp client
+        this.udp = new MediaUdp(this);
+        this.guildId = guildId;
+        this.channelId = channelId;
+        this.botId = botId;
+        this.ready = callback;
+    }
+    get type() {
+        return this.guildId ? "guild" : "call";
+    }
+    get transportEncryptor() {
+        return this._transportEncryptor;
+    }
+    get streamer() {
+        return this._streamer;
+    }
+    stop() {
+        this.interval && clearInterval(this.interval);
+        this.status.started = false;
+        this.ws?.close();
+        this.udp?.stop();
+    }
+    setSession(session_id) {
+        this.session_id = session_id;
+        this.status.hasSession = true;
+        this.start();
+    }
+    setTokens(server, token) {
+        this.token = token;
+        this.server = server;
+        this.status.hasToken = true;
+        this.start();
+    }
+    start() {
+        /*
+        ** Connection can only start once both
+        ** session description and tokens have been gathered
+        */
+        if (this.status.hasSession && this.status.hasToken) {
+            if (this.status.started)
+                return;
+            this.status.started = true;
+            this.ws = new WebSocket(`wss://${this.server}/?v=8`, {
+                followRedirects: true
+            });
+            this.ws.on("open", () => {
+                if (this.status.resuming) {
+                    this.status.resuming = false;
+                    this.resume();
+                }
+                else {
+                    this.identify();
+                }
+            });
+            this.ws.on("error", (err) => {
+                console.error(err);
+            });
+            this.ws.on("close", (code) => {
+                const wasStarted = this.status.started;
+                this.status.started = false;
+                this.udp.ready = false;
+                const canResume = code === 4_015 || code < 4_000;
+                if (canResume && wasStarted) {
+                    this.status.resuming = true;
+                    this.start();
+                }
+            });
+            this.setupEvents();
+        }
+    }
+    handleReady(d) {
+        // we hardcoded the STREAMS_SIMULCAST, which will always be array of 1
+        const stream = d.streams[0];
+        this.webRtcParams = {
+            address: d.ip,
+            port: d.port,
+            audioSsrc: d.ssrc,
+            videoSsrc: stream.ssrc,
+            rtxSsrc: stream.rtx_ssrc,
+            supportedEncryptionModes: d.modes
+        };
+    }
+    handleProtocolAck(d) {
+        const secretKey = Buffer.from(d.secret_key);
+        switch (d.mode) {
+            case SupportedEncryptionModes.AES256:
+                this._transportEncryptor = new AES256TransportEncryptor(secretKey);
+                break;
+            case SupportedEncryptionModes.XCHACHA20:
+                this._transportEncryptor = new Chacha20TransportEncryptor(secretKey);
+                break;
+        }
+        this.emit("select_protocol_ack");
+    }
+    setupEvents() {
+        this.ws?.on('message', (data, isBinary) => {
+            if (isBinary)
+                return;
+            const { op, d, seq } = JSON.parse(data.toString());
+            if (seq)
+                this._sequenceNumber = seq;
+            if (op === VoiceOpCodes.READY) { // ready
+                this.handleReady(d);
+                this.sendVoice().then(() => this.ready(this.udp));
+                this.setVideoAttributes(false);
+            }
+            else if (op >= 4000) {
+                console.error(`Error ${this.constructor.name} connection`, d);
+            }
+            else if (op === VoiceOpCodes.HELLO) {
+                this.setupHeartbeat(d.heartbeat_interval);
+            }
+            else if (op === VoiceOpCodes.SELECT_PROTOCOL_ACK) { // session description
+                this.handleProtocolAck(d);
+            }
+            else if (op === VoiceOpCodes.SPEAKING) {
+                // ignore speaking updates
+            }
+            else if (op === VoiceOpCodes.HEARTBEAT_ACK) {
+                // ignore heartbeat acknowledgements
+            }
+            else if (op === VoiceOpCodes.RESUMED) {
+                this.status.started = true;
+                this.udp.ready = true;
+            }
+            else {
+                //console.log("unhandled voice event", {op, d});
+            }
+        });
+    }
+    setupHeartbeat(interval) {
+        if (this.interval) {
+            clearInterval(this.interval);
+        }
+        this.interval = setInterval(() => {
+            this.sendOpcode(VoiceOpCodes.HEARTBEAT, {
+                t: Date.now(),
+                seq_ack: this._sequenceNumber
+            });
+        }, interval);
+    }
+    sendOpcode(code, data) {
+        this.ws?.send(JSON.stringify({
+            op: code,
+            d: data
+        }));
+    }
+    /*
+    ** identifies with media server with credentials
+    */
+    identify() {
+        if (!this.serverId)
+            throw new Error("Server ID is null or empty");
+        if (!this.session_id)
+            throw new Error("Session ID is null or empty");
+        if (!this.token)
+            throw new Error("Token is null or empty");
+        this.sendOpcode(VoiceOpCodes.IDENTIFY, {
+            server_id: this.serverId,
+            user_id: this.botId,
+            session_id: this.session_id,
+            token: this.token,
+            video: true,
+            streams: STREAMS_SIMULCAST
+        });
+    }
+    resume() {
+        if (!this.serverId)
+            throw new Error("Server ID is null or empty");
+        if (!this.session_id)
+            throw new Error("Session ID is null or empty");
+        if (!this.token)
+            throw new Error("Token is null or empty");
+        this.sendOpcode(VoiceOpCodes.RESUME, {
+            server_id: this.serverId,
+            session_id: this.session_id,
+            token: this.token,
+            seq_ack: this._sequenceNumber
+        });
+    }
+    /*
+    ** Sets protocols and ip data used for video and audio.
+    ** Uses vp8 for video
+    ** Uses opus for audio
+    */
+    setProtocols() {
+        const { ip, port } = this.udp;
+        if (!ip || !port)
+            throw new Error("IP or port is undefined (this shouldn't happen!!!)");
+        // select encryption mode
+        // From Discord docs: 
+        // You must support aead_xchacha20_poly1305_rtpsize. You should prefer to use aead_aes256_gcm_rtpsize when it is available.
+        let encryptionMode;
+        if (!this.webRtcParams)
+            throw new Error("WebRTC connection not ready");
+        if (this.webRtcParams.supportedEncryptionModes.includes(SupportedEncryptionModes.AES256) &&
+            !this._streamer.opts.forceChacha20Encryption) {
+            encryptionMode = SupportedEncryptionModes.AES256;
+        }
+        else {
+            encryptionMode = SupportedEncryptionModes.XCHACHA20;
+        }
+        return new Promise((resolve) => {
+            this.sendOpcode(VoiceOpCodes.SELECT_PROTOCOL, {
+                protocol: "udp",
+                codecs: Object.values(CodecPayloadType),
+                data: {
+                    address: ip,
+                    port: port,
+                    mode: encryptionMode
+                }
+            });
+            this.once("select_protocol_ack", () => resolve());
+        });
+    }
+    setVideoAttributes(enabled, attr) {
+        if (!this.webRtcParams)
+            throw new Error("WebRTC connection not ready");
+        const { audioSsrc, videoSsrc, rtxSsrc } = this.webRtcParams;
+        if (!enabled) {
+            this.sendOpcode(VoiceOpCodes.VIDEO, {
+                audio_ssrc: audioSsrc,
+                video_ssrc: 0,
+                rtx_ssrc: 0,
+                streams: []
+            });
+        }
+        else {
+            if (!attr)
+                throw new Error("Need to specify video attributes");
+            this.sendOpcode(VoiceOpCodes.VIDEO, {
+                audio_ssrc: audioSsrc,
+                video_ssrc: videoSsrc,
+                rtx_ssrc: rtxSsrc,
+                streams: [
+                    {
+                        type: "video",
+                        rid: "100",
+                        ssrc: videoSsrc,
+                        active: true,
+                        quality: 100,
+                        rtx_ssrc: rtxSsrc,
+                        // hardcode the max bitrate because we don't really know anyway
+                        max_bitrate: 10000 * 1000,
+                        max_framerate: enabled ? attr.fps : 0,
+                        max_resolution: {
+                            type: "fixed",
+                            width: attr.width,
+                            height: attr.height
+                        }
+                    }
+                ]
+            });
+        }
+    }
+    /*
+    ** Set speaking status
+    ** speaking -> speaking status on or off
+    */
+    setSpeaking(speaking) {
+        if (!this.webRtcParams)
+            throw new Error("WebRTC connection not ready");
+        this.sendOpcode(VoiceOpCodes.SPEAKING, {
+            delay: 0,
+            speaking: speaking ? 1 : 0,
+            ssrc: this.webRtcParams.audioSsrc
+        });
+    }
+    /*
+    ** Start media connection
+    */
+    sendVoice() {
+        return this.udp.createUdp().then(() => this.setProtocols());
+    }
+}

package/dist/client/voice/MediaUdp.d.ts

@@ -0,0 +1,26 @@
+import type { BaseMediaPacketizer } from '../packet/BaseMediaPacketizer.js';
+import type { BaseMediaConnection } from './BaseMediaConnection.js';
+export declare class MediaUdp {
+    private _mediaConnection;
+    private _socket;
+    private _ready;
+    private _audioPacketizer?;
+    private _videoPacketizer?;
+    private _ip?;
+    private _port?;
+    constructor(voiceConnection: BaseMediaConnection);
+    get audioPacketizer(): BaseMediaPacketizer | undefined;
+    get videoPacketizer(): BaseMediaPacketizer | undefined;
+    get mediaConnection(): BaseMediaConnection;
+    get ip(): string | undefined;
+    get port(): number | undefined;
+    sendAudioFrame(frame: Buffer, frametime: number): Promise<void>;
+    sendVideoFrame(frame: Buffer, frametime: number): Promise<void>;
+    setPacketizer(videoCodec: string): void;
+    sendPacket(packet: Buffer): Promise<void>;
+    handleIncoming(buf: unknown): void;
+    get ready(): boolean;
+    set ready(val: boolean);
+    stop(): void;
+    createUdp(): Promise<void>;
+}

package/dist/client/voice/MediaUdp.js

@@ -0,0 +1,140 @@
+import udpCon from 'node:dgram';
+import { isIP } from 'node:net';
+import { AudioPacketizer } from '../packet/AudioPacketizer.js';
+import { VideoPacketizerH264, VideoPacketizerH265 } from '../packet/VideoPacketizerAnnexB.js';
+import { VideoPacketizerVP8 } from '../packet/VideoPacketizerVP8.js';
+import { normalizeVideoCodec } from '../../utils.js';
+// credit to discord.js
+function parseLocalPacket(message) {
+    const packet = Buffer.from(message);
+    const ip = packet.subarray(8, packet.indexOf(0, 8)).toString('utf8');
+    if (!isIP(ip)) {
+        throw new Error('Malformed IP address');
+    }
+    const port = packet.readUInt16BE(packet.length - 2);
+    return { ip, port };
+}
+export class MediaUdp {
+    constructor(voiceConnection) {
+        this._socket = null;
+        this._ready = false;
+        this._mediaConnection = voiceConnection;
+    }
+    get audioPacketizer() {
+        return this._audioPacketizer;
+    }
+    get videoPacketizer() {
+        // This will never be undefined anyway, so it's safe
+        return this._videoPacketizer;
+    }
+    get mediaConnection() {
+        return this._mediaConnection;
+    }
+    get ip() {
+        return this._ip;
+    }
+    get port() {
+        return this._port;
+    }
+    async sendAudioFrame(frame, frametime) {
+        if (!this.ready)
+            return;
+        await this.audioPacketizer?.sendFrame(frame, frametime);
+    }
+    async sendVideoFrame(frame, frametime) {
+        if (!this.ready)
+            return;
+        await this.videoPacketizer?.sendFrame(frame, frametime);
+    }
+    setPacketizer(videoCodec) {
+        if (!this.mediaConnection.webRtcParams)
+            throw new Error("WebRTC connection not ready");
+        const { audioSsrc, videoSsrc } = this.mediaConnection.webRtcParams;
+        this._audioPacketizer = new AudioPacketizer(this, audioSsrc);
+        switch (normalizeVideoCodec(videoCodec)) {
+            case "H264":
+                this._videoPacketizer = new VideoPacketizerH264(this, videoSsrc);
+                break;
+            case "H265":
+                this._videoPacketizer = new VideoPacketizerH265(this, videoSsrc);
+                break;
+            case "VP8":
+                this._videoPacketizer = new VideoPacketizerVP8(this, videoSsrc);
+                break;
+            default:
+                throw new Error(`Packetizer not implemented for ${videoCodec}`);
+        }
+    }
+    sendPacket(packet) {
+        if (!this.mediaConnection.webRtcParams)
+            throw new Error("WebRTC connection not ready");
+        const { address, port } = this.mediaConnection.webRtcParams;
+        return new Promise((resolve, reject) => {
+            try {
+                this._socket?.send(packet, 0, packet.length, port, address, (error, bytes) => {
+                    if (error) {
+                        console.log("ERROR", error);
+                        reject(error);
+                    }
+                    resolve();
+                });
+            }
+            catch (e) {
+                reject(e);
+            }
+        });
+    }
+    handleIncoming(buf) {
+        //console.log("RECEIVED PACKET", buf);
+    }
+    get ready() {
+        return this._ready;
+    }
+    set ready(val) {
+        this._ready = val;
+    }
+    stop() {
+        try {
+            this.ready = false;
+            this._socket?.disconnect();
+        }
+        catch (e) { }
+    }
+    createUdp() {
+        if (!this.mediaConnection.webRtcParams)
+            throw new Error("WebRTC connection not ready");
+        const { audioSsrc, address, port } = this.mediaConnection.webRtcParams;
+        return new Promise((resolve, reject) => {
+            this._socket = udpCon.createSocket('udp4');
+            this._socket.on('error', (error) => {
+                console.error("Error connecting to media udp server", error);
+                reject(error);
+            });
+            this._socket.once('message', (message) => {
+                if (message.readUInt16BE(0) !== 2) {
+                    reject('wrong handshake packet for udp');
+                }
+                try {
+                    const packet = parseLocalPacket(message);
+                    this._ip = packet.ip;
+                    this._port = packet.port;
+                    this._ready = true;
+                }
+                catch (e) {
+                    reject(e);
+                }
+                resolve();
+                this._socket?.on('message', this.handleIncoming);
+            });
+            const blank = Buffer.alloc(74);
+            blank.writeUInt16BE(1, 0);
+            blank.writeUInt16BE(70, 2);
+            blank.writeUInt32BE(audioSsrc, 4);
+            this._socket.send(blank, 0, blank.length, port, address, (error, bytes) => {
+                if (error) {
+                    reject(error);
+                }
+            });
+        });
+    }
+}

package/dist/client/voice/StreamConnection.d.ts

@@ -0,0 +1,10 @@
+import { BaseMediaConnection } from "./BaseMediaConnection.js";
+export declare class StreamConnection extends BaseMediaConnection {
+    private _streamKey;
+    private _serverId;
+    setSpeaking(speaking: boolean): void;
+    get serverId(): string | null;
+    set serverId(id: string);
+    get streamKey(): string | null;
+    set streamKey(value: string);
+}

package/dist/client/voice/StreamConnection.js

@@ -0,0 +1,30 @@
+import { VoiceOpCodes } from "../voice/VoiceOpCodes.js";
+import { BaseMediaConnection } from "./BaseMediaConnection.js";
+export class StreamConnection extends BaseMediaConnection {
+    constructor() {
+        super(...arguments);
+        this._streamKey = null;
+        this._serverId = null;
+    }
+    setSpeaking(speaking) {
+        if (!this.webRtcParams)
+            throw new Error("WebRTC connection not ready");
+        this.sendOpcode(VoiceOpCodes.SPEAKING, {
+            delay: 0,
+            speaking: speaking ? 2 : 0,
+            ssrc: this.webRtcParams.audioSsrc
+        });
+    }
+    get serverId() {
+        return this._serverId;
+    }
+    set serverId(id) {
+        this._serverId = id;
+    }
+    get streamKey() {
+        return this._streamKey;
+    }
+    set streamKey(value) {
+        this._streamKey = value;
+    }
+}

package/dist/client/voice/VoiceConnection.d.ts

@@ -0,0 +1,7 @@
+import type { StreamConnection } from './StreamConnection.js';
+import { BaseMediaConnection } from './BaseMediaConnection.js';
+export declare class VoiceConnection extends BaseMediaConnection {
+    streamConnection?: StreamConnection;
+    get serverId(): string;
+    stop(): void;
+}

package/dist/client/voice/VoiceConnection.js

@@ -0,0 +1,10 @@
+import { BaseMediaConnection } from './BaseMediaConnection.js';
+export class VoiceConnection extends BaseMediaConnection {
+    get serverId() {
+        return this.guildId ?? this.channelId; // for guild vc it is the guild id, for dm voice it is the channel id
+    }
+    stop() {
+        super.stop();
+        this.streamConnection?.stop();
+    }
+}

package/dist/client/voice/VoiceMessageTypes.d.ts

@@ -0,0 +1,136 @@
+import type { VoiceOpCodes } from "./VoiceOpCodes.js";
+import type { SupportedEncryptionModes } from "../../utils.js";
+type StreamInfo = {
+    active: boolean;
+    quality: number;
+    rid: string;
+    ssrc: number;
+    rtx_ssrc: number;
+    /**
+     * always "video" from what I observed
+     */
+    type: string;
+};
+type SimulcastInfo = {
+    type: string;
+    rid: string;
+    quality: number;
+};
+type CodecPayloadType = {
+    name: string;
+    type: "audio";
+    priority: number;
+    payload_type: number;
+} | {
+    name: string;
+    type: "video";
+    priority: number;
+    payload_type: number;
+    rtx_payload_type: number;
+    encode: boolean;
+    decode: boolean;
+};
+export declare namespace Message {
+    type Identify = {
+        server_id: string;
+        user_id: string;
+        session_id: string;
+        token: string;
+        video: boolean;
+        streams: SimulcastInfo[];
+    };
+    type Resume = {
+        server_id: string;
+        session_id: string;
+        token: string;
+        seq_ack: number;
+    };
+    type Heartbeat = {
+        t: number;
+        seq_ack?: number;
+    };
+    type SelectProtocol = {
+        protocol: string;
+        codecs: CodecPayloadType[];
+        data: {
+            address: string;
+            port: number;
+            mode: SupportedEncryptionModes;
+        };
+    };
+    type Video = {
+        audio_ssrc: number;
+        video_ssrc: number;
+        rtx_ssrc: number;
+        streams: {
+            type: "video";
+            rid: string;
+            ssrc: number;
+            active: boolean;
+            quality: number;
+            rtx_ssrc: number;
+            max_bitrate: number;
+            max_framerate: number;
+            max_resolution: {
+                type: "fixed";
+                width: number;
+                height: number;
+            };
+        }[];
+    };
+    type Hello = {
+        heartbeat_interval: number;
+    };
+    type Ready = {
+        ssrc: number;
+        ip: string;
+        port: number;
+        modes: SupportedEncryptionModes[];
+        experiments: string[];
+        streams: StreamInfo[];
+    };
+    type Speaking = {
+        speaking: 0 | 1 | 2;
+        delay: number;
+        ssrc: number;
+    };
+    type SelectProtocolAck = {
+        secret_key: number[];
+        audio_codec: string;
+        video_codec: string;
+        mode: string;
+    };
+    type HeartbeatAck = {
+        t: number;
+    };
+}
+export declare namespace GatewayResponse {
+    type Generic<Op extends VoiceOpCodes, T extends Record<string, unknown> | null> = {
+        op: Op;
+        d: T;
+        seq?: number;
+    };
+    export type Hello = Generic<VoiceOpCodes.HELLO, Message.Hello>;
+    export type Ready = Generic<VoiceOpCodes.READY, Message.Ready>;
+    export type Resumed = Generic<VoiceOpCodes.RESUMED, null>;
+    export type Speaking = Generic<VoiceOpCodes.SPEAKING, Message.Speaking>;
+    export type SelectProtocolAck = Generic<VoiceOpCodes.SELECT_PROTOCOL_ACK, Message.SelectProtocolAck>;
+    export type HeartbeatAck = Generic<VoiceOpCodes.HEARTBEAT_ACK, Message.HeartbeatAck>;
+    export {};
+}
+export type GatewayResponse = GatewayResponse.Hello | GatewayResponse.Ready | GatewayResponse.Resumed | GatewayResponse.Speaking | GatewayResponse.SelectProtocolAck | GatewayResponse.HeartbeatAck;
+export declare namespace GatewayRequest {
+    type Generic<Op extends VoiceOpCodes, T extends Record<string, unknown> | null> = {
+        op: Op;
+        d: T;
+    };
+    export type Identify = Generic<VoiceOpCodes.IDENTIFY, Message.Identify>;
+    export type Resume = Generic<VoiceOpCodes.RESUME, Message.Resume>;
+    export type Heartbeat = Generic<VoiceOpCodes.HEARTBEAT, Message.Heartbeat>;
+    export type SelectProtocol = Generic<VoiceOpCodes.SELECT_PROTOCOL, Message.SelectProtocol>;
+    export type Video = Generic<VoiceOpCodes.VIDEO, Message.Video>;
+    export type Speaking = Generic<VoiceOpCodes.SPEAKING, Message.Speaking>;
+    export {};
+}
+export type GatewayRequest = GatewayRequest.Identify | GatewayRequest.Resume | GatewayRequest.Heartbeat | GatewayRequest.SelectProtocol | GatewayRequest.Video | GatewayRequest.Speaking;
+export {};

package/dist/client/voice/VoiceMessageTypes.js

@@ -0,0 +1 @@
+export {};