@m7mdxzx1/discord-video-strem 0.0.1

package/src/client/voice/VoiceMessageTypes.ts

@@ -0,0 +1,164 @@
+import type { VoiceOpCodes } from "./VoiceOpCodes.js"
+import type { SupportedEncryptionModes } from "../../utils.js"
+
+type StreamInfo = {
+    active: boolean,
+    quality: number,
+    rid: string,
+    ssrc: number,
+    rtx_ssrc: number,
+    /**
+     * always "video" from what I observed
+     */
+    type: string
+}
+
+type SimulcastInfo = {
+    type: string,
+    rid: string,
+    quality: number
+}
+
+type CodecPayloadType = {
+    name: string,
+    type: "audio",
+    priority: number,
+    payload_type: number
+} | {
+    name: string,
+    type: "video",
+    priority: number,
+    payload_type: number,
+    rtx_payload_type: number,
+    encode: boolean,
+    decode: boolean
+}
+
+export namespace Message {
+    // Request messages
+    export type Identify = {
+        server_id: string,
+        user_id: string,
+        session_id: string,
+        token: string,
+        video: boolean,
+        streams: SimulcastInfo[]
+    }
+
+    export type Resume = {
+        server_id: string,
+        session_id: string,
+        token: string,
+        seq_ack: number
+    }
+
+    export type Heartbeat = {
+        t: number,
+        seq_ack?: number
+    }
+
+    export type SelectProtocol = {
+        protocol: string,
+        codecs: CodecPayloadType[],
+        data: {
+            address: string,
+            port: number,
+            mode: SupportedEncryptionModes
+        }
+    }
+
+    export type Video = {
+        audio_ssrc: number,
+        video_ssrc: number,
+        rtx_ssrc: number,
+        streams: {
+            type: "video",
+            rid: string,
+            ssrc: number,
+            active: boolean,
+            quality: number,
+            rtx_ssrc: number,
+            max_bitrate: number,
+            max_framerate: number,
+            max_resolution: {
+                type: "fixed",
+                width: number,
+                height: number
+            }
+        }[]
+    }
+
+    // Response messages
+    export type Hello = {
+        heartbeat_interval: number
+    }
+
+    export type Ready = {
+        ssrc: number,
+        ip: string,
+        port: number,
+        modes: SupportedEncryptionModes[],
+        experiments: string[],
+        streams: StreamInfo[]
+    }
+
+    export type Speaking = {
+        speaking: 0 | 1 | 2,
+        delay: number,
+        ssrc: number
+    }
+
+    export type SelectProtocolAck = {
+        secret_key: number[],
+        audio_codec: string,
+        video_codec: string,
+        mode: string,
+    }
+
+    export type HeartbeatAck = {
+        t: number
+    }
+}
+
+export namespace GatewayResponse {
+    type Generic<Op extends VoiceOpCodes, T extends Record<string, unknown> | null> = {
+        op: Op,
+        d: T,
+        seq?: number
+    }
+    export type Hello = Generic<VoiceOpCodes.HELLO, Message.Hello>
+    export type Ready = Generic<VoiceOpCodes.READY, Message.Ready>
+    export type Resumed = Generic<VoiceOpCodes.RESUMED, null>
+    export type Speaking = Generic<VoiceOpCodes.SPEAKING, Message.Speaking>
+    export type SelectProtocolAck = Generic<VoiceOpCodes.SELECT_PROTOCOL_ACK, Message.SelectProtocolAck>
+    export type HeartbeatAck = Generic<VoiceOpCodes.HEARTBEAT_ACK, Message.HeartbeatAck>
+}
+
+export type GatewayResponse =
+    GatewayResponse.Hello |
+    GatewayResponse.Ready |
+    GatewayResponse.Resumed |
+    GatewayResponse.Speaking |
+    GatewayResponse.SelectProtocolAck |
+    GatewayResponse.HeartbeatAck
+
+export namespace GatewayRequest {
+    type Generic<Op extends VoiceOpCodes, T extends Record<string, unknown> | null> = {
+        op: Op,
+        d: T
+    }
+    export type Identify = Generic<VoiceOpCodes.IDENTIFY, Message.Identify>
+    export type Resume = Generic<VoiceOpCodes.RESUME, Message.Resume>
+    export type Heartbeat = Generic<VoiceOpCodes.HEARTBEAT, Message.Heartbeat>
+    export type SelectProtocol = Generic<VoiceOpCodes.SELECT_PROTOCOL, Message.SelectProtocol>
+    export type Video = Generic<VoiceOpCodes.VIDEO, Message.Video>
+    export type Speaking = Generic<VoiceOpCodes.SPEAKING, Message.Speaking>
+}
+
+export type GatewayRequest =
+    GatewayRequest.Identify |
+    GatewayRequest.Resume |
+    GatewayRequest.Heartbeat |
+    GatewayRequest.SelectProtocol |
+    GatewayRequest.Video |
+    GatewayRequest.Speaking

package/src/client/voice/VoiceOpCodes.ts

@@ -0,0 +1,21 @@
+export enum VoiceOpCodes {
+    IDENTIFY = 0,
+    SELECT_PROTOCOL = 1,
+    READY = 2,
+    HEARTBEAT = 3,
+    SELECT_PROTOCOL_ACK = 4,
+    SPEAKING = 5,
+    HEARTBEAT_ACK = 6,
+    RESUME = 7,
+    HELLO = 8,
+    RESUMED = 9,
+    VIDEO = 12,
+    CLIENT_DISCONNECT = 13,
+    SESSION_UPDATE = 14,
+    MEDIA_SINK_WANTS = 15,
+    VOICE_BACKEND_VERSION = 16,
+    CHANNEL_OPTIONS_UPDATE = 17,
+    FLAGS = 18,
+    SPEED_TEST = 19,
+    PLATFORM = 20
+}

package/src/client/voice/index.ts

@@ -0,0 +1,5 @@
+export * from './VoiceConnection.js';
+export * from './VoiceOpCodes.js';
+export * from './MediaUdp.js';
+export * from './StreamConnection.js';
+export * from './BaseMediaConnection.js';

package/src/index.ts

@@ -0,0 +1,5 @@
+export * from './client/index.js';
+export * from './media/index.js';
+export * from './media/newApi.js';
+export * as NewApi from './media/newApi.js';
+export * as Utils from './utils.js';

package/src/media/AudioStream.ts

@@ -0,0 +1,81 @@
+import OpusScript from "opusscript";
+import type { MediaUdp } from "../client/voice/MediaUdp.js";
+import { BaseMediaStream } from "./BaseMediaStream.js";
+
+export class AudioStream extends BaseMediaStream {
+    public udp: MediaUdp;
+    private _isMuted: boolean; // Internal state for muting
+    private _volume: number = 1.0;
+    private _opusEncoder: OpusScript;
+
+    
+
+    constructor(udp: MediaUdp, noSleep = false, initialMuted: boolean = false) {
+        super("audio", noSleep);
+        this.udp = udp;
+        this._isMuted = initialMuted; // Initialize mute state
+        this._opusEncoder = new OpusScript(48000, 2, 2048);
+    }
+
+    /**
+     * Mutes the audio stream. No audio frames will be sent.
+     */
+    public mute(): void {
+        this._isMuted = true;
+    }
+
+    /**
+     * Unmutes the audio stream. Audio frames will resume sending.
+     */
+    public unmute(): void {
+        this._isMuted = false;
+    }
+
+    /**
+     * Checks if the audio stream is currently muted.
+     * @returns True if muted, false otherwise.
+     */
+    public isMuted(): boolean {
+        return this._isMuted;
+    }
+
+    public setVolume(volume: number): void {
+        if (volume < 0) {
+            this._volume = 0;
+        } else {
+            this._volume = Math.min(volume, 2.0);
+        }
+    }
+
+    public getVolume(): number {
+        return this._volume;
+    }
+
+    protected override async _sendFrame(frame: Buffer, frametime: number): Promise<void> {
+        if (this._isMuted) {
+            return;
+        }
+
+        let processedFrame = frame;
+
+        if (this._volume !== 1.0) {
+            try {
+                const pcmData = this._opusEncoder.decode(frame);
+
+                for (let i = 0; i < pcmData.length; i += 2) {
+                    const sample = pcmData.readInt16LE(i);
+                    const newSample = Math.max(-32768, Math.min(32767, Math.floor(sample * this._volume)));
+                    pcmData.writeInt16LE(newSample, i);
+                }
+
+                processedFrame = this._opusEncoder.encode(pcmData, 960);
+            } catch (error) {
+                console.error("[AudioStream] Error processing audio frame:", error);
+                processedFrame = frame;
+            }
+        }
+
+        await this.udp.sendAudioFrame(processedFrame, frametime);
+    }
+
+}

package/src/media/BaseMediaStream.ts

@@ -0,0 +1,173 @@
+import { Log } from "debug-level";
+import { setTimeout } from "node:timers/promises";
+import { Writable } from "node:stream";
+import { combineLoHi } from "./utils.js";
+import type { Packet } from "@lng2004/libav.js-variant-webcodecs-avf-with-decoders";
+
+export class BaseMediaStream extends Writable {
+    private _pts?: number;
+    private _syncTolerance = 20;
+    private _loggerSend: Log;
+    private _loggerSync: Log;
+    private _loggerSleep: Log;
+
+    private _noSleep: boolean;
+    private _startTime?: number;
+    private _startPts?: number;
+    private _sync = true;
+    private _syncStream?: BaseMediaStream;
+
+    constructor(type: string, noSleep = false) {
+        super({ objectMode: true, highWaterMark: 0 });
+        this._loggerSend = new Log(`stream:${type}:send`);
+        this._loggerSync = new Log(`stream:${type}:sync`);
+        this._loggerSleep = new Log(`stream:${type}:sleep`);
+        this._noSleep = noSleep;
+    }
+
+    get sync(): boolean {
+        return this._sync;
+    }
+    set sync(val: boolean) {
+        this._sync = val;
+        if (val)
+            this._loggerSync.debug("Sync enabled");
+        else
+            this._loggerSync.debug("Sync disabled");
+    }
+    get syncStream() {
+        return this._syncStream;
+    }
+    set syncStream(stream: BaseMediaStream | undefined)
+    {
+        if (stream === this)
+            throw new Error("A stream cannot sync with itself");
+        this._syncStream = stream;
+    }
+    get noSleep(): boolean {
+        return this._noSleep;
+    }
+    set noSleep(val: boolean) {
+        this._noSleep = val;
+        if (!val)
+            this.resetTimingCompensation();
+    }
+    get pts(): number | undefined {
+        return this._pts;
+    }
+    get syncTolerance() {
+        return this._syncTolerance;
+    }
+    set syncTolerance(n: number) {
+        if (n < 0)
+            return;
+        this._syncTolerance = n;
+    }
+    protected async _sendFrame(frame: Buffer, frametime: number): Promise<void>
+    {
+        throw new Error("Not implemented");
+    }
+    private ptsDelta() {
+        if (this.pts !== undefined && this.syncStream?.pts !== undefined)
+            return this.pts - this.syncStream.pts;
+        return undefined;
+    }
+    private isAhead() {
+        const delta = this.ptsDelta();
+        return this.syncStream?.writableEnded === false && delta !== undefined && delta > this.syncTolerance;
+    }
+    private isBehind() {
+        const delta = this.ptsDelta();
+        return this.syncStream?.writableEnded === false && delta !== undefined && delta < -this.syncTolerance;
+    }
+    private resetTimingCompensation() {
+        this._startTime = this._startPts = undefined;
+    }
+    async _write(frame: Packet, _: BufferEncoding, callback: (error?: Error | null) => void) {
+        const { data, ptshi, pts, durationhi, duration, time_base_num, time_base_den } = frame;
+        // biome-ignore lint/style/noNonNullAssertion: this will never happen with our media stream
+        const frametime = combineLoHi(durationhi!, duration!) / time_base_den! * time_base_num! * 1000;
+
+        const start_sendFrame = performance.now();
+        await this._sendFrame(Buffer.from(data), frametime);
+        const end_sendFrame = performance.now();
+
+        // biome-ignore lint/style/noNonNullAssertion: this will never happen with our media stream
+        this._pts = combineLoHi(ptshi!, pts!) / time_base_den! * time_base_num! * 1000;
+        this.emit("pts", this._pts);
+
+        const sendTime = end_sendFrame - start_sendFrame;
+        const ratio = sendTime / frametime;
+        this._loggerSend.debug({
+            stats: {
+                pts: this._pts,
+                frame_size: data.length,
+                duration: sendTime,
+                frametime
+            }
+        }, `Frame sent in ${sendTime.toFixed(2)}ms (${(ratio * 100).toFixed(2)}% frametime)`);
+        if (ratio > 1)
+        {
+            this._loggerSend.warn({
+                frame_size: data.length,
+                duration: sendTime,
+                frametime
+            }, `Frame takes too long to send (${(ratio * 100).toFixed(2)}% frametime)`)
+        }
+
+        this._startTime ??= start_sendFrame;
+        this._startPts ??= this._pts;
+        const sleep = Math.max(
+            0, this._pts - this._startPts + frametime - (end_sendFrame - this._startTime)
+        );
+        if (this._noSleep || sleep === 0)
+        {
+            callback(null);
+        }
+        else if (this.sync && this.isBehind())
+        {
+            this._loggerSync.debug({
+                stats: {
+                    pts: this.pts,
+                    pts_other: this.syncStream?.pts
+                }
+            }, "Stream is behind. Not sleeping for this frame");
+            this.resetTimingCompensation();
+            callback(null);
+        }
+        else if (this.sync && this.isAhead())
+        {
+            do
+            {
+                this._loggerSync.debug({
+                    stats: {
+                        pts: this.pts,
+                        pts_other: this.syncStream?.pts,
+                        frametime
+                    }
+                }, `Stream is ahead. Waiting for ${frametime}ms`);
+                await setTimeout(frametime);
+            }
+            while (this.sync && this.isAhead());
+            this.resetTimingCompensation();
+            callback(null);
+        }
+        else
+        {
+            this._loggerSleep.debug({
+                stats: {
+                    pts: this._pts,
+                    startPts: this._startPts,
+                    time: end_sendFrame,
+                    startTime: this._startTime,
+                    frametime
+                }
+            }, `Sleeping for ${sleep}ms`);
+            setTimeout(sleep).then(() => callback(null));
+        }
+    }
+    _destroy(error: Error | null, callback: (error?: Error | null) => void): void {
+        super._destroy(error, callback);
+        this.syncStream = undefined;
+    }
+}