@m7mdxzx1/discord-video-strem 0.0.1

package/src/media/newApi.ts

@@ -0,0 +1,618 @@
+import ffmpeg from 'fluent-ffmpeg';
+import pDebounce from 'p-debounce';
+import sharp from 'sharp';
+import Log from 'debug-level';
+import { PassThrough, type Readable } from "node:stream";
+import { demux } from './LibavDemuxer.js';
+import { setTimeout as delay } from 'node:timers/promises';
+import { VideoStream } from './VideoStream.js';
+import { AudioStream } from './AudioStream.js';
+import { isFiniteNonZero } from '../utils.js';
+import { AVCodecID } from './LibavCodecId.js';
+import { createDecoder } from './LibavDecoder.js';
+
+import LibAV from '@lng2004/libav.js-variant-webcodecs-avf-with-decoders';
+import type { SupportedVideoCodec } from '../utils.js';
+import type { MediaUdp, Streamer } from '../client/index.js';
+
+export type EncoderOptions = {
+    /**
+     * Disable video transcoding
+     * If enabled, all video related settings have no effects, and the input
+     * video stream is used as-is.
+     *
+     * You need to ensure that the video stream has the right properties
+     * (keyframe every 1s, B-frames disabled). Failure to do so will result in
+     * a glitchy stream, or degraded performance
+     */
+    noTranscoding: boolean,
+
+    /**
+     * Video width
+     */
+    width: number,
+
+    /**
+     * Video height
+     */
+    height: number,
+
+    /**
+     * Video frame rate
+     */
+    frameRate?: number,
+
+    /**
+     * Video codec
+     */
+    videoCodec: SupportedVideoCodec,
+
+    /**
+     * Video average bitrate in kbps
+     */
+    bitrateVideo: number,
+
+    /**
+     * Video max bitrate in kbps
+     */
+    bitrateVideoMax: number,
+
+    /**
+     * Audio bitrate in kbps
+     */
+    bitrateAudio: number,
+
+    /**
+     * Enable audio output
+     */
+    includeAudio: boolean,
+
+    /**
+     * Enable hardware accelerated decoding
+     */
+    hardwareAcceleratedDecoding: boolean,
+
+    /**
+     * Add some options to minimize latency
+     */
+    minimizeLatency: boolean,
+
+    /**
+     * Preset for x264 and x265
+     */
+    h26xPreset: "ultrafast" | "superfast" | "veryfast" | "faster" | "fast" | "medium" | "slow" | "slower" | "veryslow" | "placebo",
+
+    /**
+     * Custom headers for HTTP requests
+     */
+    customHeaders: Record<string, string>,
+
+    /**
+     * Input for seeking time
+     */
+    seekTime?: number;
+
+    /**
+     * Custom ffmpeg flags/options to pass directly to ffmpeg
+     * These will be added to the command after other options
+     */
+    customFfmpegFlags: string[]
+}
+
+export interface AudioController {
+    mute(): void;
+    unmute(): void;
+    isMuted(): boolean;
+    setVolume(volume: number): void;
+    getVolume(): number;
+}
+
+export function prepareStream(
+    input: string | Readable,
+    options: Partial<EncoderOptions> = {},
+    cancelSignal?: AbortSignal
+) {
+    cancelSignal?.throwIfAborted();
+    const defaultOptions = {
+        noTranscoding: false,
+        // negative values = resize by aspect ratio, see https://trac.ffmpeg.org/wiki/Scaling
+        width: -2,
+        height: -2,
+        frameRate: undefined,
+        videoCodec: "H264",
+        bitrateVideo: 5000,
+        bitrateVideoMax: 7000,
+        bitrateAudio: 128,
+        includeAudio: true,
+        hardwareAcceleratedDecoding: false,
+        minimizeLatency: false,
+        h26xPreset: "ultrafast",
+        customHeaders: {
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.3",
+            "Connection": "keep-alive",
+        },
+        seekTime: 0,
+        customFfmpegFlags: []
+    } satisfies EncoderOptions;
+
+    function mergeOptions(opts: Partial<EncoderOptions>) {
+        return {
+            noTranscoding:
+                opts.noTranscoding ?? defaultOptions.noTranscoding,
+
+            width:
+                isFiniteNonZero(opts.width) ? Math.round(opts.width) : defaultOptions.width,
+
+            height:
+                isFiniteNonZero(opts.height) ? Math.round(opts.height) : defaultOptions.height,
+
+            frameRate:
+                isFiniteNonZero(opts.frameRate) && opts.frameRate > 0
+                    ? opts.frameRate
+                    : defaultOptions.frameRate,
+
+            videoCodec:
+                opts.videoCodec ?? defaultOptions.videoCodec,
+
+            bitrateVideo:
+                isFiniteNonZero(opts.bitrateVideo) && opts.bitrateVideo > 0
+                    ? Math.round(opts.bitrateVideo)
+                    : defaultOptions.bitrateVideo,
+
+            bitrateVideoMax:
+                isFiniteNonZero(opts.bitrateVideoMax) && opts.bitrateVideoMax > 0
+                    ? Math.round(opts.bitrateVideoMax)
+                    : defaultOptions.bitrateVideoMax,
+
+            bitrateAudio:
+                isFiniteNonZero(opts.bitrateAudio) && opts.bitrateAudio > 0
+                    ? Math.round(opts.bitrateAudio)
+                    : defaultOptions.bitrateAudio,
+
+            includeAudio:
+                opts.includeAudio ?? defaultOptions.includeAudio,
+
+            hardwareAcceleratedDecoding:
+                opts.hardwareAcceleratedDecoding ?? defaultOptions.hardwareAcceleratedDecoding,
+
+            minimizeLatency:
+                opts.minimizeLatency ?? defaultOptions.minimizeLatency,
+
+            h26xPreset:
+                opts.h26xPreset ?? defaultOptions.h26xPreset,
+
+            customHeaders: {
+                ...defaultOptions.customHeaders, ...opts.customHeaders
+            },
+
+            seekTime: opts.seekTime ?? defaultOptions.seekTime,
+            customFfmpegFlags:
+                opts.customFfmpegFlags ?? defaultOptions.customFfmpegFlags
+        } satisfies EncoderOptions
+    }
+
+    const mergedOptions = mergeOptions(options);
+
+    let isHttpUrl = false;
+    let isHls = false;
+
+    if (typeof input === "string") {
+        isHttpUrl = input.startsWith('http') || input.startsWith('https');
+        isHls = input.includes('m3u');
+    }
+
+    const output = new PassThrough();
+
+    // command creation
+    const command = ffmpeg();
+
+    // Seeking if applicable (Must be applied before input for fast seek)
+    if (mergedOptions.seekTime && mergedOptions.seekTime > 0) {
+        command.inputOption('-ss', String(mergedOptions.seekTime));
+    }
+
+    command.input(input)
+        .addOption('-loglevel', 'info');
+
+    // input options
+    const { hardwareAcceleratedDecoding, minimizeLatency, customHeaders } = mergedOptions;
+    if (hardwareAcceleratedDecoding)
+        command.inputOption('-hwaccel', 'auto');
+
+    if (minimizeLatency) {
+        command.addOptions([
+            '-fflags nobuffer',
+            '-analyzeduration 0'
+        ]);
+    }
+
+    if (isHttpUrl) {
+        command.inputOption('-headers',
+            Object.entries(customHeaders).map(([k, v]) => `${k}: ${v}`).join("\\r\\n") // Corrected escape sequence
+        );
+        if (!isHls) {
+            command.inputOptions([
+                '-reconnect 1',
+                '-reconnect_at_eof 1',
+                '-reconnect_streamed 1',
+                '-reconnect_delay_max 4294'
+            ]);
+        }
+    }
+
+    // general output options
+    command
+        .output(output)
+        .outputFormat("matroska");
+
+    // video setup
+    const {
+        noTranscoding, width, height, frameRate, bitrateVideo, bitrateVideoMax, videoCodec, h26xPreset
+    } = mergedOptions;
+    command.addOutputOption("-map 0:v");
+
+    if (noTranscoding)
+    {
+        command.videoCodec("copy");
+    }
+    else
+    {
+        command.videoFilter(`scale=${width}:${height}`);
+
+        if (frameRate)
+            command.fpsOutput(frameRate);
+
+        command.addOutputOption([
+            "-b:v", `${bitrateVideo}k`,
+            "-maxrate:v", `${bitrateVideoMax}k`,
+            "-bf", "0",
+            "-pix_fmt", "yuv420p",
+            "-force_key_frames", "expr:gte(t,n_forced*1)"
+        ]);
+
+        switch (videoCodec) {
+            case 'AV1':
+                command
+                    .videoCodec("libsvtav1");
+                break;
+            case 'VP8':
+                command
+                    .videoCodec("libvpx")
+                    .outputOption('-deadline', 'realtime');
+                break;
+            case 'VP9':
+                command
+                    .videoCodec("libvpx-vp9")
+                    .outputOption('-deadline', 'realtime');
+                break;
+            case 'H264':
+                command
+                    .videoCodec("libx264")
+                    .outputOptions([
+                        '-tune zerolatency',
+                        `-preset ${h26xPreset}`,
+                        '-profile:v baseline',
+                    ]);
+                break;
+            case 'H265':
+                command
+                    .videoCodec("libx265")
+                    .outputOptions([
+                        '-tune zerolatency',
+                        `-preset ${h26xPreset}`,
+                        '-profile:v main',
+                    ]);
+                break;
+        }
+    }
+
+    // audio setup
+    const { includeAudio, bitrateAudio } = mergedOptions;
+    if (includeAudio)
+        command
+            .addOutputOption("-map 0:a?")
+            .audioChannels(2)
+            /*
+             * I don't have much surround sound material to test this with,
+             * if you do and you have better settings for this, feel free to
+             * contribute!
+             */
+            .addOutputOption("-lfe_mix_level 1")
+            .audioFrequency(48000)
+            .audioCodec("libopus")
+            .audioBitrate(`${bitrateAudio}k`)
+            .audioFilters("volume@internal_lib=1.0")
+
+    // Add custom ffmpeg flags
+    if (mergedOptions.customFfmpegFlags && mergedOptions.customFfmpegFlags.length > 0) {
+        command.addOptions(mergedOptions.customFfmpegFlags);
+    }
+
+    // exit handling
+    const promise = new Promise<void>((resolve, reject) => {
+        command.on("error", (err) => {
+            if (cancelSignal?.aborted)
+                /**
+                 * fluent-ffmpeg might throw an error when SIGTERM is sent to
+                 * the process, so we check if the abort signal is triggered
+                 * and throw that instead
+                 */
+                reject(cancelSignal.reason);
+            else
+                reject(err);
+        });
+        command.on("end", () => resolve());
+    });
+    promise.catch(() => {});
+    cancelSignal?.addEventListener("abort", () => command.kill("SIGTERM"), { once: true });
+    command.run();
+
+    return { command, output, promise };
+}
+
+
+export type PlayStreamOptions = {
+    /**
+     * Set stream type as \"Go Live\" or camera stream
+     */
+    type: "go-live" | "camera",
+
+    /**
+     * Override video width sent to Discord.
+     *
+     * DO NOT SPECIFY UNLESS YOU KNOW WHAT YOU'RE DOING!
+     */
+    width: number,
+
+    /**
+     * Override video height sent to Discord.
+     *
+     * DO NOT SPECIFY UNLESS YOU KNOW WHAT YOU'RE DOING!
+     */
+    height: number,
+
+    /**
+     * Override video frame rate sent to Discord.
+     *
+     * DO NOT SPECIFY UNLESS YOU KNOW WHAT YOU'RE DOING!
+     */
+    frameRate: number,
+
+    /**
+     * Same as ffmpeg's `readrate_initial_burst` command line flag
+     *
+     * See https://ffmpeg.org/ffmpeg.html#:~:text=%2Dreadrate_initial_burst
+     */
+    readrateInitialBurst: number | undefined,
+
+    /**
+     * Enable stream preview from input stream (experimental)
+     */
+    streamPreview: boolean,
+    seekTime?: number;
+    customHeaders?: Record<string, string>;
+    initialMuted?: boolean;
+}
+
+export async function playStream(
+    input: Readable, streamer: Streamer,
+    options: Partial<PlayStreamOptions> = {},
+    cancelSignal?: AbortSignal,
+    existingUdp: MediaUdp | null = null,
+): Promise<{ audioController?: AudioController; done: Promise<void> }> 
+{
+    const logger = new Log("playStream");
+    cancelSignal?.throwIfAborted();
+    if (!streamer.voiceConnection)
+        throw new Error("Bot is not connected to a voice channel");
+
+    logger.debug("Initializing demuxer");
+    const { video, audio } = await demux(input);
+    cancelSignal?.throwIfAborted();
+
+    if (!video)
+        throw new Error("No video stream in media");
+
+    const cleanupFuncs: (() => unknown)[] = [];
+    const videoCodecMap: Record<number, SupportedVideoCodec> = {
+        [AVCodecID.AV_CODEC_ID_H264]: "H264",
+        [AVCodecID.AV_CODEC_ID_H265]: "H265",
+        [AVCodecID.AV_CODEC_ID_VP8]: "VP8",
+        [AVCodecID.AV_CODEC_ID_VP9]: "VP9",
+        [AVCodecID.AV_CODEC_ID_AV1]: "AV1"
+    };
+    const defaultOptions = {
+        type: "go-live",
+        width: video.width,
+        height: video.height,
+        frameRate: video.framerate_num / video.framerate_den,
+        readrateInitialBurst: undefined,
+        streamPreview: false,
+        seekTime: 0,
+        customHeaders: {},
+        initialMuted: false,
+    } satisfies PlayStreamOptions;
+
+    function mergeOptions(opts: Partial<PlayStreamOptions>)
+    {
+        return {
+            type:
+                opts.type ?? defaultOptions.type,
+
+            width:
+                isFiniteNonZero(opts.width) && opts.width > 0
+                    ? Math.round(opts.width)
+                    : defaultOptions.width,
+
+            height:
+                isFiniteNonZero(opts.height) && opts.height > 0
+                    ? Math.round(opts.height)
+                    : defaultOptions.height,
+
+            frameRate: Math.round(
+                isFiniteNonZero(opts.frameRate) && opts.frameRate > 0
+                    ? Math.round(opts.frameRate)
+                    : defaultOptions.frameRate
+            ),
+
+            readrateInitialBurst:
+                isFiniteNonZero(opts.readrateInitialBurst) && opts.readrateInitialBurst > 0
+                    ? opts.readrateInitialBurst
+                    : defaultOptions.readrateInitialBurst,
+
+            streamPreview:
+                opts.streamPreview ?? defaultOptions.streamPreview,
+
+            seekTime: opts.seekTime ?? defaultOptions.seekTime,
+            customHeaders: {
+                        ...defaultOptions.customHeaders, ...opts.customHeaders
+            },
+            initialMuted:
+                opts.initialMuted ?? defaultOptions.initialMuted,
+        } satisfies PlayStreamOptions;
+    }
+
+    const mergedOptions = mergeOptions(options);
+    logger.debug({ options: mergedOptions }, "Merged options");
+
+    let udp: MediaUdp;
+    let stopStream: () => unknown;
+    if (!existingUdp) {
+        udp = await streamer.createStream();
+    } else {
+        udp = existingUdp;
+        udp.mediaConnection.setSpeaking(false);
+        udp.mediaConnection.setVideoAttributes(false);
+        await delay(250);
+    }
+    // stopStream = () => streamer.stopStream();
+    stopStream = () => console.log("Aborted")
+    udp.setPacketizer(videoCodecMap[video.codec]);
+    udp.mediaConnection.setSpeaking(true);
+    udp.mediaConnection.setVideoAttributes(true, {
+        width: mergedOptions.width,
+        height: mergedOptions.height,
+        fps: mergedOptions.frameRate
+    });
+
+    const vStream = new VideoStream(udp);
+    video.stream.pipe(vStream);
+    let audioStreamInstance: AudioStream | undefined;
+
+    if (audio)
+    {
+        audioStreamInstance = new AudioStream(udp, false, mergedOptions.initialMuted);
+        audio.stream.pipe(audioStreamInstance);
+        vStream.syncStream = audioStreamInstance;
+        audioStreamInstance.syncStream = vStream;
+
+        const burstTime = mergedOptions.readrateInitialBurst;
+        if (typeof burstTime === "number")
+        {
+            vStream.sync = false;
+            vStream.noSleep = audioStreamInstance.noSleep = true;
+            const stopBurst = (pts: number) => {
+                if (pts < burstTime * 1000)
+                    return;
+                vStream.sync = true;
+                if (audioStreamInstance) {
+                    vStream.noSleep = audioStreamInstance.noSleep = false;
+                }
+                vStream.off("pts", stopBurst);
+            };
+            vStream.on("pts", stopBurst);
+        }
+    }
+    if (mergedOptions.streamPreview && mergedOptions.type === "go-live")
+    {
+        (async () => {
+            const logger = new Log("playStream:preview");
+            logger.debug("Initializing decoder for stream preview");
+            const decoder = await createDecoder(video.codec, video.codecpar);
+            if (!decoder)
+            {
+                logger.warn("Failed to initialize decoder. Stream preview will be disabled");
+                return;
+            }
+            cleanupFuncs.push(() => {
+                logger.debug("Freeing decoder");
+                decoder.free();
+            });
+            const updatePreview = pDebounce.promise(async (packet: LibAV.Packet) => {
+                if (!(packet.flags !== undefined && packet.flags & LibAV.AV_PKT_FLAG_KEY))
+                    return;
+                const decodeStart = performance.now();
+                const [frame] = await decoder.decode([packet]).catch(() => []);
+                if (!frame)
+                    return;
+                const decodeEnd = performance.now();
+                logger.debug(`Decoding a frame took ${decodeEnd - decodeStart}ms`);
+
+                return sharp(frame.data, {
+                    raw: {
+                        width: frame.width ?? 0,
+                        height: frame.height ?? 0,
+                        channels: 4
+                    }
+                })
+                .resize(1024, 576, { fit: "inside" })
+                .jpeg()
+                .toBuffer()
+                .then(image => streamer.setStreamPreview(image))
+                .catch(() => {});
+            });
+            video.stream.on("data", updatePreview);
+            cleanupFuncs.push(() => video.stream.off("data", updatePreview));
+        })();
+    }
+    const streamPromise = new Promise<void>((resolve, reject) => {
+        cleanupFuncs.push(() => {
+            stopStream();
+            udp.mediaConnection.setSpeaking(false);
+            udp.mediaConnection.setVideoAttributes(false);
+        });
+        let cleanedUp = false;
+        const cleanup = () => {
+            if (cleanedUp)
+                return;
+            cleanedUp = true;
+            for (const f of cleanupFuncs)
+                f();
+        }
+        cancelSignal?.addEventListener("abort", () => {
+            cleanup();
+            reject(cancelSignal.reason);
+        }, { once: true })
+        vStream.once("finish", () => {
+            if (cancelSignal?.aborted)
+                return;
+            cleanup();
+            resolve();
+        });
+         vStream.once("error", (err) => {
+            if (cancelSignal?.aborted)
+                 return;
+            cleanup();
+            reject(err);
+         });
+         if (audio) {
+             audio.stream.once("error", (err) => {
+                 if (cancelSignal?.aborted)
+                     return;
+                 cleanup();
+                 reject(err);
+             });
+         }
+    }).catch((err) => {
+        if (!cancelSignal?.aborted) {
+             logger.error("Error during playStream:", err);
+        }
+        if (err !== cancelSignal?.reason) {
+             throw err;
+        }
+   });
+      return {
+       audioController: audioStreamInstance,
+       done: streamPromise,
+   };
+}

package/src/media/utils.ts

@@ -0,0 +1,6 @@
+import LibAV from "@lng2004/libav.js-variant-webcodecs-avf-with-decoders";
+
+export function combineLoHi(hi: number, lo: number): number
+{
+    return LibAV.i64tof64(lo, hi);
+}

package/src/utils.ts

@@ -0,0 +1,77 @@
+import type { AnyChannel, DMChannel, GroupDMChannel, VoiceBasedChannel } from "discord.js-selfbot-v13";
+
+export function normalizeVideoCodec(
+    codec: string
+): "H264" | "H265" | "VP8" | "VP9" | "AV1" {
+    if (/H\.?264|AVC/i.test(codec)) return "H264";
+    if (/H\.?265|HEVC/i.test(codec)) return "H265";
+    if (/VP(8|9)/i.test(codec)) return codec.toUpperCase() as "VP8" | "VP9";
+    if (/AV1/i.test(codec)) return "AV1";
+    throw new Error(`Unknown codec: ${codec}`);
+}
+
+// The available video streams are sent by client on connection to voice gateway using OpCode Identify (0)
+// The server then replies with the ssrc and rtxssrc for each available stream using OpCode Ready (2)
+// RID is used specifically to distinguish between different simulcast streams of the same video source,
+// but we don't really care about sending multiple quality streams, so we hardcode a single one
+export const STREAMS_SIMULCAST = [{ type: "screen", rid: "100", quality: 100 }];
+
+export enum SupportedEncryptionModes {
+    AES256 = "aead_aes256_gcm_rtpsize",
+    XCHACHA20 = "aead_xchacha20_poly1305_rtpsize",
+}
+
+export type SupportedVideoCodec = "H264" | "H265" | "VP8" | "VP9" | "AV1";
+
+// RTP extensions
+export const extensions = [{ id: 5, len: 2, val: 0 }];
+
+export const max_int16bit = 2 ** 16;
+export const max_int32bit = 2 ** 32;
+
+export function isFiniteNonZero(n: number | undefined): n is number {
+    return !!n && Number.isFinite(n);
+}
+
+export function parseStreamKey(
+    streamKey: string
+): {
+          type: "guild" | "call";
+          channelId: string;
+          guildId: string | null;
+          userId: string;
+      }
+{
+    const streamKeyArray = streamKey.split(":");
+     
+    const type = streamKeyArray.shift();
+
+    if (type !== "guild" && type !== "call") {
+        throw new Error(`Invalid stream key type: ${type}`);
+    }
+
+    if ((type === "guild" && streamKeyArray.length < 3) || (type === "call" && streamKey.length < 2)) 
+        throw new Error(`Invalid stream key: ${streamKey}`); // invalid stream key
+   
+    let guildId: string | null = null;
+    if (type === "guild") {
+        guildId = streamKeyArray.shift() ?? null;
+    }
+    const channelId = streamKeyArray.shift();
+    const userId = streamKeyArray.shift();
+
+    if (!channelId || !userId) {
+        throw new Error(`Invalid stream key: ${streamKey}`);
+    }
+    return { type, channelId, guildId, userId };
+}
+
+export function generateStreamKey(type: "guild" | "call", guildId: string | null, channelId: string, userId: string): string {
+    const streamKey = `${type}${type === "guild" ? `:${guildId}` : ""}:${channelId}:${userId}`;
+
+    return streamKey;
+}
+
+export function isVoiceChannel(channel: AnyChannel): channel is DMChannel | GroupDMChannel | VoiceBasedChannel {
+    return (channel.type === "DM" || channel.type === "GROUP_DM" || channel.type === "GUILD_STAGE_VOICE" || channel.type === "GUILD_VOICE")
+}