@lmna22/aio-downloader 1.0.1

package/src/lib/twitter.js

@@ -0,0 +1,239 @@
+const axios = require('axios');
+const querystring = require('querystring');
+
+const defaultFeatures = {
+    "creator_subscriptions_tweet_preview_api_enabled": true,
+    "premium_content_api_read_enabled": false,
+    "communities_web_enable_tweet_community_results_fetch": true,
+    "c9s_tweet_anatomy_moderator_badge_enabled": true,
+    "responsive_web_grok_analyze_button_fetch_trends_enabled": false,
+    "responsive_web_grok_analyze_post_followups_enabled": false,
+    "responsive_web_jetfuel_frame": false,
+    "responsive_web_grok_share_attachment_enabled": true,
+    "articles_preview_enabled": true,
+    "responsive_web_edit_tweet_api_enabled": true,
+    "graphql_is_translatable_rweb_tweet_is_translatable_enabled": true,
+    "view_counts_everywhere_api_enabled": true,
+    "longform_notetweets_consumption_enabled": true,
+    "responsive_web_twitter_article_tweet_consumption_enabled": true,
+    "tweet_awards_web_tipping_enabled": false,
+    "responsive_web_grok_show_grok_translated_post": false,
+    "responsive_web_grok_analysis_button_from_backend": false,
+    "creator_subscriptions_quote_tweet_preview_enabled": false,
+    "freedom_of_speech_not_reach_fetch_enabled": true,
+    "standardized_nudges_misinfo": true,
+    "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true,
+    "longform_notetweets_rich_text_read_enabled": true,
+    "longform_notetweets_inline_media_enabled": true,
+    "profile_label_improvements_pcf_label_in_post_enabled": true,
+    "rweb_tipjar_consumption_enabled": true,
+    "responsive_web_graphql_exclude_directive_enabled": true,
+    "verified_phone_label_enabled": false,
+    "responsive_web_grok_image_annotation_enabled": true,
+    "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
+    "responsive_web_graphql_timeline_navigation_enabled": true,
+    "responsive_web_enhance_cards_enabled": false
+};
+
+const defaultVariables = {
+    "withCommunity": false,
+    "includePromotedContent": false,
+    "withVoice": false
+};
+
+const USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:84.0) Gecko/20100101 Firefox/84.0';
+
+const DEFAULT_HEADERS = {
+    'User-Agent': USER_AGENT,
+    'Accept': '*/*',
+    'Accept-Language': 'en-US, en, *;q=0.5',
+    'Accept-Encoding': 'gzip, deflate, br',
+    'TE': 'trailers',
+};
+
+const tokenCache = new Map();
+
+async function getTokens(tweetUrl) {
+    if (tokenCache.has(tweetUrl)) {
+        return tokenCache.get(tweetUrl);
+    }
+
+    const { data: redirectPageData } = await axios.get(tweetUrl, { headers: DEFAULT_HEADERS });
+    const mainJsUrlMatch = redirectPageData.match(/https:\/\/abs.twimg.com\/responsive-web\/client-web-legacy\/main\.[^\.]+\.js/);
+
+    if (!mainJsUrlMatch) {
+        throw new Error('Failed to find main JS URL.');
+    }
+
+    const mainJsUrl = mainJsUrlMatch[0];
+    const bearerToken = await extractBearerToken(mainJsUrl);
+    const guestToken = await getGuestToken(redirectPageData);
+    const tokens = { bearer: bearerToken, guest: guestToken };
+    tokenCache.set(tweetUrl, tokens);
+    return tokens;
+}
+
+async function extractBearerToken(mainJsUrl) {
+    const { data: mainJsData } = await axios.get(mainJsUrl);
+    const match = mainJsData.match(/:"Bearer ([^"]+)"/);
+    if (!match) throw new Error('Failed to find bearer token.');
+    return match[1];
+}
+
+async function getGuestToken(data) {
+    const match = data.match(/gt=(\d+)/);
+    if (!match) throw new Error('Failed to find guest token.');
+    return match[1];
+}
+
+function extractTweetId(tweetUrl) {
+    const match = tweetUrl.match(/(?<=status\/)\d+/);
+    if (!match) throw new Error('Could not parse tweet id from URL. Make sure you are using the correct URL.');
+    return match[0];
+}
+
+function updateFeaturesAndVariables(errorData, features, variables) {
+    const neededVariablesPattern = /Variable '([^']+)'/;
+    const neededFeaturesPattern = /The following features cannot be null: ([^"]+)/;
+
+    if (!errorData || !errorData.errors || !Array.isArray(errorData.errors)) {
+        return;
+    }
+
+    errorData.errors.forEach(error => {
+        if (!error || !error.message) return;
+
+        const neededVars = error.message.match(neededVariablesPattern);
+        if (neededVars) {
+            neededVars.slice(1).forEach(v => variables[v] = true);
+        }
+
+        const neededFeatures = error.message.match(neededFeaturesPattern);
+        if (neededFeatures) {
+            neededFeatures[1].split(',').forEach(feature => {
+                features[feature.trim()] = true;
+            });
+        }
+    });
+}
+
+function getDetailsUrl(tID, features, variables) {
+    const variablesCopy = { ...variables, tweetId: tID };
+    const variablesEncoded = querystring.escape(JSON.stringify(variablesCopy));
+    const featuresEncoded = querystring.escape(JSON.stringify(features));
+    return `https://api.x.com/graphql/Vg2Akr5FzUmF0sTplA5k6g/TweetResultByRestId?variables=${variablesEncoded}&features=${featuresEncoded}`;
+}
+
+function extractMp4s(data) {
+    if (!data || !data.core || !data.legacy || !data.views) {
+        return {
+            creator: '@fort.kun',
+            status: false,
+            message: 'Invalid data format from Twitter API'
+        };
+    }
+
+    const { core, legacy, views } = data;
+    const author = core.user_results.result.legacy.screen_name;
+    let description = legacy.full_text.replace(/https:\/\/t\.co\/[a-zA-Z0-9_-]+\s*$/, '').trim();
+    const listUrl = legacy.entities?.media?.map(media => {
+        if (media.type === 'video' || media.type === 'animated_gif') {
+            const variants = media.video_info?.variants;
+
+            if (variants && variants.length > 0) {
+                const mp4Variants = variants.filter(v => v.bitrate !== undefined);
+
+                if (mp4Variants.length > 0) {
+                    const bestQualityVariant = mp4Variants.reduce((prev, curr) => (
+                        curr.bitrate > prev.bitrate ? curr : prev
+                    ), mp4Variants[0]);
+
+                    return {
+                        type: media.type === 'video' ? 'video' : 'gif',
+                        thumb: media.media_url_https,
+                        url: bestQualityVariant.url
+                    };
+                }
+            }
+            return null;
+        } else if (media.type === 'photo') {
+            return {
+                type: 'image',
+                url: `${media.media_url_https}?format=png&name=large`
+            };
+        }
+        return null;
+    }).filter(Boolean) || [];
+
+    return {
+        creator: '@fort.kun',
+        status: true,
+        data: {
+            author,
+            like: legacy.favorite_count,
+            view: views.count,
+            retweet: legacy.retweet_count,
+            description,
+            sensitiveContent: legacy.possibly_sensitive,
+            result: listUrl,
+        }
+    };
+}
+
+async function twitterDownloader(tweetUrl) {
+    let features = { ...defaultFeatures };
+    let variables = { ...defaultVariables };
+
+    try {
+        const token = await getTokens(tweetUrl);
+        if (!token.bearer || !token.guest) throw new Error('Failed to get bearer token.');
+        const { guest, bearer } = token;
+        const tweetId = extractTweetId(tweetUrl);
+        let url = getDetailsUrl(tweetId, features, variables);
+        let details;
+
+        for (let retries = 0; retries < 3; retries++) {
+            try {
+                const { data, status } = await axios.get(url, {
+                    headers: {
+                        "Authorization": `Bearer ${bearer}`,
+                        "X-Guest-Token": guest,
+                    },
+                });
+
+                if (status === 200 && data?.data?.tweetResult?.result) {
+                    details = data.data.tweetResult.result;
+                    break;
+                } else {
+                    throw new Error(`Request failed with status ${status}`);
+                }
+            } catch (error) {
+                if (error.response?.status === 400 && error.response.data) {
+                    updateFeaturesAndVariables(error.response.data, features, variables);
+                    url = getDetailsUrl(tweetId, features, variables);
+                } else {
+                    throw error;
+                }
+            }
+        }
+
+        if (!details) {
+            return {
+                creator: '@fort.kun',
+                status: false,
+                message: 'Failed to retrieve tweet details after multiple retries.'
+            };
+        }
+
+        return extractMp4s(details);
+    } catch (error) {
+        tokenCache.delete(tweetUrl);
+        return {
+            creator: '@fort.kun',
+            status: false,
+            message: error.message || 'An unexpected error occurred'
+        };
+    }
+}
+
+module.exports = twitterDownloader;

package/src/lib/youtube.js

@@ -0,0 +1,257 @@
+const fs = require('fs');
+const fsPromises = require('fs').promises;
+const { spawn } = require('child_process');
+const path = require('path');
+const os = require('os');
+const { join } = require('path');
+
+let ffmpegPath;
+try {
+    ffmpegPath = require('ffmpeg-static');
+} catch (err) { }
+
+let youtubedl;
+try {
+    youtubedl = require('youtube-dl-exec');
+} catch (err) { }
+
+const tempDir = os.tmpdir();
+
+const QUALITY_MAP = {
+    1: '160',
+    2: '134',
+    3: '135',
+    4: '136',
+    5: '137',
+    6: '264',
+    7: '266',
+    8: 'bestaudio',
+    9: 'bitrateList'
+};
+
+async function youtubeDownloader(link, qualityIndex) {
+    if (!youtubedl) {
+        return {
+            status: false,
+            message: `youtube-dl-exec not found, can't download video`,
+        };
+    }
+    try {
+        let quality = QUALITY_MAP[qualityIndex] || QUALITY_MAP[2];
+
+        const info = await youtubedl(link, {
+            dumpSingleJson: true,
+            noCheckCertificates: true,
+            noWarnings: true,
+            preferFreeFormats: true,
+            addHeader: ['referer:youtube.com', 'user-agent:googlebot']
+        });
+
+        const videoDetails = info;
+        const thumb = info.thumbnail;
+
+        const tempInfoFile = path.join(tempDir, `info_${Date.now()}.json`);
+        await fsPromises.writeFile(tempInfoFile, JSON.stringify(info));
+
+        let result;
+        if (quality === 'bitrateList') {
+            result = getBitrateList(info);
+        } else if (qualityIndex > 7 || quality === 'bestaudio') {
+            result = await downloadAudioOnly(tempInfoFile, quality, videoDetails, thumb);
+        } else {
+            result = await downloadVideoWithAudio(tempInfoFile, quality, videoDetails, thumb);
+        }
+
+        await fsPromises.unlink(tempInfoFile);
+
+        return result;
+
+    } catch (err) {
+        return {
+            creator: '@fort.kun',
+            status: false,
+            message: err.message,
+        };
+    }
+}
+
+function getBitrateList(info) {
+    const bitrateList = info.formats
+        .filter(element => element.acodec !== 'none' && element.vcodec === 'none')
+        .map(element => ({
+            codec: element.acodec,
+            bitrate: element.abr,
+            format_id: element.format_id
+        }))
+        .sort((a, b) => b.bitrate - a.bitrate);
+
+    return {
+        creator: '@fort.kun',
+        status: true,
+        data: { bitrateList }
+    };
+}
+
+async function downloadAudioOnly(infoFile, quality, videoDetails, thumb) {
+    const tempMp3 = path.join(tempDir, `temp_audio_${Date.now()}.mp3`);
+
+    await youtubedl.exec('', {
+        loadInfoJson: infoFile,
+        extractAudio: true,
+        audioFormat: 'mp3',
+        audioQuality: '0',
+        output: tempMp3
+    });
+
+    const mp3Buffer = await fsPromises.readFile(tempMp3);
+    await fsPromises.unlink(tempMp3);
+
+    return createResponse(videoDetails, mp3Buffer, quality, thumb, 'mp3');
+}
+
+async function downloadVideoWithAudio(infoFile, quality, videoDetails, thumb) {
+    const baseName = `temp_video_${Date.now()}`;
+    const videoOutput = path.join(tempDir, `${baseName}.fvideo.mp4`);
+    const audioOutput = path.join(tempDir, `${baseName}.faudio.m4a`);
+    const finalOutput = path.join(tempDir, `${baseName}.mp4`);
+
+    try {
+        await youtubedl.exec('', {
+            loadInfoJson: infoFile,
+            format: quality,
+            output: videoOutput
+        });
+
+        await youtubedl.exec('', {
+            loadInfoJson: infoFile,
+            format: 'bestaudio',
+            output: audioOutput
+        });
+
+        await new Promise((resolve, reject) => {
+            const ffmpeg = spawn(ffmpegPath, [
+                '-i', videoOutput,
+                '-i', audioOutput,
+                '-c:v', 'copy',
+                '-c:a', 'aac',
+                '-strict', 'experimental',
+                finalOutput
+            ]);
+
+            ffmpeg.on('close', (code) => {
+                if (code === 0) {
+                    resolve();
+                } else {
+                    reject(new Error(`ffmpeg exited with code ${code}`));
+                }
+            });
+        });
+
+        const mp4Buffer = await fsPromises.readFile(finalOutput);
+
+        await fsPromises.unlink(videoOutput);
+        await fsPromises.unlink(audioOutput);
+        await fsPromises.unlink(finalOutput);
+
+        return createResponse(videoDetails, mp4Buffer, quality, thumb, 'mp4');
+
+    } catch (err) {
+        throw err;
+    }
+}
+
+function createResponse(videoDetails, buffer, quality, thumb, type) {
+    return {
+        creator: '@fort.kun',
+        status: true,
+        data: {
+            title: videoDetails.title,
+            result: buffer,
+            size: buffer.length,
+            quality,
+            desc: videoDetails.description,
+            views: videoDetails.view_count,
+            likes: videoDetails.like_count,
+            dislikes: 0,
+            channel: videoDetails.uploader,
+            uploadDate: videoDetails.upload_date,
+            thumb,
+            type
+        },
+    };
+}
+
+function sanitizeTitle(title) {
+    return title
+        .replace(/[\/\\:*?"<>|]/g, '_')
+        .trim();
+}
+
+async function youtubePlaylistDownloader(url, quality, folderPath = join(process.cwd() + '/temp')) {
+    let playlistId;
+    try {
+        playlistId = url.slice(url.indexOf("list="), url.indexOf("&index"));
+    } catch {
+        return {
+            creator: '@fort.kun',
+            status: false,
+            message: 'Invalid Playlist URL'
+        };
+    }
+    try {
+        const axios = require('axios');
+        const { data } = await axios.get(url);
+        const htmlStr = data;
+
+        let arr = htmlStr.split('"watchEndpoint":{"videoId":"');
+        var db = {};
+
+        for (var i = 1; i < arr.length; i++) {
+            let str = arr[i];
+            let eI = str.indexOf('"');
+            if (str.slice(eI, eI + 13) != '","playlistId') continue;
+            let sstr = str.slice(0, eI);
+            db[sstr] = 1;
+        }
+
+        let title = htmlStr.match(/property="og:title" content="(.+?)"/)?.[1];
+
+        let resultPath = [];
+        let metadata = [];
+
+        if (!fs.existsSync(folderPath)) {
+            fs.mkdirSync(folderPath);
+        }
+
+        for (const key of Object.keys(db)) {
+            const res = await youtubeDownloader(`https://www.youtube.com/watch?v=${key}`, quality);
+            if (res.status && res.data) {
+                const filePath = join(folderPath, `${sanitizeTitle(res.data.title)}.${res.data.type}`);
+                fs.writeFileSync(filePath, res.data.result);
+                resultPath.push(filePath);
+                metadata.push(res.data);
+            }
+        }
+
+        return {
+            creator: '@fort.kun',
+            status: true,
+            data: {
+                title,
+                resultPath,
+                metadata
+            }
+        };
+    } catch (e) {
+        return {
+            creator: '@fort.kun',
+            status: false,
+            message: e.message || 'Something went wrong.'
+        };
+    }
+}
+
+module.exports = {
+    youtubeDownloader,
+    youtubePlaylistDownloader
+};

package/src/utils.js

@@ -0,0 +1,93 @@
+const axios = require("axios");
+const path = require("path");
+
+const DEFAULT_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36";
+
+function delay(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function cleanText(text) {
+    if (!text) return null;
+    const t = String(text).replace(/\s+/g, " ").trim();
+    return t || null;
+}
+
+function sanitizeFileName(name) {
+    return String(name || "untitled")
+        .replace(/[<>:"/\\|?*\x00-\x1F]/g, "_")
+        .replace(/\s+/g, " ")
+        .trim()
+        .slice(0, 120);
+}
+
+function getExtFromUrl(url, fallback = ".mp4") {
+    try {
+        const pathname = new URL(url).pathname;
+        const ext = path.extname(pathname);
+        if (ext) return ext.split("?")[0];
+        return fallback;
+    } catch {
+        return fallback;
+    }
+}
+
+function formatNumber(num) {
+    return new Intl.NumberFormat("en-US").format(num || 0);
+}
+
+function isValidUrl(url) {
+    if (!url) return false;
+    return /^https?:\/\//i.test(url);
+}
+
+function safeJsonParse(text) {
+    try {
+        return JSON.parse(text);
+    } catch {
+        return null;
+    }
+}
+
+function uniqBy(arr, keyFn) {
+    const seen = new Set();
+    const out = [];
+    for (const item of arr) {
+        const key = keyFn(item);
+        if (!key || seen.has(key)) continue;
+        seen.add(key);
+        out.push(item);
+    }
+    return out;
+}
+
+async function makeRequest(url, options = {}) {
+    const config = {
+        headers: {
+            "User-Agent": DEFAULT_UA,
+            ...options.headers,
+        },
+        timeout: options.timeout || 30000,
+        maxRedirects: options.maxRedirects || 5,
+        ...options,
+    };
+    delete config.headers;
+    config.headers = {
+        "User-Agent": DEFAULT_UA,
+        ...options.headers,
+    };
+    return axios(url, config);
+}
+
+module.exports = {
+    DEFAULT_UA,
+    delay,
+    cleanText,
+    sanitizeFileName,
+    getExtFromUrl,
+    formatNumber,
+    isValidUrl,
+    safeJsonParse,
+    uniqBy,
+    makeRequest,
+};