flux-dl 1.0.0

package/src/platforms/youtube.js

@@ -0,0 +1,449 @@
+const axios = require('axios');
+const YouTubeInnerTube = require('../utils/youtubeInnerTube');
+const BrowserEmulation = require('../utils/browserEmulation');
+
+module.exports = {
+  name: 'YouTube',
+  innerTube: new YouTubeInnerTube(),
+  browser: new BrowserEmulation(),
+
+  canHandle(url) {
+    return url.includes('youtube.com') || url.includes('youtu.be');
+  },
+
+  async extractInfo(url, options) {
+    try {
+      const videoId = this.extractVideoId(url);
+      if (!videoId) {
+        throw new Error('Invalid YouTube URL');
+      }
+
+      console.log(`\n=== Extracting YouTube Video: ${videoId} ===`);
+
+      // Starte Browser-Session (wie echter User)
+      await this.browser.startSession();
+
+      // Methode 1: InnerTube API mit verschiedenen Clients
+      const clientPriority = ['android', 'ios', 'tvEmbedded', 'web'];
+      
+      for (const clientType of clientPriority) {
+        try {
+          console.log(`[InnerTube] Trying ${clientType} client...`);
+          const data = await this.innerTube.getVideoInfo(videoId, clientType);
+
+          if (data.videoDetails && data.streamingData) {
+            const formats = data.streamingData.formats || [];
+            const adaptiveFormats = data.streamingData.adaptiveFormats || [];
+            const allFormats = [...formats, ...adaptiveFormats];
+            
+            const formatsWithUrl = allFormats.filter(f => f.url);
+            
+            if (formatsWithUrl.length > 0) {
+              console.log(`✓ ${clientType} client success! ${formatsWithUrl.length} formats available`);
+              
+              const bestFormat = this.selectBestFormat(formatsWithUrl);
+              
+              return {
+                title: data.videoDetails.title,
+                videoId: videoId,
+                duration: parseInt(data.videoDetails.lengthSeconds),
+                thumbnail: data.videoDetails.thumbnail.thumbnails[0].url,
+                author: data.videoDetails.author,
+                viewCount: parseInt(data.videoDetails.viewCount || 0),
+                videoUrl: bestFormat.url,
+                quality: bestFormat.qualityLabel || bestFormat.quality || 'unknown',
+                format: bestFormat,
+                allFormats: formatsWithUrl,
+                platform: this.name,
+                clientUsed: clientType
+              };
+            }
+          }
+        } catch (innerTubeError) {
+          console.log(`✗ ${clientType} client failed: ${innerTubeError.message}`);
+          continue;
+        }
+      }
+
+      // Methode 2: Web-Scraping (Fallback)
+      console.log('[Web Scraping] Trying web scraping method...');
+      
+      const watchUrl = `https://www.youtube.com/watch?v=${videoId}`;
+      
+      await this.browser.humanDelay(800, 1500);
+      
+      const response = await this.browser.makeRequest(watchUrl, {
+        referer: 'https://www.youtube.com/'
+      });
+
+      const html = response.data;
+      const playerResponse = this.extractPlayerResponse(html);
+      
+      if (!playerResponse || !playerResponse.videoDetails) {
+        throw new Error('Could not extract video details');
+      }
+
+      const videoDetails = playerResponse.videoDetails;
+      const streamingData = playerResponse.streamingData;
+
+      if (!streamingData) {
+        throw new Error('No streaming data available');
+      }
+
+      let allFormats = [
+        ...(streamingData.formats || []),
+        ...(streamingData.adaptiveFormats || [])
+      ];
+
+      console.log(`Found ${allFormats.length} total formats`);
+
+      // Entschlüsselung falls nötig
+      const needsDecipher = allFormats.some(f => !f.url);
+      
+      if (needsDecipher) {
+        console.log('[Decryption] Decrypting signatures...');
+        
+        const playerUrl = this.extractPlayerUrl(html);
+        if (!playerUrl) {
+          throw new Error('Could not find player URL');
+        }
+        
+        await this.browser.humanDelay(300, 700);
+        
+        const playerCode = await this.loadPlayerCode(playerUrl);
+        allFormats = this.decipherAllFormats(allFormats, playerCode);
+      }
+
+      const validFormats = allFormats.filter(f => f.url);
+      console.log(`✓ ${validFormats.length} formats ready for download`);
+      
+      if (validFormats.length === 0) {
+        throw new Error('No valid formats found');
+      }
+
+      const bestFormat = this.selectBestFormat(validFormats);
+      console.log(`Selected: ${bestFormat.qualityLabel || 'unknown'} quality\n`);
+
+      return {
+        title: videoDetails.title,
+        videoId: videoId,
+        duration: parseInt(videoDetails.lengthSeconds),
+        thumbnail: videoDetails.thumbnail.thumbnails[0].url,
+        author: videoDetails.author,
+        viewCount: parseInt(videoDetails.viewCount || 0),
+        videoUrl: bestFormat.url,
+        quality: bestFormat.qualityLabel || 'unknown',
+        format: bestFormat,
+        allFormats: validFormats,
+        platform: this.name,
+        clientUsed: 'web-scraping'
+      };
+
+    } catch (error) {
+      throw new Error(`YouTube extraction failed: ${error.message}`);
+    }
+  },
+
+  decipherAllFormats(formats, playerCode) {
+    const vm = require('vm');
+    
+    console.log('Extracting decipher and N-transform functions...');
+    
+    // Extrahiere beide Funktionen
+    const decipherFunc = this.extractDecipherFunc(playerCode);
+    const nFunc = this.extractNFunc(playerCode);
+    
+    if (decipherFunc) console.log('✓ Decipher function found');
+    if (nFunc) console.log('✓ N-transform function found');
+    
+    let successCount = 0;
+    
+    for (const format of formats) {
+      // Schritt 1: URL aus signatureCipher extrahieren
+      if (!format.url && (format.signatureCipher || format.cipher)) {
+        const params = new URLSearchParams(format.signatureCipher || format.cipher);
+        let url = params.get('url');
+        const s = params.get('s');
+        const sp = params.get('sp') || 'signature';
+        
+        if (s && decipherFunc) {
+          try {
+            const sig = decipherFunc(s);
+            url = `${url}&${sp}=${encodeURIComponent(sig)}`;
+          } catch (e) {
+            console.warn('Signature decipher failed:', e.message);
+          }
+        }
+        
+        format.url = url;
+      }
+      
+      // Schritt 2: N-Parameter transformieren (KRITISCH!)
+      if (format.url) {
+        const nMatch = format.url.match(/[&?]n=([^&]+)/);
+        if (nMatch) {
+          if (nFunc) {
+            try {
+              const n = decodeURIComponent(nMatch[1]);
+              const newN = nFunc(n);
+              
+              if (newN && newN !== n) {
+                format.url = format.url.replace(
+                  /([&?])n=[^&]+/,
+                  `$1n=${encodeURIComponent(newN)}`
+                );
+                format.nTransformed = true;
+                successCount++;
+              } else {
+                // Transformation gab gleichen Wert zurück - behalte Original
+                format.nTransformed = false;
+              }
+            } catch (e) {
+              console.warn('N-transform failed:', e.message);
+              // WICHTIG: Behalte den originalen N-Parameter!
+              format.nTransformed = false;
+            }
+          } else {
+            // Kein N-Transform gefunden - behalte Original-URL
+            console.warn('No N-transform function, keeping original N parameter');
+            format.nTransformed = false;
+          }
+        }
+      }
+    }
+    
+    console.log(`N-parameter transformed for ${successCount} formats`);
+    
+    return formats;
+  },
+
+  extractDecipherFunc(code) {
+    try {
+      const vm = require('vm');
+      
+      // Finde Funktion
+      const patterns = [
+        /([a-zA-Z0-9$]+)=function\([a-zA-Z]\)\{[a-zA-Z]=\1\.split\(""\)/,
+        /\b([a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*\{\s*a\s*=\s*a\.split\(\s*""\s*\)/
+      ];
+
+      let funcName = null;
+      for (const p of patterns) {
+        const m = code.match(p);
+        if (m) {
+          funcName = m[1];
+          break;
+        }
+      }
+
+      if (!funcName) return null;
+
+      // Extrahiere Funktion und Helper
+      const funcRe = new RegExp(`${funcName.replace(/\$/g, '\\$')}=function\\([a-zA-Z]\\)\\{[^}]+\\}`);
+      const funcMatch = code.match(funcRe);
+      if (!funcMatch) return null;
+
+      const funcBody = funcMatch[0];
+      const helperMatch = funcBody.match(/;([a-zA-Z0-9$]+)\./);
+      if (!helperMatch) return null;
+
+      const helperName = helperMatch[1];
+      const helperRe = new RegExp(`var ${helperName.replace(/\$/g, '\\$')}=\\{[\\s\\S]+?\\}\\};`);
+      const helperMatch2 = code.match(helperRe);
+      if (!helperMatch2) return null;
+
+      const fullCode = `${helperMatch2[0]}\n${funcBody}\n${funcName};`;
+      
+      const ctx = {};
+      vm.createContext(ctx);
+      return vm.runInContext(fullCode, ctx);
+    } catch (e) {
+      return null;
+    }
+  },
+
+  extractNFunc(code) {
+    try {
+      const vm = require('vm');
+      
+      // Erweiterte Patterns für N-Funktion (2024 Update)
+      const patterns = [
+        // Pattern 1: Enhanced throttling function
+        /\.get\("n"\)\)&&\(b=([a-zA-Z0-9$]+)(?:\[(\d+)\])?\(b\)/,
+        
+        // Pattern 2: Direct assignment
+        /&&\(b=([a-zA-Z0-9$]+)\(decodeURIComponent\(b\)\)\)/,
+        
+        // Pattern 3: Array access
+        /([a-zA-Z0-9$]+)\[(\d+)\]\(b\)/,
+        
+        // Pattern 4: Simple function call
+        /\b([a-zA-Z0-9$]{2,})\s*=\s*function\([a-zA-Z]\)\{[^}]*\.split\(""\)[^}]*\.join\(""\)\}/,
+        
+        // Pattern 5: With var declaration
+        /var\s+([a-zA-Z0-9$]+)=\{[^}]*\bfunction\([a-zA-Z]\)\{[^}]*\.split\(""\)/
+      ];
+
+      let funcName = null;
+      let arrayIndex = null;
+
+      for (let i = 0; i < patterns.length; i++) {
+        const m = code.match(patterns[i]);
+        if (m) {
+          funcName = m[1];
+          arrayIndex = m[2] || null;
+          console.log(`✓ Found N-function pattern ${i + 1}: ${funcName}${arrayIndex ? `[${arrayIndex}]` : ''}`);
+          break;
+        }
+      }
+
+      if (!funcName) {
+        console.warn('Could not find N-transform function');
+        return null;
+      }
+
+      // Versuche die Funktion zu extrahieren
+      let funcCode = null;
+      
+      // Methode 1: Array-Zugriff (z.B. yt.a[42])
+      if (arrayIndex) {
+        const arrayPattern = new RegExp(`var ${funcName.replace(/\$/g, '\\$')}=\\[([\\s\\S]*?)\\];`, 'g');
+        const arrayMatch = code.match(arrayPattern);
+        
+        if (arrayMatch) {
+          try {
+            const arrayContent = arrayMatch[0];
+            const ctx = {};
+            vm.createContext(ctx);
+            vm.runInContext(arrayContent, ctx);
+            
+            if (ctx[funcName] && ctx[funcName][arrayIndex]) {
+              return ctx[funcName][arrayIndex];
+            }
+          } catch (e) {
+            console.warn('Array extraction failed:', e.message);
+          }
+        }
+      }
+      
+      // Methode 2: Direkte Funktionsextraktion
+      const funcPatterns = [
+        new RegExp(`${funcName.replace(/\$/g, '\\$')}\\s*=\\s*function\\([^)]+\\)\\{[\\s\\S]{1,2000}?\\}`, 'g'),
+        new RegExp(`var ${funcName.replace(/\$/g, '\\$')}\\s*=\\s*function\\([^)]+\\)\\{[\\s\\S]{1,2000}?\\}`, 'g'),
+        new RegExp(`function ${funcName.replace(/\$/g, '\\$')}\\([^)]+\\)\\{[\\s\\S]{1,2000}?\\}`, 'g')
+      ];
+
+      for (const pattern of funcPatterns) {
+        const matches = code.match(pattern);
+        if (matches && matches.length > 0) {
+          funcCode = matches[0];
+          break;
+        }
+      }
+
+      if (!funcCode) {
+        console.warn('Could not extract N-function body');
+        return null;
+      }
+
+      // Versuche die Funktion auszuführen
+      try {
+        const fullCode = `${funcCode}\n${funcName};`;
+        
+        const ctx = {};
+        vm.createContext(ctx);
+        const func = vm.runInContext(fullCode, ctx);
+        
+        // Teste die Funktion
+        try {
+          const testResult = func('test_string_123');
+          if (testResult && typeof testResult === 'string') {
+            console.log('✓ N-transform function validated successfully');
+            return func;
+          }
+        } catch (testError) {
+          console.warn('N-function test failed:', testError.message);
+        }
+        
+        return func;
+      } catch (vmError) {
+        console.warn('Failed to execute N-function:', vmError.message);
+        return null;
+      }
+
+    } catch (e) {
+      console.warn('N-function extraction error:', e.message);
+      return null;
+    }
+  },
+
+  extractPlayerResponse(html) {
+    const patterns = [
+      /var ytInitialPlayerResponse\s*=\s*({.+?});/,
+      /ytInitialPlayerResponse\s*=\s*({.+?});/
+    ];
+
+    for (const p of patterns) {
+      const m = html.match(p);
+      if (m) {
+        try {
+          return JSON.parse(m[1]);
+        } catch (e) {
+          continue;
+        }
+      }
+    }
+    return null;
+  },
+
+  extractPlayerUrl(html) {
+    const m = html.match(/"jsUrl":"([^"]+)"/);
+    if (!m) return null;
+    
+    let url = m[1].replace(/\\\//g, '/');
+    if (url.startsWith('//')) url = 'https:' + url;
+    else if (url.startsWith('/')) url = 'https://www.youtube.com' + url;
+    
+    return url;
+  },
+
+  async loadPlayerCode(url) {
+    const res = await axios.get(url, {
+      headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36' },
+      httpsAgent: new (require('https').Agent)({ rejectUnauthorized: false })
+    });
+    return res.data;
+  },
+
+  selectBestFormat(formats) {
+    // Mit Audio + Video
+    const withBoth = formats.filter(f => 
+      f.url && f.mimeType && f.mimeType.includes('video') && f.audioQuality
+    );
+    if (withBoth.length > 0) {
+      withBoth.sort((a, b) => (b.height || 0) - (a.height || 0));
+      return withBoth[0];
+    }
+
+    // Nur Video
+    const videoOnly = formats.filter(f => 
+      f.url && f.mimeType && f.mimeType.includes('video')
+    );
+    if (videoOnly.length > 0) {
+      videoOnly.sort((a, b) => (b.height || 0) - (a.height || 0));
+      return videoOnly[0];
+    }
+
+    return formats[0];
+  },
+
+  extractVideoId(url) {
+    let m = url.match(/[?&]v=([^&]+)/);
+    if (m) return m[1];
+    m = url.match(/youtu\.be\/([^?]+)/);
+    if (m) return m[1];
+    m = url.match(/\/embed\/([^?]+)/);
+    if (m) return m[1];
+    return null;
+  }
+};

package/src/utils/browserEmulation.js

@@ -0,0 +1,241 @@
+const axios = require('axios');
+const CookieManager = require('./cookieManager');
+
+class BrowserEmulation {
+  constructor() {
+    this.cookies = {};
+    this.sessionStarted = false;
+    this.lastRequestTime = 0;
+    this.cookieManager = new CookieManager();
+    
+    // Versuche Cookies aus Datei zu laden
+    this.cookieManager.loadFromFile('cookies.txt');
+  }
+
+  // Simuliere echten Browser-Start
+  async startSession() {
+    if (this.sessionStarted) return;
+    
+    console.log('Starting browser session...');
+    
+    // Wenn wir Cookies aus Datei haben, überspringe Homepage-Laden
+    if (this.cookieManager.hasCookies()) {
+      console.log('✓ Using cookies from cookies.txt (authenticated session)');
+      this.sessionStarted = true;
+      return;
+    }
+    
+    // Schritt 1: Lade YouTube Homepage (wie echter User)
+    try {
+      const response = await axios.get('https://www.youtube.com', {
+        headers: this.getInitialHeaders(),
+        httpsAgent: new (require('https').Agent)({
+          rejectUnauthorized: false
+        })
+      });
+      
+      // Extrahiere Cookies
+      if (response.headers['set-cookie']) {
+        this.parseCookies(response.headers['set-cookie']);
+      }
+      
+      // Warte wie echter User
+      await this.humanDelay(500, 1500);
+      
+      this.sessionStarted = true;
+      console.log('✓ Browser session established');
+    } catch (error) {
+      console.warn('Session start failed, continuing anyway...');
+    }
+  }
+
+  parseCookies(setCookieHeaders) {
+    for (const cookie of setCookieHeaders) {
+      const parts = cookie.split(';')[0].split('=');
+      if (parts.length === 2) {
+        this.cookies[parts[0]] = parts[1];
+      }
+    }
+  }
+
+  getCookieString() {
+    // Kombiniere Session-Cookies und geladene Cookies
+    const sessionCookies = Object.entries(this.cookies)
+      .map(([key, value]) => `${key}=${value}`)
+      .join('; ');
+    
+    const fileCookies = this.cookieManager.getCookieString('youtube.com');
+    
+    if (sessionCookies && fileCookies) {
+      return `${fileCookies}; ${sessionCookies}`;
+    }
+    
+    return fileCookies || sessionCookies;
+  }
+
+  getInitialHeaders() {
+    return {
+      'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+      'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
+      'Accept-Language': 'en-US,en;q=0.9',
+      'Accept-Encoding': 'gzip, deflate, br',
+      'Connection': 'keep-alive',
+      'Upgrade-Insecure-Requests': '1',
+      'Sec-Fetch-Dest': 'document',
+      'Sec-Fetch-Mode': 'navigate',
+      'Sec-Fetch-Site': 'none',
+      'Sec-Fetch-User': '?1',
+      'sec-ch-ua': '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
+      'sec-ch-ua-mobile': '?0',
+      'sec-ch-ua-platform': '"Windows"',
+      'Cache-Control': 'max-age=0'
+    };
+  }
+
+  getVideoPageHeaders(referer = null) {
+    const headers = {
+      'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+      'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
+      'Accept-Language': 'en-US,en;q=0.9',
+      'Accept-Encoding': 'gzip, deflate, br',
+      'Connection': 'keep-alive',
+      'Upgrade-Insecure-Requests': '1',
+      'Sec-Fetch-Dest': 'document',
+      'Sec-Fetch-Mode': 'navigate',
+      'Sec-Fetch-Site': referer ? 'same-origin' : 'none',
+      'Sec-Fetch-User': '?1',
+      'sec-ch-ua': '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
+      'sec-ch-ua-mobile': '?0',
+      'sec-ch-ua-platform': '"Windows"',
+      'Cache-Control': 'max-age=0'
+    };
+
+    if (referer) {
+      headers['Referer'] = referer;
+    }
+
+    const cookieStr = this.getCookieString();
+    if (cookieStr) {
+      headers['Cookie'] = cookieStr;
+    }
+
+    return headers;
+  }
+
+  getVideoDownloadHeaders(videoUrl, referer) {
+    // KRITISCH: Range-Header für YouTube Downloads
+    const headers = {
+      'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+      'Accept': '*/*',
+      'Accept-Language': 'en-US,en;q=0.9',
+      'Connection': 'keep-alive',
+      'Range': 'bytes=0-',
+      'Referer': referer,
+      'Sec-Fetch-Dest': 'video',
+      'Sec-Fetch-Mode': 'no-cors',
+      'Sec-Fetch-Site': 'cross-site',
+      'sec-ch-ua': '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
+      'sec-ch-ua-mobile': '?0',
+      'sec-ch-ua-platform': '"Windows"'
+    };
+
+    const cookieStr = this.getCookieString();
+    if (cookieStr) {
+      headers['Cookie'] = cookieStr;
+    }
+
+    return headers;
+  }
+
+  // Simuliere menschliche Verzögerungen
+  async humanDelay(min = 300, max = 800) {
+    const delay = Math.floor(Math.random() * (max - min + 1)) + min;
+    await new Promise(resolve => setTimeout(resolve, delay));
+  }
+
+  // Rate-Limiting wie echter User
+  async respectRateLimit() {
+    const now = Date.now();
+    const timeSinceLastRequest = now - this.lastRequestTime;
+    
+    // Mindestens 200ms zwischen Requests
+    if (timeSinceLastRequest < 200) {
+      await new Promise(resolve => setTimeout(resolve, 200 - timeSinceLastRequest));
+    }
+    
+    this.lastRequestTime = Date.now();
+  }
+
+  async makeRequest(url, options = {}) {
+    await this.respectRateLimit();
+    
+    const config = {
+      url: url,
+      method: options.method || 'GET',
+      headers: options.headers || this.getVideoPageHeaders(options.referer),
+      timeout: options.timeout || 30000,
+      httpsAgent: new (require('https').Agent)({
+        rejectUnauthorized: false
+      }),
+      maxRedirects: 5,
+      validateStatus: (status) => status < 500
+    };
+
+    if (options.responseType) {
+      config.responseType = options.responseType;
+    }
+
+    const response = await axios(config);
+    
+    // Speichere neue Cookies
+    if (response.headers['set-cookie']) {
+      this.parseCookies(response.headers['set-cookie']);
+    }
+
+    return response;
+  }
+
+  async downloadStream(url, referer, onProgress) {
+    await this.respectRateLimit();
+    
+    const headers = this.getVideoDownloadHeaders(url, referer);
+    
+    const config = {
+      url: url,
+      method: 'GET',
+      headers: headers,
+      responseType: 'stream',
+      httpsAgent: new (require('https').Agent)({
+        rejectUnauthorized: false
+      }),
+      maxRedirects: 10,
+      timeout: 0,
+      decompress: false, // WICHTIG: Keine automatische Dekompression
+      validateStatus: (status) => status >= 200 && status < 400
+    };
+
+    const response = await axios(config);
+    
+    if (response.status === 403) {
+      // Nur loggen, nicht werfen - manchmal funktioniert es trotzdem
+      console.warn('Warning: Received 403 status, but continuing...');
+    }
+
+    if (onProgress) {
+      const totalLength = response.headers['content-length'];
+      let downloadedLength = 0;
+
+      response.data.on('data', (chunk) => {
+        downloadedLength += chunk.length;
+        if (totalLength) {
+          const percent = Math.round((downloadedLength / totalLength) * 100);
+          onProgress(percent, downloadedLength, totalLength);
+        }
+      });
+    }
+
+    return response.data;
+  }
+}
+
+module.exports = BrowserEmulation;