UrlCategorise 0.1.7 → 0.1.11

data/bin/generate_video_lists

@@ -7,31 +7,40 @@ require 'json'
 require 'fileutils'
 require 'time'
 require 'set'
+require 'optparse'
+require 'thread'
 
 module VideoListGenerator
   class ExtractorParser
     YT_DLP_API_URL = 'https://api.github.com/repos/yt-dlp/yt-dlp/contents/yt_dlp/extractor'
-    
-    def initialize
+
+    attr_accessor :max_download_workers, :max_process_workers, :parallel_enabled
+
+    def initialize(options = {})
       @video_domains = Set.new
       @video_regexes = []
+      @domains_mutex = Mutex.new
+      @regexes_mutex = Mutex.new
+      @max_download_workers = options[:max_download_workers] || 16
+      @max_process_workers = options[:max_process_workers] || 8
+      @parallel_enabled = options.fetch(:parallel_enabled, true)
     end
 
     def generate_lists
-      puts "🎬 Generating video hosting lists from yt-dlp extractors..."
-      
-      puts "📝 Adding manually curated video hosting domains..."
+      puts '🎬 Generating video hosting lists from yt-dlp extractors...'
+
+      puts '📝 Adding manually curated video hosting domains...'
       add_manual_domains
-      
-      puts "📡 Fetching yt-dlp extractor data..."
+
+      puts '📡 Fetching yt-dlp extractor data...'
       fetch_extractor_data
-      
+
       create_lists_directory
       generate_hosts_file
       generate_regex_file
-      
-      puts "✅ Video hosting lists generated successfully!"
-      puts "📁 Files created in lists/ directory:"
+
+      puts '✅ Video hosting lists generated successfully!'
+      puts '📁 Files created in lists/ directory:'
       puts "   - video_hosting_domains.hosts (#{@video_domains.size} domains)"
       puts "   - video_url_patterns.txt (#{@video_regexes.size} patterns)"
     end
@@ -55,177 +64,310 @@ module VideoListGenerator
         'streamable.com',
         'wistia.com', 'fast.wistia.com', 'fast.wistia.net'
       ]
-      
+
       core_domains.each { |domain| @video_domains.add(domain) }
     end
 
     def fetch_extractor_data
       uri = URI(YT_DLP_API_URL)
       response = Net::HTTP.get_response(uri)
-      
+
       unless response.code == '200'
-        puts "⚠️  Could not fetch from yt-dlp API, using manual domains only"
+        puts '⚠️  Could not fetch from yt-dlp API, using manual domains only'
         return
       end
-      
+
       files = JSON.parse(response.body)
-      
+
       # Process extractor files to find domains and patterns
       extractor_files = files.select { |file| file['name'].end_with?('.py') && file['name'] != '__init__.py' }
-      
+
       puts "📝 Processing #{extractor_files.length} extractor files..."
-      
-      extractor_files.each_with_index do |file, index|
-        print "\r\e[K"  # Clear the entire line
-        print "Processing #{index + 1}/#{extractor_files.length}: #{file['name']}"
-        process_extractor_file(file)
+
+      puts "⬇️  Downloading #{extractor_files.length} files with #{@max_download_workers} threads..."
+      file_contents = download_files_parallel(extractor_files)
+
+      if @parallel_enabled
+        puts "🚀 Processing #{file_contents.length} files with #{@max_process_workers} threads..."
+        process_content_parallel(file_contents)
+      else
+        puts "📥 Processing #{file_contents.length} files sequentially..."
+        process_content_sequential(file_contents)
       end
-      
+
       puts "\n🎯 Found #{@video_domains.size} unique video hosting domains"
       puts "🔍 Extracted #{@video_regexes.size} URL pattern regexes"
     end
 
-    def process_extractor_file(file_info)
-      begin
-        response = Net::HTTP.get_response(URI(file_info['download_url']))
-        return unless response.code == '200'
-        
-        content = response.body
-        
-        extract_domains_from_content(content, file_info['name'])
-        extract_regexes_from_content(content, file_info['name'])
-      rescue StandardError => e
-        # Silently skip errors to avoid spam
+    def download_files_parallel(extractor_files)
+      file_contents = []
+      mutex = Mutex.new
+      download_count = 0
+      total = extractor_files.length
+
+      queue = Queue.new
+      extractor_files.each { |f| queue << f }
+
+      workers = @max_download_workers.times.map do
+        Thread.new do
+          while (file = begin; queue.pop(true); rescue ThreadError; nil; end)
+            begin
+              response = Net::HTTP.get_response(URI(file['download_url']))
+              if response.code == '200'
+                mutex.synchronize do
+                  file_contents << { name: file['name'], content: response.body }
+                  download_count += 1
+                  print "\r\e[K"
+                  print "Downloaded #{download_count}/#{total} files"
+                end
+              else
+                mutex.synchronize { download_count += 1 }
+              end
+            rescue StandardError
+              mutex.synchronize { download_count += 1 }
+            end
+          end
+        end
+      end
+
+      workers.each(&:join)
+
+      puts "\n✅ Downloaded #{file_contents.length}/#{total} files successfully"
+      file_contents
+    end
+
+    def process_content_sequential(file_contents)
+      file_contents.each_with_index do |file_data, index|
+        print "\r\e[K"
+        print "Processing #{index + 1}/#{file_contents.length}: #{file_data[:name]}"
+        extract_domains_from_content(file_data[:content], file_data[:name])
+        extract_regexes_from_content(file_data[:content], file_data[:name])
+      end
+    end
+
+    def process_content_parallel(file_contents)
+      queue = Queue.new
+      file_contents.each { |f| queue << f }
+      process_count = 0
+      total = file_contents.length
+      count_mutex = Mutex.new
+
+      workers = @max_process_workers.times.map do
+        Thread.new do
+          while (file_data = begin; queue.pop(true); rescue ThreadError; nil; end)
+            # Each thread collects its own results to minimize lock contention
+            local_domains = Set.new
+            local_regexes = []
+
+            process_file_content(file_data[:content], file_data[:name], local_domains, local_regexes)
+
+            # Merge results under lock
+            @domains_mutex.synchronize { local_domains.each { |d| @video_domains.add(d) } }
+            @regexes_mutex.synchronize { @video_regexes.concat(local_regexes) }
+
+            count_mutex.synchronize do
+              process_count += 1
+              print "\r\e[K"
+              print "Processed #{process_count}/#{total} files"
+            end
+          end
+        end
+      end
+
+      workers.each(&:join)
+    end
+
+    def process_file_content(content, filename, domains, regexes)
+      # Look for _VALID_URL patterns and extract domains
+      valid_url_patterns = content.scan(/_VALID_URL\s*=\s*r?['"]([^'"]+)['"]/m)
+
+      valid_url_patterns.each do |pattern|
+        pattern = pattern[0] if pattern.is_a?(Array)
+        extracted = extract_domains_from_regex(pattern)
+        extracted.each { |domain| domains.add(domain) }
+
+        cleaned_pattern = clean_regex_pattern(pattern)
+        if cleaned_pattern && !cleaned_pattern.empty? && is_useful_pattern?(cleaned_pattern)
+          regexes << {
+            source: filename.gsub('.py', ''),
+            pattern: cleaned_pattern,
+            original: pattern
+          }
+        end
+      end
+
+      # Look for IE_NAME patterns
+      ie_name_patterns = content.scan(/IE_NAME\s*=\s*['"]([^'"]+)['"]/m)
+      ie_name_patterns.each do |name|
+        name = name[0] if name.is_a?(Array)
+        next unless name && name.include?('.') && valid_domain?(name)
+
+        domains.add(name.downcase)
+      end
+
+      # Look for hardcoded URLs in the content
+      url_patterns = content.scan(%r{['"]https?://(?:www\.)?([a-zA-Z0-9-]+\.[a-zA-Z]{2,})}i)
+      url_patterns.each do |match|
+        domain = match[0] if match.is_a?(Array)
+        next unless domain && valid_domain?(domain.downcase)
+
+        domains.add(domain.downcase)
       end
+
+      # Look for class-based domain hints from filename
+      return unless filename.end_with?('.py')
+
+      base_name = filename.gsub('.py', '').downcase
+      if base_name.include?('.')
+        potential_domain = base_name
+      elsif base_name.length > 3 && !%w[common generic base].include?(base_name)
+        potential_domain = "#{base_name}.com"
+      end
+
+      return unless potential_domain && valid_domain?(potential_domain)
+
+      domains.add(potential_domain)
+    end
+
+    def process_extractor_file(file_info)
+      response = Net::HTTP.get_response(URI(file_info['download_url']))
+      return unless response.code == '200'
+
+      content = response.body
+
+      extract_domains_from_content(content, file_info['name'])
+      extract_regexes_from_content(content, file_info['name'])
+    rescue StandardError
+      # Silently skip errors to avoid spam
     end
 
     def extract_domains_from_content(content, filename)
       # Look for _VALID_URL patterns and extract domains
       valid_url_patterns = content.scan(/_VALID_URL\s*=\s*r?['"]([^'"]+)['"]/m)
-      
+
       valid_url_patterns.each do |pattern|
         pattern = pattern[0] if pattern.is_a?(Array)
         domains = extract_domains_from_regex(pattern)
         domains.each { |domain| @video_domains.add(domain) }
       end
-      
+
       # Look for IE_NAME patterns
       ie_name_patterns = content.scan(/IE_NAME\s*=\s*['"]([^'"]+)['"]/m)
       ie_name_patterns.each do |name|
         name = name[0] if name.is_a?(Array)
         next unless name && name.include?('.') && valid_domain?(name)
+
         @video_domains.add(name.downcase)
       end
-      
+
       # Look for hardcoded URLs in the content
-      url_patterns = content.scan(/['"]https?:\/\/(?:www\.)?([a-zA-Z0-9-]+\.[a-zA-Z]{2,})/i)
+      url_patterns = content.scan(%r{['"]https?://(?:www\.)?([a-zA-Z0-9-]+\.[a-zA-Z]{2,})}i)
       url_patterns.each do |match|
         domain = match[0] if match.is_a?(Array)
         next unless domain && valid_domain?(domain.downcase)
+
         @video_domains.add(domain.downcase)
       end
-      
+
       # Look for class-based domain hints from filename
-      if filename.end_with?('.py')
-        base_name = filename.gsub('.py', '').downcase
-        # Try to infer domain from extractor name
-        if base_name.include?('.')
-          potential_domain = base_name
-        elsif base_name.length > 3 && !%w[common generic base].include?(base_name)
-          # Common patterns: youtube -> youtube.com, vimeo -> vimeo.com
-          potential_domain = "#{base_name}.com"
-        end
-        
-        if potential_domain && valid_domain?(potential_domain)
-          @video_domains.add(potential_domain)
-        end
+      return unless filename.end_with?('.py')
+
+      base_name = filename.gsub('.py', '').downcase
+      # Try to infer domain from extractor name
+      if base_name.include?('.')
+        potential_domain = base_name
+      elsif base_name.length > 3 && !%w[common generic base].include?(base_name)
+        # Common patterns: youtube -> youtube.com, vimeo -> vimeo.com
+        potential_domain = "#{base_name}.com"
       end
+
+      return unless potential_domain && valid_domain?(potential_domain)
+
+      @video_domains.add(potential_domain)
     end
 
     def extract_regexes_from_content(content, filename)
       # Extract _VALID_URL patterns for video detection
       valid_url_patterns = content.scan(/_VALID_URL\s*=\s*r?['"]([^'"]+)['"]/m)
-      
+
       valid_url_patterns.each do |pattern|
         pattern = pattern[0] if pattern.is_a?(Array)
-        
+
         cleaned_pattern = clean_regex_pattern(pattern)
-        if cleaned_pattern && !cleaned_pattern.empty? && is_useful_pattern?(cleaned_pattern)
-          @video_regexes << {
-            source: filename.gsub('.py', ''),
-            pattern: cleaned_pattern,
-            original: pattern
-          }
-        end
+        next unless cleaned_pattern && !cleaned_pattern.empty? && is_useful_pattern?(cleaned_pattern)
+
+        @video_regexes << {
+          source: filename.gsub('.py', ''),
+          pattern: cleaned_pattern,
+          original: pattern
+        }
       end
     end
 
     def extract_domains_from_regex(pattern)
       domains = Set.new
-      
+
       # Extract domains from common regex patterns - more comprehensive
       patterns_to_try = [
         # Standard URLs
-        /https?:\/\/(?:www\.)?([a-zA-Z0-9-]+\.[a-zA-Z]{2,})/i,
+        %r{https?://(?:www\.)?([a-zA-Z0-9-]+\.[a-zA-Z]{2,})}i,
         # Escaped URLs with backslashes
-        /https?:\\\/\\\/(?:www\\.)?([a-zA-Z0-9-]+\\.[a-zA-Z]{2,})/i,
+        %r{https?:\\/\\/(?:www\\.)?([a-zA-Z0-9-]+\\.[a-zA-Z]{2,})}i,
         # Domain patterns with optional www (escaped)
-        /\(\?\:www\\\.\)\?([a-zA-Z0-9-]+\.[a-zA-Z]{2,})/i,
+        /\(\?:www\\\.\)\?([a-zA-Z0-9-]+\.[a-zA-Z]{2,})/i,
         # Simple domain references
         /([a-zA-Z0-9-]+\.[a-zA-Z]{2,})/i
       ]
-      
+
       patterns_to_try.each do |regex|
         matches = pattern.scan(regex)
         matches.each do |match|
           domain = match.is_a?(Array) ? match[0] : match
           domain = clean_domain(domain) if domain
-          if domain && valid_domain?(domain.downcase)
-            domains.add(domain.downcase)
-          end
+          domains.add(domain.downcase) if domain && valid_domain?(domain.downcase)
         end
       end
-      
+
       domains.to_a
     end
 
     def clean_domain(domain)
       return nil unless domain
-      
+
       # Remove regex escapes and clean up
-      cleaned = domain.gsub(/\\\./, '.').gsub(/[\(\)\?\:\|\\]/, '')
+      cleaned = domain.gsub(/\\\./, '.').gsub(/[()?:|\\]/, '')
       cleaned = cleaned.strip
-      
+
       # Remove trailing regex syntax
-      cleaned = cleaned.split(/[\(\)\[\]]/)[0] if cleaned.include?('(') || cleaned.include?('[')
-      
+      cleaned = cleaned.split(/[()\[\]]/)[0] if cleaned.include?('(') || cleaned.include?('[')
+
       cleaned
     end
 
     def clean_regex_pattern(pattern)
       return nil unless pattern
-      
+
       # Simplify common patterns for practical use
       cleaned = pattern.dup
-      
+
       # Remove complex lookaheads/lookbehinds
-      cleaned = cleaned.gsub(/\(\?\![^)]*\)/, '')
-      cleaned = cleaned.gsub(/\(\?\<\![^)]*\)/, '')
-      
+      cleaned = cleaned.gsub(/\(\?![^)]*\)/, '')
+      cleaned = cleaned.gsub(/\(\?<![^)]*\)/, '')
+
       # Simplify common patterns
-      cleaned = cleaned.gsub(/\(\?\:www\\\.\)\?/, '(?:www\.)?')
+      cleaned = cleaned.gsub(/\(\?:www\\\.\)\?/, '(?:www\.)?')
       cleaned = cleaned.gsub(/\\\./, '\.')
-      
+
       # Validate regex with warning suppression
       begin
         original_warning = $-w
-        $-w = nil  # Disable warnings
+        $-w = nil # Disable warnings
         Regexp.new(cleaned)
         cleaned
       rescue RegexpError
         nil
       ensure
-        $-w = original_warning  # Restore warnings
+        $-w = original_warning # Restore warnings
       end
     end
 
@@ -235,7 +377,7 @@ module VideoListGenerator
       return false if pattern.length < 10
       return false if pattern.include?('blob:')
       return false if pattern.match?(/\$\s*$/) # ends with just $
-      
+
       true
     end
 
@@ -246,6 +388,7 @@ module VideoListGenerator
       return false if domain.include?('example.') || domain.include?('test.')
       return false if domain.include?('localhost')
       return false if domain.end_with?('.html') || domain.end_with?('.htm')
+
       true
     end
 
@@ -255,10 +398,10 @@ module VideoListGenerator
 
     def generate_hosts_file
       hosts_file_path = 'lists/video_hosting_domains.hosts'
-      
+
       File.open(hosts_file_path, 'w') do |file|
         write_hosts_header(file)
-        
+
         @video_domains.sort.each do |domain|
           file.puts "0.0.0.0 #{domain}"
           file.puts "0.0.0.0 www.#{domain}" unless domain.start_with?('www.')
@@ -268,34 +411,34 @@ module VideoListGenerator
 
     def generate_regex_file
       regex_file_path = 'lists/video_url_patterns.txt'
-      
+
       File.open(regex_file_path, 'w') do |file|
         write_regex_header(file)
-        
+
         # Add manual high-priority patterns first
         add_manual_video_patterns(file)
-        
+
         @video_regexes.each do |regex_info|
           file.puts "# Source: #{regex_info[:source]}"
           file.puts "# Pattern: #{regex_info[:pattern]}"
           file.puts "# Original: #{regex_info[:original]}"
           file.puts regex_info[:pattern]
-          file.puts ""
+          file.puts ''
         end
       end
     end
 
     def write_hosts_header(file)
-      file.puts "# Video Hosting Domains - PiHole Compatible Hosts File"
+      file.puts '# Video Hosting Domains - PiHole Compatible Hosts File'
       file.puts "# Generated on: #{Time.now.strftime('%Y-%m-%d %H:%M:%S UTC')}"
-      file.puts "# Source: yt-dlp extractors (https://github.com/yt-dlp/yt-dlp)"
-      file.puts "# Purpose: Block access to video hosting websites"
-      file.puts "# Format: 0.0.0.0 domain.com"
-      file.puts "#"
-      file.puts "# This file contains domains extracted from yt-dlp video extractors"
-      file.puts "# to help identify and categorize video hosting websites."
-      file.puts "#"
-      file.puts ""
+      file.puts '# Source: yt-dlp extractors (https://github.com/yt-dlp/yt-dlp)'
+      file.puts '# Purpose: Block access to video hosting websites'
+      file.puts '# Format: 0.0.0.0 domain.com'
+      file.puts '#'
+      file.puts '# This file contains domains extracted from yt-dlp video extractors'
+      file.puts '# to help identify and categorize video hosting websites.'
+      file.puts '#'
+      file.puts ''
     end
 
     def add_manual_video_patterns(file)
@@ -303,12 +446,12 @@ module VideoListGenerator
       manual_patterns = [
         {
           source: 'manual_youtube',
-          pattern: 'https?://(?:www\\.)?(?:youtube\\.com/watch\\?v=|youtu\\.be/)[a-zA-Z0-9_-]{11}',
+          pattern: 'https?://(?:www\\.)?(?:youtube\\.com/watch\\?.*[&?]?v=|youtu\\.be/)[a-zA-Z0-9_-]{11}',
           description: 'YouTube video watch URLs'
         },
         {
           source: 'manual_youtube_shorts',
-          pattern: 'https?://(?:www\\.)?youtube\\.com/shorts/[a-zA-Z0-9_-]{11}',
+          pattern: 'https?://(?:www\\.)?youtube\\.com/shorts/[a-zA-Z0-9_-]{11}(?:\\?.*)?$',
           description: 'YouTube Shorts URLs'
         },
         {
@@ -332,42 +475,78 @@ module VideoListGenerator
           description: 'TikTok video URLs'
         }
       ]
-      
-      file.puts "# ===== MANUAL HIGH-PRIORITY PATTERNS ====="
-      file.puts ""
-      
+
+      file.puts '# ===== MANUAL HIGH-PRIORITY PATTERNS ====='
+      file.puts ''
+
       manual_patterns.each do |pattern_info|
         file.puts "# Source: #{pattern_info[:source]}"
         file.puts "# Description: #{pattern_info[:description]}"
         file.puts "# Pattern: #{pattern_info[:pattern]}"
         file.puts pattern_info[:pattern]
-        file.puts ""
+        file.puts ''
       end
-      
-      file.puts "# ===== EXTRACTED PATTERNS FROM YT-DLP ====="
-      file.puts ""
+
+      file.puts '# ===== EXTRACTED PATTERNS FROM YT-DLP ====='
+      file.puts ''
     end
 
     def write_regex_header(file)
-      file.puts "# Video URL Detection Patterns"
+      file.puts '# Video URL Detection Patterns'
       file.puts "# Generated on: #{Time.now.strftime('%Y-%m-%d %H:%M:%S UTC')}"
-      file.puts "# Source: yt-dlp extractors (https://github.com/yt-dlp/yt-dlp) + manual patterns"
-      file.puts "# Purpose: Regex patterns to detect video URLs vs other content"
-      file.puts "#"
-      file.puts "# These patterns help distinguish between:"
-      file.puts "# - Direct video content URLs"
-      file.puts "# - Homepage, playlist, user profile, community content URLs"
-      file.puts "#"
-      file.puts "# Usage: Use these patterns to categorize URLs from video hosting domains"
-      file.puts "#        to determine if they contain actual video content or other resources"
-      file.puts "#"
-      file.puts ""
+      file.puts '# Source: yt-dlp extractors (https://github.com/yt-dlp/yt-dlp) + manual patterns'
+      file.puts '# Purpose: Regex patterns to detect video URLs vs other content'
+      file.puts '#'
+      file.puts '# These patterns help distinguish between:'
+      file.puts '# - Direct video content URLs'
+      file.puts '# - Homepage, playlist, user profile, community content URLs'
+      file.puts '#'
+      file.puts '# Usage: Use these patterns to categorize URLs from video hosting domains'
+      file.puts '#        to determine if they contain actual video content or other resources'
+      file.puts '#'
+      file.puts ''
     end
   end
 end
 
 # Run the generator if this file is executed directly
 if __FILE__ == $0
-  generator = VideoListGenerator::ExtractorParser.new
+  # Default configuration
+  options = {
+    max_download_workers: 16,
+    max_process_workers: 8,
+    verbose: false
+  }
+
+  # Parse command-line options
+  OptionParser.new do |opts|
+    opts.banner = "Usage: #{$0} [options]"
+    opts.separator ''
+    opts.separator 'Generate video hosting domain lists and URL patterns from yt-dlp extractors'
+    opts.separator ''
+
+    opts.on('-d', '--download-workers NUM', Integer, "Number of download threads (default: #{options[:max_download_workers]})") do |num|
+      options[:max_download_workers] = num
+    end
+
+    opts.on('-p', '--process-workers NUM', Integer, "Number of processing threads (default: #{options[:max_process_workers]})") do |num|
+      options[:max_process_workers] = num
+    end
+
+    opts.on('-v', '--verbose', 'Verbose output') do
+      options[:verbose] = true
+    end
+
+    opts.on('--no-parallel', 'Disable parallel processing (use sequential only)') do
+      options[:parallel_enabled] = false
+    end
+
+    opts.on('-h', '--help', 'Show this help message') do
+      puts opts
+      exit
+    end
+  end.parse!
+
+  generator = VideoListGenerator::ExtractorParser.new(options)
   generator.generate_lists
-end
+end