UrlCategorise 0.1.7 → 0.1.11

data/bin/check_lists

@@ -1,9 +1,112 @@
 #!/usr/bin/env ruby
 
 require 'bundler/setup'
+require 'thread'
+require 'optparse'
 require_relative '../lib/url_categorise'
 
+# Default configuration
+options = {
+  parallel_enabled: false,
+  max_threads: 8,
+  verbose: false
+}
+
+# Parse command-line options
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{$0} [options]"
+  opts.separator ''
+  opts.separator 'Check all URLs in constants for availability'
+  opts.separator ''
+
+  opts.on('-p', '--parallel', 'Enable parallel URL checking') do
+    options[:parallel_enabled] = true
+  end
+
+  opts.on('-t', '--threads NUM', Integer, "Number of threads to use (default: #{options[:max_threads]})") do |num|
+    options[:max_threads] = num
+  end
+
+  opts.on('-v', '--verbose', 'Verbose output') do
+    options[:verbose] = true
+  end
+
+  opts.on('-h', '--help', 'Show this help message') do
+    puts opts
+    exit
+  end
+end.parse!
+
 puts '=== CHECKING ALL URLs IN CONSTANTS ==='
+puts "🚀 Using parallel processing with #{options[:max_threads]} threads" if options[:parallel_enabled]
+
+def check_url(url)
+  begin
+    response = HTTParty.head(url, timeout: 10)
+    case response.code
+    when 200
+      { url: url, status: '✅ OK', code: response.code }
+    when 404
+      { url: url, status: '❌ 404 Not Found', code: response.code }
+    when 403
+      { url: url, status: '❌ 403 Forbidden', code: response.code }
+    when 500..599
+      { url: url, status: "❌ Server Error (#{response.code})", code: response.code }
+    else
+      { url: url, status: "⚠️ HTTP #{response.code}", code: response.code }
+    end
+  rescue Net::TimeoutError, HTTParty::TimeoutError
+    { url: url, status: '❌ Timeout', code: nil }
+  rescue SocketError, Errno::ECONNREFUSED
+    { url: url, status: '❌ DNS/Network Error', code: nil }
+  rescue StandardError => e
+    { url: url, status: "❌ Error: #{e.class}", code: nil }
+  end
+end
+
+def check_urls_parallel(urls, max_threads)
+  results = []
+  results_mutex = Mutex.new
+  
+  # Create work queue
+  queue = Queue.new
+  urls.each { |url| queue << url }
+  
+  # Create worker threads
+  threads = []
+  max_threads.times do
+    threads << Thread.new do
+      while !queue.empty?
+        begin
+          url = queue.pop(true) # non-blocking pop
+          result = check_url(url)
+          
+          results_mutex.synchronize do
+            results << result
+          end
+        rescue ThreadError
+          # Queue is empty, thread can exit
+          break
+        end
+      end
+    end
+  end
+  
+  # Wait for all threads to complete
+  threads.each(&:join)
+  
+  # Sort results to maintain original order
+  urls.map { |url| results.find { |r| r[:url] == url } }
+end
+
+def check_urls_sequential(urls)
+  urls.map do |url|
+    print "  Testing #{url}... "
+    result = check_url(url)
+    puts result[:status]
+    result
+  end
+end
 
 UrlCategorise::Constants::DEFAULT_HOST_URLS.each do |category, urls|
   puts "\n#{category.upcase}:"
@@ -20,28 +123,27 @@ UrlCategorise::Constants::DEFAULT_HOST_URLS.each do |category, urls|
     next
   end
 
-  actual_urls.each do |url|
-    print "  Testing #{url}... "
-    begin
-      response = HTTParty.head(url, timeout: 10)
-      case response.code
-      when 200
-        puts '✅ OK'
-      when 404
-        puts '❌ 404 Not Found'
-      when 403
-        puts '❌ 403 Forbidden'
-      when 500..599
-        puts "❌ Server Error (#{response.code})"
-      else
-        puts "⚠️ HTTP #{response.code}"
-      end
-    rescue Net::TimeoutError, HTTParty::TimeoutError
-      puts '❌ Timeout'
-    rescue SocketError, Errno::ECONNREFUSED
-      puts '❌ DNS/Network Error'
-    rescue StandardError => e
-      puts "❌ Error: #{e.class}"
+  if options[:parallel_enabled] && actual_urls.length > 1
+    puts "  ⚡ Checking #{actual_urls.length} URLs in parallel..."
+    start_time = Time.now
+    results = check_urls_parallel(actual_urls, [options[:max_threads], actual_urls.length].min)
+    end_time = Time.now
+    
+    # Display results
+    results.each do |result|
+      puts "  #{result[:url]} → #{result[:status]}"
     end
+    
+    puts "  ⏱️ Completed in #{'%.2f' % ((end_time - start_time) * 1000)}ms"
+  else
+    results = check_urls_sequential(actual_urls)
   end
+  
+  # Summary for this category
+  ok_count = results.count { |r| r[:status].include?('✅') }
+  error_count = results.length - ok_count
+  puts "  📊 Summary: #{ok_count} OK, #{error_count} errors" if results.length > 1
 end
+
+puts "\n🎯 Health check completed!"
+puts "💡 Tip: Use --parallel (-p) for faster checking and --threads (-t) NUM to control thread count"

data/bin/export_csv

@@ -7,55 +7,64 @@ require_relative '../lib/url_categorise'
 options = {
   output_path: nil,
   cache_dir: nil,
-  verbose: false,
+  verbose: true,  # Verbose by default as requested
   iab_compliance: false,
   smart_categorization: false,
   auto_load_datasets: false,
-  kaggle_credentials_file: nil
+  kaggle_credentials_file: nil,
+  parallel_loading: true # Enable by default for better performance
 }
 
 OptionParser.new do |opts|
   opts.banner = "Usage: #{$0} [options]"
-  opts.separator ""
-  opts.separator "Export all categorized domains and dataset content as a single CSV file for AI training"
-  opts.separator ""
-  
-  opts.on("-o", "--output PATH", "Output directory path (default: cache_dir/exports/csv or ./exports/csv)") do |path|
+  opts.separator ''
+  opts.separator 'Export all categorized domains and dataset content as a single CSV file for AI training'
+  opts.separator ''
+
+  opts.on('-o', '--output PATH', 'Output directory path (default: cache_dir/exports/csv or ./exports/csv)') do |path|
     options[:output_path] = path
   end
-  
-  opts.on("-c", "--cache-dir PATH", "Cache directory path for client initialization") do |path|
+
+  opts.on('-c', '--cache-dir PATH', 'Cache directory path for client initialization') do |path|
     options[:cache_dir] = path
   end
-  
-  opts.on("--iab-compliance", "Enable IAB compliance for category mapping") do
+
+  opts.on('--iab-compliance', 'Enable IAB compliance for category mapping') do
     options[:iab_compliance] = true
   end
-  
-  opts.on("--smart-categorization", "Enable smart categorization") do
+
+  opts.on('--smart-categorization', 'Enable smart categorization') do
     options[:smart_categorization] = true
   end
-  
-  opts.on("--auto-load-datasets", "Auto-load datasets from constants for rich content export") do
+
+  opts.on('--auto-load-datasets', 'Auto-load datasets from constants for rich content export') do
     options[:auto_load_datasets] = true
   end
-  
-  opts.on("--kaggle-credentials FILE", "Path to Kaggle credentials file (default: ~/.kaggle/kaggle.json)") do |file|
+
+  opts.on('--kaggle-credentials FILE', 'Path to Kaggle credentials file (default: ~/.kaggle/kaggle.json)') do |file|
     options[:kaggle_credentials_file] = file
   end
-  
-  opts.on("-v", "--verbose", "Verbose output") do
+
+  opts.on('-v', '--verbose', 'Force verbose output (default behavior)') do
     options[:verbose] = true
   end
-  
-  opts.on("-h", "--help", "Show this help message") do
+
+  opts.on('-q', '--quiet', '--silence', 'Disable verbose output') do
+    options[:verbose] = false
+  end
+
+  opts.on('--no-parallel', 'Disable parallel loading (use for debugging or reduced memory usage)') do
+    options[:parallel_loading] = false
+  end
+
+  opts.on('-h', '--help', 'Show this help message') do
     puts opts
     exit
   end
 end.parse!
 
-puts "=== UrlCategorise CSV Data Export ===" if options[:verbose]
-puts "Initializing client..." if options[:verbose]
+puts '=== UrlCategorise CSV Data Export ===' if options[:verbose]
+puts 'Initializing client...' if options[:verbose]
 
 begin
   # Build dataset config if datasets should be loaded
@@ -65,7 +74,7 @@ begin
       cache_path: options[:cache_dir] ? File.join(options[:cache_dir], 'datasets') : './url_cache/datasets',
       download_path: options[:cache_dir] ? File.join(options[:cache_dir], 'downloads') : './url_cache/downloads'
     }
-    
+
     # Add Kaggle credentials if provided
     if options[:kaggle_credentials_file]
       dataset_config[:kaggle] = { credentials_file: options[:kaggle_credentials_file] }
@@ -79,42 +88,42 @@ begin
     iab_compliance: options[:iab_compliance],
     smart_categorization: options[:smart_categorization],
     auto_load_datasets: options[:auto_load_datasets],
-    dataset_config: dataset_config
+    dataset_config: dataset_config,
+    parallel_loading: options[:parallel_loading]
   )
-  
+
   if options[:verbose] && options[:auto_load_datasets]
-    puts "Client initialized with dataset loading enabled"
-    puts "Dataset statistics:"
+    puts 'Client initialized with dataset loading enabled'
+    puts 'Dataset statistics:'
     puts "  Dataset categories: #{client.count_of_dataset_categories}"
     puts "  Dataset hosts: #{client.count_of_dataset_hosts.to_s.reverse.gsub(/(\\d{3})(?=\\d)/, '\\1,').reverse}"
     puts "  Dataset data size: #{client.size_of_dataset_data.round(2)} MB" if client.respond_to?(:size_of_dataset_data)
   end
-  
-  puts "Exporting CSV data..." if options[:verbose]
-  
+
+  puts 'Exporting CSV data...' if options[:verbose]
+
   result = client.export_csv_data(options[:output_path])
-  
+
   puts "\n✅ Export completed successfully!"
   puts "📁 Export directory: #{result[:export_directory]}"
   puts "📄 CSV file: #{result[:csv_file]}"
   puts "📄 Summary file: #{result[:summary_file]}"
-  
+
   puts "\n📊 Data Summary:"
   puts "  Total entries: #{result[:total_entries]}"
   puts "  Domain categorizations: #{result[:summary][:domain_categorization_entries]}"
   puts "  Dataset content entries: #{result[:summary][:dataset_content_entries]}"
   puts "  Total categories: #{result[:summary][:total_categories]}"
   puts "  Has dataset content: #{result[:summary][:has_dataset_content]}"
-  
+
   if options[:verbose]
     puts "\n🏷️  Categories included:"
     result[:summary][:categories].each do |category|
       puts "    - #{category}"
     end
   end
-  
 rescue StandardError => e
   puts "❌ Error: #{e.message}"
   puts e.backtrace if options[:verbose]
   exit 1
-end
+end

data/bin/export_hosts

@@ -7,62 +7,72 @@ require_relative '../lib/url_categorise'
 options = {
   output_path: nil,
   cache_dir: nil,
-  verbose: false
+  verbose: true,  # Verbose by default as requested
+  parallel_loading: true # Enable by default for better performance
 }
 
 OptionParser.new do |opts|
   opts.banner = "Usage: #{$0} [options]"
-  opts.separator ""
-  opts.separator "Export all categorized domains as separate hosts files per category"
-  opts.separator ""
-  
-  opts.on("-o", "--output PATH", "Output directory path (default: cache_dir/exports/hosts or ./exports/hosts)") do |path|
+  opts.separator ''
+  opts.separator 'Export all categorized domains as separate hosts files per category'
+  opts.separator ''
+
+  opts.on('-o', '--output PATH',
+          'Output directory path (default: cache_dir/exports/hosts or ./exports/hosts)') do |path|
     options[:output_path] = path
   end
-  
-  opts.on("-c", "--cache-dir PATH", "Cache directory path for client initialization") do |path|
+
+  opts.on('-c', '--cache-dir PATH', 'Cache directory path for client initialization') do |path|
     options[:cache_dir] = path
   end
-  
-  opts.on("-v", "--verbose", "Verbose output") do
+
+  opts.on('-v', '--verbose', 'Force verbose output (default behavior)') do
     options[:verbose] = true
   end
-  
-  opts.on("-h", "--help", "Show this help message") do
+
+  opts.on('-q', '--quiet', '--silence', 'Disable verbose output') do
+    options[:verbose] = false
+  end
+
+  opts.on('--no-parallel', 'Disable parallel loading (use for debugging or reduced memory usage)') do
+    options[:parallel_loading] = false
+  end
+
+  opts.on('-h', '--help', 'Show this help message') do
     puts opts
     exit
   end
 end.parse!
 
-puts "=== UrlCategorise Hosts Export ===" if options[:verbose]
-puts "Initializing client..." if options[:verbose]
+puts '=== UrlCategorise Hosts Export ===' if options[:verbose]
+puts 'Initializing client...' if options[:verbose]
 
 begin
   client = UrlCategorise::Client.new(
-    cache_dir: options[:cache_dir]
+    cache_dir: options[:cache_dir],
+    parallel_loading: options[:parallel_loading]
   )
-  
-  puts "Exporting hosts files..." if options[:verbose]
-  
+
+  puts 'Exporting hosts files...' if options[:verbose]
+
   result = client.export_hosts_files(options[:output_path])
-  
+
   summary = result.delete(:_summary)
-  
+
   puts "\n✅ Export completed successfully!"
   puts "📁 Export directory: #{summary[:export_directory]}"
   puts "📊 Total categories exported: #{summary[:total_categories]}"
   puts "🌐 Total domains exported: #{summary[:total_domains]}"
   puts "📄 Summary file: #{summary[:path]}"
-  
+
   if options[:verbose]
     puts "\n📋 Files created:"
-    result.each do |category, info|
+    result.each do |_category, info|
       puts "  #{info[:filename]} - #{info[:count]} domains"
     end
   end
-  
 rescue StandardError => e
   puts "❌ Error: #{e.message}"
   puts e.backtrace if options[:verbose]
   exit 1
-end
+end