UrlCategorise 0.1.7 → 0.1.11

data/lib/url_categorise/client.rb

@@ -1,5 +1,6 @@
-require 'set'
-require 'digest'
+require "set"
+require "digest"
+require "etc"
 
 module UrlCategorise
   class Client < ApiPattern::Client
@@ -7,17 +8,23 @@ module UrlCategorise
     include ActiveAttr::Model
 
     def self.compatible_api_version
-      'v2'
+      "v2"
     end
 
     def self.api_version
-      'v2 2025-08-23'
+      "v2 2025-08-23"
+    end
+
+    def self.ractor_available?
+      defined?(Ractor) && Ractor.respond_to?(:new)
+    rescue
+      false
     end
 
     attribute :host_urls, default: -> { DEFAULT_HOST_URLS }
     attribute :cache_dir
     attribute :force_download, type: Boolean, default: false
-    attribute :dns_servers, default: ['1.1.1.1', '1.0.0.1']
+    attribute :dns_servers, default: [ "1.1.1.1", "1.0.0.1" ]
     attribute :request_timeout, type: Integer, default: 10
     attribute :iab_compliance_enabled, type: Boolean, default: false
     attribute :iab_version, default: :v3
@@ -26,18 +33,22 @@ module UrlCategorise
     attribute :smart_rules, default: -> { {} }
     attribute :regex_categorization_enabled, type: Boolean, default: false
     attribute :regex_patterns_file, default: -> { VIDEO_URL_PATTERNS_FILE }
+    attribute :debug_enabled, type: Boolean, default: false
+    attribute :parallel_loading_enabled, type: Boolean, default: -> { self.class.ractor_available? }
+    attribute :max_threads, type: Integer, default: 8
+    attribute :max_ractor_workers, type: Integer, default: -> { [ 4, Etc.nprocessors ].max }
 
     attr_reader :hosts, :metadata, :dataset_processor, :dataset_categories, :regex_patterns
 
     def initialize(**kwargs)
       # Extract dataset_config for later use
       dataset_config = kwargs.fetch(:dataset_config, {})
-      
+
       # Set ActiveAttr attributes - preserve explicitly passed values including nil
       self.host_urls = kwargs.key?(:host_urls) ? kwargs[:host_urls] : DEFAULT_HOST_URLS
       self.cache_dir = kwargs[:cache_dir] # will be nil if not provided or explicitly nil
       self.force_download = kwargs.key?(:force_download) ? kwargs[:force_download] : false
-      self.dns_servers = kwargs.key?(:dns_servers) ? kwargs[:dns_servers] : ['1.1.1.1', '1.0.0.1']
+      self.dns_servers = kwargs.key?(:dns_servers) ? kwargs[:dns_servers] : [ "1.1.1.1", "1.0.0.1" ]
       self.request_timeout = kwargs.key?(:request_timeout) ? kwargs[:request_timeout] : 10
       self.iab_compliance_enabled = kwargs.key?(:iab_compliance) ? kwargs[:iab_compliance] : false
       self.iab_version = kwargs.key?(:iab_version) ? kwargs[:iab_version] : :v3
@@ -46,6 +57,10 @@ module UrlCategorise
       self.smart_rules = initialize_smart_rules(kwargs.key?(:smart_rules) ? kwargs[:smart_rules] : {})
       self.regex_categorization_enabled = kwargs.key?(:regex_categorization) ? kwargs[:regex_categorization] : false
       self.regex_patterns_file = kwargs.key?(:regex_patterns_file) ? kwargs[:regex_patterns_file] : VIDEO_URL_PATTERNS_FILE
+      self.debug_enabled = kwargs.key?(:debug) ? kwargs[:debug] : false
+      self.parallel_loading_enabled = kwargs.key?(:parallel_loading) ? kwargs[:parallel_loading] : self.class.ractor_available?
+      self.max_threads = kwargs.key?(:max_threads) ? kwargs[:max_threads] : 8
+      self.max_ractor_workers = kwargs.key?(:max_ractor_workers) ? kwargs[:max_ractor_workers] : [ 4, Etc.nprocessors ].max
 
       @metadata = {}
       @dataset_categories = Set.new # Track which categories come from datasets
@@ -54,35 +69,86 @@ module UrlCategorise
       # Initialize dataset processor if config provided
       @dataset_processor = initialize_dataset_processor(dataset_config) unless dataset_config.empty?
 
-      @hosts = fetch_and_build_host_lists
+      debug_log("Initializing UrlCategorise Client with debug enabled")
+
+      initialization_start_time = Time.now
+
+      debug_log("Loading host lists from #{(host_urls || {}).keys.size} categories")
+
+      @hosts = debug_time("Host lists loading") do
+        fetch_and_build_host_lists
+      end
 
       # Load regex patterns if enabled
-      load_regex_patterns if regex_categorization_enabled
+      if regex_categorization_enabled
+        debug_log("Regex categorization enabled, loading patterns from #{regex_patterns_file}")
+        debug_time("Regex patterns loading") { load_regex_patterns }
+      end
 
       # Auto-load datasets from constants if enabled
-      load_datasets_from_constants if auto_load_datasets && @dataset_processor
+      if auto_load_datasets && @dataset_processor
+        debug_log("Auto-loading datasets from constants")
+        debug_time("Datasets auto-loading") { load_datasets_from_constants }
+      end
+
+      # Calculate and log total initialization time
+      if debug_enabled
+        total_time = ((Time.now - initialization_start_time) * 1000).round(2)
+        debug_log("🎯 Client initialization completed in #{total_time}ms")
+      else
+        debug_log("Client initialization completed")
+      end
     end
 
     def categorise(url)
-      host = (URI.parse(url).host || url).downcase
-      host = host.gsub('www.', '')
+      return [] if url.nil? || (url.respond_to?(:strip) && url.strip.empty?)
 
-      categories = @hosts.keys.select do |category|
-        @hosts[category].any? do |blocked_host|
-          host == blocked_host || host.end_with?(".#{blocked_host}")
+      debug_time("Categorizing '#{url}'") do
+        debug_log("🔍 Starting categorization for URL: #{url}")
+
+        host = begin
+          (URI.parse(url).host || url).downcase
+        rescue URI::InvalidURIError
+          url.to_s.downcase
         end
-      end
+        host = host.gsub("www.", "")
+        debug_log("📌 Extracted host: #{host}")
 
-      # Apply smart categorization if enabled
-      categories = apply_smart_categorization(url, categories) if smart_categorization_enabled
+        categories = @hosts.keys.select do |category|
+          @hosts[category].any? do |blocked_host|
+            host == blocked_host || host.end_with?(".#{blocked_host}")
+          end
+        end
+        debug_log("📋 Basic categorization matches: #{categories}")
 
-      # Apply regex categorization if enabled
-      categories = apply_regex_categorization(url, categories) if regex_categorization_enabled
+        # Apply smart categorisation if enabled
+        if smart_categorization_enabled
+          debug_log("🧠 Applying smart categorization")
+          categories = apply_smart_categorisation(url, categories)
+          debug_log("📋 After smart categorization: #{categories}")
+        end
 
-      if iab_compliance_enabled
-        IabCompliance.get_iab_categories(categories, iab_version)
-      else
-        categories
+        # Apply regex categorisation if enabled
+        if regex_categorization_enabled
+          debug_log("🔗 Applying regex categorization")
+          categories = apply_regex_categorisation(url, categories)
+          debug_log("📋 After regex categorization: #{categories}")
+        end
+
+        # Check if blog
+        categories += [:blog] if blog_url?(url)
+
+        final_categories = if iab_compliance_enabled
+          debug_log("🏢 Applying IAB compliance mapping")
+          iab_categories = IabCompliance.get_iab_categories(categories, iab_version)
+          debug_log("📋 Final IAB categories: #{iab_categories}")
+          iab_categories
+        else
+          debug_log("✅ Final categories: #{categories}")
+          categories
+        end
+
+        final_categories
       end
     end
 
@@ -121,8 +187,8 @@ module UrlCategorise
 
       # First check if it's from a video hosting domain
       categories = categorise(url)
-      video_hosting_categories = categories & [:video, :video_hosting, :youtube, :vimeo, :tiktok, :dailymotion, :twitch]
-      
+      video_hosting_categories = categories & %i[video_hosting youtube vimeo tiktok dailymotion twitch]
+
       return false unless video_hosting_categories.any?
 
       # Then check if it matches video content patterns
@@ -138,6 +204,163 @@ module UrlCategorise
       false
     end
 
+    def shorts_url?(url)
+      return false unless url && !url.empty?
+      return false unless regex_categorization_enabled && @regex_patterns.any?
+
+      # Check for video hosting domain
+      categories = categorise(url)
+      return false unless (categories & %i[video_hosting youtube vimeo tiktok dailymotion twitch]).any?
+
+      # Check for shorts patterns
+      shorts_patterns = [
+        %r{https?://(?:www\.)?youtube\.com/shorts/[a-zA-Z0-9_-]{11}}i,
+        %r{https?://(?:www\.)?tiktok\.com/@[^/]+/video/\d+}i,
+        %r{https?://(?:www\.)?instagram\.com/(?:reel|reels)/[a-zA-Z0-9_-]+}i,
+        %r{https?://(?:www\.)?facebook\.com/.*/videos/\d+}i
+      ]
+
+      shorts_patterns.any? { |pattern| url.match?(pattern) }
+    rescue StandardError
+      false
+    end
+
+    def playlist_url?(url)
+      return false unless url && !url.empty?
+      return false unless regex_categorization_enabled && @regex_patterns.any?
+
+      # Check for video hosting domain
+      categories = categorise(url)
+      return false unless (categories & %i[video_hosting youtube vimeo tiktok dailymotion twitch]).any?
+
+      # Check for playlist patterns
+      playlist_patterns = [
+        %r{https?://(?:www\.)?youtube\.com/(?:playlist\?list=|watch\?.*[&?]list=)[a-zA-Z0-9_-]+}i,
+        %r{https?://(?:www\.)?vimeo\.com/(?:album|showcase)/\d+}i,
+        %r{https?://(?:www\.)?dailymotion\.com/playlist/[a-zA-Z0-9_-]+}i,
+        %r{https?://(?:www\.)?twitch\.tv/collections/[a-zA-Z0-9_-]+}i
+      ]
+
+      playlist_patterns.any? { |pattern| url.match?(pattern) }
+    rescue StandardError
+      false
+    end
+
+    def music_url?(url)
+      return false unless url && !url.empty?
+
+      # Check for dedicated music platforms
+      music_categories = categorise(url)
+      music_domains = music_categories & %i[music spotify soundcloud apple_music youtube_music]
+      return true if music_domains.any?
+
+      # Check for music-specific patterns on video platforms
+      return false unless regex_categorization_enabled && @regex_patterns.any?
+
+      video_categories = music_categories & %i[video_hosting youtube vimeo]
+      return false unless video_categories.any?
+
+      # Check for music patterns in URLs
+      music_patterns = [
+        %r{https?://(?:www\.)?youtube\.com/watch\?.*v=[a-zA-Z0-9_-]{11}.*[&?]list=(?:PL|RD|UU)[a-zA-Z0-9_-]+}i, # YouTube music playlists
+        %r{https?://music\.youtube\.com}i,
+        %r{https?://(?:www\.)?youtube\.com/(?:c/|channel/|user/)?.*music}i,
+        %r{https?://(?:www\.)?vimeo\.com/channels/music}i,
+        %r{https?://(?:www\.)?dailymotion\.com/.*music}i,
+        %r{/music[/_-]|music[/_-]|\bmusic\b}i # Generic music indicators in path
+      ]
+
+      music_patterns.any? { |pattern| url.match?(pattern) }
+    rescue StandardError
+      false
+    end
+
+    def channel_url?(url)
+      return false unless url && !url.empty?
+      return false unless regex_categorization_enabled && @regex_patterns.any?
+
+      # Check for video hosting domain
+      categories = categorise(url)
+      return false unless (categories & %i[video_hosting youtube vimeo tiktok dailymotion twitch]).any?
+
+      # Check for channel/profile patterns
+      channel_patterns = [
+        %r{https?://(?:www\.)?youtube\.com/(?:@[a-zA-Z0-9_-]+|c/[a-zA-Z0-9_-]+|channel/[a-zA-Z0-9_-]+|user/[a-zA-Z0-9_-]+)(?:/|$)}i,
+        %r{https?://(?:www\.)?tiktok\.com/@[a-zA-Z0-9_.-]+$}i, # TikTok channels - must end after username
+        %r{https?://(?:www\.)?twitch\.tv/[a-zA-Z0-9_-]+(?:/|$)}i,
+        %r{https?://(?:www\.)?vimeo\.com/user\d+}i,
+        %r{https?://(?:www\.)?dailymotion\.com/[a-zA-Z0-9_-]+(?:/|$)}i
+      ]
+
+      channel_patterns.any? { |pattern| url.match?(pattern) }
+    rescue StandardError
+      false
+    end
+
+    def live_stream_url?(url)
+      return false unless url && !url.empty?
+      return false unless regex_categorization_enabled && @regex_patterns.any?
+
+      # Check for video hosting domain
+      categories = categorise(url)
+      return false unless (categories & %i[video_hosting youtube vimeo tiktok dailymotion twitch]).any?
+
+      # Check for live stream patterns
+      live_patterns = [
+        %r{https?://(?:www\.)?youtube\.com/watch\?.*v=[a-zA-Z0-9_-]{11}.*[&?]live=1}i,
+        %r{https?://(?:www\.)?youtube\.com/live/[a-zA-Z0-9_-]+}i,
+        %r{https?://(?:www\.)?twitch\.tv/[a-zA-Z0-9_-]+(?:/|$)}i, # Twitch channels are typically live
+        %r{https?://(?:www\.)?facebook\.com/.*/live}i,
+        %r{https?://(?:www\.)?instagram\.com/[^/]+/live}i,
+        %r{/live[/_-]|live[/_-]|\blive\b}i # Generic live indicators
+      ]
+
+      live_patterns.any? { |pattern| url.match?(pattern) }
+    rescue StandardError
+      false
+    end
+
+    def blog_url?(url)
+      return false unless url && !url.empty?
+
+      # Simple string matching for blog-related keywords
+      blog_indicators = [
+        /\/blog[\/\?]?/i,        # /blog/ or /blog? in path
+        /\/blogs[\/\?]?/i,       # /blogs/ or /blogs? in path
+        /blog\./i,               # blog.domain.com subdomain
+        /blog-/i,                # blog-something in domain or path
+        /wordpress/i,            # WordPress blogs
+        /blogspot/i,             # Blogspot blogs
+        /medium\.com/i,          # Medium articles
+        /substack\.com/i,        # Substack newsletters/blogs
+        /post[\/\?]/i,           # /post/ in path (common blog pattern)
+        /posts[\/\?]/i,          # /posts/ in path (common blog pattern)
+        /article[\/\?]/i,        # /article/ in path
+        /articles[\/\?]/i,       # /articles/ in path
+        /diary/i,                # Personal diary/blog
+        /journal/i,              # Journal entries
+        /\bblog\b/i              # The word "blog" as whole word
+      ]
+
+      # Exclude search engines and common false positives
+      search_exclusions = [
+        /google\.com/i,
+        /bing\.com/i,
+        /yahoo\.com/i,
+        /duckduckgo\.com/i,
+        /search\..*\?.*q=/i,     # Generic search queries
+        /\?.*q=.*blog/i          # Search queries containing "blog"
+      ]
+
+      # Return false if it matches search exclusions
+      return false if search_exclusions.any? { |pattern| url.match?(pattern) }
+
+      # Check for blog indicators
+      blog_indicators.any? { |pattern| url.match?(pattern) }
+    rescue StandardError
+      false
+    end
+
     def count_of_hosts
       @hosts.keys.map do |category|
         @hosts[category].size
@@ -209,7 +432,7 @@ module UrlCategorise
     end
 
     def check_all_lists
-      puts 'Checking all lists in constants...'
+      puts "Checking all lists in constants..."
 
       unreachable_lists = {}
       missing_categories = []
@@ -220,7 +443,7 @@ module UrlCategorise
 
         if urls.empty?
           missing_categories << category
-          puts '  ❌ No URLs defined for category'
+          puts "  ❌ No URLs defined for category"
           next
         end
 
@@ -235,7 +458,7 @@ module UrlCategorise
           end
 
           unless url_valid?(url)
-            unreachable_lists[category] << { url: url, error: 'Invalid URL format' }
+            unreachable_lists[category] << { url: url, error: "Invalid URL format" }
             puts "  ❌ Invalid URL format: #{url}"
             next
           end
@@ -247,18 +470,18 @@ module UrlCategorise
 
             case response.code
             when 200
-              puts '✅ OK'
+              puts "✅ OK"
               successful_lists[category] << url
             when 301, 302, 307, 308
               puts "↗️  Redirect (#{response.code})"
-              puts "      Redirects to: #{response.headers['location']}" if response.headers['location']
+              puts "      Redirects to: #{response.headers['location']}" if response.headers["location"]
               successful_lists[category] << url
             when 404
-              puts '❌ Not Found (404)'
-              unreachable_lists[category] << { url: url, error: '404 Not Found' }
+              puts "❌ Not Found (404)"
+              unreachable_lists[category] << { url: url, error: "404 Not Found" }
             when 403
-              puts '❌ Forbidden (403)'
-              unreachable_lists[category] << { url: url, error: '403 Forbidden' }
+              puts "❌ Forbidden (403)"
+              unreachable_lists[category] << { url: url, error: "403 Forbidden" }
             when 500..599
               puts "❌ Server Error (#{response.code})"
               unreachable_lists[category] << { url: url, error: "Server Error #{response.code}" }
@@ -267,13 +490,13 @@ module UrlCategorise
               unreachable_lists[category] << { url: url, error: "HTTP #{response.code}" }
             end
           rescue Timeout::Error
-            puts '❌ Timeout'
-            unreachable_lists[category] << { url: url, error: 'Request timeout' }
+            puts "❌ Timeout"
+            unreachable_lists[category] << { url: url, error: "Request timeout" }
           rescue SocketError => e
-            puts '❌ DNS/Network Error'
+            puts "❌ DNS/Network Error"
             unreachable_lists[category] << { url: url, error: "DNS/Network: #{e.message}" }
           rescue HTTParty::Error, Net::HTTPError => e
-            puts '❌ HTTP Error'
+            puts "❌ HTTP Error"
             unreachable_lists[category] << { url: url, error: "HTTP Error: #{e.message}" }
           rescue StandardError => e
             puts "❌ Error: #{e.class}"
@@ -290,9 +513,9 @@ module UrlCategorise
       end
 
       # Generate summary report
-      puts "\n" + '=' * 80
-      puts 'LIST HEALTH REPORT'
-      puts '=' * 80
+      puts "\n" + "=" * 80
+      puts "LIST HEALTH REPORT"
+      puts "=" * 80
 
       puts "\n📊 SUMMARY:"
       total_categories = (host_urls || {}).keys.length
@@ -327,7 +550,7 @@ module UrlCategorise
         puts "  - #{category} (#{url_count} URL#{'s' if url_count != 1})"
       end
 
-      puts "\n" + '=' * 80
+      puts "\n" + "=" * 80
 
       # Return structured data for programmatic use
       {
@@ -343,31 +566,57 @@ module UrlCategorise
     end
 
     def load_kaggle_dataset(dataset_owner, dataset_name, options = {})
-      raise Error, 'Dataset processor not configured' unless @dataset_processor
+      raise Error, "Dataset processor not configured" unless @dataset_processor
 
+      debug_log("Loading Kaggle dataset: #{dataset_owner}/#{dataset_name}")
       default_options = { use_cache: true, integrate_data: true }
       merged_options = default_options.merge(options)
 
-      dataset = @dataset_processor.process_kaggle_dataset(dataset_owner, dataset_name, merged_options)
+      dataset = debug_time("Kaggle dataset processing: #{dataset_owner}/#{dataset_name}") do
+        @dataset_processor.process_kaggle_dataset(dataset_owner, dataset_name, merged_options)
+      end
 
       if merged_options[:integrate_data]
-        integrate_dataset(dataset, merged_options[:category_mappings] || {})
+        debug_log("Integrating Kaggle dataset into categorization")
+        result = debug_time("Dataset integration") do
+          integrate_dataset(dataset, merged_options[:category_mappings] || {})
+        end
+        if result
+          debug_log("✅ Successfully integrated Kaggle dataset #{dataset_owner}/#{dataset_name}")
+        else
+          debug_log("⚠️  Integration returned nil for Kaggle dataset #{dataset_owner}/#{dataset_name}")
+        end
+        result
       else
+        debug_log("Returning raw Kaggle dataset (not integrated)")
         dataset
       end
     end
 
     def load_csv_dataset(url, options = {})
-      raise Error, 'Dataset processor not configured' unless @dataset_processor
+      raise Error, "Dataset processor not configured" unless @dataset_processor
 
+      debug_log("Loading CSV dataset: #{url}")
       default_options = { use_cache: true, integrate_data: true }
       merged_options = default_options.merge(options)
 
-      dataset = @dataset_processor.process_csv_dataset(url, merged_options)
+      dataset = debug_time("CSV dataset processing: #{url}") do
+        @dataset_processor.process_csv_dataset(url, merged_options)
+      end
 
       if merged_options[:integrate_data]
-        integrate_dataset(dataset, merged_options[:category_mappings] || {})
+        debug_log("Integrating CSV dataset into categorization")
+        result = debug_time("Dataset integration") do
+          integrate_dataset(dataset, merged_options[:category_mappings] || {})
+        end
+        if result
+          debug_log("✅ Successfully integrated CSV dataset #{url}")
+        else
+          debug_log("⚠️  Integration returned nil for CSV dataset #{url}")
+        end
+        result
       else
+        debug_log("Returning raw CSV dataset (not integrated)")
         dataset
       end
     end
@@ -402,39 +651,44 @@ module UrlCategorise
     end
 
     def export_hosts_files(output_path = nil)
-      export_dir = output_path || (cache_dir ? File.join(cache_dir, 'exports', 'hosts') : File.join(Dir.pwd, 'exports', 'hosts'))
-      
+      export_dir = output_path || (if cache_dir
+                                     File.join(cache_dir, "exports",
+                                               "hosts")
+                                   else
+                                     File.join(Dir.pwd, "exports", "hosts")
+                                   end)
+
       FileUtils.mkdir_p(export_dir) unless Dir.exist?(export_dir)
-      
+
       exported_files = {}
-      
+
       @hosts.each do |category, domains|
         next if domains.empty?
-        
+
         filename = "#{category}.hosts"
         file_path = File.join(export_dir, filename)
-        
-        File.open(file_path, 'w') do |file|
+
+        File.open(file_path, "w") do |file|
           file.puts "# #{category.to_s.gsub('_', ' ').split.map(&:capitalize).join(' ')} - Generated by UrlCategorise"
           file.puts "# Generated at: #{Time.now}"
           file.puts "# Total entries: #{domains.length}"
           file.puts ""
-          
+
           domains.sort.each do |domain|
             file.puts "0.0.0.0 #{domain}"
           end
         end
-        
+
         exported_files[category] = {
           path: file_path,
           filename: filename,
           count: domains.length
         }
       end
-      
+
       # Create summary file
-      summary_path = File.join(export_dir, '_export_summary.txt')
-      File.open(summary_path, 'w') do |file|
+      summary_path = File.join(export_dir, "_export_summary.txt")
+      File.open(summary_path, "w") do |file|
         file.puts "UrlCategorise Hosts Export Summary"
         file.puts "=================================="
         file.puts "Generated at: #{Time.now}"
@@ -443,56 +697,61 @@ module UrlCategorise
         file.puts "Total domains: #{@hosts.values.map(&:length).sum}"
         file.puts ""
         file.puts "Files created:"
-        
-        exported_files.each do |category, info|
+
+        exported_files.each do |_category, info|
           file.puts "  #{info[:filename]} - #{info[:count]} domains"
         end
       end
-      
+
       exported_files[:_summary] = {
         path: summary_path,
         total_categories: exported_files.keys.length,
         total_domains: @hosts.values.map(&:length).sum,
         export_directory: export_dir
       }
-      
+
       exported_files
     end
 
     def export_csv_data(output_path = nil)
-      require 'csv'
-      
-      export_dir = output_path || (cache_dir ? File.join(cache_dir, 'exports', 'csv') : File.join(Dir.pwd, 'exports', 'csv'))
-      
+      require "csv"
+
+      export_dir = output_path || (if cache_dir
+                                     File.join(cache_dir, "exports",
+                                               "csv")
+                                   else
+                                     File.join(Dir.pwd, "exports", "csv")
+                                   end)
+
       FileUtils.mkdir_p(export_dir) unless Dir.exist?(export_dir)
-      
+
       # Create single comprehensive CSV with ALL data
-      timestamp = Time.now.strftime('%Y%m%d_%H%M%S')
+      timestamp = Time.now.strftime("%Y%m%d_%H%M%S")
       filename = "url_categorise_comprehensive_export_#{timestamp}.csv"
       file_path = File.join(export_dir, filename)
-      
+
       # Collect all available data
       all_data = collect_all_export_data
-      
+
       # Create CSV with dynamic headers
       headers = determine_comprehensive_headers(all_data)
-      
-      CSV.open(file_path, 'w', headers: true) do |csv|
+
+      CSV.open(file_path, "w", headers: true) do |csv|
         csv << headers
-        
+
         all_data.each do |entry|
-          row = headers.map { |header| entry[header] || entry[header.to_sym] || '' }
+          row = headers.map { |header| entry[header] || entry[header.to_sym] || "" }
           csv << row
         end
       end
-      
+
       # Create summary file
       summary_filename = "export_summary_#{timestamp}.json"
       summary_file_path = File.join(export_dir, summary_filename)
-      
+
       summary = create_comprehensive_export_summary(file_path, all_data, export_dir)
       File.write(summary_file_path, JSON.pretty_generate(summary))
-      
+
       {
         csv_file: file_path,
         summary_file: summary_file_path,
@@ -504,6 +763,45 @@ module UrlCategorise
 
     private
 
+    def debug_log(message)
+      puts "[UrlCategorise DEBUG] #{message}" if debug_enabled
+    end
+
+    def debug_time(description)
+      return yield unless debug_enabled
+
+      start_time = Time.now
+      result = yield
+      end_time = Time.now
+      elapsed = ((end_time - start_time) * 1000).round(2)
+      debug_log("#{description} completed in #{elapsed}ms")
+      result
+    end
+
+    def test_environment?
+      # Check environment variables
+      return true if ENV["RAILS_ENV"] == "test" || ENV["RACK_ENV"] == "test"
+
+      # Check if MiniTest is defined (after it's loaded)
+      return true if defined?(Minitest) || defined?(MiniTest)
+
+      # Check caller stack for test-related files
+      caller_locations = caller_locations(1, 10) || []
+      test_patterns = [ /test\//, /_test\.rb$/, /minitest/, /rake.*test/ ]
+      return true if caller_locations.any? { |loc|
+        test_patterns.any? { |pattern| loc.path.match?(pattern) }
+      }
+
+      # Check if we're running under rake test
+      return true if defined?(Rake) && ARGV.include?("test")
+
+      # Check for common test runners
+      return true if $PROGRAM_NAME.include?("rake") && ARGV.include?("test")
+      return true if $PROGRAM_NAME.include?("minitest")
+
+      false
+    end
+
     def load_regex_patterns
       return unless regex_patterns_file
 
@@ -519,9 +817,9 @@ module UrlCategorise
 
         # Check if this line is a source comment
         if line.match(/^# Source: (.+)$/)
-          current_category = $1.downcase
+          current_category = ::Regexp.last_match(1).downcase
           @regex_patterns[current_category] = [] unless @regex_patterns[current_category]
-        elsif current_category && !line.start_with?('#') && !line.empty?
+        elsif current_category && !line.start_with?("#") && !line.empty?
           # This is a regex pattern
           begin
             regex = Regexp.new(line)
@@ -535,11 +833,13 @@ module UrlCategorise
         end
       end
 
-      puts "Loaded #{@regex_patterns.values.flatten.size} regex patterns from #{@regex_patterns.keys.size} categories" if @regex_patterns.any?
+      return unless @regex_patterns.any?
+
+      debug_log("Loaded #{@regex_patterns.values.flatten.size} regex patterns from #{@regex_patterns.keys.size} categories")
     end
 
     def fetch_regex_patterns_content
-      if regex_patterns_file.start_with?('http://', 'https://')
+      if regex_patterns_file.start_with?("http://", "https://")
         # Remote URL
         begin
           response = HTTParty.get(regex_patterns_file, timeout: request_timeout)
@@ -548,9 +848,9 @@ module UrlCategorise
           puts "Warning: Failed to fetch regex patterns from #{regex_patterns_file}: #{e.message}"
           return nil
         end
-      elsif regex_patterns_file.start_with?('file://')
+      elsif regex_patterns_file.start_with?("file://")
         # Local file URL
-        file_path = regex_patterns_file.sub('file://', '')
+        file_path = regex_patterns_file.sub("file://", "")
         return File.read(file_path) if File.exist?(file_path)
       elsif File.exist?(regex_patterns_file)
         # Direct file path
@@ -561,21 +861,23 @@ module UrlCategorise
       nil
     end
 
-    def apply_regex_categorization(url, existing_categories)
+    def apply_regex_categorisation(url, existing_categories)
       return existing_categories unless @regex_patterns.any?
 
       # If we have existing categories that match domains, check if the URL matches video patterns
-      video_categories = existing_categories & [:video, :video_hosting, :youtube, :vimeo, :tiktok]
-      
+      video_categories = existing_categories & %i[video_hosting youtube vimeo tiktok]
+
       if video_categories.any?
         # Check if this URL matches any video patterns
-        @regex_patterns.each do |category, patterns|
+        @regex_patterns.each do |_category, patterns|
           patterns.each do |pattern_info|
-            if url.match?(pattern_info[:pattern])
-              # This is a video content URL, add a more specific categorization
-              existing_categories << "#{video_categories.first}_content".to_sym unless existing_categories.include?("#{video_categories.first}_content".to_sym)
-              break
+            next unless url.match?(pattern_info[:pattern])
+
+            # This is a video content URL, add a more specific categorisation
+            unless existing_categories.include?("#{video_categories.first}_content".to_sym)
+              existing_categories << "#{video_categories.first}_content".to_sym
             end
+            break
           end
         end
       end
@@ -585,13 +887,13 @@ module UrlCategorise
 
     def collect_all_export_data
       all_data = []
-      
+
       # 1. Add all processed domain/category mappings
       @hosts.each do |category, domains|
         domains.each do |domain|
-          source_type = @dataset_categories.include?(category) ? 'dataset' : 'blocklist'
+          source_type = @dataset_categories.include?(category) ? "dataset" : "blocklist"
           is_dataset_category = @dataset_categories.include?(category)
-          
+
           # Get IAB mappings if compliance is enabled
           iab_v2 = nil
           iab_v3 = nil
@@ -599,36 +901,36 @@ module UrlCategorise
             iab_v2 = IabCompliance.map_category_to_iab(category, :v2)
             iab_v3 = IabCompliance.map_category_to_iab(category, :v3)
           end
-          
+
           entry = {
-            'data_type' => 'domain_categorization',
-            'domain' => domain,
-            'url' => domain, # For compatibility
-            'category' => category.to_s,
-            'source_type' => source_type,
-            'is_dataset_category' => is_dataset_category,
-            'iab_category_v2' => iab_v2,
-            'iab_category_v3' => iab_v3,
-            'export_timestamp' => Time.now.iso8601,
-            'smart_categorization_enabled' => smart_categorization_enabled
+            "data_type" => "domain_categorization",
+            "domain" => domain,
+            "url" => domain, # For compatibility
+            "category" => category.to_s,
+            "source_type" => source_type,
+            "is_dataset_category" => is_dataset_category,
+            "iab_category_v2" => iab_v2,
+            "iab_category_v3" => iab_v3,
+            "export_timestamp" => Time.now.iso8601,
+            "smart_categorization_enabled" => smart_categorization_enabled
           }
-          
+
           all_data << entry
         end
       end
-      
+
       # 2. Add raw dataset content from cache
       collect_cached_dataset_content.each do |entry|
-        entry['data_type'] = 'raw_dataset_content'
+        entry["data_type"] = "raw_dataset_content"
         all_data << entry
       end
-      
+
       # 3. Try to collect currently loaded dataset data if available
       collect_current_dataset_content.each do |entry|
-        entry['data_type'] = 'current_dataset_content'  
+        entry["data_type"] = "current_dataset_content"
         all_data << entry
       end
-      
+
       all_data
     end
 
@@ -638,25 +940,27 @@ module UrlCategorise
 
       # Collect from cached datasets if available
       (@dataset_metadata || {}).each do |data_hash, metadata|
-        cache_key = @dataset_processor.send(:generate_cache_key, metadata[:source_identifier] || data_hash, metadata[:source_type]&.to_sym || :unknown)
+        cache_key = @dataset_processor.send(:generate_cache_key, metadata[:source_identifier] || data_hash,
+                                            metadata[:source_type]&.to_sym || :unknown)
         cached_result = @dataset_processor.send(:load_from_cache, cache_key)
-        
-        if cached_result && cached_result.is_a?(Hash) && cached_result['raw_content']
-          cached_result['raw_content'].each do |entry|
+
+        if cached_result && cached_result.is_a?(Hash) && cached_result["raw_content"]
+          cached_result["raw_content"].each do |entry|
             enhanced_entry = entry.dup
-            enhanced_entry['dataset_source'] = metadata[:source_identifier] || 'unknown'
-            enhanced_entry['dataset_type'] = metadata[:source_type] || 'unknown'
-            enhanced_entry['processed_at'] = metadata[:processed_at]
+            enhanced_entry["dataset_source"] = metadata[:source_identifier] || "unknown"
+            enhanced_entry["dataset_type"] = metadata[:source_type] || "unknown"
+            enhanced_entry["processed_at"] = metadata[:processed_at]
             cached_data << enhanced_entry
           end
         elsif cached_result.is_a?(Array)
           # Legacy format - array of entries
           cached_result.each do |entry|
             next unless entry.is_a?(Hash)
+
             enhanced_entry = entry.dup
-            enhanced_entry['dataset_source'] = metadata[:source_identifier] || 'unknown' 
-            enhanced_entry['dataset_type'] = metadata[:source_type] || 'unknown'
-            enhanced_entry['processed_at'] = metadata[:processed_at]
+            enhanced_entry["dataset_source"] = metadata[:source_identifier] || "unknown"
+            enhanced_entry["dataset_type"] = metadata[:source_type] || "unknown"
+            enhanced_entry["processed_at"] = metadata[:processed_at]
             cached_data << enhanced_entry
           end
         end
@@ -666,7 +970,7 @@ module UrlCategorise
     end
 
     def collect_current_dataset_content
-      # This is a placeholder - in practice, the original dataset content 
+      # This is a placeholder - in practice, the original dataset content
       # is processed and only domain mappings are kept in @hosts.
       # The raw content should come from cache, but if we want to be more
       # aggressive, we could re-process datasets here or store them differently.
@@ -683,36 +987,36 @@ module UrlCategorise
 
       # Core headers that should appear first
       core_headers = %w[data_type domain url category]
-      
+
       # Standard categorization headers
       categorization_headers = %w[source_type is_dataset_category iab_category_v2 iab_category_v3]
-      
+
       # Dataset content headers
       content_headers = %w[title description text content summary body]
-      
+
       # Metadata headers
       metadata_headers = %w[dataset_source dataset_type processed_at export_timestamp smart_categorization_enabled]
-      
+
       # Build final header order
       ordered_headers = []
       ordered_headers += (core_headers & all_keys_array)
       ordered_headers += (categorization_headers & all_keys_array)
       ordered_headers += (content_headers & all_keys_array)
-      
+
       # Add any remaining headers (alphabetically sorted)
       remaining_headers = (all_keys_array - ordered_headers - metadata_headers).sort
       ordered_headers += remaining_headers
-      
+
       # Add metadata headers at the end
       ordered_headers += (metadata_headers & all_keys_array)
-      
+
       ordered_headers
     end
 
     def create_comprehensive_export_summary(file_path, all_data, export_dir)
-      domain_entries = all_data.select { |entry| entry['data_type'] == 'domain_categorization' }
-      dataset_entries = all_data.select { |entry| entry['data_type']&.include?('dataset') }
-      
+      domain_entries = all_data.select { |entry| entry["data_type"] == "domain_categorization" }
+      dataset_entries = all_data.select { |entry| entry["data_type"]&.include?("dataset") }
+
       {
         export_info: {
           timestamp: Time.now.iso8601,
@@ -741,13 +1045,10 @@ module UrlCategorise
       }
     end
 
-
-    private
-
     def initialize_dataset_processor(config)
       processor_config = {
-        download_path: config[:download_path] || cache_dir&.+(File::SEPARATOR + 'downloads'),
-        cache_path: config[:cache_path] || cache_dir&.+(File::SEPARATOR + 'datasets'),
+        download_path: config[:download_path] || cache_dir&.+(File::SEPARATOR + "downloads"),
+        cache_path: config[:cache_path] || cache_dir&.+(File::SEPARATOR + "datasets"),
         timeout: config[:timeout] || request_timeout,
         enable_kaggle: config.fetch(:enable_kaggle, true) # Default to true for backwards compatibility
       }
@@ -765,7 +1066,7 @@ module UrlCategorise
       DatasetProcessor.new(**processor_config)
     rescue Error => e
       # Dataset processor failed to initialize, but client can still work without it
-      puts "Warning: Dataset processor initialization failed: #{e.message}" if ENV['DEBUG']
+      puts "Warning: Dataset processor initialization failed: #{e.message}" if ENV["DEBUG"]
       nil
     end
 
@@ -773,37 +1074,51 @@ module UrlCategorise
       return dataset unless @dataset_processor
       return nil unless dataset # Handle nil datasets gracefully
 
+      debug_log("Processing dataset for integration...")
       processed_result = @dataset_processor.integrate_dataset_into_categorization(dataset, category_mappings)
 
       # Handle new data structure with categories and raw_content
-      if processed_result.is_a?(Hash) && processed_result['categories']
-        categorized_data = processed_result['categories']
-        metadata = processed_result['_metadata']
+      if processed_result.is_a?(Hash) && processed_result["categories"]
+        categorised_data = processed_result["categories"]
+        metadata = processed_result["_metadata"]
+        debug_log("Received new-format dataset with #{categorised_data.keys.length} categories")
       else
-        # Legacy format - assume the whole result is categorized data
-        categorized_data = processed_result
-        metadata = categorized_data[:_metadata] if categorized_data.respond_to?(:delete)
+        # Legacy format - assume the whole result is categorised data
+        categorised_data = processed_result
+        metadata = categorised_data[:_metadata] if categorised_data.respond_to?(:delete)
+        debug_log("Received legacy-format dataset with #{categorised_data.keys.length} categories")
       end
 
       # Store metadata
       if metadata
         @dataset_metadata ||= {}
         @dataset_metadata[metadata[:data_hash]] = metadata
+        debug_log("Stored dataset metadata with hash: #{metadata[:data_hash]}")
       end
 
       # Merge with existing host data
-      categorized_data.each do |category, domains|
-        next if category.to_s.start_with?('_') # Skip internal keys
+      total_domains_added = 0
+      categories_processed = 0
+      categorised_data.each do |category, domains|
+        next if category.to_s.start_with?("_") # Skip internal keys
 
         # Convert category to symbol for consistency
         category_sym = category.to_sym
         @hosts[category_sym] ||= []
+        before_size = @hosts[category_sym].size
         @hosts[category_sym].concat(domains).uniq!
+        after_size = @hosts[category_sym].size
+        domains_added = after_size - before_size
+
+        debug_log("  Category #{category_sym}: added #{domains_added} new domains (#{domains.size} total in dataset)")
+        total_domains_added += domains_added
+        categories_processed += 1
 
         # Track this as a dataset category
         @dataset_categories.add(category_sym)
       end
 
+      debug_log("✅ Dataset integration completed: #{categories_processed} categories, #{total_domains_added} new domains added")
       dataset
     end
 
@@ -811,78 +1126,90 @@ module UrlCategorise
       return unless defined?(CATEGORIY_DATABASES) && CATEGORIY_DATABASES.is_a?(Array)
       return unless @dataset_processor
 
-      puts "Loading #{CATEGORIY_DATABASES.length} datasets from constants..." if ENV['DEBUG']
+      debug_log("Loading #{CATEGORIY_DATABASES.length} datasets from constants...")
       loaded_count = 0
 
       CATEGORIY_DATABASES.each do |dataset_config|
-        begin
-          case dataset_config[:type]
-          when :kaggle
-            # Parse the kaggle path to get owner and dataset name
-            path_parts = dataset_config[:path].split('/')
-            next unless path_parts.length == 2
-
-            dataset_owner, dataset_name = path_parts
-            
-            # Check if dataset is already cached before attempting to load
-            cache_key = @dataset_processor.send(:generate_cache_key, "#{dataset_owner}/#{dataset_name}", :kaggle)
-            cache_file = File.join(@dataset_processor.cache_path, cache_key)
-            
-            if File.exist?(cache_file)
-              puts "Loading cached Kaggle dataset: #{dataset_owner}/#{dataset_name}" if ENV['DEBUG']
+        case dataset_config[:type]
+        when :kaggle
+          # Parse the kaggle path to get owner and dataset name
+          path_parts = dataset_config[:path].split("/")
+          next unless path_parts.length == 2
+
+          dataset_owner, dataset_name = path_parts
+          dataset_identifier = "#{dataset_owner}/#{dataset_name}"
+
+          # Check if dataset is already cached before attempting to load
+          cache_key = @dataset_processor.send(:generate_cache_key, dataset_identifier, :kaggle)
+          cache_file = File.join(@dataset_processor.cache_path, cache_key)
+
+          if File.exist?(cache_file)
+            debug_log("✅ Dataset cache HIT for Kaggle dataset: #{dataset_identifier}")
+            debug_time("Loading cached Kaggle dataset: #{dataset_identifier}") do
               load_kaggle_dataset(dataset_owner, dataset_name, {
                                     use_cache: true,
                                     integrate_data: true
                                   })
-              loaded_count += 1
-            else
-              puts "Attempting to download missing Kaggle dataset: #{dataset_owner}/#{dataset_name}" if ENV['DEBUG']
-              begin
+            end
+            loaded_count += 1
+          else
+            debug_log("❌ Dataset cache MISS for Kaggle dataset: #{dataset_identifier}")
+            debug_log("Attempting to download missing Kaggle dataset: #{dataset_identifier}")
+            begin
+              debug_time("Downloading Kaggle dataset: #{dataset_identifier}") do
                 load_kaggle_dataset(dataset_owner, dataset_name, {
                                       use_cache: true,
                                       integrate_data: true
                                     })
-                loaded_count += 1
-              rescue Error => e
-                puts "Warning: Failed to download Kaggle dataset #{dataset_owner}/#{dataset_name}: #{e.message}" if ENV['DEBUG']
               end
+              debug_log("💾 Successfully downloaded and cached Kaggle dataset: #{dataset_identifier}")
+              loaded_count += 1
+            rescue Error => e
+              debug_log("⚠️  Failed to download Kaggle dataset #{dataset_identifier}: #{e.message}")
             end
+          end
 
-          when :csv
-            # Check if CSV dataset is cached
-            cache_key = @dataset_processor.send(:generate_cache_key, dataset_config[:path], :csv)
-            cache_file = File.join(@dataset_processor.cache_path, cache_key)
-            
-            if File.exist?(cache_file)
-              puts "Loading cached CSV dataset: #{dataset_config[:path]}" if ENV['DEBUG']
-              load_csv_dataset(dataset_config[:path], {
+        when :csv
+          # Check if CSV dataset is cached
+          dataset_path = dataset_config[:path]
+          cache_key = @dataset_processor.send(:generate_cache_key, dataset_path, :csv)
+          cache_file = File.join(@dataset_processor.cache_path, cache_key)
+
+          if File.exist?(cache_file)
+            debug_log("✅ Dataset cache HIT for CSV dataset: #{dataset_path}")
+            debug_time("Loading cached CSV dataset: #{dataset_path}") do
+              load_csv_dataset(dataset_path, {
                                  use_cache: true,
                                  integrate_data: true
                                })
-              loaded_count += 1
-            else
-              puts "Attempting to download missing CSV dataset: #{dataset_config[:path]}" if ENV['DEBUG']
-              begin
-                load_csv_dataset(dataset_config[:path], {
+            end
+            loaded_count += 1
+          else
+            debug_log("❌ Dataset cache MISS for CSV dataset: #{dataset_path}")
+            debug_log("Attempting to download missing CSV dataset: #{dataset_path}")
+            begin
+              debug_time("Downloading CSV dataset: #{dataset_path}") do
+                load_csv_dataset(dataset_path, {
                                    use_cache: true,
                                    integrate_data: true
                                  })
-                loaded_count += 1
-              rescue Error => e
-                puts "Warning: Failed to download CSV dataset #{dataset_config[:path]}: #{e.message}" if ENV['DEBUG']
               end
+              debug_log("💾 Successfully downloaded and cached CSV dataset: #{dataset_path}")
+              loaded_count += 1
+            rescue Error => e
+              debug_log("⚠️  Failed to download CSV dataset #{dataset_path}: #{e.message}")
             end
           end
-        rescue Error => e
-          puts "Warning: Failed to load dataset #{dataset_config[:path]}: #{e.message}" if ENV['DEBUG']
-          # Continue loading other datasets even if one fails
-        rescue StandardError => e
-          puts "Warning: Unexpected error loading dataset #{dataset_config[:path]}: #{e.message}" if ENV['DEBUG']
-          # Continue loading other datasets even if one fails
         end
+      rescue Error => e
+        debug_log("⚠️  Failed to load dataset #{dataset_config[:path]}: #{e.message}")
+        # Continue loading other datasets even if one fails
+      rescue StandardError => e
+        debug_log("⚠️  Unexpected error loading dataset #{dataset_config[:path]}: #{e.message}")
+        # Continue loading other datasets even if one fails
       end
 
-      puts "Finished loading datasets from constants (#{loaded_count}/#{CATEGORIY_DATABASES.length} loaded)" if ENV['DEBUG']
+      debug_log("Finished loading datasets from constants (#{loaded_count}/#{CATEGORIY_DATABASES.length} loaded)")
     end
 
     def hash_size_in_mb(hash)
@@ -903,8 +1230,16 @@ module UrlCategorise
     def fetch_and_build_host_lists
       @hosts = {}
 
-      (host_urls || {}).keys.each do |category|
-        @hosts[category] = build_host_data((host_urls || {})[category])
+      # Disable parallel loading for tests to avoid freezing issues
+      if test_environment?
+        debug_log("📥 Using sequential loading (test environment detected)")
+        fetch_and_build_host_lists_sequential
+      elsif parallel_loading_enabled && self.class.ractor_available? && (host_urls || {}).size > 1
+        debug_log("🚀 Using parallel loading with Ractors for #{(host_urls || {}).keys.size} categories")
+        fetch_and_build_host_lists_parallel
+      else
+        debug_log("📥 Using sequential loading for #{(host_urls || {}).keys.size} categories")
+        fetch_and_build_host_lists_sequential
       end
 
       sub_category_values = categories_with_keys
@@ -922,6 +1257,276 @@ module UrlCategorise
       @hosts
     end
 
+    def fetch_and_build_host_lists_sequential
+      (host_urls || {}).keys.each do |category|
+        @hosts[category] = build_host_data((host_urls || {})[category])
+      end
+    end
+
+    def fetch_and_build_host_lists_parallel
+      # Hybrid approach: HTTP requests serially, file processing with Ractors in parallel
+      # First, download all content serially to avoid Ractor HTTP issues
+      debug_log("📡 Phase 1: Downloading content serially")
+
+      all_download_tasks = []
+      (host_urls || {}).each do |category, urls|
+        urls.each do |url|
+          next unless url_valid?(url)
+          all_download_tasks << { category: category, url: url }
+        end
+      end
+
+      # Download all content serially
+      downloaded_content = {}
+      all_download_tasks.each do |task|
+        url = task[:url]
+        category = task[:category]
+
+        debug_log("📥 Downloading #{url}")
+        content = nil
+
+        # Try cache first
+        if cache_dir && !force_download
+          cached_data = debug_time("Cache lookup for #{url}") do
+            read_from_cache(url)
+          end
+          if cached_data && !cached_data.empty?
+            debug_log("✅ Cache HIT for #{url} - loaded #{cached_data.size} hosts")
+            downloaded_content["#{category}:#{url}"] = { hosts: cached_data, from_cache: true }
+            next
+          else
+            debug_log("❌ Cache MISS for #{url}")
+          end
+        else
+          debug_log("Cache disabled (cache_dir: #{cache_dir.inspect}, force_download: #{force_download})")
+        end
+
+        # Download content
+        if url.start_with?("file://")
+          file_path = url.sub("file://", "")
+          if File.exist?(file_path)
+            content = File.read(file_path)
+          end
+        else
+          begin
+            raw_data = HTTParty.get(url, timeout: request_timeout)
+            content = raw_data&.body
+          rescue => e
+            debug_log("⚠️ Download failed for #{url}: #{e.message}")
+            next
+          end
+        end
+
+        downloaded_content["#{category}:#{url}"] = { content: content, from_cache: false }
+        debug_log("📥 Downloaded #{content&.length || 0} bytes from #{url}")
+      end
+
+      # Phase 2: Process content in parallel using Ractors (if available) or threads
+      debug_log("⚡ Phase 2: Processing content in parallel")
+
+      if self.class.ractor_available? && !test_environment?
+        process_content_with_ractors(downloaded_content)
+      else
+        process_content_with_threads(downloaded_content)
+      end
+    end
+
+    def process_content_with_ractors(downloaded_content)
+      debug_time("Parallel content processing with Ractors") do
+        # Create a work queue
+        # Suppress Ractor experimental warnings
+        original_verbose = $VERBOSE
+        $VERBOSE = nil
+        work_queue = Ractor.new do
+          # :nocov:
+          tasks = []
+          no_more_tasks = false
+
+          while (task = Ractor.receive) != :done
+            if task == :get_task
+              if tasks.empty? && no_more_tasks
+                Ractor.yield(nil) # No more tasks available
+              else
+                Ractor.yield(tasks.shift) # May return nil if empty, but more tasks might come
+              end
+            elsif task == :no_more_tasks
+              no_more_tasks = true
+            else
+              tasks << task
+            end
+          end
+          # :nocov:
+        end
+
+        # Add all tasks to queue
+        downloaded_content.each do |key, data|
+          work_queue.send([ key, data ])
+        end
+        
+        # Signal that no more tasks will be added
+        work_queue.send(:no_more_tasks)
+
+        # Create worker Ractors (limited by max_ractor_workers)
+        num_workers = [ max_ractor_workers, downloaded_content.size ].min
+        debug_log("🔧 Creating #{num_workers} Ractor workers")
+
+        workers = num_workers.times.map do
+          Ractor.new(work_queue, cache_dir) do |queue, cache_path|
+            # :nocov:
+            require "digest"
+            require "fileutils"
+
+            results = []
+            loop do
+              queue.send(:get_task)
+              task = queue.take
+              break if task.nil?
+
+              key, data = task
+              category, url = key.split(":", 2)
+
+              if data[:from_cache]
+                # Already processed, just return
+                results << [ category, data[:hosts] ]
+              else
+                # Parse the content
+                content = data[:content]
+                if content.nil? || content.empty?
+                  results << [ category, [] ]
+                else
+                  # Parse content (simplified version for Ractor)
+                  lines = content.split("\n").reject { |line| line.empty? || line.strip.start_with?("#") }
+                  sample_lines = lines.first(20)
+
+                  parsed_hosts = if sample_lines.any? { |line| line.match(/^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\s+/) }
+                    # hosts format
+                    lines.map do |line|
+                      parts = line.split(" ")
+                      parts.length >= 2 ? parts[1].strip : nil
+                    end.compact.reject(&:empty?)
+                  elsif sample_lines.any? { |line| line.include?("address=/") }
+                    # dnsmasq format
+                    lines.map do |line|
+                      match = line.match(%r{address=/(.+?)/})
+                      match ? match[1] : nil
+                    end.compact
+                  elsif sample_lines.any? { |line| line.match(/^\|\|/) }
+                    # ublock format
+                    lines.map { |line| line.gsub(/^\|\|/, "").gsub(/[$\^].*$/, "").strip }.reject(&:empty?)
+                  else
+                    # plain format
+                    lines.map(&:strip)
+                  end.compact.sort.uniq
+
+                  # Cache the results if cache is enabled
+                  if cache_path && !parsed_hosts.empty?
+                    begin
+                      FileUtils.mkdir_p(cache_path) unless Dir.exist?(cache_path)
+                      filename = Digest::MD5.hexdigest(url) + ".cache"
+                      cache_file = File.join(cache_path, filename)
+
+                      cache_data = {
+                        hosts: parsed_hosts,
+                        cached_at: Time.now
+                      }
+
+                      File.write(cache_file, Marshal.dump(cache_data))
+                    rescue
+                      # Cache save failed, continue
+                    end
+                  end
+
+                  results << [ category, parsed_hosts ]
+                end
+              end
+            end
+
+            results
+            # :nocov:
+          end
+        end
+
+        # Collect all results from workers
+        all_results = workers.flat_map(&:take)
+        work_queue.send(:done)
+
+        # Merge results into @hosts
+        all_results.each do |category, hosts|
+          category_sym = category.to_sym
+          @hosts[category_sym] ||= []
+          @hosts[category_sym].concat(hosts)
+          @hosts[category_sym].uniq!
+        end
+
+        # Final consolidation
+        @hosts.each do |category, hosts|
+          debug_log("🏁 Category #{category}: loaded #{hosts.size} hosts with Ractor processing")
+        end
+
+        # Restore original verbose setting
+        $VERBOSE = original_verbose
+      end
+    end
+
+    def process_content_with_threads(downloaded_content)
+      debug_time("Parallel content processing with Threads") do
+        require "thread"
+
+        # Create a work queue
+        work_queue = Queue.new
+        downloaded_content.each { |key, data| work_queue << [ key, data ] }
+
+        # Create a limited number of worker threads
+        num_workers = [ max_threads, downloaded_content.size ].min
+        debug_log("🔧 Creating #{num_workers} thread workers")
+
+        threads = []
+        results_mutex = Mutex.new
+
+        num_workers.times do
+          thread = Thread.new do
+            while (task = work_queue.pop(true) rescue nil)
+              key, data = task
+              category, url = key.split(":", 2)
+
+              if data[:from_cache]
+                # Already processed
+                parsed_hosts = data[:hosts]
+              else
+                # Parse the content
+                content = data[:content]
+                next if content.nil? || content.empty?
+
+                parsed_hosts = parse_list_content(content, detect_list_format(content))
+
+                # Save to cache
+                if cache_dir && parsed_hosts && !parsed_hosts.empty?
+                  save_to_cache(url, parsed_hosts)
+                end
+              end
+
+              # Thread-safe result storage
+              results_mutex.synchronize do
+                category_sym = category.to_sym
+                @hosts[category_sym] ||= []
+                @hosts[category_sym].concat(parsed_hosts || [])
+                @hosts[category_sym].uniq!
+              end
+            end
+          end
+
+          threads << thread
+        end
+
+        # Wait for all threads to complete
+        threads.each(&:join)
+
+        @hosts.each do |category, hosts|
+          debug_log("🏁 Category #{category}: loaded #{hosts.size} hosts with Thread processing")
+        end
+      end
+    end
+
     def initialize_smart_rules(custom_rules)
       custom_rules = {} if custom_rules.nil?
       default_rules = {
@@ -950,7 +1555,7 @@ module UrlCategorise
       default_rules.merge(custom_rules)
     end
 
-    def apply_smart_categorization(url, categories)
+    def apply_smart_categorisation(url, categories)
       return categories unless smart_categorization_enabled
 
       host = extract_host(url)
@@ -990,9 +1595,11 @@ module UrlCategorise
     end
 
     def extract_host(url)
-      (URI.parse(url).host || url).downcase.gsub('www.', '')
+      return nil if url.nil? || (url.respond_to?(:strip) && url.strip.empty?)
+
+      (URI.parse(url).host || url).downcase.gsub("www.", "")
     rescue URI::InvalidURIError
-      url.downcase.gsub('www.', '')
+      url.to_s.downcase.gsub("www.", "")
     end
 
     def build_host_data(urls)
@@ -1001,52 +1608,75 @@ module UrlCategorise
       urls.each do |url|
         next unless url_valid?(url)
 
+        debug_log("Processing host list: #{url}")
         hosts_data = nil
 
-        hosts_data = read_from_cache(url) if cache_dir && !force_download
+        if cache_dir && !force_download
+          debug_log("Attempting to load from cache: #{url}")
+          hosts_data = debug_time("Cache lookup for #{url}") do
+            read_from_cache(url)
+          end
+          if hosts_data
+            debug_log("✅ Cache HIT for #{url} - loaded #{hosts_data.size} hosts")
+          else
+            debug_log("❌ Cache MISS for #{url}")
+          end
+        else
+          debug_log("Cache disabled (cache_dir: #{cache_dir.inspect}, force_download: #{force_download})")
+        end
 
         if hosts_data.nil?
-          hosts_data = download_and_parse_list(url)
-          save_to_cache(url, hosts_data) if cache_dir
+          debug_time("Downloading and parsing #{url}") do
+            hosts_data = download_and_parse_list(url)
+            debug_log("Downloaded #{hosts_data&.size || 0} hosts from #{url}")
+          end
+          if cache_dir && hosts_data
+            debug_time("Saving to cache for #{url}") do
+              save_to_cache(url, hosts_data)
+            end
+            debug_log("💾 Saved #{hosts_data.size} hosts to cache for #{url}")
+          end
         end
 
         all_hosts.concat(hosts_data) if hosts_data
       end
 
-      all_hosts.compact.sort.uniq
+      result = all_hosts.compact.sort.uniq
+      debug_log("Total unique hosts collected: #{result.size}")
+      result
     end
 
     def download_and_parse_list(url)
-      if url.start_with?('file://')
+      if url.start_with?("file://")
         # Handle local file URLs
-        file_path = url.sub('file://', '')
+        file_path = url.sub("file://", "")
         return [] unless File.exist?(file_path)
-        
+
         content = File.read(file_path)
         return [] if content.nil? || content.empty?
-        
+
         # Store metadata
         @metadata[url] = {
           last_updated: Time.now,
           content_hash: Digest::SHA256.hexdigest(content),
-          status: 'success'
+          status: "success"
         }
-        
+
         return parse_list_content(content, detect_list_format(content))
       end
-      
+
       raw_data = HTTParty.get(url, timeout: request_timeout)
       return [] if raw_data.body.nil? || raw_data.body.empty?
 
       # Store metadata
-      etag = raw_data.headers['etag']
-      last_modified = raw_data.headers['last-modified']
+      etag = raw_data.headers["etag"]
+      last_modified = raw_data.headers["last-modified"]
       @metadata[url] = {
         last_updated: Time.now,
         etag: etag,
         last_modified: last_modified,
         content_hash: Digest::SHA256.hexdigest(raw_data.body),
-        status: 'success'
+        status: "success"
       }
 
       parse_list_content(raw_data.body, detect_list_format(raw_data.body))
@@ -1055,18 +1685,18 @@ module UrlCategorise
       @metadata[url] = {
         last_updated: Time.now,
         error: e.message,
-        status: 'failed'
+        status: "failed"
       }
       []
     end
 
     def parse_list_content(content, format)
-      lines = content.split("\n").reject { |line| line.empty? || line.strip.start_with?('#') }
+      lines = content.split("\n").reject { |line| line.empty? || line.strip.start_with?("#") }
 
       case format
       when :hosts
         lines.map do |line|
-          parts = line.split(' ')
+          parts = line.split(" ")
           # Extract domain from hosts format: "0.0.0.0 domain.com" -> "domain.com"
           parts.length >= 2 ? parts[1].strip : nil
         end.compact.reject(&:empty?)
@@ -1078,7 +1708,7 @@ module UrlCategorise
           match ? match[1] : nil
         end.compact
       when :ublock
-        lines.map { |line| line.gsub(/^\|\|/, '').gsub(/[$\^].*$/, '').strip }.reject(&:empty?)
+        lines.map { |line| line.gsub(/^\|\|/, "").gsub(/[$\^].*$/, "").strip }.reject(&:empty?)
       else
         lines.map(&:strip)
       end
@@ -1087,11 +1717,11 @@ module UrlCategorise
     def detect_list_format(content)
       # Skip comments and empty lines, then look at first 20 non-comment lines
       sample_lines = content.split("\n")
-                            .reject { |line| line.empty? || line.strip.start_with?('#') }
+                            .reject { |line| line.empty? || line.strip.start_with?("#") }
                             .first(20)
 
       return :hosts if sample_lines.any? { |line| line.match(/^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\s+/) }
-      return :dnsmasq if sample_lines.any? { |line| line.include?('address=/') }
+      return :dnsmasq if sample_lines.any? { |line| line.include?("address=/") }
       return :ublock if sample_lines.any? { |line| line.match(/^\|\|/) }
 
       :plain
@@ -1101,7 +1731,7 @@ module UrlCategorise
       return nil unless cache_dir
 
       FileUtils.mkdir_p(cache_dir) unless Dir.exist?(cache_dir)
-      filename = Digest::MD5.hexdigest(url) + '.cache'
+      filename = Digest::MD5.hexdigest(url) + ".cache"
       File.join(cache_dir, filename)
     end
 
@@ -1145,8 +1775,8 @@ module UrlCategorise
       # Check if remote content has changed
       begin
         head_response = HTTParty.head(url, timeout: request_timeout)
-        remote_etag = head_response.headers['etag']
-        remote_last_modified = head_response.headers['last-modified']
+        remote_etag = head_response.headers["etag"]
+        remote_last_modified = head_response.headers["last-modified"]
 
         cached_metadata = cache_data[:metadata]
 
@@ -1182,10 +1812,10 @@ module UrlCategorise
 
     def url_valid?(url)
       return false if url.nil? || url.empty?
-      return true if url.start_with?('file://')
-      
+      return true if url.start_with?("file://")
+
       uri = URI.parse(url)
-      uri.is_a?(URI::HTTP) && !uri.host.nil?
+      uri.is_a?(URI::HTTP) && !uri.host.nil? && !uri.host.empty?
     rescue URI::InvalidURIError
       false
     end