scraper_utils 0.1.0 → 0.3.0

data/lib/scraper_utils/fiber_scheduler.rb

@@ -0,0 +1,229 @@
+# frozen_string_literal: true
+
+require "fiber"
+
+module ScraperUtils
+  # A utility module for interleaving multiple scraping operations
+  # using fibers during connection delay periods. This allows efficient
+  # use of wait time by switching between operations.
+  module FiberScheduler
+    # @return [Array<Fiber>] List of active fibers managed by the scheduler
+    def self.registry
+      @registry ||= []
+    end
+
+    # Checks if the current code is running within a registered fiber
+    #
+    # @return [Boolean] true if running in a registered fiber, false otherwise
+    def self.in_fiber?
+      !Fiber.current.nil? && registry.include?(Fiber.current)
+    end
+
+    # Gets the authority associated with the current fiber
+    #
+    # @return [String, nil] the authority name or nil if not in a fiber
+    def self.current_authority
+      return nil unless in_fiber?
+
+      Fiber.current.instance_variable_get(:@authority)
+    end
+
+    # Logs a message, automatically prefixing with authority name if in a fiber
+    #
+    # @param message [String] the message to log
+    # @return [void]
+    def self.log(message)
+      authority = current_authority
+      $stderr.flush
+      if authority
+        puts "[#{authority}] #{message}"
+      else
+        puts message
+      end
+      $stdout.flush
+    end
+
+    # Returns a hash of exceptions encountered during processing, indexed by authority
+    #
+    # @return [Hash{Symbol => Exception}] exceptions by authority
+    def self.exceptions
+      @exceptions ||= {}
+    end
+
+    # Returns a hash of the yielded / block values
+    #
+    # @return [Hash{Symbol => Any}] values by authority
+    def self.values
+      @values ||= {}
+    end
+
+    # Checks if fiber scheduling is currently enabled
+    #
+    # @return [Boolean] true if enabled, false otherwise
+    def self.enabled?
+      @enabled ||= false
+    end
+
+    # Enables fiber scheduling
+    #
+    # @return [void]
+    def self.enable!
+      reset! unless enabled?
+      @enabled = true
+    end
+
+    # Disables fiber scheduling
+    #
+    # @return [void]
+    def self.disable!
+      @enabled = false
+    end
+
+    # Resets the scheduler state, and disables. Use before retrying failed authorities.
+    #
+    # @return [void]
+    def self.reset!
+      @registry = []
+      @exceptions = {}
+      @values = {}
+      @enabled = false
+      @delay_requested = 0.0
+      @time_slept = 0.0
+      @resume_count = 0
+      @initial_resume_at = Time.now - 60.0 # one minute ago
+    end
+
+    # Registers a block to scrape for a specific authority
+    #
+    # @param authority [String] the name of the authority being processed
+    # @yield to the block containing the scraping operation to be run in the fiber
+    # @return [Fiber] a fiber that calls the block. With @authority and @resume_at instance vars
+    def self.register_operation(authority, &block)
+      # Automatically enable fiber scheduling when operations are registered
+      enable!
+
+      fiber = Fiber.new do
+        values[authority] = block.call
+      rescue StandardError => e
+        # Store exception against the authority
+        exceptions[authority] = e
+      ensure
+        # Remove itself when done regardless of success/failure
+        registry.delete(Fiber.current)
+      end
+
+      # Start fibres in registration order
+      @initial_resume_at += 0.1
+      fiber.instance_variable_set(:@resume_at, @initial_resume_at)
+      fiber.instance_variable_set(:@authority, authority)
+      registry << fiber
+
+      if DebugUtils.basic?
+        FiberScheduler.log "Registered #{authority} operation with fiber: #{fiber.object_id} for interleaving"
+      end
+      # Process immediately when testing
+      fiber.resume if ScraperUtils::RandomizeUtils.sequential?
+      fiber
+    end
+
+    # Run all registered fibers until completion
+    #
+    # @return [Hash] Exceptions that occurred during execution
+    def self.run_all
+      count = registry.size
+      while (fiber = find_earliest_fiber)
+        if fiber.alive?
+          authority = begin
+            fiber.instance_variable_get(:@authority)
+          rescue StandardError
+            nil
+          end
+          @resume_count ||= 0
+          @resume_count += 1
+          values[authority] = fiber.resume
+        else
+          FiberScheduler.log "WARNING: fiber is dead but did not remove itself from registry! #{fiber.object_id}"
+          registry.delete(fiber)
+        end
+      end
+
+      if @time_slept&.positive? && @delay_requested&.positive?
+        percent_slept = (100.0 * @time_slept / @delay_requested).round(1)
+      end
+      puts
+      FiberScheduler.log "FiberScheduler processed #{@resume_count} calls to delay for #{count} registrations, " \
+           "sleeping #{percent_slept}% (#{@time_slept&.round(1)}) of the " \
+           "#{@delay_requested&.round(1)} seconds requested."
+      puts
+
+      exceptions
+    end
+
+    # Delays the current fiber and potentially runs another one
+    # Falls back to regular sleep if fiber scheduling is not enabled
+    #
+    # @param seconds [Numeric] the number of seconds to delay
+    # @return [Integer] return from sleep operation or 0
+    def self.delay(seconds)
+      seconds = 0.0 unless seconds&.positive?
+      @delay_requested ||= 0.0
+      @delay_requested += seconds
+
+      current_fiber = Fiber.current
+
+      if !enabled? || !current_fiber || registry.size <= 1
+        @time_slept ||= 0.0
+        @time_slept += seconds
+        log("Sleeping #{seconds.round(3)} seconds") if DebugUtils.basic?
+        return sleep(seconds)
+      end
+
+      now = Time.now
+      resume_at = now + seconds
+
+      # Don't resume at the same time as someone else,
+      # FIFO queue if seconds == 0
+      @other_resumes ||= []
+      @other_resumes = @other_resumes.delete_if { |t| t < now }
+      while @other_resumes.include?(resume_at) && resume_at
+        resume_at += 0.01
+      end
+
+      # Used to compare when other fibers need to be resumed
+      current_fiber.instance_variable_set(:@resume_at, resume_at)
+
+      # Yield control back to the scheduler so another fiber can run
+      Fiber.yield
+
+      # When we get control back, check if we need to sleep more
+      remaining = resume_at - Time.now
+      if remaining.positive?
+        @time_slept ||= 0.0
+        @time_slept += remaining
+        log("Sleeping remaining #{remaining.round(3)} seconds") if DebugUtils.basic?
+        sleep(remaining)
+      end || 0
+    end
+
+    # Finds the fiber with the earliest wake-up time
+    #
+    # @return [Fiber, nil] the fiber with the earliest wake-up time or nil if none found
+    def self.find_earliest_fiber
+      earliest_time = nil
+      earliest_fiber = nil
+
+      registry.each do |fiber|
+        resume_at = fiber.instance_variable_get(:@resume_at)
+        if earliest_time.nil? || resume_at < earliest_time
+          earliest_time = resume_at
+          earliest_fiber = fiber
+        end
+      end
+
+      earliest_fiber
+    end
+
+    # Mark methods as private
+    private_class_method :find_earliest_fiber
+  end
+end

data/lib/scraper_utils/log_utils.rb

@@ -13,13 +13,10 @@ module ScraperUtils
     # @param start_time [Time] When this scraping attempt was started
     # @param attempt [Integer] 1 for first run, 2 for first retry, 3 for last retry (without proxy)
     # @param authorities [Array<Symbol>] List of authorities attempted to scrape
-    # @param results [Hash] Results for each authority containing:
-    #   - :records_scraped [Integer] Number of records successfully scraped
-    #   - :unprocessable_records [Integer] Optional Number of unprocessable record like regions
-    #   - :error [Exception, nil] Any exception that occurred during scraping
-    #   - :proxy_used [Boolean] Whether a proxy was used
+    # @param exceptions [Hash{Symbol => Exception}] Any exceptions that occurred during scraping
+    # `DataQualityMonitor.stats` is checked for :saved and :unprocessed entries
     # @return [void]
-    def self.log_scraping_run(start_time, attempt, authorities, results)
+    def self.log_scraping_run(start_time, attempt, authorities, exceptions)
       raise ArgumentError, "Invalid start time" unless start_time.is_a?(Time)
       raise ArgumentError, "Authorities must be a non-empty array" if authorities.empty?
 
@@ -31,10 +28,11 @@ module ScraperUtils
       interrupted = []
 
       authorities.each do |authority_label|
-        result = results[authority_label] || {}
+        stats = ScraperUtils::DataQualityMonitor.stats&.fetch(authority_label, nil) || {}
 
-        status = if result[:records_scraped]&.positive?
-                   result[:error] ? :interrupted : :successful
+        exception = exceptions[authority_label]
+        status = if stats[:saved]&.positive?
+                   exception ? :interrupted : :successful
                  else
                    :failed
                  end
@@ -51,13 +49,12 @@ module ScraperUtils
           "run_at" => start_time.iso8601,
           "attempt" => attempt,
           "authority_label" => authority_label.to_s,
-          "records_scraped" => result[:records_scraped] || 0,
-          "unprocessable_records" => result[:unprocessable_records] || 0,
-          "used_proxy" => result[:proxy_used] ? 1 : 0,
+          "records_saved" => stats[:saved] || 0,
+          "unprocessable_records" => stats[:unprocessed] || 0,
           "status" => status.to_s,
-          "error_message" => result[:error]&.message,
-          "error_class" => result[:error]&.class&.to_s,
-          "error_backtrace" => extract_meaningful_backtrace(result[:error])
+          "error_message" => exception&.message,
+          "error_class" => exception&.class&.to_s,
+          "error_backtrace" => extract_meaningful_backtrace(exception)
         }
 
         save_log_record(record)
@@ -76,37 +73,72 @@ module ScraperUtils
       cleanup_old_records
     end
 
-    def self.report_on_results(authorities, results)
-      expect_bad = ENV["MORPH_EXPECT_BAD"]&.split(",")&.map(&:to_sym) || []
+    # Report on the results
+    # @param authorities [Array<Symbol>] List of authorities attempted to scrape
+    # @param exceptions [Hash{Symbol => Exception}] Any exceptions that occurred during scraping
+    # `DataQualityMonitor.stats` is checked for :saved and :unprocessed entries
+    # @return [void]
+    def self.report_on_results(authorities, exceptions)
+      if ENV["MORPH_EXPECT_BAD"]
+        expect_bad = ENV["MORPH_EXPECT_BAD"].split(",").map(&:strip).map(&:to_sym)
+      end
+      expect_bad ||= []
+
+      $stderr.flush
+      puts "MORPH_EXPECT_BAD=#{ENV.fetch('MORPH_EXPECT_BAD', nil)}"
+
+      # Print summary table
+      puts "\nScraping Summary:"
+      summary_format = "%-20s %6s %6s %s"
 
-      puts "MORPH_EXPECT_BAD=#{ENV['MORPH_EXPECT_BAD']}" if expect_bad.any?
+      puts format(summary_format, 'Authority', 'OK', 'Bad', 'Exception')
+      puts format(summary_format, "-" * 20, "-" * 6, "-" * 6, "-" * 50)
+
+      authorities.each do |authority|
+        stats = ScraperUtils::DataQualityMonitor.stats&.fetch(authority, {}) || {}
+
+        ok_records = stats[:saved] || 0
+        bad_records = stats[:unprocessed] || 0
+
+        expect_bad_prefix = expect_bad.include?(authority) ? "[EXPECT BAD] " : ""
+        exception_msg = if exceptions[authority]
+                          "#{exceptions[authority].class} - #{exceptions[authority].message}"
+                        else
+                          "-"
+                        end
+        puts format(summary_format, authority.to_s, ok_records, bad_records,
+                    "#{expect_bad_prefix}#{exception_msg}".slice(0, 70))
+      end
+      puts
 
       errors = []
 
       # Check for authorities that were expected to be bad but are now working
       unexpected_working = expect_bad.select do |authority|
-        result = results[authority]
-        result && result[:records_scraped]&.positive? && result[:error].nil?
+        stats = ScraperUtils::DataQualityMonitor.stats&.fetch(authority, {}) || {}
+        stats[:saved]&.positive? && !exceptions[authority]
       end
 
       if unexpected_working.any?
-        errors << "WARNING: Remove #{unexpected_working.join(',')} from EXPECT_BAD as it now works!"
+        errors <<
+          "WARNING: Remove #{unexpected_working.join(',')} from MORPH_EXPECT_BAD as it now works!"
       end
 
       # Check for authorities with unexpected errors
       unexpected_errors = authorities
-                          .select { |authority| results[authority]&.dig(:error) }
+                          .select { |authority| exceptions[authority] }
                           .reject { |authority| expect_bad.include?(authority) }
 
       if unexpected_errors.any?
         errors << "ERROR: Unexpected errors in: #{unexpected_errors.join(',')} " \
-          "(Add to MORPH_EXPECT_BAD?)"
+                  "(Add to MORPH_EXPECT_BAD?)"
         unexpected_errors.each do |authority|
-          error = results[authority][:error]
+          error = exceptions[authority]
           errors << "  #{authority}: #{error.class} - #{error.message}"
         end
       end
 
+      $stdout.flush
       if errors.any?
         errors << "See earlier output for details"
         raise errors.join("\n")
@@ -134,7 +166,8 @@ module ScraperUtils
         "interrupted" => interrupted.join(","),
         "successful_count" => successful.size,
         "interrupted_count" => interrupted.size,
-        "failed_count" => failed.size
+        "failed_count" => failed.size,
+        "public_ip" => ScraperUtils::MechanizeUtils.public_ip
       }
 
       ScraperWiki.save_sqlite(

data/lib/scraper_utils/mechanize_utils/agent_config.rb

@@ -0,0 +1,276 @@
+# frozen_string_literal: true
+
+require "mechanize"
+require "ipaddr"
+
+module ScraperUtils
+  module MechanizeUtils
+    # Configuration for a Mechanize agent with sensible defaults and configurable settings.
+    # Supports global configuration through {.configure} and per-instance overrides.
+    #
+    # @example Setting global defaults
+    #   ScraperUtils::MechanizeUtils::AgentConfig.configure do |config|
+    #     config.default_timeout = 90
+    #     config.default_random_delay = 5
+    #   end
+    #
+    # @example Creating an instance with defaults
+    #   config = ScraperUtils::MechanizeUtils::AgentConfig.new
+    #
+    # @example Overriding specific settings
+    #   config = ScraperUtils::MechanizeUtils::AgentConfig.new(
+    #     timeout: 120,
+    #     random_delay: 10
+    #   )
+    class AgentConfig
+      DEFAULT_TIMEOUT = 60
+      DEFAULT_RANDOM_DELAY = 5
+      DEFAULT_MAX_LOAD = 33.3
+      MAX_LOAD_CAP = 50.0
+
+      # Class-level defaults that can be modified
+      class << self
+        # @return [Integer] Default timeout in seconds for agent connections
+        attr_accessor :default_timeout
+
+        # @return [Boolean] Default setting for compliance with headers and robots.txt
+        attr_accessor :default_compliant_mode
+
+        # @return [Integer, nil] Default average random delay in seconds
+        attr_accessor :default_random_delay
+
+        # @return [Float, nil] Default maximum server load percentage (nil = no response delay)
+        attr_accessor :default_max_load
+
+        # @return [Boolean] Default setting for SSL certificate verification
+        attr_accessor :default_disable_ssl_certificate_check
+
+        # @return [Boolean] Default flag for Australian proxy preference
+        attr_accessor :default_australian_proxy
+
+        # @return [String, nil] Default Mechanize user agent
+        attr_accessor :default_user_agent
+
+        # Configure default settings for all AgentConfig instances
+        # @yield [self] Yields self for configuration
+        # @example
+        #   AgentConfig.configure do |config|
+        #     config.default_timeout = 90
+        #     config.default_random_delay = 5
+        #     config.default_max_load = 15
+        #   end
+        # @return [void]
+        def configure
+          yield self if block_given?
+        end
+
+        # Reset all configuration options to their default values
+        # @return [void]
+        def reset_defaults!
+          @default_timeout = ENV.fetch('MORPH_TIMEOUT', DEFAULT_TIMEOUT).to_i # 60
+          @default_compliant_mode = ENV.fetch('MORPH_NOT_COMPLIANT', nil).to_s.empty? # true
+          @default_random_delay = ENV.fetch('MORPH_RANDOM_DELAY', DEFAULT_RANDOM_DELAY).to_i # 33
+          @default_max_load = ENV.fetch('MORPH_MAX_LOAD', DEFAULT_MAX_LOAD).to_f # 20
+          @default_disable_ssl_certificate_check = !ENV.fetch('MORPH_DISABLE_SSL_CHECK', nil).to_s.empty? # false
+          @default_australian_proxy = !ENV.fetch('MORPH_USE_PROXY', nil).to_s.empty? # false
+          @default_user_agent = ENV.fetch('MORPH_USER_AGENT', nil) # Uses Mechanize user agent
+        end
+      end
+
+      # Set defaults on load
+      reset_defaults!
+
+      # @return [String] User agent string
+      attr_reader :user_agent
+
+      # Give access for testing
+
+      attr_reader :max_load, :min_random, :max_random
+
+      # Creates Mechanize agent configuration with sensible defaults overridable via configure
+      # @param timeout [Integer, nil] Timeout for agent connections (default: 60)
+      # @param compliant_mode [Boolean, nil] Comply with headers and robots.txt (default: true)
+      # @param random_delay [Integer, nil] Average random delay in seconds (default: 3)
+      # @param max_load [Float, nil] Maximum server load percentage (nil = no delay, default: 20%)
+      #                              When compliant_mode is true, max_load is capped at 33%
+      # @param disable_ssl_certificate_check [Boolean, nil] Skip SSL verification (default: false)
+      # @param australian_proxy [Boolean, nil] Use proxy if available (default: false)
+      # @param user_agent [String, nil] Configure Mechanize user agent
+      def initialize(timeout: nil,
+                     compliant_mode: nil,
+                     random_delay: nil,
+                     max_load: nil,
+                     disable_ssl_certificate_check: nil,
+                     australian_proxy: nil,
+                     user_agent: nil)
+        @timeout = timeout.nil? ? self.class.default_timeout : timeout
+        @compliant_mode = compliant_mode.nil? ? self.class.default_compliant_mode : compliant_mode
+        @random_delay = random_delay.nil? ? self.class.default_random_delay : random_delay
+        @max_load = max_load.nil? ? self.class.default_max_load : max_load
+        @max_load = [@max_load || DEFAULT_MAX_LOAD, MAX_LOAD_CAP].min if @compliant_mode
+        @user_agent = user_agent.nil? ? self.class.default_user_agent : user_agent
+
+        @disable_ssl_certificate_check = if disable_ssl_certificate_check.nil?
+                                           self.class.default_disable_ssl_certificate_check
+                                         else
+                                           disable_ssl_certificate_check
+                                         end
+        @australian_proxy = if australian_proxy.nil?
+                              self.class.default_australian_proxy
+                            else
+                              australian_proxy
+                            end
+
+        # Validate proxy URL format if proxy will be used
+        @australian_proxy &&= !ScraperUtils.australian_proxy.to_s.empty?
+        if @australian_proxy
+          uri = begin
+            URI.parse(ScraperUtils.australian_proxy.to_s)
+          rescue URI::InvalidURIError => e
+            raise URI::InvalidURIError, "Invalid proxy URL format: #{e.message}"
+          end
+          unless uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS)
+            raise URI::InvalidURIError, "Proxy URL must start with http:// or https://"
+          end
+          unless uri.host && uri.port
+            raise URI::InvalidURIError, "Proxy URL must include host and port"
+          end
+        end
+
+        if @random_delay
+          @min_random = Math.sqrt(@random_delay * 3.0 / 13.0).round(3)
+          @max_random = (3 * @min_random).round(3)
+        end
+
+        today = Date.today.strftime("%Y-%m-%d")
+        @user_agent = ENV.fetch("MORPH_USER_AGENT", nil)&.sub("TODAY", today)
+        if @compliant_mode
+          version = ScraperUtils::VERSION
+          @user_agent ||= "Mozilla/5.0 (compatible; ScraperUtils/#{version} #{today}; +https://github.com/ianheggie-oaf/scraper_utils)"
+        end
+
+        @robots_checker = RobotsChecker.new(@user_agent) if @user_agent
+        @adaptive_delay = AdaptiveDelay.new(max_load: @max_load) if @max_load
+        display_options
+      end
+
+      # Configures a Mechanize agent with these settings
+      # @param agent [Mechanize] The agent to configure
+      # @return [void]
+      def configure_agent(agent)
+        agent.verify_mode = OpenSSL::SSL::VERIFY_NONE if @disable_ssl_certificate_check
+
+        if @timeout
+          agent.open_timeout = @timeout
+          agent.read_timeout = @timeout
+        end
+        if @compliant_mode
+          agent.user_agent = user_agent
+          agent.request_headers ||= {}
+          agent.request_headers["Accept"] =
+            "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+          agent.request_headers["Upgrade-Insecure-Requests"] = "1"
+        end
+        if @australian_proxy
+          agent.agent.set_proxy(ScraperUtils.australian_proxy)
+          agent.request_headers["Accept-Language"] = "en-AU,en-US;q=0.9,en;q=0.8"
+          verify_proxy_works(agent)
+        end
+
+        @connection_started_at = nil
+        agent.pre_connect_hooks << method(:pre_connect_hook)
+        agent.post_connect_hooks << method(:post_connect_hook)
+      end
+
+      private
+
+      def display_options
+        display_args = []
+        display_args << "timeout=#{@timeout}" if @timeout
+        display_args << if ScraperUtils.australian_proxy.to_s.empty? && !@australian_proxy
+                          "#{ScraperUtils::AUSTRALIAN_PROXY_ENV_VAR} not set"
+                        else
+                          "australian_proxy=#{@australian_proxy.inspect}"
+                        end
+        display_args << "compliant_mode" if @compliant_mode
+        display_args << "random_delay=#{@random_delay}" if @random_delay
+        display_args << "max_load=#{@max_load}%" if @max_load
+        display_args << "disable_ssl_certificate_check" if @disable_ssl_certificate_check
+        display_args << "default args" if display_args.empty?
+        ScraperUtils::FiberScheduler.log(
+          "Configuring Mechanize agent with #{display_args.join(', ')}"
+        )
+      end
+
+      def pre_connect_hook(_agent, request)
+        @connection_started_at = Time.now
+        return unless DebugUtils.verbose?
+
+        ScraperUtils::FiberScheduler.log(
+          "Pre Connect request: #{request.inspect} at #{@connection_started_at}"
+        )
+      end
+
+      def post_connect_hook(_agent, uri, response, _body)
+        raise ArgumentError, "URI must be present in post-connect hook" unless uri
+
+        response_time = Time.now - @connection_started_at
+        if DebugUtils.basic?
+          ScraperUtils::FiberScheduler.log(
+            "Post Connect uri: #{uri.inspect}, response: #{response.inspect} " \
+            "after #{response_time} seconds"
+          )
+        end
+
+        if @robots_checker&.disallowed?(uri)
+          raise ScraperUtils::UnprocessableSite,
+                "URL is disallowed by robots.txt specific rules: #{uri}"
+        end
+
+        delays = {
+          robot_txt: @robots_checker&.crawl_delay&.round(3),
+          max_load: @adaptive_delay&.next_delay(uri, response_time)&.round(3),
+          random: (@min_random ? (rand(@min_random..@max_random)**2).round(3) : nil)
+        }
+        @delay = delays.values.compact.max
+        if @delay&.positive?
+          $stderr.flush
+          ScraperUtils::FiberScheduler.log("Delaying #{@delay} seconds, max of #{delays.inspect}") if ENV["DEBUG"]
+          $stdout.flush
+          ScraperUtils::FiberScheduler.delay(@delay)
+        end
+
+        response
+      end
+
+      def verify_proxy_works(agent)
+        $stderr.flush
+        $stdout.flush
+        FiberScheduler.log "Checking proxy works..."
+        my_ip = MechanizeUtils.public_ip(agent)
+        begin
+          IPAddr.new(my_ip)
+        rescue IPAddr::InvalidAddressError => e
+          raise "Invalid public IP address returned by proxy check: #{my_ip.inspect}: #{e}"
+        end
+        ScraperUtils::FiberScheduler.log "Proxy is using IP address: #{my_ip.inspect}"
+        my_headers = MechanizeUtils.public_headers(agent)
+        begin
+          # Check response is JSON just to be safe!
+          headers = JSON.parse(my_headers)
+          puts "Proxy is passing headers:"
+          puts JSON.pretty_generate(headers["headers"])
+        rescue JSON::ParserError => e
+          puts "Couldn't parse public_headers: #{e}! Raw response:"
+          puts my_headers.inspect
+        end
+      rescue Timeout::Error => e # Includes Net::OpenTimeout
+        raise "Proxy check timed out: #{e}"
+      rescue Errno::ECONNREFUSED, Net::HTTP::Persistent::Error => e
+        raise "Failed to connect to proxy: #{e}"
+      rescue Mechanize::ResponseCodeError => e
+        raise "Proxy check error: #{e}"
+      end
+    end
+  end
+end