scraper_utils 0.2.0 → 0.4.0

data/lib/scraper_utils/log_utils.rb

@@ -13,8 +13,8 @@ module ScraperUtils
     # @param start_time [Time] When this scraping attempt was started
     # @param attempt [Integer] 1 for first run, 2 for first retry, 3 for last retry (without proxy)
     # @param authorities [Array<Symbol>] List of authorities attempted to scrape
-    # @param exceptions [Hash > Exception] Any exception that occurred during scraping
-    # DataQualityMonitor.stats is checked for :saved and :unprocessed entries
+    # @param exceptions [Hash{Symbol => Exception}] Any exceptions that occurred during scraping
+    # `DataQualityMonitor.stats` is checked for :saved and :unprocessed entries
     # @return [void]
     def self.log_scraping_run(start_time, attempt, authorities, exceptions)
       raise ArgumentError, "Invalid start time" unless start_time.is_a?(Time)
@@ -75,39 +75,39 @@ module ScraperUtils
 
     # Report on the results
     # @param authorities [Array<Symbol>] List of authorities attempted to scrape
-    # @param exceptions [Hash > Exception] Any exception that occurred during scraping
-    # DataQualityMonitor.stats is checked for :saved and :unprocessed entries
+    # @param exceptions [Hash{Symbol => Exception}] Any exceptions that occurred during scraping
+    # `DataQualityMonitor.stats` is checked for :saved and :unprocessed entries
     # @return [void]
     def self.report_on_results(authorities, exceptions)
-      expect_bad = ENV["MORPH_EXPECT_BAD"]&.split(",")&.map(&:strip)&.map(&:to_sym) || []
+      if ENV["MORPH_EXPECT_BAD"]
+        expect_bad = ENV["MORPH_EXPECT_BAD"].split(",").map(&:strip).map(&:to_sym)
+      end
+      expect_bad ||= []
 
-      puts "MORPH_EXPECT_BAD=#{ENV.fetch('MORPH_EXPECT_BAD', nil)}" if expect_bad.any?
+      $stderr.flush
+      puts "MORPH_EXPECT_BAD=#{ENV.fetch('MORPH_EXPECT_BAD', nil)}"
 
       # Print summary table
       puts "\nScraping Summary:"
       summary_format = "%-20s %6s %6s %s"
 
-      puts summary_format % %w[Authority OK Bad Exception]
-      puts summary_format % ['-' * 20, '-' * 6, '-' * 6, '-' * 50]
+      puts format(summary_format, 'Authority', 'OK', 'Bad', 'Exception')
+      puts format(summary_format, "-" * 20, "-" * 6, "-" * 6, "-" * 50)
 
       authorities.each do |authority|
         stats = ScraperUtils::DataQualityMonitor.stats&.fetch(authority, {}) || {}
-    
+
         ok_records = stats[:saved] || 0
         bad_records = stats[:unprocessed] || 0
-      
+
         expect_bad_prefix = expect_bad.include?(authority) ? "[EXPECT BAD] " : ""
         exception_msg = if exceptions[authority]
                           "#{exceptions[authority].class} - #{exceptions[authority].message}"
                         else
                           "-"
                         end
-        puts summary_format % [
-          authority.to_s, 
-          ok_records, 
-          bad_records,
-          "#{expect_bad_prefix}#{exception_msg}".slice(0, 70)
-        ]
+        puts format(summary_format, authority.to_s, ok_records, bad_records,
+                    "#{expect_bad_prefix}#{exception_msg}".slice(0, 70))
       end
       puts
 
@@ -120,7 +120,8 @@ module ScraperUtils
       end
 
       if unexpected_working.any?
-        errors << "WARNING: Remove #{unexpected_working.join(',')} from MORPH_EXPECT_BAD as it now works!"
+        errors <<
+          "WARNING: Remove #{unexpected_working.join(',')} from MORPH_EXPECT_BAD as it now works!"
       end
 
       # Check for authorities with unexpected errors
@@ -137,6 +138,7 @@ module ScraperUtils
         end
       end
 
+      $stdout.flush
       if errors.any?
         errors << "See earlier output for details"
         raise errors.join("\n")

data/lib/scraper_utils/mechanize_utils/agent_config.rb

@@ -23,6 +23,11 @@ module ScraperUtils
     #     random_delay: 10
     #   )
     class AgentConfig
+      DEFAULT_TIMEOUT = 60
+      DEFAULT_RANDOM_DELAY = 5
+      DEFAULT_MAX_LOAD = 33.3
+      MAX_LOAD_CAP = 50.0
+
       # Class-level defaults that can be modified
       class << self
         # @return [Integer] Default timeout in seconds for agent connections
@@ -62,65 +67,68 @@ module ScraperUtils
         # Reset all configuration options to their default values
         # @return [void]
         def reset_defaults!
-          @default_timeout = 60
-          @default_compliant_mode = true
-          @default_random_delay = 3
-          @default_max_load = 20.0
-          @default_disable_ssl_certificate_check = false
-          @default_australian_proxy = nil
-          @default_user_agent = nil
+          @default_timeout = ENV.fetch('MORPH_TIMEOUT', DEFAULT_TIMEOUT).to_i # 60
+          @default_compliant_mode = ENV.fetch('MORPH_NOT_COMPLIANT', nil).to_s.empty? # true
+          @default_random_delay = ENV.fetch('MORPH_RANDOM_DELAY', DEFAULT_RANDOM_DELAY).to_i # 5
+          @default_max_load = ENV.fetch('MORPH_MAX_LOAD', DEFAULT_MAX_LOAD).to_f # 33.3
+          @default_disable_ssl_certificate_check = !ENV.fetch('MORPH_DISABLE_SSL_CHECK', nil).to_s.empty? # false
+          @default_australian_proxy = !ENV.fetch('MORPH_USE_PROXY', nil).to_s.empty? # false
+          @default_user_agent = ENV.fetch('MORPH_USER_AGENT', nil) # Uses Mechanize user agent
         end
       end
 
       # Set defaults on load
       reset_defaults!
 
-
       # @return [String] User agent string
       attr_reader :user_agent
 
       # Give access for testing
 
-      attr_reader :max_load
-      attr_reader :min_random
-      attr_reader :max_random
+      attr_reader :max_load, :min_random, :max_random
 
-      # Creates configuration for a Mechanize agent with sensible defaults
-      # @param timeout [Integer, nil] Timeout for agent connections (default: 60 unless changed)
-      # @param compliant_mode [Boolean, nil] Comply with headers and robots.txt (default: true unless changed)
-      # @param random_delay [Integer, nil] Average random delay in seconds (default: 3 unless changed)
-      # @param max_load [Float, nil] Maximum server load percentage (nil = no response delay, default: 20%)
+      # Creates Mechanize agent configuration with sensible defaults overridable via configure
+      # @param timeout [Integer, nil] Timeout for agent connections (default: 60)
+      # @param compliant_mode [Boolean, nil] Comply with headers and robots.txt (default: true)
+      # @param random_delay [Integer, nil] Average random delay in seconds (default: 3)
+      # @param max_load [Float, nil] Maximum server load percentage (nil = no delay, default: 20%)
       #                              When compliant_mode is true, max_load is capped at 33%
-      # @param disable_ssl_certificate_check [Boolean, nil] Skip SSL verification (default: false unless changed)
-      # @param australian_proxy [Boolean, nil] Use proxy if available (default: false unless changed)
+      # @param disable_ssl_certificate_check [Boolean, nil] Skip SSL verification (default: false)
+      # @param australian_proxy [Boolean, nil] Use proxy if available (default: false)
       # @param user_agent [String, nil] Configure Mechanize user agent
       def initialize(timeout: nil,
                      compliant_mode: nil,
                      random_delay: nil,
                      max_load: nil,
                      disable_ssl_certificate_check: nil,
-                     australian_proxy: false,
+                     australian_proxy: nil,
                      user_agent: nil)
         @timeout = timeout.nil? ? self.class.default_timeout : timeout
         @compliant_mode = compliant_mode.nil? ? self.class.default_compliant_mode : compliant_mode
         @random_delay = random_delay.nil? ? self.class.default_random_delay : random_delay
         @max_load = max_load.nil? ? self.class.default_max_load : max_load
-        @max_load = [@max_load || 20.0, 33.0].min if @compliant_mode
+        @max_load = [@max_load || DEFAULT_MAX_LOAD, MAX_LOAD_CAP].min if @compliant_mode
         @user_agent = user_agent.nil? ? self.class.default_user_agent : user_agent
 
-        @disable_ssl_certificate_check = disable_ssl_certificate_check.nil? ?
-                                           self.class.default_disable_ssl_certificate_check :
+        @disable_ssl_certificate_check = if disable_ssl_certificate_check.nil?
+                                           self.class.default_disable_ssl_certificate_check
+                                         else
                                            disable_ssl_certificate_check
-        @australian_proxy = australian_proxy.nil? ? self.class.default_australian_proxy : australian_proxy
+                                         end
+        @australian_proxy = if australian_proxy.nil?
+                              self.class.default_australian_proxy
+                            else
+                              australian_proxy
+                            end
 
         # Validate proxy URL format if proxy will be used
         @australian_proxy &&= !ScraperUtils.australian_proxy.to_s.empty?
         if @australian_proxy
           uri = begin
-                  URI.parse(ScraperUtils.australian_proxy.to_s)
-                rescue URI::InvalidURIError => e
-                  raise URI::InvalidURIError, "Invalid proxy URL format: #{e.message}"
-                end
+            URI.parse(ScraperUtils.australian_proxy.to_s)
+          rescue URI::InvalidURIError => e
+            raise URI::InvalidURIError, "Invalid proxy URL format: #{e.message}"
+          end
           unless uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS)
             raise URI::InvalidURIError, "Proxy URL must start with http:// or https://"
           end
@@ -135,7 +143,7 @@ module ScraperUtils
         end
 
         today = Date.today.strftime("%Y-%m-%d")
-        @user_agent = ENV['MORPH_USER_AGENT']&.sub("TODAY", today)
+        @user_agent = ENV.fetch("MORPH_USER_AGENT", nil)&.sub("TODAY", today)
         if @compliant_mode
           version = ScraperUtils::VERSION
           @user_agent ||= "Mozilla/5.0 (compatible; ScraperUtils/#{version} #{today}; +https://github.com/ianheggie-oaf/scraper_utils)"
@@ -159,7 +167,8 @@ module ScraperUtils
         if @compliant_mode
           agent.user_agent = user_agent
           agent.request_headers ||= {}
-          agent.request_headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+          agent.request_headers["Accept"] =
+            "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
           agent.request_headers["Upgrade-Insecure-Requests"] = "1"
         end
         if @australian_proxy
@@ -178,32 +187,39 @@ module ScraperUtils
       def display_options
         display_args = []
         display_args << "timeout=#{@timeout}" if @timeout
-        if @australian_proxy
-          display_args << "australian_proxy=#{@australian_proxy.inspect}"
-        elsif ScraperUtils.australian_proxy.to_s.empty?
-          display_args << "#{ScraperUtils::AUSTRALIAN_PROXY_ENV_VAR} not set"
-        else
-          display_args << "australian_proxy=#{@australian_proxy.inspect}"
-        end
+        display_args << if ScraperUtils.australian_proxy.to_s.empty? && !@australian_proxy
+                          "#{ScraperUtils::AUSTRALIAN_PROXY_ENV_VAR} not set"
+                        else
+                          "australian_proxy=#{@australian_proxy.inspect}"
+                        end
         display_args << "compliant_mode" if @compliant_mode
         display_args << "random_delay=#{@random_delay}" if @random_delay
         display_args << "max_load=#{@max_load}%" if @max_load
         display_args << "disable_ssl_certificate_check" if @disable_ssl_certificate_check
         display_args << "default args" if display_args.empty?
-        ScraperUtils::FiberScheduler.log "Configuring Mechanize agent with #{display_args.join(', ')}"
+        ScraperUtils::FiberScheduler.log(
+          "Configuring Mechanize agent with #{display_args.join(', ')}"
+        )
       end
 
       def pre_connect_hook(_agent, request)
         @connection_started_at = Time.now
-        ScraperUtils::FiberScheduler.log "Pre Connect request: #{request.inspect} at #{@connection_started_at}" if ENV["DEBUG"]
+        return unless DebugUtils.verbose?
+
+        ScraperUtils::FiberScheduler.log(
+          "Pre Connect request: #{request.inspect} at #{@connection_started_at}"
+        )
       end
 
       def post_connect_hook(_agent, uri, response, _body)
         raise ArgumentError, "URI must be present in post-connect hook" unless uri
 
         response_time = Time.now - @connection_started_at
-        if ENV["DEBUG"]
-          ScraperUtils::FiberScheduler.log "Post Connect uri: #{uri.inspect}, response: #{response.inspect} after #{response_time} seconds"
+        if DebugUtils.basic?
+          ScraperUtils::FiberScheduler.log(
+            "Post Connect uri: #{uri.inspect}, response: #{response.inspect} " \
+            "after #{response_time} seconds"
+          )
         end
 
         if @robots_checker&.disallowed?(uri)
@@ -214,18 +230,23 @@ module ScraperUtils
         delays = {
           robot_txt: @robots_checker&.crawl_delay&.round(3),
           max_load: @adaptive_delay&.next_delay(uri, response_time)&.round(3),
-          random: (@min_random ? (rand(@min_random..@max_random) ** 2).round(3) : nil)
+          random: (@min_random ? (rand(@min_random..@max_random)**2).round(3) : nil)
         }
         @delay = delays.values.compact.max
         if @delay&.positive?
-          puts "Delaying #{@delay} seconds, max of #{delays.inspect}" if ENV["DEBUG"]
-          sleep(@delay)
+          $stderr.flush
+          ScraperUtils::FiberScheduler.log("Delaying #{@delay} seconds, max of #{delays.inspect}") if ENV["DEBUG"]
+          $stdout.flush
+          ScraperUtils::FiberScheduler.delay(@delay)
         end
 
         response
       end
 
       def verify_proxy_works(agent)
+        $stderr.flush
+        $stdout.flush
+        FiberScheduler.log "Checking proxy works..."
         my_ip = MechanizeUtils.public_ip(agent)
         begin
           IPAddr.new(my_ip)
@@ -233,17 +254,17 @@ module ScraperUtils
           raise "Invalid public IP address returned by proxy check: #{my_ip.inspect}: #{e}"
         end
         ScraperUtils::FiberScheduler.log "Proxy is using IP address: #{my_ip.inspect}"
-        my_headers = MechanizeUtils::public_headers(agent)
+        my_headers = MechanizeUtils.public_headers(agent)
         begin
           # Check response is JSON just to be safe!
           headers = JSON.parse(my_headers)
           puts "Proxy is passing headers:"
-          puts JSON.pretty_generate(headers['headers'])
+          puts JSON.pretty_generate(headers["headers"])
         rescue JSON::ParserError => e
           puts "Couldn't parse public_headers: #{e}! Raw response:"
           puts my_headers.inspect
         end
-      rescue Net::OpenTimeout, Timeout::Error => e
+      rescue Timeout::Error => e # Includes Net::OpenTimeout
         raise "Proxy check timed out: #{e}"
       rescue Errno::ECONNREFUSED, Net::HTTP::Persistent::Error => e
         raise "Failed to connect to proxy: #{e}"

data/lib/scraper_utils/mechanize_utils.rb

@@ -45,20 +45,28 @@ module ScraperUtils
     #
     # @param agent [Mechanize, nil] Mechanize agent to use for IP lookup or nil when clearing cache
     # @param force [Boolean] Force a new IP lookup, by clearing cache first
-    # @return [String] The public IP address
+    # @return [String, nil] The public IP address
     def self.public_ip(agent = nil, force: false)
       @public_ip = nil if force
-      @public_ip ||= agent&.get(PUBLIC_IP_URL)&.body&.strip if agent
+      @public_ip ||= begin
+        response = agent&.get(PUBLIC_IP_URL)
+        response&.body&.strip
+      end
+      @public_ip
     end
 
     # Retrieves and logs the headers that make it through the proxy
     #
     # @param agent [Mechanize, nil] Mechanize agent to use for IP lookup or nil when clearing cache
     # @param force [Boolean] Force a new IP lookup, by clearing cache first
-    # @return [String] The list of headers in json format
+    # @return [String, nil] The list of headers in json format
     def self.public_headers(agent = nil, force: false)
       @public_headers = nil if force
-      @public_headers ||= agent&.get(HEADERS_ECHO_URL)&.body&.strip if agent
+      @public_headers ||= begin
+        response = agent&.get(HEADERS_ECHO_URL)
+        response&.body&.strip
+      end
+      @public_headers
     end
   end
 end

data/lib/scraper_utils/randomize_utils.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module ScraperUtils
+  # Provides utilities for randomizing processing order in scrapers,
+  # particularly helpful for distributing load and avoiding predictable patterns
+  module RandomizeUtils
+    # Returns a randomized version of the input collection when in production mode,
+    # or the original collection when in test/sequential mode
+    #
+    # @param collection [Array, Enumerable] Collection of items to potentially randomize
+    # @return [Array] Randomized or original collection depending on environment
+    def self.randomize_order(collection)
+      return collection.to_a if sequential?
+
+      collection.to_a.shuffle
+    end
+
+    # Checks if sequential processing is enabled
+    #
+    # @return [Boolean] true when in test mode or MORPH_PROCESS_SEQUENTIALLY is set
+    def self.sequential?
+      @sequential = !ENV["MORPH_PROCESS_SEQUENTIALLY"].to_s.empty? if @sequential.nil?
+      @sequential || false
+    end
+
+    # Explicitly set sequential mode for testing
+    #
+    # @param value [Boolean, nil] true to enable sequential mode, false to disable, nil to clear cache
+    # @return [Boolean, nil]
+    def self.sequential=(value)
+      @sequential = value
+    end
+  end
+end

data/lib/scraper_utils/robots_checker.rb

@@ -14,8 +14,10 @@ module ScraperUtils
     # * Crawl-delay from either User-agent: bot name or * (default)
     def initialize(user_agent)
       @user_agent = extract_user_agent(user_agent).downcase
-      if ENV["DEBUG"]
-        ScraperUtils::FiberScheduler.log "Checking robots.txt for user agent prefix: #{@user_agent} (case insensitive)"
+      if DebugUtils.basic?
+        ScraperUtils::FiberScheduler.log(
+          "Checking robots.txt for user agent prefix: #{@user_agent} (case insensitive)"
+        )
       end
       @rules = {} # domain -> {rules: [], delay: int}
       @delay = nil # Delay from last robots.txt check
@@ -73,7 +75,11 @@ module ScraperUtils
         @rules[domain] = rules
         rules
       rescue StandardError => e
-        ScraperUtils::FiberScheduler.log "Warning: Failed to fetch robots.txt for #{domain}: #{e.message}" if ENV["DEBUG"]
+        if DebugUtils.basic?
+          ScraperUtils::FiberScheduler.log(
+            "WARNING: Failed to fetch robots.txt for #{domain}: #{e.message}"
+          )
+        end
         nil
       end
     end
@@ -141,4 +147,3 @@ module ScraperUtils
     end
   end
 end
-

data/lib/scraper_utils/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ScraperUtils
-  VERSION = "0.2.0"
+  VERSION = "0.4.0"
 end

data/lib/scraper_utils.rb

@@ -5,8 +5,11 @@ require "scraper_utils/authority_utils"
 require "scraper_utils/data_quality_monitor"
 require "scraper_utils/db_utils"
 require "scraper_utils/debug_utils"
+require "scraper_utils/fiber_scheduler"
 require "scraper_utils/log_utils"
+require "scraper_utils/mechanize_utils/agent_config"
 require "scraper_utils/mechanize_utils"
+require "scraper_utils/randomize_utils"
 require "scraper_utils/robots_checker"
 require "scraper_utils/version"
 
@@ -15,9 +18,6 @@ module ScraperUtils
   # Constants for configuration on Morph.io
   AUSTRALIAN_PROXY_ENV_VAR = "MORPH_AUSTRALIAN_PROXY"
 
-  # Enable debug locally, not on morph.io
-  DEBUG_ENV_VAR = "DEBUG"
-
   # Fatal Error
   class Error < StandardError
   end
@@ -31,13 +31,6 @@ module ScraperUtils
   class UnprocessableRecord < Error
   end
 
-  # Check if debug mode is enabled
-  #
-  # @return [Boolean] Whether debug mode is active
-  def self.debug?
-    !ENV[DEBUG_ENV_VAR].to_s.empty?
-  end
-
   def self.australian_proxy
     ap = ENV[AUSTRALIAN_PROXY_ENV_VAR].to_s
     ap.empty? ? nil : ap

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scraper_utils
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Ian Heggie
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-02-27 00:00:00.000000000 Z
+date: 2025-03-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -74,16 +74,21 @@ files:
 - SPECS.md
 - bin/console
 - bin/setup
+- docs/example_scrape_with_fibers.rb
+- docs/example_scraper.rb
 - lib/scraper_utils.rb
 - lib/scraper_utils/adaptive_delay.rb
 - lib/scraper_utils/authority_utils.rb
+- lib/scraper_utils/cycle_utils.rb
 - lib/scraper_utils/data_quality_monitor.rb
+- lib/scraper_utils/date_range_utils.rb
 - lib/scraper_utils/db_utils.rb
 - lib/scraper_utils/debug_utils.rb
 - lib/scraper_utils/fiber_scheduler.rb
 - lib/scraper_utils/log_utils.rb
 - lib/scraper_utils/mechanize_utils.rb
 - lib/scraper_utils/mechanize_utils/agent_config.rb
+- lib/scraper_utils/randomize_utils.rb
 - lib/scraper_utils/robots_checker.rb
 - lib/scraper_utils/version.rb
 - scraper_utils.gemspec