scraper_utils 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b4291b6994419c04851935fe4aa4e047eb4069cab3fecf451bf65f8e91acb48d
-  data.tar.gz: 2e3a657ce230f9c6bc9defe042cf7babb9e52e2130d32f0ec8312571f5dcb26a
+  metadata.gz: a2082c406ad96266f644fc1dfc588046aa43494e9e79b8fec38fe59252c09f06
+  data.tar.gz: 5100907cdcc8c55ddd59b25cf393d212f681b573ef874c5a6c65f748a8c852ec
 SHA512:
-  metadata.gz: 51c29aea77f43a8c7de8e874a2601b4e0b9e9c36ae512180f10dcd182b2ecc899cc08944face686e0f993b02338975672d9eaf06d8a0185ee222cfc263993244
-  data.tar.gz: ec63009a4f10677a8e9500b5ca15c68b432081fa995d5bee2eda2b2cff88cdb7090595be7ad64a0dfeacf09decbc292313b47f9f8af795a3b95c718c77f59339
+  metadata.gz: feabea23ac5b14f6b642769db303bcab954b22cfd2d95d7694af51afd661c73f1ff70a0c17b54557a730fe811275c126c0fdd4473e2ae6d2b83cfc495aa52bc6
+  data.tar.gz: d91154c0dbccfb4271fd4830a82316676c1ad5665773c30d212bcc1ab6178a09d4fee97b2b2a32480c32b8ee69da08fc9a78575460d49f8233fb91b74bf7df66

data/.gitignore

@@ -9,6 +9,9 @@
 /test/tmp/
 /test/version_tmp/
 
+# Ignore log files
+/log/
+
 # Temp files
 ,*
 *.bak

data/.rubocop.yml

@@ -1,3 +1,7 @@
+plugins:
+  - rubocop-rake
+  - rubocop-rspec
+
 AllCops:
   NewCops: enable
 

data/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+
+## 0.2.1 - 2025-02-28
+
+Fixed broken v0.2.0
+
+## 0.2.0 - 2025-02-28
+
+Added FiberScheduler, enabled complient mode with delays by default and simplified usage removing third retry without proxy
+
 ## 0.1.0 - 2025-02-23
 
-`First release for development`
+First release for development

data/Gemfile

@@ -22,12 +22,15 @@ gem "sqlite3", platform && (platform == :heroku16 ? "~> 1.4.0" : "~> 1.6.3")
 gem "scraperwiki", git: "https://github.com/openaustralia/scraperwiki-ruby.git",
                    branch: "morph_defaults"
 
-# development and test test gems
+# development and test gems
 gem "rake", platform && (platform == :heroku16 ? "~> 12.3.3" : "~> 13.0")
 gem "rspec", platform && (platform == :heroku16 ? "~> 3.9.0" : "~> 3.12")
-gem "rubocop", platform && (platform == :heroku16 ? "~> 0.80.0" : "~> 1.57")
+gem "rubocop", platform && (platform == :heroku16 ? "~> 1.28.2" : "~> 1.73")
+gem "rubocop-rake", platform && (platform == :heroku16 ? "~> 0.6.0" : "~> 0.7")
+gem "rubocop-rspec", platform && (platform == :heroku16 ? "~> 2.10.0" : "~> 3.5")
 gem "simplecov", platform && (platform == :heroku16 ? "~> 0.18.0" : "~> 0.22.0")
 gem "simplecov-console"
+gem "terminal-table"
 gem "webmock", platform && (platform == :heroku16 ? "~> 3.14.0" : "~> 3.19.0")
 
 gemspec

data/README.md

@@ -13,11 +13,12 @@ our scraper accessing your systems, here's what you should know:
 
 ### How to Control Our Behavior
 
-Our scraper utilities respect the standard server **robots.txt** control mechanisms (by default). To control our access:
+Our scraper utilities respect the standard server **robots.txt** control mechanisms (by default).
+To control our access:
 
 - Add a section for our user agent: `User-agent: ScraperUtils` (default)
-- Set a crawl delay: `Crawl-delay: 5`
-- If needed specify disallowed paths: `Disallow: /private/`
+- Set a crawl delay, eg: `Crawl-delay: 20`
+- If needed specify disallowed paths*: `Disallow: /private/`
 
 ### Built-in Politeness Features
 
@@ -26,13 +27,22 @@ Even without specific configuration, our scrapers will, by default:
 - **Identify themselves**: Our user agent clearly indicates who we are and provides a link to the project repository:
   `Mozilla/5.0 (compatible; ScraperUtils/0.2.0 2025-02-22; +https://github.com/ianheggie-oaf/scraper_utils)`
 
-- **Limit server load**: We introduce delays to avoid undue load on your server's by default based on your response
-  time.
-  The slower your server is running, the longer the delay we add between requests to help you.
-  In the default "compliant mode" this defaults to 20% and custom settings are capped at 33% maximum.
+- **Limit server load**: We slow down our requests so we should never be a significant load to your server, let alone
+  overload it.
+  The slower your server is running, the longer the delay we add between requests to help.
+  In the default "compliant mode" this defaults to a max load of 20% and is capped at 33%.
 
-- **Add randomized delays**: We add random delays between requests to avoid creating regular traffic patterns that might
-  impact server performance (enabled by default).
+- **Add randomized delays**: We add random delays between requests to further reduce our impact on servers, which should
+  bring us
+- down to the load of a single industrious person.
+
+Extra utilities provided for scrapers to further reduce your server load:
+
+- **Interleave requests**: This spreads out the requests to your server rather than focusing on one scraper at a time.
+
+- **Intelligent Date Range selection**: This reduces server load by over 60% by a smarter choice of date range searches,
+  checking the recent 4 days each day and reducing down to checking each 3 days by the end of the 33-day mark. This
+  replaces the simplistic check of the last 30 days each day.
 
 Our goal is to access public planning information without negatively impacting your services.
 
@@ -50,10 +60,6 @@ And then execute:
 
     $ bundle
 
-Or install it yourself for testing:
-
-    $ gem install scraper_utils
-
 Usage
 -----
 
@@ -101,12 +107,12 @@ export DEBUG=1
 Add `client_options` to your AUTHORITIES configuration and move any of the following settings into it:
 
 * `timeout: Integer` - Timeout for agent connections in case the server is slower than normal
-* `australian_proxy: true` - Use the MORPH_AUSTRALIAN_PROXY as proxy url if the site is geo-locked
+* `australian_proxy: true` - Use the proxy url in the `MORPH_AUSTRALIAN_PROXY` env variable if the site is geo-locked
 * `disable_ssl_certificate_check: true` - Disabled SSL verification for old / incorrect certificates
 
 See the documentation on `ScraperUtils::MechanizeUtils::AgentConfig` for more options
 
-Then adjust your code to accept client_options and pass then through to:
+Then adjust your code to accept `client_options` and pass then through to:
 `ScraperUtils::MechanizeUtils.mechanize_agent(client_options || {})`
 to receive a `Mechanize::Agent` configured accordingly.
 
@@ -115,130 +121,48 @@ The agent returned is configured using Mechanize hooks to implement the desired
 ### Default Configuration
 
 By default, the Mechanize agent is configured with the following settings.
+As you can see, the defaults can be changed using env variables.
+
+Note - compliant mode forces max_load to be set to a value no greater than 33.
+PLEASE don't use our user agent string with a max_load higher than 33!
 
 ```ruby
 ScraperUtils::MechanizeUtils::AgentConfig.configure do |config|
-  config.default_timeout = 60
-  config.default_compliant_mode = true
-  config.default_random_delay = 3
-  config.default_max_load = 20 # percentage
-  config.default_disable_ssl_certificate_check = false
-  config.default_australian_proxy = false
+  config.default_timeout = ENV.fetch('MORPH_TIMEOUT', 60).to_i # 60
+  config.default_compliant_mode = ENV.fetch('MORPH_NOT_COMPLIANT', nil).to_s.empty? # true
+  config.default_random_delay = ENV.fetch('MORPH_RANDOM_DELAY', 15).to_i # 15
+  config.default_max_load = ENV.fetch('MORPH_MAX_LOAD', 20.0).to_f # 20
+  config.default_disable_ssl_certificate_check = !ENV.fetch('MORPH_DISABLE_SSL_CHECK', nil).to_s.empty? # false
+  config.default_australian_proxy = !ENV.fetch('MORPH_USE_PROXY', nil).to_s.empty? # false
+  config.default_user_agent = ENV.fetch('MORPH_USER_AGENT', nil) # Uses Mechanize user agent
 end
 ```
 
 You can modify these global defaults before creating any Mechanize agents. These settings will be used for all Mechanize
 agents created by `ScraperUtils::MechanizeUtils.mechanize_agent` unless overridden by passing parameters to that method.
 
-### Example updated `scraper.rb` file
-
-Update your `scraper.rb` as per the following example for basic utilities:
+To speed up testing, set the following in `spec_helper.rb`:
 
 ```ruby
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-$LOAD_PATH << "./lib"
-
-require "scraper_utils"
-require "technology_one_scraper"
-
-# Main Scraper class
-class Scraper
-  AUTHORITIES = YourScraper::AUTHORITIES
-
-  # ADD: attempt argument
-  def scrape(authorities, attempt)
-    exceptions = {}
-    # ADD: Report attempt number
-    authorities.each do |authority_label|
-      puts "\nCollecting feed data for #{authority_label}, attempt: #{attempt}..."
-
-      begin
-        # REPLACE:
-        # YourScraper.scrape(authority_label) do |record|
-        #   record["authority_label"] = authority_label.to_s
-        #   YourScraper.log(record)
-        #   ScraperWiki.save_sqlite(%w[authority_label council_reference], record)
-        # end
-        # WITH:
-        ScraperUtils::DataQualityMonitor.start_authority(authority_label)
-        YourScraper.scrape(authority_label) do |record|
-          begin
-            record["authority_label"] = authority_label.to_s
-            ScraperUtils::DbUtils.save_record(record)
-          rescue ScraperUtils::UnprocessableRecord => e
-            ScraperUtils::DataQualityMonitor.log_unprocessable_record(e, record)
-            exceptions[authority_label] = e
-          end
-        end
-        # END OF REPLACE
-      end
-    rescue StandardError => e
-      warn "#{authority_label}: ERROR: #{e}"
-      warn e.backtrace
-      exceptions[authority_label] = e
-    end
-
-    exceptions
-  end
-
-  def self.selected_authorities
-    ScraperUtils::AuthorityUtils.selected_authorities(AUTHORITIES.keys)
-  end
-
-  def self.run(authorities)
-    puts "Scraping authorities: #{authorities.join(', ')}"
-    start_time = Time.now
-    exceptions = scrape(authorities, 1)
-    # Set start_time and attempt to the call above and log run below
-    ScraperUtils::LogUtils.log_scraping_run(
-      start_time,
-      1,
-      authorities,
-      exceptions
-    )
-
-    unless exceptions.empty?
-      puts "\n***************************************************"
-      puts "Now retrying authorities which earlier had failures"
-      puts exceptions.keys.join(", ").to_s
-      puts "***************************************************"
-
-      start_time = Time.now
-      exceptions = scrape(exceptions.keys, 2)
-      # Set start_time and attempt to the call above and log run below
-      ScraperUtils::LogUtils.log_scraping_run(
-        start_time,
-        2,
-        authorities,
-        exceptions
-      )
-    end
-
-    # Report on results, raising errors for unexpected conditions
-    ScraperUtils::LogUtils.report_on_results(authorities, exceptions)
-  end
+ScraperUtils::MechanizeUtils::AgentConfig.configure do |config|
+  config.default_random_delay = nil
+  config.default_max_load = 33
 end
+```
 
-if __FILE__ == $PROGRAM_NAME
-  # Default to list of authorities we can't or won't fix in code, explain why
-  # wagga: url redirects and then reports Application error
+### Example updated `scraper.rb` file
 
-  ENV["MORPH_EXPECT_BAD"] ||= "wagga"
-  Scraper.run(Scraper.selected_authorities)
-end
-```
+Update your `scraper.rb` as per the [example scraper](docs/example_scraper.rb).
 
 Your code should raise ScraperUtils::UnprocessableRecord when there is a problem with the data presented on a page for a
 record.
 Then just before you would normally yield a record for saving, rescue that exception and:
 
-* Call ScraperUtils::DataQualityMonitor.log_unprocessable_record(e, record)
+* Call `ScraperUtils::DataQualityMonitor.log_unprocessable_record(e, record)`
 * NOT yield the record for saving
 
 In your code update where create a mechanize agent (often `YourScraper.scrape_period`) and the `AUTHORITIES` hash
-to move mechanize_agent options (like `australian_proxy` and `timeout`) to a hash under a new key: `client_options`.
+to move Mechanize agent options (like `australian_proxy` and `timeout`) to a hash under a new key: `client_options`.
 For example:
 
 ```ruby
@@ -297,44 +221,73 @@ The `ScraperUtils::FiberScheduler` provides a lightweight utility that:
 * thus optimizing the total scraper run time
 * allows you to increase the random delay for authorities without undue effect on total run time
 * For the curious, it uses [ruby fibers](https://ruby-doc.org/core-2.5.8/Fiber.html) rather than threads as that is
-  simpler to get right and debug!
+  a simpler system and thus easier to get right, understand and debug!
+* Cycles around the authorities when compliant_mode, max_load and random_delay are disabled
 
-To enable change the scrape method in the example above to;
+To enable change the scrape method to be like [example scrape method using fibers](docs/example_scrape_with_fibers.rb)
 
-```ruby
+And use `ScraperUtils::FiberScheduler.log` instead of `puts` when logging within the authority processing code.
+This will prefix the output lines with the authority name, which is needed since the system will interleave the work and
+thus the output.
+
+This uses `ScraperUtils::RandomizeUtils` as described below. Remember to add the recommended line to
+`spec/spec_heper.rb`.
+
+Intelligent Date Range Selection
+--------------------------------                                                                                                                                                                                   
+
+To further reduce server load and speed up scrapers, we provide an intelligent date range selection mechanism
+that can reduce server requests by 60% without significantly impacting delay in picking up changes.
+
+The `ScraperUtils::DateRangeUtils#calculate_date_ranges` method provides a smart approach to searching historical
+records:
+
+- Always checks the most recent 4 days daily (configurable)
+- Progressively reduces search frequency for older records
+- Uses a Fibonacci-like progression to create natural, efficient search intervals
+- Configurable `max_period` (default is 3 days)
+- merges adjacent search ranges and handles the changeover in search frequency by extending some searches
 
-def scrape(authorities, attempt)
-  ScraperUtils::FiberScheduler.reset!
-  exceptions = {}
-  authorities.each do |authority_label|
-    ScraperUtils::FiberScheduler.register_operation(authority_label) do
-      ScraperUtils::FiberScheduler.log "Collecting feed data for #{authority_label}, attempt: #{attempt}..."
-      begin
-        ScraperUtils::DataQualityMonitor.start_authority(authority_label)
-        YourScraper.scrape(authority_label) do |record|
-          begin
-            record["authority_label"] = authority_label.to_s
-            ScraperUtils::DbUtils.save_record(record)
-          rescue ScraperUtils::UnprocessableRecord => e
-            ScraperUtils::DataQualityMonitor.log_unprocessable_record(e, record)
-            exceptions[authority_label] = e
-          end
-        end
-      rescue StandardError => e
-        warn "#{authority_label}: ERROR: #{e}"
-        warn e.backtrace
-        exceptions[authority_label] = e
-      end
-    end # end of register_operation block
+Example usage in your scraper:
+
+ ```ruby                                                                                                                                                                                                            
+date_ranges = ScraperUtils::DateRangeUtils.new.calculate_date_ranges
+date_ranges.each do |from_date, to_date, _debugging_comment|
+  # Adjust your normal search code to use for this date range                                                                                                                                                                             
+  your_search_records(from_date: from_date, to_date: to_date) do |record|
+    # process as normal
   end
-  ScraperUtils::FiberScheduler.run_all
-  exceptions
 end
 ```
 
-And use `ScraperUtils::FiberScheduler.log` instead of `puts` when logging within the authority processing code.
-This will prefix the output lines with the authority name, which is needed since the system will interleave the work and
-thus the output.
+Typical server load reductions:
+
+* Max period 2 days : ~42% of the 33 days selected
+* Max period 3 days : ~37% of the 33 days selected (default)
+* Max period 5 days : ~35% (or ~31% when days = 45)
+
+See the class documentation for customizing defaults and passing options.
+
+Randomizing Requests
+--------------------
+
+Pass a `Collection` or `Array` to `ScraperUtils::RandomizeUtils.randomize_order` to randomize it in production, but
+receive in as is when testing.
+
+Use this with the list of records scraped from an index to randomise any requests for further information to be less Bot like.
+
+### Spec setup
+
+You should enforce sequential mode when testing by adding the following code to `spec/spec_helper.rb` :
+
+```
+ScraperUtils::RandomizeUtils.sequential = true
+```
+
+Note:
+
+* You can also force sequential mode by setting the env variable `MORPH_PROCESS_SEQUENTIALLY` to `1` (any non blank)
+* testing using VCR requires sequential mode
 
 Development
 -----------
@@ -356,7 +309,7 @@ NOTE: You need to use ruby 3.2.2 instead of 2.5.8 to release to OTP protected ac
 Contributing
 ------------
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/ianheggie-oaf/scraper_utils
+Bug reports and pull requests with working tests are welcome on [GitHub](https://github.com/ianheggie-oaf/scraper_utils)
 
 CHANGELOG.md is maintained by the author aiming to follow https://github.com/vweevers/common-changelog
 
@@ -364,3 +317,4 @@ License
 -------
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+

data/docs/example_scrape_with_fibers.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+# Example scrape method updated to use ScraperUtils::FibreScheduler
+
+def scrape(authorities, attempt)
+  ScraperUtils::FiberScheduler.reset!
+  exceptions = {}
+  authorities.each do |authority_label|
+    ScraperUtils::FiberScheduler.register_operation(authority_label) do
+      ScraperUtils::FiberScheduler.log(
+        "Collecting feed data for #{authority_label}, attempt: #{attempt}..."
+      )
+      ScraperUtils::DataQualityMonitor.start_authority(authority_label)
+      YourScraper.scrape(authority_label) do |record|
+        record["authority_label"] = authority_label.to_s
+        ScraperUtils::DbUtils.save_record(record)
+      rescue ScraperUtils::UnprocessableRecord => e
+        ScraperUtils::DataQualityMonitor.log_unprocessable_record(e, record)
+        exceptions[authority_label] = e
+        # Continues processing other records
+      end
+    rescue StandardError => e
+      warn "#{authority_label}: ERROR: #{e}"
+      warn e.backtrace || "No backtrace available"
+      exceptions[authority_label] = e
+    end
+    # end of register_operation block
+  end
+  ScraperUtils::FiberScheduler.run_all
+  exceptions
+end

data/docs/example_scraper.rb

@@ -0,0 +1,93 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+$LOAD_PATH << "./lib"
+
+require "scraper_utils"
+require "technology_one_scraper"
+
+# Main Scraper class
+class Scraper
+  AUTHORITIES = YourScraper::AUTHORITIES
+
+  # ADD: attempt argument
+  def scrape(authorities, attempt)
+    exceptions = {}
+    # ADD: Report attempt number
+    authorities.each do |authority_label|
+      puts "\nCollecting feed data for #{authority_label}, attempt: #{attempt}..."
+
+      begin
+        # REPLACE:
+        # YourScraper.scrape(authority_label) do |record|
+        #   record["authority_label"] = authority_label.to_s
+        #   YourScraper.log(record)
+        #   ScraperWiki.save_sqlite(%w[authority_label council_reference], record)
+        # end
+        # WITH:
+        ScraperUtils::DataQualityMonitor.start_authority(authority_label)
+        YourScraper.scrape(authority_label) do |record|
+          begin
+            record["authority_label"] = authority_label.to_s
+            ScraperUtils::DbUtils.save_record(record)
+          rescue ScraperUtils::UnprocessableRecord => e
+            ScraperUtils::DataQualityMonitor.log_unprocessable_record(e, record)
+            exceptions[authority_label] = e
+          end
+        end
+        # END OF REPLACE
+      end
+    rescue StandardError => e
+      warn "#{authority_label}: ERROR: #{e}"
+      warn e.backtrace
+      exceptions[authority_label] = e
+    end
+
+    exceptions
+  end
+
+  def self.selected_authorities
+    ScraperUtils::AuthorityUtils.selected_authorities(AUTHORITIES.keys)
+  end
+
+  def self.run(authorities)
+    puts "Scraping authorities: #{authorities.join(', ')}"
+    start_time = Time.now
+    exceptions = scrape(authorities, 1)
+    # Set start_time and attempt to the call above and log run below
+    ScraperUtils::LogUtils.log_scraping_run(
+      start_time,
+      1,
+      authorities,
+      exceptions
+    )
+
+    unless exceptions.empty?
+      puts "\n***************************************************"
+      puts "Now retrying authorities which earlier had failures"
+      puts exceptions.keys.join(", ").to_s
+      puts "***************************************************"
+
+      start_time = Time.now
+      exceptions = scrape(exceptions.keys, 2)
+      # Set start_time and attempt to the call above and log run below
+      ScraperUtils::LogUtils.log_scraping_run(
+        start_time,
+        2,
+        authorities,
+        exceptions
+      )
+    end
+
+    # Report on results, raising errors for unexpected conditions
+    ScraperUtils::LogUtils.report_on_results(authorities, exceptions)
+  end
+end
+
+if __FILE__ == $PROGRAM_NAME
+  # Default to list of authorities we can't or won't fix in code, explain why
+  # wagga: url redirects and then reports Application error
+
+  ENV["MORPH_EXPECT_BAD"] ||= "wagga"
+  Scraper.run(Scraper.selected_authorities)
+end

data/lib/scraper_utils/adaptive_delay.rb

@@ -2,64 +2,69 @@
 
 require "uri"
 
-# Adapts delays between requests based on server response times.
-# Target delay is proportional to response time based on max_load setting.
-# Uses an exponential moving average to smooth variations in response times.
-class AdaptiveDelay
-  DEFAULT_MIN_DELAY = 0.0
-  DEFAULT_MAX_DELAY = 30.0 # Presumed default timeout for Mechanize
+module ScraperUtils
+  # Adapts delays between requests based on server response times.
+  # Target delay is proportional to response time based on max_load setting.
+  # Uses an exponential moving average to smooth variations in response times.
+  class AdaptiveDelay
+    DEFAULT_MIN_DELAY = 0.0
+    DEFAULT_MAX_DELAY = 30.0 # Presumed default timeout for Mechanize
 
-  attr_reader :min_delay, :max_delay, :max_load
+    attr_reader :min_delay, :max_delay, :max_load
 
-  # Creates a new adaptive delay calculator
-  #
-  # @param min_delay [Float] Minimum delay between requests in seconds
-  # @param max_delay [Float] Maximum delay between requests in seconds
-  # @param max_load [Float] Maximum load percentage (1-99) we aim to place on the server
-  #                         Lower values are more conservative (e.g., 20% = 4x response time delay)
-  def initialize(min_delay: DEFAULT_MIN_DELAY, max_delay: DEFAULT_MAX_DELAY, max_load: 20.0)
-    @delays = {} # domain -> last delay used
-    @min_delay = min_delay.to_f
-    @max_delay = max_delay.to_f
-    @max_load = max_load.to_f.clamp(1.0, 99.0)
-    @response_multiplier = (100.0 - @max_load) / @max_load
+    # Creates a new adaptive delay calculator
+    #
+    # @param min_delay [Float] Minimum delay between requests in seconds
+    # @param max_delay [Float] Maximum delay between requests in seconds
+    # @param max_load [Float] Maximum load percentage (1-99) we aim to place on the server
+    #                         Lower values are more conservative (e.g., 20% = 4x response time delay)
+    def initialize(min_delay: DEFAULT_MIN_DELAY, max_delay: DEFAULT_MAX_DELAY, max_load: 20.0)
+      @delays = {} # domain -> last delay used
+      @min_delay = min_delay.to_f
+      @max_delay = max_delay.to_f
+      @max_load = max_load.to_f.clamp(1.0, 99.0)
+      @response_multiplier = (100.0 - @max_load) / @max_load
 
-    if ENV["DEBUG"]
-      ScraperUtils::FiberScheduler.log "AdaptiveDelay initialized with delays between #{@min_delay} and #{@max_delay} seconds"
-      ScraperUtils::FiberScheduler.log "Using max_load of #{@max_load}% (response time multiplier: #{@response_multiplier.round(2)}x)"
+      return unless DebugUtils.basic?
+
+      ScraperUtils::FiberScheduler.log(
+        "AdaptiveDelay initialized with delays between #{@min_delay} and #{@max_delay} seconds, " \
+          "Max_load #{@max_load}% thus response multiplier: #{@response_multiplier.round(2)}x"
+      )
     end
-  end
 
-  # @param uri [URI::Generic, String] The URL to extract the domain from
-  # @return [String] The domain in the format "scheme://host"
-  def domain(uri)
-    uri = URI(uri) unless uri.is_a?(URI)
-    "#{uri.scheme}://#{uri.host}".downcase
-  end
+    # @param uri [URI::Generic, String] The URL to extract the domain from
+    # @return [String] The domain in the format "scheme://host"
+    def domain(uri)
+      uri = URI(uri) unless uri.is_a?(URI)
+      "#{uri.scheme}://#{uri.host}".downcase
+    end
 
-  # @param uri [URI::Generic, String] URL to get delay for
-  # @return [Float] Current delay for the domain, or min_delay if no delay set
-  def delay(uri)
-    @delays[domain(uri)] || @min_delay
-  end
+    # @param uri [URI::Generic, String] URL to get delay for
+    # @return [Float] Current delay for the domain, or min_delay if no delay set
+    def delay(uri)
+      @delays[domain(uri)] || @min_delay
+    end
 
-  # @param uri [URI::Generic, String] URL the response came from
-  # @param response_time [Float] Time in seconds the server took to respond
-  # @return [Float] The calculated delay to use with the next request
-  def next_delay(uri, response_time)
-    uris_domain = domain(uri)
-    target_delay = (response_time * @response_multiplier).clamp(0.0, @max_delay)
-    current_delay = @delays[uris_domain] || target_delay
-    delay = ((9.0 * current_delay) + target_delay) / 10.0
-    delay = delay.clamp(@min_delay, @max_delay)
+    # @param uri [URI::Generic, String] URL the response came from
+    # @param response_time [Float] Time in seconds the server took to respond
+    # @return [Float] The calculated delay to use with the next request
+    def next_delay(uri, response_time)
+      uris_domain = domain(uri)
+      target_delay = (response_time * @response_multiplier).clamp(0.0, @max_delay)
+      current_delay = @delays[uris_domain] || target_delay
+      delay = ((9.0 * current_delay) + target_delay) / 10.0
+      delay = delay.clamp(@min_delay, @max_delay)
 
-    if ENV["DEBUG"]
-      ScraperUtils::FiberScheduler.log "Adaptive delay for #{uris_domain} updated to " \
-             "#{delay.round(2)}s (target: #{@response_multiplier.round(1)}x " \
-             "response_time of #{response_time.round(2)}s)"
-    end
+      if DebugUtils.basic?
+        ScraperUtils::FiberScheduler.log(
+          "Adaptive delay for #{uris_domain} updated to #{delay.round(2)}s (target: " \
+            "#{@response_multiplier.round(1)}x response_time of #{response_time.round(2)}s)"
+        )
+      end
 
-    @delays[uris_domain] = delay
-    delay
+      @delays[uris_domain] = delay
+      delay
+    end
   end
 end