scraper_utils 0.2.0 → 0.4.0

data/lib/scraper_utils/adaptive_delay.rb

@@ -2,64 +2,69 @@
 
 require "uri"
 
-# Adapts delays between requests based on server response times.
-# Target delay is proportional to response time based on max_load setting.
-# Uses an exponential moving average to smooth variations in response times.
-class AdaptiveDelay
-  DEFAULT_MIN_DELAY = 0.0
-  DEFAULT_MAX_DELAY = 30.0 # Presumed default timeout for Mechanize
+module ScraperUtils
+  # Adapts delays between requests based on server response times.
+  # Target delay is proportional to response time based on max_load setting.
+  # Uses an exponential moving average to smooth variations in response times.
+  class AdaptiveDelay
+    DEFAULT_MIN_DELAY = 0.0
+    DEFAULT_MAX_DELAY = 30.0 # Presumed default timeout for Mechanize
 
-  attr_reader :min_delay, :max_delay, :max_load
+    attr_reader :min_delay, :max_delay, :max_load
 
-  # Creates a new adaptive delay calculator
-  #
-  # @param min_delay [Float] Minimum delay between requests in seconds
-  # @param max_delay [Float] Maximum delay between requests in seconds
-  # @param max_load [Float] Maximum load percentage (1-99) we aim to place on the server
-  #                         Lower values are more conservative (e.g., 20% = 4x response time delay)
-  def initialize(min_delay: DEFAULT_MIN_DELAY, max_delay: DEFAULT_MAX_DELAY, max_load: 20.0)
-    @delays = {} # domain -> last delay used
-    @min_delay = min_delay.to_f
-    @max_delay = max_delay.to_f
-    @max_load = max_load.to_f.clamp(1.0, 99.0)
-    @response_multiplier = (100.0 - @max_load) / @max_load
+    # Creates a new adaptive delay calculator
+    #
+    # @param min_delay [Float] Minimum delay between requests in seconds
+    # @param max_delay [Float] Maximum delay between requests in seconds
+    # @param max_load [Float] Maximum load percentage (1-99) we aim to place on the server
+    #                         Lower values are more conservative (e.g., 20% = 4x response time delay)
+    def initialize(min_delay: DEFAULT_MIN_DELAY, max_delay: DEFAULT_MAX_DELAY, max_load: 20.0)
+      @delays = {} # domain -> last delay used
+      @min_delay = min_delay.to_f
+      @max_delay = max_delay.to_f
+      @max_load = max_load.to_f.clamp(1.0, 99.0)
+      @response_multiplier = (100.0 - @max_load) / @max_load
 
-    if ENV["DEBUG"]
-      ScraperUtils::FiberScheduler.log "AdaptiveDelay initialized with delays between #{@min_delay} and #{@max_delay} seconds"
-      ScraperUtils::FiberScheduler.log "Using max_load of #{@max_load}% (response time multiplier: #{@response_multiplier.round(2)}x)"
+      return unless DebugUtils.basic?
+
+      ScraperUtils::FiberScheduler.log(
+        "AdaptiveDelay initialized with delays between #{@min_delay} and #{@max_delay} seconds, " \
+          "Max_load #{@max_load}% thus response multiplier: #{@response_multiplier.round(2)}x"
+      )
     end
-  end
 
-  # @param uri [URI::Generic, String] The URL to extract the domain from
-  # @return [String] The domain in the format "scheme://host"
-  def domain(uri)
-    uri = URI(uri) unless uri.is_a?(URI)
-    "#{uri.scheme}://#{uri.host}".downcase
-  end
+    # @param uri [URI::Generic, String] The URL to extract the domain from
+    # @return [String] The domain in the format "scheme://host"
+    def domain(uri)
+      uri = URI(uri) unless uri.is_a?(URI)
+      "#{uri.scheme}://#{uri.host}".downcase
+    end
 
-  # @param uri [URI::Generic, String] URL to get delay for
-  # @return [Float] Current delay for the domain, or min_delay if no delay set
-  def delay(uri)
-    @delays[domain(uri)] || @min_delay
-  end
+    # @param uri [URI::Generic, String] URL to get delay for
+    # @return [Float] Current delay for the domain, or min_delay if no delay set
+    def delay(uri)
+      @delays[domain(uri)] || @min_delay
+    end
 
-  # @param uri [URI::Generic, String] URL the response came from
-  # @param response_time [Float] Time in seconds the server took to respond
-  # @return [Float] The calculated delay to use with the next request
-  def next_delay(uri, response_time)
-    uris_domain = domain(uri)
-    target_delay = (response_time * @response_multiplier).clamp(0.0, @max_delay)
-    current_delay = @delays[uris_domain] || target_delay
-    delay = ((9.0 * current_delay) + target_delay) / 10.0
-    delay = delay.clamp(@min_delay, @max_delay)
+    # @param uri [URI::Generic, String] URL the response came from
+    # @param response_time [Float] Time in seconds the server took to respond
+    # @return [Float] The calculated delay to use with the next request
+    def next_delay(uri, response_time)
+      uris_domain = domain(uri)
+      target_delay = (response_time * @response_multiplier).clamp(0.0, @max_delay)
+      current_delay = @delays[uris_domain] || target_delay
+      delay = ((9.0 * current_delay) + target_delay) / 10.0
+      delay = delay.clamp(@min_delay, @max_delay)
 
-    if ENV["DEBUG"]
-      ScraperUtils::FiberScheduler.log "Adaptive delay for #{uris_domain} updated to " \
-             "#{delay.round(2)}s (target: #{@response_multiplier.round(1)}x " \
-             "response_time of #{response_time.round(2)}s)"
-    end
+      if DebugUtils.basic?
+        ScraperUtils::FiberScheduler.log(
+          "Adaptive delay for #{uris_domain} updated to #{delay.round(2)}s (target: " \
+            "#{@response_multiplier.round(1)}x response_time of #{response_time.round(2)}s)"
+        )
+      end
 
-    @delays[uris_domain] = delay
-    delay
+      @delays[uris_domain] = delay
+      delay
+    end
   end
 end

data/lib/scraper_utils/cycle_utils.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module ScraperUtils
+  # Provides utilities for cycling through a range of options day by day
+  module CycleUtils
+    # Returns position in cycle from zero onwards
+    # @param cycle [Integer] Cycle size (2 onwards)
+    # @param date [Date, nil] Optional date to use instead of today
+    # @return [Integer] position in cycle progressing from zero to cycle-1 and then repeating day by day
+    # Can override using CYCLE_POSITION ENV variable
+    def self.position(cycle, date: nil)
+      day = ENV.fetch('CYCLE_POSITION', (date || Date.today).jd).to_i
+      day % cycle
+    end
+
+    # Returns one value per day, cycling through all possible values in order
+    # @param values [Array] Values to cycle through
+    # @param date [Date, nil] Optional date to use instead of today to calculate position
+    # @return value from array
+    # Can override using CYCLE_POSITION ENV variable
+    def self.pick(values, date: nil)
+      values[position(values.size, date: date)]
+    end
+  end
+end

data/lib/scraper_utils/data_quality_monitor.rb

@@ -6,8 +6,8 @@ module ScraperUtils
   class DataQualityMonitor
     # Get the statistics for all authorities
     # @return [Hash, nil] Hash of statistics per authority or nil if none started
-    def self.stats
-      @stats
+    class << self
+      attr_reader :stats
     end
 
     # Notes the start of processing an authority and clears any previous stats
@@ -16,28 +16,39 @@ module ScraperUtils
     # @return [void]
     def self.start_authority(authority_label)
       @stats ||= {}
-      @authority_label = authority_label
-      @stats[@authority_label] = { saved: 0, unprocessed: 0}
+      @stats[authority_label] = { saved: 0, unprocessed: 0 }
     end
 
-    def self.threshold
-      5.01 + @stats[@authority_label][:saved] * 0.1 if @stats&.fetch(@authority_label, nil)
+    # Extracts authority label and ensures stats are setup for record
+    def self.extract_authority(record)
+      authority_label = (record&.key?("authority_label") ? record["authority_label"] : "").to_sym
+      @stats ||= {}
+      @stats[authority_label] ||= { saved: 0, unprocessed: 0 }
+      authority_label
+    end
+
+    def self.threshold(authority_label)
+      5.01 + (@stats[authority_label][:saved] * 0.1) if @stats&.fetch(authority_label, nil)
     end
 
     # Logs an unprocessable record and raises an exception if error threshold is exceeded
     # The threshold is 5 + 10% of saved records
     #
-    # @param e [Exception] The exception that caused the record to be unprocessable
+    # @param exception [Exception] The exception that caused the record to be unprocessable
     # @param record [Hash, nil] The record that couldn't be processed
     # @raise [ScraperUtils::UnprocessableSite] When too many records are unprocessable
     # @return [void]
-    def self.log_unprocessable_record(e, record)
-      start_authority(:"") unless @stats
-      @stats[@authority_label][:unprocessed] += 1
-      ScraperUtils::FiberScheduler.log "Erroneous record #{@authority_label} - #{record&.fetch('address', nil) || record.inspect}: #{e}"
-      if @stats[@authority_label][:unprocessed] > threshold
-        raise ScraperUtils::UnprocessableSite, "Too many unprocessable_records for #{@authority_label}: #{@stats[@authority_label].inspect} - aborting processing of site!"
-      end
+    def self.log_unprocessable_record(exception, record)
+      authority_label = extract_authority(record)
+      @stats[authority_label][:unprocessed] += 1
+      ScraperUtils::FiberScheduler.log "Erroneous record #{authority_label} - #{record&.fetch(
+        'address', nil
+      ) || record.inspect}: #{exception}"
+      return unless @stats[authority_label][:unprocessed] > threshold(authority_label)
+
+      raise ScraperUtils::UnprocessableSite,
+            "Too many unprocessable_records for #{authority_label}: " \
+            "#{@stats[authority_label].inspect} - aborting processing of site!"
     end
 
     # Logs a successfully saved record
@@ -45,9 +56,9 @@ module ScraperUtils
     # @param record [Hash] The record that was saved
     # @return [void]
     def self.log_saved_record(record)
-      start_authority(:"") unless @stats
-      @stats[@authority_label][:saved] += 1
-      ScraperUtils::FiberScheduler.log "Saving record #{@authority_label} - #{record['address']}"
+      authority_label = extract_authority(record)
+      @stats[authority_label][:saved] += 1
+      ScraperUtils::FiberScheduler.log "Saving record #{authority_label} - #{record['address']}"
     end
   end
 end

data/lib/scraper_utils/date_range_utils.rb

@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+
+module ScraperUtils
+  class DateRangeUtils
+    MERGE_ADJACENT_RANGES = true
+    PERIODS = [2, 3, 5, 8].freeze
+
+    class << self
+      # @return [Integer] Default number of days to cover
+      attr_accessor :default_days
+
+      # @return [Integer] Default days to always include in ranges
+      attr_accessor :default_everytime
+
+      # @return [Integer, nil] Default max days between any one date being in a range
+      attr_accessor :default_max_period
+
+      # Configure default settings for all DateRangeUtils instances
+      # @yield [self] Yields self for configuration
+      # @example
+      #   AgentConfig.configure do |config|
+      #     config.default_everytime = 3
+      #     config.default_days = 35
+      #     config.default_max_period = 5
+      #   end
+      # @return [void]
+      def configure
+        yield self if block_given?
+      end
+
+      # Reset all configuration options to their default values
+      # @return [void]
+      def reset_defaults!
+        @default_days = ENV.fetch('MORPH_DAYS', 33).to_i # 33
+        @default_everytime = ENV.fetch('MORPH_EVERYTIME', 4).to_i # 4
+        @default_max_period = ENV.fetch('MORPH_MAX_PERIOD', 3).to_i # 3
+      end
+    end
+
+    # Set defaults on load
+    reset_defaults!
+
+    attr_reader :max_period_used
+    attr_reader :extended_max_period
+
+    # Generates one or more date ranges to check the most recent daily through to checking each max_period
+    # There is a graduated schedule from the latest `everytime` days through to the oldest of `days` dates which is checked each `max_period` days.
+    # @param days [Integer, nil] create ranges that cover the last `days` dates
+    # @param everytime [Integer, nil] Always include the latest `everytime` out of `days` dates
+    # @param max_period [Integer, nil] the last `days` dates must be checked at least every `max_period` days
+    # @param today [Date, nil] overrides the default determination of today at UTC+09:30 (middle of Australia)
+    # @return [Array{[Date, Date, String]}] being from_date, to_date and a comment
+    #
+    # Uses a Fibonacci sequence to create a natural progression of check frequencies.
+    # Newer data is checked more frequently, with periods between checks growing
+    # according to the Fibonacci sequence (2, 3, 5, 8, 13...) until reaching max_period.
+    # This creates an efficient schedule that mimics natural information decay patterns.
+    def calculate_date_ranges(days: nil, everytime: nil, max_period: nil, today: nil)
+      _calculate_date_ranges(
+        Integer(days || self.class.default_days),
+        Integer(everytime || self.class.default_everytime),
+        Integer(max_period || self.class.default_max_period),
+        today || Time.now(in: '+09:30').to_date
+      )
+    end
+
+    private
+
+    def _calculate_date_ranges(days, everytime, max_period, today)
+      @max_period_used = 1
+      to_date = today
+      valid_periods = PERIODS.select { |p| p <= max_period }
+      if !max_period.positive? || !days.positive?
+        return []
+      elsif valid_periods.empty? || everytime >= days
+        # cover everything everytime
+        return [[today + 1 - days, today, "everything"]]
+      end
+
+      max_period = valid_periods.max
+
+      run_number = today.to_date.jd
+      ranges = []
+      if everytime.positive?
+        ranges << [to_date + 1 - everytime, to_date, "everytime"]
+        days -= everytime
+        to_date -= everytime
+      end
+
+      periods = valid_periods.dup
+      loop do
+        period = periods.shift
+        break if period.nil? || period >= max_period || !days.positive?
+
+        if DebugUtils.trace?
+          FiberScheduler.log "DEBUG: #{period} day periods started #{(today - to_date).to_i} days in."
+        end
+        period.times do |index|
+          break unless days.positive?
+
+          this_period = [days, period].min
+          break if this_period <= 0
+
+          earliest_from = to_date - days
+          # we are working from the oldest back towards today
+          if run_number % period == index
+            from = to_date - index - (this_period - 1)
+            from = earliest_from if from < earliest_from
+            to = [today, to_date - index].min
+            break if from > to
+
+            @max_period_used = [this_period, @max_period_used].max
+            if ranges.any? && ranges.last[0] <= to + 1 && MERGE_ADJACENT_RANGES
+              # extend adjacent range
+              ranges.last[0] = [from, ranges.last[0]].min
+              ranges.last[2] = "#{period}\##{index},#{ranges.last[2]}"
+            else
+              to = ranges.last[0] - 1 if ranges.any? && to >= ranges.last[0]
+              ranges << [from, to, "#{period}\##{index}"]
+            end
+          end
+          days -= this_period
+          to_date -= this_period
+        end
+      end
+      # remainder of range at max_period, whatever that is
+      if days.positive? && ScraperUtils::DebugUtils.trace?
+        FiberScheduler.log "DEBUG: #{max_period} day periods started #{(today - to_date).to_i} days in."
+      end
+      index = -1
+      while days.positive?
+        index += 1
+        this_period = [days, max_period].min
+        break if this_period <= 0
+
+        earliest_from = to_date - days
+        if (run_number % max_period) == (index % max_period)
+          from = to_date - index - (this_period - 1)
+          from = earliest_from if from < earliest_from
+          to = to_date - index
+          break if from > to
+
+          @max_period_used = [this_period, @max_period_used].max
+          if ranges.any? && ranges.last[0] <= to + 1 && MERGE_ADJACENT_RANGES
+            # extend adjacent range
+            ranges.last[0] = [from, ranges.last[0]].min
+            ranges.last[2] = "#{this_period}\##{index},#{ranges.last[2]}"
+          else
+            to = ranges.last[0] - 1 if ranges.any? && to >= ranges.last[0]
+            ranges << [from, to, "#{this_period}\##{index}"]
+          end
+        end
+        days -= this_period
+        to_date -= this_period
+      end
+      ranges.reverse
+    end
+  end
+end

data/lib/scraper_utils/db_utils.rb

@@ -33,8 +33,6 @@ module ScraperUtils
                     else
                       ["council_reference"]
                     end
-
-
       ScraperWiki.save_sqlite(primary_key, record)
       ScraperUtils::DataQualityMonitor.log_saved_record(record)
     end

data/lib/scraper_utils/debug_utils.rb

@@ -5,6 +5,50 @@ require "json"
 module ScraperUtils
   # Utilities for debugging web scraping processes
   module DebugUtils
+    DEBUG_ENV_VAR = "DEBUG"
+    MORPH_DEBUG_ENV_VAR = "MORPH_DEBUG"
+
+    # Debug level constants
+    DISABLED_LEVEL = 0
+    BASIC_LEVEL = 1
+    VERBOSE_LEVEL = 2
+    TRACE_LEVEL = 3
+
+    # Get current debug level (0 = disabled, 1 = basic, 2 = verbose, 3 = trace)
+    # Checks DEBUG and MORPH_DEBUG env variables
+    # @return [Integer] Debug level
+    def self.debug_level
+      debug = ENV.fetch(DEBUG_ENV_VAR, ENV.fetch(MORPH_DEBUG_ENV_VAR, '0'))
+      debug =~ /^\d/ ? debug.to_i : BASIC_LEVEL
+    end
+
+    # Check if debug is enabled at specified level or higher
+    #
+    # @param level [Integer] Minimum debug level to check for
+    # @return [Boolean] true if debugging at specified level is enabled
+    def self.debug?(level = BASIC_LEVEL)
+      debug_level >= level
+    end
+
+    # Check if basic debug output or higher is enabled
+    # @return [Boolean] true if debugging is enabled
+    def self.basic?
+      debug?(BASIC_LEVEL)
+    end
+
+    # Check if verbose debug output or higher is enabled
+    # @return [Boolean] true if verbose debugging is enabled
+    def self.verbose?
+      debug?(VERBOSE_LEVEL)
+    end
+
+    # Check if debug tracing or higher is enabled
+    # @return [Boolean] true if debugging is enabled at trace level
+    def self.trace?
+      debug?(TRACE_LEVEL)
+    end
+
+
     # Logs details of an HTTP request when debug mode is enabled
     #
     # @param method [String] HTTP method (GET, POST, etc.)
@@ -14,13 +58,14 @@ module ScraperUtils
     # @param body [Hash, nil] Optional request body
     # @return [void]
     def self.debug_request(method, url, parameters: nil, headers: nil, body: nil)
-      return unless ScraperUtils.debug?
+      return unless basic?
 
       puts
-      ScraperUtils::FiberScheduler.log "🔍 #{method.upcase} #{url}"
+      FiberScheduler.log "🔍 #{method.upcase} #{url}"
       puts "Parameters:", JSON.pretty_generate(parameters) if parameters
       puts "Headers:", JSON.pretty_generate(headers) if headers
       puts "Body:", JSON.pretty_generate(body) if body
+      $stdout.flush
     end
 
     # Logs details of a web page when debug mode is enabled
@@ -29,10 +74,10 @@ module ScraperUtils
     # @param message [String] Context or description for the debug output
     # @return [void]
     def self.debug_page(page, message)
-      return unless ScraperUtils.debug?
+      return unless trace?
 
       puts
-      ScraperUtils::FiberScheduler.log "🔍 DEBUG: #{message}"
+      FiberScheduler.log "🔍 DEBUG: #{message}"
       puts "Current URL: #{page.uri}"
       puts "Page title: #{page.at('title').text.strip}" if page.at("title")
       puts "",
@@ -40,6 +85,7 @@ module ScraperUtils
            "-" * 40,
            page.body,
            "-" * 40
+      $stdout.flush
     end
 
     # Logs details about a specific page selector when debug mode is enabled
@@ -49,10 +95,10 @@ module ScraperUtils
     # @param message [String] Context or description for the debug output
     # @return [void]
     def self.debug_selector(page, selector, message)
-      return unless ScraperUtils.debug?
+      return unless trace?
 
       puts
-      ScraperUtils::FiberScheduler.log "🔍 DEBUG: #{message}"
+      FiberScheduler.log "🔍 DEBUG: #{message}"
       puts "Looking for selector: #{selector}"
       element = page.at(selector)
       if element
@@ -64,6 +110,7 @@ module ScraperUtils
         puts page.body
         puts "-" * 40
       end
+      $stdout.flush
     end
   end
 end

data/lib/scraper_utils/fiber_scheduler.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'fiber'
+require "fiber"
 
 module ScraperUtils
   # A utility module for interleaving multiple scraping operations
@@ -24,6 +24,7 @@ module ScraperUtils
     # @return [String, nil] the authority name or nil if not in a fiber
     def self.current_authority
       return nil unless in_fiber?
+
       Fiber.current.instance_variable_get(:@authority)
     end
 
@@ -33,11 +34,13 @@ module ScraperUtils
     # @return [void]
     def self.log(message)
       authority = current_authority
+      $stderr.flush
       if authority
         puts "[#{authority}] #{message}"
       else
         puts message
       end
+      $stdout.flush
     end
 
     # Returns a hash of exceptions encountered during processing, indexed by authority
@@ -47,7 +50,7 @@ module ScraperUtils
       @exceptions ||= {}
     end
 
-    # Returns a hash of values which will be the values yielded along the way then the block value when it completes
+    # Returns a hash of the yielded / block values
     #
     # @return [Hash{Symbol => Any}] values by authority
     def self.values
@@ -76,7 +79,7 @@ module ScraperUtils
       @enabled = false
     end
 
-    # Resets the scheduler state, and disables the scheduler. Use this before retrying failed authorities.
+    # Resets the scheduler state, and disables. Use before retrying failed authorities.
     #
     # @return [void]
     def self.reset!
@@ -94,21 +97,19 @@ module ScraperUtils
     #
     # @param authority [String] the name of the authority being processed
     # @yield to the block containing the scraping operation to be run in the fiber
-    # @return [Fiber] the created fiber that calls the block. With @authority and @resume_at instance variables
+    # @return [Fiber] a fiber that calls the block. With @authority and @resume_at instance vars
     def self.register_operation(authority, &block)
       # Automatically enable fiber scheduling when operations are registered
       enable!
 
       fiber = Fiber.new do
-        begin
-          values[authority] = block.call
-        rescue StandardError => e
-          # Store exception against the authority
-          exceptions[authority] = e
-        ensure
-          # Remove itself when done regardless of success/failure
-          registry.delete(Fiber.current)
-        end
+        values[authority] = block.call
+      rescue StandardError => e
+        # Store exception against the authority
+        exceptions[authority] = e
+      ensure
+        # Remove itself when done regardless of success/failure
+        registry.delete(Fiber.current)
       end
 
       # Start fibres in registration order
@@ -117,9 +118,11 @@ module ScraperUtils
       fiber.instance_variable_set(:@authority, authority)
       registry << fiber
 
-      puts "Registered #{authority} operation with fiber: #{fiber.object_id} for interleaving" if ENV['DEBUG']
-      # Important: Don't immediately resume the fiber here
-      # Let the caller decide when to start or coordinate fibers
+      if DebugUtils.basic?
+        FiberScheduler.log "Registered #{authority} operation with fiber: #{fiber.object_id} for interleaving"
+      end
+      # Process immediately when testing
+      fiber.resume if ScraperUtils::RandomizeUtils.sequential?
       fiber
     end
 
@@ -130,19 +133,28 @@ module ScraperUtils
       count = registry.size
       while (fiber = find_earliest_fiber)
         if fiber.alive?
-          authority = fiber.instance_variable_get(:@authority) rescue nil
+          authority = begin
+            fiber.instance_variable_get(:@authority)
+          rescue StandardError
+            nil
+          end
           @resume_count ||= 0
           @resume_count += 1
           values[authority] = fiber.resume
         else
-          puts "WARNING: fiber is dead but did not remove itself from registry! #{fiber.object_id}"
+          FiberScheduler.log "WARNING: fiber is dead but did not remove itself from registry! #{fiber.object_id}"
           registry.delete(fiber)
         end
       end
 
-      percent_slept = (100.0 * @time_slept / @delay_requested).round(1) if @time_slept&.positive? && @delay_requested&.positive?
-      puts "FiberScheduler processed #{@resume_count} calls to delay for #{count} registrations, sleeping " \
-             "#{percent_slept}% (#{@time_slept&.round(1)}) of the #{@delay_requested&.round(1)} seconds requested."
+      if @time_slept&.positive? && @delay_requested&.positive?
+        percent_slept = (100.0 * @time_slept / @delay_requested).round(1)
+      end
+      puts
+      FiberScheduler.log "FiberScheduler processed #{@resume_count} calls to delay for #{count} registrations, " \
+           "sleeping #{percent_slept}% (#{@time_slept&.round(1)}) of the " \
+           "#{@delay_requested&.round(1)} seconds requested."
+      puts
 
       exceptions
     end
@@ -162,10 +174,20 @@ module ScraperUtils
       if !enabled? || !current_fiber || registry.size <= 1
         @time_slept ||= 0.0
         @time_slept += seconds
+        log("Sleeping #{seconds.round(3)} seconds") if DebugUtils.basic?
         return sleep(seconds)
       end
 
-      resume_at = Time.now + seconds
+      now = Time.now
+      resume_at = now + seconds
+
+      # Don't resume at the same time as someone else,
+      # FIFO queue if seconds == 0
+      @other_resumes ||= []
+      @other_resumes = @other_resumes.delete_if { |t| t < now }
+      while @other_resumes.include?(resume_at) && resume_at
+        resume_at += 0.01
+      end
 
       # Used to compare when other fibers need to be resumed
       current_fiber.instance_variable_set(:@resume_at, resume_at)
@@ -178,6 +200,7 @@ module ScraperUtils
       if remaining.positive?
         @time_slept ||= 0.0
         @time_slept += remaining
+        log("Sleeping remaining #{remaining.round(3)} seconds") if DebugUtils.basic?
         sleep(remaining)
       end || 0
     end