scraper_utils 0.1.0 → 0.2.0

data/lib/scraper_utils/adaptive_delay.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "uri"
+
+# Adapts delays between requests based on server response times.
+# Target delay is proportional to response time based on max_load setting.
+# Uses an exponential moving average to smooth variations in response times.
+class AdaptiveDelay
+  DEFAULT_MIN_DELAY = 0.0
+  DEFAULT_MAX_DELAY = 30.0 # Presumed default timeout for Mechanize
+
+  attr_reader :min_delay, :max_delay, :max_load
+
+  # Creates a new adaptive delay calculator
+  #
+  # @param min_delay [Float] Minimum delay between requests in seconds
+  # @param max_delay [Float] Maximum delay between requests in seconds
+  # @param max_load [Float] Maximum load percentage (1-99) we aim to place on the server
+  #                         Lower values are more conservative (e.g., 20% = 4x response time delay)
+  def initialize(min_delay: DEFAULT_MIN_DELAY, max_delay: DEFAULT_MAX_DELAY, max_load: 20.0)
+    @delays = {} # domain -> last delay used
+    @min_delay = min_delay.to_f
+    @max_delay = max_delay.to_f
+    @max_load = max_load.to_f.clamp(1.0, 99.0)
+    @response_multiplier = (100.0 - @max_load) / @max_load
+
+    if ENV["DEBUG"]
+      ScraperUtils::FiberScheduler.log "AdaptiveDelay initialized with delays between #{@min_delay} and #{@max_delay} seconds"
+      ScraperUtils::FiberScheduler.log "Using max_load of #{@max_load}% (response time multiplier: #{@response_multiplier.round(2)}x)"
+    end
+  end
+
+  # @param uri [URI::Generic, String] The URL to extract the domain from
+  # @return [String] The domain in the format "scheme://host"
+  def domain(uri)
+    uri = URI(uri) unless uri.is_a?(URI)
+    "#{uri.scheme}://#{uri.host}".downcase
+  end
+
+  # @param uri [URI::Generic, String] URL to get delay for
+  # @return [Float] Current delay for the domain, or min_delay if no delay set
+  def delay(uri)
+    @delays[domain(uri)] || @min_delay
+  end
+
+  # @param uri [URI::Generic, String] URL the response came from
+  # @param response_time [Float] Time in seconds the server took to respond
+  # @return [Float] The calculated delay to use with the next request
+  def next_delay(uri, response_time)
+    uris_domain = domain(uri)
+    target_delay = (response_time * @response_multiplier).clamp(0.0, @max_delay)
+    current_delay = @delays[uris_domain] || target_delay
+    delay = ((9.0 * current_delay) + target_delay) / 10.0
+    delay = delay.clamp(@min_delay, @max_delay)
+
+    if ENV["DEBUG"]
+      ScraperUtils::FiberScheduler.log "Adaptive delay for #{uris_domain} updated to " \
+             "#{delay.round(2)}s (target: #{@response_multiplier.round(1)}x " \
+             "response_time of #{response_time.round(2)}s)"
+    end
+
+    @delays[uris_domain] = delay
+    delay
+  end
+end

data/lib/scraper_utils/authority_utils.rb

@@ -3,13 +3,13 @@
 module ScraperUtils
   # Utilities for managing and selecting authorities
   module AuthorityUtils
+    AUTHORITIES_ENV_VAR = "MORPH_AUTHORITIES"
+
     # Selects authorities based on environment variable or returns all authorities
     #
     # @param all_authorities [Array<Symbol>] Full list of available authorities
     # @return [Array<Symbol>] Selected subset of authorities or all authorities
     # @raise [ScraperUtils::Error] If invalid authorities are specified in MORPH_AUTHORITIES
-    AUTHORITIES_ENV_VAR = "MORPH_AUTHORITIES"
-
     def self.selected_authorities(all_authorities)
       if ENV[AUTHORITIES_ENV_VAR]
         authorities = ENV[AUTHORITIES_ENV_VAR].split(",").map(&:strip).map(&:to_sym)

data/lib/scraper_utils/data_quality_monitor.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module ScraperUtils
+  # Monitors data quality during scraping by tracking successful vs failed record processing
+  # Automatically triggers an exception if the error rate exceeds a threshold
+  class DataQualityMonitor
+    # Get the statistics for all authorities
+    # @return [Hash, nil] Hash of statistics per authority or nil if none started
+    def self.stats
+      @stats
+    end
+
+    # Notes the start of processing an authority and clears any previous stats
+    #
+    # @param authority_label [Symbol] The authority we are processing
+    # @return [void]
+    def self.start_authority(authority_label)
+      @stats ||= {}
+      @authority_label = authority_label
+      @stats[@authority_label] = { saved: 0, unprocessed: 0}
+    end
+
+    def self.threshold
+      5.01 + @stats[@authority_label][:saved] * 0.1 if @stats&.fetch(@authority_label, nil)
+    end
+
+    # Logs an unprocessable record and raises an exception if error threshold is exceeded
+    # The threshold is 5 + 10% of saved records
+    #
+    # @param e [Exception] The exception that caused the record to be unprocessable
+    # @param record [Hash, nil] The record that couldn't be processed
+    # @raise [ScraperUtils::UnprocessableSite] When too many records are unprocessable
+    # @return [void]
+    def self.log_unprocessable_record(e, record)
+      start_authority(:"") unless @stats
+      @stats[@authority_label][:unprocessed] += 1
+      ScraperUtils::FiberScheduler.log "Erroneous record #{@authority_label} - #{record&.fetch('address', nil) || record.inspect}: #{e}"
+      if @stats[@authority_label][:unprocessed] > threshold
+        raise ScraperUtils::UnprocessableSite, "Too many unprocessable_records for #{@authority_label}: #{@stats[@authority_label].inspect} - aborting processing of site!"
+      end
+    end
+
+    # Logs a successfully saved record
+    #
+    # @param record [Hash] The record that was saved
+    # @return [void]
+    def self.log_saved_record(record)
+      start_authority(:"") unless @stats
+      @stats[@authority_label][:saved] += 1
+      ScraperUtils::FiberScheduler.log "Saving record #{@authority_label} - #{record['address']}"
+    end
+  end
+end

data/lib/scraper_utils/db_utils.rb

@@ -34,8 +34,9 @@ module ScraperUtils
                       ["council_reference"]
                     end
 
-      puts "Saving record #{record['council_reference']} - #{record['address']}"
+
       ScraperWiki.save_sqlite(primary_key, record)
+      ScraperUtils::DataQualityMonitor.log_saved_record(record)
     end
   end
 end

data/lib/scraper_utils/debug_utils.rb

@@ -16,19 +16,11 @@ module ScraperUtils
     def self.debug_request(method, url, parameters: nil, headers: nil, body: nil)
       return unless ScraperUtils.debug?
 
-      puts "\n🔍 #{method.upcase} #{url}"
-      if parameters
-        puts "Parameters:"
-        puts JSON.pretty_generate(parameters)
-      end
-      if headers
-        puts "Headers:"
-        puts JSON.pretty_generate(headers)
-      end
-      return unless body
-
-      puts "Body:"
-      puts JSON.pretty_generate(body)
+      puts
+      ScraperUtils::FiberScheduler.log "🔍 #{method.upcase} #{url}"
+      puts "Parameters:", JSON.pretty_generate(parameters) if parameters
+      puts "Headers:", JSON.pretty_generate(headers) if headers
+      puts "Body:", JSON.pretty_generate(body) if body
     end
 
     # Logs details of a web page when debug mode is enabled
@@ -39,15 +31,15 @@ module ScraperUtils
     def self.debug_page(page, message)
       return unless ScraperUtils.debug?
 
-      puts "",
-           "🔍 DEBUG: #{message}"
+      puts
+      ScraperUtils::FiberScheduler.log "🔍 DEBUG: #{message}"
       puts "Current URL: #{page.uri}"
       puts "Page title: #{page.at('title').text.strip}" if page.at("title")
       puts "",
-           "Page content:"
-      puts "-" * 40
-      puts page.body
-      puts "-" * 40
+           "Page content:",
+           "-" * 40,
+           page.body,
+           "-" * 40
     end
 
     # Logs details about a specific page selector when debug mode is enabled
@@ -59,7 +51,8 @@ module ScraperUtils
     def self.debug_selector(page, selector, message)
       return unless ScraperUtils.debug?
 
-      puts "\n🔍 DEBUG: #{message}"
+      puts
+      ScraperUtils::FiberScheduler.log "🔍 DEBUG: #{message}"
       puts "Looking for selector: #{selector}"
       element = page.at(selector)
       if element

data/lib/scraper_utils/fiber_scheduler.rb

@@ -0,0 +1,206 @@
+# frozen_string_literal: true
+
+require 'fiber'
+
+module ScraperUtils
+  # A utility module for interleaving multiple scraping operations
+  # using fibers during connection delay periods. This allows efficient
+  # use of wait time by switching between operations.
+  module FiberScheduler
+    # @return [Array<Fiber>] List of active fibers managed by the scheduler
+    def self.registry
+      @registry ||= []
+    end
+
+    # Checks if the current code is running within a registered fiber
+    #
+    # @return [Boolean] true if running in a registered fiber, false otherwise
+    def self.in_fiber?
+      !Fiber.current.nil? && registry.include?(Fiber.current)
+    end
+
+    # Gets the authority associated with the current fiber
+    #
+    # @return [String, nil] the authority name or nil if not in a fiber
+    def self.current_authority
+      return nil unless in_fiber?
+      Fiber.current.instance_variable_get(:@authority)
+    end
+
+    # Logs a message, automatically prefixing with authority name if in a fiber
+    #
+    # @param message [String] the message to log
+    # @return [void]
+    def self.log(message)
+      authority = current_authority
+      if authority
+        puts "[#{authority}] #{message}"
+      else
+        puts message
+      end
+    end
+
+    # Returns a hash of exceptions encountered during processing, indexed by authority
+    #
+    # @return [Hash{Symbol => Exception}] exceptions by authority
+    def self.exceptions
+      @exceptions ||= {}
+    end
+
+    # Returns a hash of values which will be the values yielded along the way then the block value when it completes
+    #
+    # @return [Hash{Symbol => Any}] values by authority
+    def self.values
+      @values ||= {}
+    end
+
+    # Checks if fiber scheduling is currently enabled
+    #
+    # @return [Boolean] true if enabled, false otherwise
+    def self.enabled?
+      @enabled ||= false
+    end
+
+    # Enables fiber scheduling
+    #
+    # @return [void]
+    def self.enable!
+      reset! unless enabled?
+      @enabled = true
+    end
+
+    # Disables fiber scheduling
+    #
+    # @return [void]
+    def self.disable!
+      @enabled = false
+    end
+
+    # Resets the scheduler state, and disables the scheduler. Use this before retrying failed authorities.
+    #
+    # @return [void]
+    def self.reset!
+      @registry = []
+      @exceptions = {}
+      @values = {}
+      @enabled = false
+      @delay_requested = 0.0
+      @time_slept = 0.0
+      @resume_count = 0
+      @initial_resume_at = Time.now - 60.0 # one minute ago
+    end
+
+    # Registers a block to scrape for a specific authority
+    #
+    # @param authority [String] the name of the authority being processed
+    # @yield to the block containing the scraping operation to be run in the fiber
+    # @return [Fiber] the created fiber that calls the block. With @authority and @resume_at instance variables
+    def self.register_operation(authority, &block)
+      # Automatically enable fiber scheduling when operations are registered
+      enable!
+
+      fiber = Fiber.new do
+        begin
+          values[authority] = block.call
+        rescue StandardError => e
+          # Store exception against the authority
+          exceptions[authority] = e
+        ensure
+          # Remove itself when done regardless of success/failure
+          registry.delete(Fiber.current)
+        end
+      end
+
+      # Start fibres in registration order
+      @initial_resume_at += 0.1
+      fiber.instance_variable_set(:@resume_at, @initial_resume_at)
+      fiber.instance_variable_set(:@authority, authority)
+      registry << fiber
+
+      puts "Registered #{authority} operation with fiber: #{fiber.object_id} for interleaving" if ENV['DEBUG']
+      # Important: Don't immediately resume the fiber here
+      # Let the caller decide when to start or coordinate fibers
+      fiber
+    end
+
+    # Run all registered fibers until completion
+    #
+    # @return [Hash] Exceptions that occurred during execution
+    def self.run_all
+      count = registry.size
+      while (fiber = find_earliest_fiber)
+        if fiber.alive?
+          authority = fiber.instance_variable_get(:@authority) rescue nil
+          @resume_count ||= 0
+          @resume_count += 1
+          values[authority] = fiber.resume
+        else
+          puts "WARNING: fiber is dead but did not remove itself from registry! #{fiber.object_id}"
+          registry.delete(fiber)
+        end
+      end
+
+      percent_slept = (100.0 * @time_slept / @delay_requested).round(1) if @time_slept&.positive? && @delay_requested&.positive?
+      puts "FiberScheduler processed #{@resume_count} calls to delay for #{count} registrations, sleeping " \
+             "#{percent_slept}% (#{@time_slept&.round(1)}) of the #{@delay_requested&.round(1)} seconds requested."
+
+      exceptions
+    end
+
+    # Delays the current fiber and potentially runs another one
+    # Falls back to regular sleep if fiber scheduling is not enabled
+    #
+    # @param seconds [Numeric] the number of seconds to delay
+    # @return [Integer] return from sleep operation or 0
+    def self.delay(seconds)
+      seconds = 0.0 unless seconds&.positive?
+      @delay_requested ||= 0.0
+      @delay_requested += seconds
+
+      current_fiber = Fiber.current
+
+      if !enabled? || !current_fiber || registry.size <= 1
+        @time_slept ||= 0.0
+        @time_slept += seconds
+        return sleep(seconds)
+      end
+
+      resume_at = Time.now + seconds
+
+      # Used to compare when other fibers need to be resumed
+      current_fiber.instance_variable_set(:@resume_at, resume_at)
+
+      # Yield control back to the scheduler so another fiber can run
+      Fiber.yield
+
+      # When we get control back, check if we need to sleep more
+      remaining = resume_at - Time.now
+      if remaining.positive?
+        @time_slept ||= 0.0
+        @time_slept += remaining
+        sleep(remaining)
+      end || 0
+    end
+
+    # Finds the fiber with the earliest wake-up time
+    #
+    # @return [Fiber, nil] the fiber with the earliest wake-up time or nil if none found
+    def self.find_earliest_fiber
+      earliest_time = nil
+      earliest_fiber = nil
+
+      registry.each do |fiber|
+        resume_at = fiber.instance_variable_get(:@resume_at)
+        if earliest_time.nil? || resume_at < earliest_time
+          earliest_time = resume_at
+          earliest_fiber = fiber
+        end
+      end
+
+      earliest_fiber
+    end
+
+    # Mark methods as private
+    private_class_method :find_earliest_fiber
+  end
+end

data/lib/scraper_utils/log_utils.rb

@@ -13,13 +13,10 @@ module ScraperUtils
     # @param start_time [Time] When this scraping attempt was started
     # @param attempt [Integer] 1 for first run, 2 for first retry, 3 for last retry (without proxy)
     # @param authorities [Array<Symbol>] List of authorities attempted to scrape
-    # @param results [Hash] Results for each authority containing:
-    #   - :records_scraped [Integer] Number of records successfully scraped
-    #   - :unprocessable_records [Integer] Optional Number of unprocessable record like regions
-    #   - :error [Exception, nil] Any exception that occurred during scraping
-    #   - :proxy_used [Boolean] Whether a proxy was used
+    # @param exceptions [Hash > Exception] Any exception that occurred during scraping
+    # DataQualityMonitor.stats is checked for :saved and :unprocessed entries
     # @return [void]
-    def self.log_scraping_run(start_time, attempt, authorities, results)
+    def self.log_scraping_run(start_time, attempt, authorities, exceptions)
       raise ArgumentError, "Invalid start time" unless start_time.is_a?(Time)
       raise ArgumentError, "Authorities must be a non-empty array" if authorities.empty?
 
@@ -31,10 +28,11 @@ module ScraperUtils
       interrupted = []
 
       authorities.each do |authority_label|
-        result = results[authority_label] || {}
+        stats = ScraperUtils::DataQualityMonitor.stats&.fetch(authority_label, nil) || {}
 
-        status = if result[:records_scraped]&.positive?
-                   result[:error] ? :interrupted : :successful
+        exception = exceptions[authority_label]
+        status = if stats[:saved]&.positive?
+                   exception ? :interrupted : :successful
                  else
                    :failed
                  end
@@ -51,13 +49,12 @@ module ScraperUtils
           "run_at" => start_time.iso8601,
           "attempt" => attempt,
           "authority_label" => authority_label.to_s,
-          "records_scraped" => result[:records_scraped] || 0,
-          "unprocessable_records" => result[:unprocessable_records] || 0,
-          "used_proxy" => result[:proxy_used] ? 1 : 0,
+          "records_saved" => stats[:saved] || 0,
+          "unprocessable_records" => stats[:unprocessed] || 0,
           "status" => status.to_s,
-          "error_message" => result[:error]&.message,
-          "error_class" => result[:error]&.class&.to_s,
-          "error_backtrace" => extract_meaningful_backtrace(result[:error])
+          "error_message" => exception&.message,
+          "error_class" => exception&.class&.to_s,
+          "error_backtrace" => extract_meaningful_backtrace(exception)
         }
 
         save_log_record(record)
@@ -76,33 +73,66 @@ module ScraperUtils
       cleanup_old_records
     end
 
-    def self.report_on_results(authorities, results)
-      expect_bad = ENV["MORPH_EXPECT_BAD"]&.split(",")&.map(&:to_sym) || []
-
-      puts "MORPH_EXPECT_BAD=#{ENV['MORPH_EXPECT_BAD']}" if expect_bad.any?
+    # Report on the results
+    # @param authorities [Array<Symbol>] List of authorities attempted to scrape
+    # @param exceptions [Hash > Exception] Any exception that occurred during scraping
+    # DataQualityMonitor.stats is checked for :saved and :unprocessed entries
+    # @return [void]
+    def self.report_on_results(authorities, exceptions)
+      expect_bad = ENV["MORPH_EXPECT_BAD"]&.split(",")&.map(&:strip)&.map(&:to_sym) || []
+
+      puts "MORPH_EXPECT_BAD=#{ENV.fetch('MORPH_EXPECT_BAD', nil)}" if expect_bad.any?
+
+      # Print summary table
+      puts "\nScraping Summary:"
+      summary_format = "%-20s %6s %6s %s"
+
+      puts summary_format % %w[Authority OK Bad Exception]
+      puts summary_format % ['-' * 20, '-' * 6, '-' * 6, '-' * 50]
+
+      authorities.each do |authority|
+        stats = ScraperUtils::DataQualityMonitor.stats&.fetch(authority, {}) || {}
+    
+        ok_records = stats[:saved] || 0
+        bad_records = stats[:unprocessed] || 0
+      
+        expect_bad_prefix = expect_bad.include?(authority) ? "[EXPECT BAD] " : ""
+        exception_msg = if exceptions[authority]
+                          "#{exceptions[authority].class} - #{exceptions[authority].message}"
+                        else
+                          "-"
+                        end
+        puts summary_format % [
+          authority.to_s, 
+          ok_records, 
+          bad_records,
+          "#{expect_bad_prefix}#{exception_msg}".slice(0, 70)
+        ]
+      end
+      puts
 
       errors = []
 
       # Check for authorities that were expected to be bad but are now working
       unexpected_working = expect_bad.select do |authority|
-        result = results[authority]
-        result && result[:records_scraped]&.positive? && result[:error].nil?
+        stats = ScraperUtils::DataQualityMonitor.stats&.fetch(authority, {}) || {}
+        stats[:saved]&.positive? && !exceptions[authority]
       end
 
       if unexpected_working.any?
-        errors << "WARNING: Remove #{unexpected_working.join(',')} from EXPECT_BAD as it now works!"
+        errors << "WARNING: Remove #{unexpected_working.join(',')} from MORPH_EXPECT_BAD as it now works!"
       end
 
       # Check for authorities with unexpected errors
       unexpected_errors = authorities
-                          .select { |authority| results[authority]&.dig(:error) }
+                          .select { |authority| exceptions[authority] }
                           .reject { |authority| expect_bad.include?(authority) }
 
       if unexpected_errors.any?
         errors << "ERROR: Unexpected errors in: #{unexpected_errors.join(',')} " \
-          "(Add to MORPH_EXPECT_BAD?)"
+                  "(Add to MORPH_EXPECT_BAD?)"
         unexpected_errors.each do |authority|
-          error = results[authority][:error]
+          error = exceptions[authority]
           errors << "  #{authority}: #{error.class} - #{error.message}"
         end
       end
@@ -134,7 +164,8 @@ module ScraperUtils
         "interrupted" => interrupted.join(","),
         "successful_count" => successful.size,
         "interrupted_count" => interrupted.size,
-        "failed_count" => failed.size
+        "failed_count" => failed.size,
+        "public_ip" => ScraperUtils::MechanizeUtils.public_ip
       }
 
       ScraperWiki.save_sqlite(