scraper_utils 0.2.0 → 0.3.0

data/lib/scraper_utils/data_quality_monitor.rb

@@ -6,8 +6,8 @@ module ScraperUtils
   class DataQualityMonitor
     # Get the statistics for all authorities
     # @return [Hash, nil] Hash of statistics per authority or nil if none started
-    def self.stats
-      @stats
+    class << self
+      attr_reader :stats
     end
 
     # Notes the start of processing an authority and clears any previous stats
@@ -16,28 +16,39 @@ module ScraperUtils
     # @return [void]
     def self.start_authority(authority_label)
       @stats ||= {}
-      @authority_label = authority_label
-      @stats[@authority_label] = { saved: 0, unprocessed: 0}
+      @stats[authority_label] = { saved: 0, unprocessed: 0 }
     end
 
-    def self.threshold
-      5.01 + @stats[@authority_label][:saved] * 0.1 if @stats&.fetch(@authority_label, nil)
+    # Extracts authority label and ensures stats are setup for record
+    def self.extract_authority(record)
+      authority_label = (record&.key?("authority_label") ? record["authority_label"] : "").to_sym
+      @stats ||= {}
+      @stats[authority_label] ||= { saved: 0, unprocessed: 0 }
+      authority_label
+    end
+
+    def self.threshold(authority_label)
+      5.01 + (@stats[authority_label][:saved] * 0.1) if @stats&.fetch(authority_label, nil)
     end
 
     # Logs an unprocessable record and raises an exception if error threshold is exceeded
     # The threshold is 5 + 10% of saved records
     #
-    # @param e [Exception] The exception that caused the record to be unprocessable
+    # @param exception [Exception] The exception that caused the record to be unprocessable
     # @param record [Hash, nil] The record that couldn't be processed
     # @raise [ScraperUtils::UnprocessableSite] When too many records are unprocessable
     # @return [void]
-    def self.log_unprocessable_record(e, record)
-      start_authority(:"") unless @stats
-      @stats[@authority_label][:unprocessed] += 1
-      ScraperUtils::FiberScheduler.log "Erroneous record #{@authority_label} - #{record&.fetch('address', nil) || record.inspect}: #{e}"
-      if @stats[@authority_label][:unprocessed] > threshold
-        raise ScraperUtils::UnprocessableSite, "Too many unprocessable_records for #{@authority_label}: #{@stats[@authority_label].inspect} - aborting processing of site!"
-      end
+    def self.log_unprocessable_record(exception, record)
+      authority_label = extract_authority(record)
+      @stats[authority_label][:unprocessed] += 1
+      ScraperUtils::FiberScheduler.log "Erroneous record #{authority_label} - #{record&.fetch(
+        'address', nil
+      ) || record.inspect}: #{exception}"
+      return unless @stats[authority_label][:unprocessed] > threshold(authority_label)
+
+      raise ScraperUtils::UnprocessableSite,
+            "Too many unprocessable_records for #{authority_label}: " \
+            "#{@stats[authority_label].inspect} - aborting processing of site!"
     end
 
     # Logs a successfully saved record
@@ -45,9 +56,9 @@ module ScraperUtils
     # @param record [Hash] The record that was saved
     # @return [void]
     def self.log_saved_record(record)
-      start_authority(:"") unless @stats
-      @stats[@authority_label][:saved] += 1
-      ScraperUtils::FiberScheduler.log "Saving record #{@authority_label} - #{record['address']}"
+      authority_label = extract_authority(record)
+      @stats[authority_label][:saved] += 1
+      ScraperUtils::FiberScheduler.log "Saving record #{authority_label} - #{record['address']}"
     end
   end
 end

data/lib/scraper_utils/date_range_utils.rb

@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+
+module ScraperUtils
+  class DateRangeUtils
+    MERGE_ADJACENT_RANGES = true
+    PERIODS = [2, 3, 5, 8].freeze
+
+    class << self
+      # @return [Integer] Default number of days to cover
+      attr_accessor :default_days
+
+      # @return [Integer] Default days to always include in ranges
+      attr_accessor :default_everytime
+
+      # @return [Integer, nil] Default max days between any one date being in a range
+      attr_accessor :default_max_period
+
+      # Configure default settings for all DateRangeUtils instances
+      # @yield [self] Yields self for configuration
+      # @example
+      #   AgentConfig.configure do |config|
+      #     config.default_everytime = 3
+      #     config.default_days = 35
+      #     config.default_max_period = 5
+      #   end
+      # @return [void]
+      def configure
+        yield self if block_given?
+      end
+
+      # Reset all configuration options to their default values
+      # @return [void]
+      def reset_defaults!
+        @default_days = ENV.fetch('MORPH_DAYS', 33).to_i # 33
+        @default_everytime = ENV.fetch('MORPH_EVERYTIME', 4).to_i # 4
+        @default_max_period = ENV.fetch('MORPH_MAX_PERIOD', 3).to_i # 3
+      end
+    end
+
+    # Set defaults on load
+    reset_defaults!
+
+    attr_reader :max_period_used
+    attr_reader :extended_max_period
+
+    # Generates one or more date ranges to check the most recent daily through to checking each max_period
+    # There is a graduated schedule from the latest `everytime` days through to the oldest of `days` dates which is checked each `max_period` days.
+    # @param days [Integer, nil] create ranges that cover the last `days` dates
+    # @param everytime [Integer, nil] Always include the latest `everytime` out of `days` dates
+    # @param max_period [Integer, nil] the last `days` dates must be checked at least every `max_period` days
+    # @param today [Date, nil] overrides the default determination of today at UTC+09:30 (middle of Australia)
+    # @return [Array{[Date, Date, String]}] being from_date, to_date and a comment
+    #
+    # Uses a Fibonacci sequence to create a natural progression of check frequencies.
+    # Newer data is checked more frequently, with periods between checks growing
+    # according to the Fibonacci sequence (2, 3, 5, 8, 13...) until reaching max_period.
+    # This creates an efficient schedule that mimics natural information decay patterns.
+    def calculate_date_ranges(days: nil, everytime: nil, max_period: nil, today: nil)
+      _calculate_date_ranges(
+        Integer(days || self.class.default_days),
+        Integer(everytime || self.class.default_everytime),
+        Integer(max_period || self.class.default_max_period),
+        today || Time.now(in: '+09:30').to_date
+      )
+    end
+
+    private
+
+    def _calculate_date_ranges(days, everytime, max_period, today)
+      @max_period_used = 1
+      to_date = today
+      valid_periods = PERIODS.select { |p| p <= max_period }
+      if !max_period.positive? || !days.positive?
+        return []
+      elsif valid_periods.empty? || everytime >= days
+        # cover everything everytime
+        return [[today + 1 - days, today, "everything"]]
+      end
+
+      max_period = valid_periods.max
+
+      run_number = today.to_date.jd
+      ranges = []
+      if everytime.positive?
+        ranges << [to_date + 1 - everytime, to_date, "everytime"]
+        days -= everytime
+        to_date -= everytime
+      end
+
+      periods = valid_periods.dup
+      loop do
+        period = periods.shift
+        break if period.nil? || period >= max_period || !days.positive?
+
+        if DebugUtils.trace?
+          FiberScheduler.log "DEBUG: #{period} day periods started #{(today - to_date).to_i} days in."
+        end
+        period.times do |index|
+          break unless days.positive?
+
+          this_period = [days, period].min
+          break if this_period <= 0
+
+          earliest_from = to_date - days
+          # we are working from the oldest back towards today
+          if run_number % period == index
+            from = to_date - index - (this_period - 1)
+            from = earliest_from if from < earliest_from
+            to = [today, to_date - index].min
+            break if from > to
+
+            @max_period_used = [this_period, @max_period_used].max
+            if ranges.any? && ranges.last[0] <= to + 1 && MERGE_ADJACENT_RANGES
+              # extend adjacent range
+              ranges.last[0] = [from, ranges.last[0]].min
+              ranges.last[2] = "#{period}\##{index},#{ranges.last[2]}"
+            else
+              to = ranges.last[0] - 1 if ranges.any? && to >= ranges.last[0]
+              ranges << [from, to, "#{period}\##{index}"]
+            end
+          end
+          days -= this_period
+          to_date -= this_period
+        end
+      end
+      # remainder of range at max_period, whatever that is
+      if days.positive? && ScraperUtils::DebugUtils.trace?
+        FiberScheduler.log "DEBUG: #{max_period} day periods started #{(today - to_date).to_i} days in."
+      end
+      index = -1
+      while days.positive?
+        index += 1
+        this_period = [days, max_period].min
+        break if this_period <= 0
+
+        earliest_from = to_date - days
+        if (run_number % max_period) == (index % max_period)
+          from = to_date - index - (this_period - 1)
+          from = earliest_from if from < earliest_from
+          to = to_date - index
+          break if from > to
+
+          @max_period_used = [this_period, @max_period_used].max
+          if ranges.any? && ranges.last[0] <= to + 1 && MERGE_ADJACENT_RANGES
+            # extend adjacent range
+            ranges.last[0] = [from, ranges.last[0]].min
+            ranges.last[2] = "#{this_period}\##{index},#{ranges.last[2]}"
+          else
+            to = ranges.last[0] - 1 if ranges.any? && to >= ranges.last[0]
+            ranges << [from, to, "#{this_period}\##{index}"]
+          end
+        end
+        days -= this_period
+        to_date -= this_period
+      end
+      ranges.reverse
+    end
+  end
+end

data/lib/scraper_utils/db_utils.rb

@@ -33,8 +33,6 @@ module ScraperUtils
                     else
                       ["council_reference"]
                     end
-
-
       ScraperWiki.save_sqlite(primary_key, record)
       ScraperUtils::DataQualityMonitor.log_saved_record(record)
     end

data/lib/scraper_utils/debug_utils.rb

@@ -5,6 +5,50 @@ require "json"
 module ScraperUtils
   # Utilities for debugging web scraping processes
   module DebugUtils
+    DEBUG_ENV_VAR = "DEBUG"
+    MORPH_DEBUG_ENV_VAR = "MORPH_DEBUG"
+
+    # Debug level constants
+    DISABLED_LEVEL = 0
+    BASIC_LEVEL = 1
+    VERBOSE_LEVEL = 2
+    TRACE_LEVEL = 3
+
+    # Get current debug level (0 = disabled, 1 = basic, 2 = verbose, 3 = trace)
+    # Checks DEBUG and MORPH_DEBUG env variables
+    # @return [Integer] Debug level
+    def self.debug_level
+      debug = ENV.fetch(DEBUG_ENV_VAR, ENV.fetch(MORPH_DEBUG_ENV_VAR, '0'))
+      debug =~ /^\d/ ? debug.to_i : BASIC_LEVEL
+    end
+
+    # Check if debug is enabled at specified level or higher
+    #
+    # @param level [Integer] Minimum debug level to check for
+    # @return [Boolean] true if debugging at specified level is enabled
+    def self.debug?(level = BASIC_LEVEL)
+      debug_level >= level
+    end
+
+    # Check if basic debug output or higher is enabled
+    # @return [Boolean] true if debugging is enabled
+    def self.basic?
+      debug?(BASIC_LEVEL)
+    end
+
+    # Check if verbose debug output or higher is enabled
+    # @return [Boolean] true if verbose debugging is enabled
+    def self.verbose?
+      debug?(VERBOSE_LEVEL)
+    end
+
+    # Check if debug tracing or higher is enabled
+    # @return [Boolean] true if debugging is enabled at trace level
+    def self.trace?
+      debug?(TRACE_LEVEL)
+    end
+
+
     # Logs details of an HTTP request when debug mode is enabled
     #
     # @param method [String] HTTP method (GET, POST, etc.)
@@ -14,13 +58,14 @@ module ScraperUtils
     # @param body [Hash, nil] Optional request body
     # @return [void]
     def self.debug_request(method, url, parameters: nil, headers: nil, body: nil)
-      return unless ScraperUtils.debug?
+      return unless basic?
 
       puts
-      ScraperUtils::FiberScheduler.log "🔍 #{method.upcase} #{url}"
+      FiberScheduler.log "🔍 #{method.upcase} #{url}"
       puts "Parameters:", JSON.pretty_generate(parameters) if parameters
       puts "Headers:", JSON.pretty_generate(headers) if headers
       puts "Body:", JSON.pretty_generate(body) if body
+      $stdout.flush
     end
 
     # Logs details of a web page when debug mode is enabled
@@ -29,10 +74,10 @@ module ScraperUtils
     # @param message [String] Context or description for the debug output
     # @return [void]
     def self.debug_page(page, message)
-      return unless ScraperUtils.debug?
+      return unless trace?
 
       puts
-      ScraperUtils::FiberScheduler.log "🔍 DEBUG: #{message}"
+      FiberScheduler.log "🔍 DEBUG: #{message}"
       puts "Current URL: #{page.uri}"
       puts "Page title: #{page.at('title').text.strip}" if page.at("title")
       puts "",
@@ -40,6 +85,7 @@ module ScraperUtils
            "-" * 40,
            page.body,
            "-" * 40
+      $stdout.flush
     end
 
     # Logs details about a specific page selector when debug mode is enabled
@@ -49,10 +95,10 @@ module ScraperUtils
     # @param message [String] Context or description for the debug output
     # @return [void]
     def self.debug_selector(page, selector, message)
-      return unless ScraperUtils.debug?
+      return unless trace?
 
       puts
-      ScraperUtils::FiberScheduler.log "🔍 DEBUG: #{message}"
+      FiberScheduler.log "🔍 DEBUG: #{message}"
       puts "Looking for selector: #{selector}"
       element = page.at(selector)
       if element
@@ -64,6 +110,7 @@ module ScraperUtils
         puts page.body
         puts "-" * 40
       end
+      $stdout.flush
     end
   end
 end

data/lib/scraper_utils/fiber_scheduler.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'fiber'
+require "fiber"
 
 module ScraperUtils
   # A utility module for interleaving multiple scraping operations
@@ -24,6 +24,7 @@ module ScraperUtils
     # @return [String, nil] the authority name or nil if not in a fiber
     def self.current_authority
       return nil unless in_fiber?
+
       Fiber.current.instance_variable_get(:@authority)
     end
 
@@ -33,11 +34,13 @@ module ScraperUtils
     # @return [void]
     def self.log(message)
       authority = current_authority
+      $stderr.flush
       if authority
         puts "[#{authority}] #{message}"
       else
         puts message
       end
+      $stdout.flush
     end
 
     # Returns a hash of exceptions encountered during processing, indexed by authority
@@ -47,7 +50,7 @@ module ScraperUtils
       @exceptions ||= {}
     end
 
-    # Returns a hash of values which will be the values yielded along the way then the block value when it completes
+    # Returns a hash of the yielded / block values
     #
     # @return [Hash{Symbol => Any}] values by authority
     def self.values
@@ -76,7 +79,7 @@ module ScraperUtils
       @enabled = false
     end
 
-    # Resets the scheduler state, and disables the scheduler. Use this before retrying failed authorities.
+    # Resets the scheduler state, and disables. Use before retrying failed authorities.
     #
     # @return [void]
     def self.reset!
@@ -94,21 +97,19 @@ module ScraperUtils
     #
     # @param authority [String] the name of the authority being processed
     # @yield to the block containing the scraping operation to be run in the fiber
-    # @return [Fiber] the created fiber that calls the block. With @authority and @resume_at instance variables
+    # @return [Fiber] a fiber that calls the block. With @authority and @resume_at instance vars
     def self.register_operation(authority, &block)
       # Automatically enable fiber scheduling when operations are registered
       enable!
 
       fiber = Fiber.new do
-        begin
-          values[authority] = block.call
-        rescue StandardError => e
-          # Store exception against the authority
-          exceptions[authority] = e
-        ensure
-          # Remove itself when done regardless of success/failure
-          registry.delete(Fiber.current)
-        end
+        values[authority] = block.call
+      rescue StandardError => e
+        # Store exception against the authority
+        exceptions[authority] = e
+      ensure
+        # Remove itself when done regardless of success/failure
+        registry.delete(Fiber.current)
       end
 
       # Start fibres in registration order
@@ -117,9 +118,11 @@ module ScraperUtils
       fiber.instance_variable_set(:@authority, authority)
       registry << fiber
 
-      puts "Registered #{authority} operation with fiber: #{fiber.object_id} for interleaving" if ENV['DEBUG']
-      # Important: Don't immediately resume the fiber here
-      # Let the caller decide when to start or coordinate fibers
+      if DebugUtils.basic?
+        FiberScheduler.log "Registered #{authority} operation with fiber: #{fiber.object_id} for interleaving"
+      end
+      # Process immediately when testing
+      fiber.resume if ScraperUtils::RandomizeUtils.sequential?
       fiber
     end
 
@@ -130,19 +133,28 @@ module ScraperUtils
       count = registry.size
       while (fiber = find_earliest_fiber)
         if fiber.alive?
-          authority = fiber.instance_variable_get(:@authority) rescue nil
+          authority = begin
+            fiber.instance_variable_get(:@authority)
+          rescue StandardError
+            nil
+          end
           @resume_count ||= 0
           @resume_count += 1
           values[authority] = fiber.resume
         else
-          puts "WARNING: fiber is dead but did not remove itself from registry! #{fiber.object_id}"
+          FiberScheduler.log "WARNING: fiber is dead but did not remove itself from registry! #{fiber.object_id}"
           registry.delete(fiber)
         end
       end
 
-      percent_slept = (100.0 * @time_slept / @delay_requested).round(1) if @time_slept&.positive? && @delay_requested&.positive?
-      puts "FiberScheduler processed #{@resume_count} calls to delay for #{count} registrations, sleeping " \
-             "#{percent_slept}% (#{@time_slept&.round(1)}) of the #{@delay_requested&.round(1)} seconds requested."
+      if @time_slept&.positive? && @delay_requested&.positive?
+        percent_slept = (100.0 * @time_slept / @delay_requested).round(1)
+      end
+      puts
+      FiberScheduler.log "FiberScheduler processed #{@resume_count} calls to delay for #{count} registrations, " \
+           "sleeping #{percent_slept}% (#{@time_slept&.round(1)}) of the " \
+           "#{@delay_requested&.round(1)} seconds requested."
+      puts
 
       exceptions
     end
@@ -162,10 +174,20 @@ module ScraperUtils
       if !enabled? || !current_fiber || registry.size <= 1
         @time_slept ||= 0.0
         @time_slept += seconds
+        log("Sleeping #{seconds.round(3)} seconds") if DebugUtils.basic?
         return sleep(seconds)
       end
 
-      resume_at = Time.now + seconds
+      now = Time.now
+      resume_at = now + seconds
+
+      # Don't resume at the same time as someone else,
+      # FIFO queue if seconds == 0
+      @other_resumes ||= []
+      @other_resumes = @other_resumes.delete_if { |t| t < now }
+      while @other_resumes.include?(resume_at) && resume_at
+        resume_at += 0.01
+      end
 
       # Used to compare when other fibers need to be resumed
       current_fiber.instance_variable_set(:@resume_at, resume_at)
@@ -178,6 +200,7 @@ module ScraperUtils
       if remaining.positive?
         @time_slept ||= 0.0
         @time_slept += remaining
+        log("Sleeping remaining #{remaining.round(3)} seconds") if DebugUtils.basic?
         sleep(remaining)
       end || 0
     end

data/lib/scraper_utils/log_utils.rb

@@ -13,8 +13,8 @@ module ScraperUtils
     # @param start_time [Time] When this scraping attempt was started
     # @param attempt [Integer] 1 for first run, 2 for first retry, 3 for last retry (without proxy)
     # @param authorities [Array<Symbol>] List of authorities attempted to scrape
-    # @param exceptions [Hash > Exception] Any exception that occurred during scraping
-    # DataQualityMonitor.stats is checked for :saved and :unprocessed entries
+    # @param exceptions [Hash{Symbol => Exception}] Any exceptions that occurred during scraping
+    # `DataQualityMonitor.stats` is checked for :saved and :unprocessed entries
     # @return [void]
     def self.log_scraping_run(start_time, attempt, authorities, exceptions)
       raise ArgumentError, "Invalid start time" unless start_time.is_a?(Time)
@@ -75,39 +75,39 @@ module ScraperUtils
 
     # Report on the results
     # @param authorities [Array<Symbol>] List of authorities attempted to scrape
-    # @param exceptions [Hash > Exception] Any exception that occurred during scraping
-    # DataQualityMonitor.stats is checked for :saved and :unprocessed entries
+    # @param exceptions [Hash{Symbol => Exception}] Any exceptions that occurred during scraping
+    # `DataQualityMonitor.stats` is checked for :saved and :unprocessed entries
     # @return [void]
     def self.report_on_results(authorities, exceptions)
-      expect_bad = ENV["MORPH_EXPECT_BAD"]&.split(",")&.map(&:strip)&.map(&:to_sym) || []
+      if ENV["MORPH_EXPECT_BAD"]
+        expect_bad = ENV["MORPH_EXPECT_BAD"].split(",").map(&:strip).map(&:to_sym)
+      end
+      expect_bad ||= []
 
-      puts "MORPH_EXPECT_BAD=#{ENV.fetch('MORPH_EXPECT_BAD', nil)}" if expect_bad.any?
+      $stderr.flush
+      puts "MORPH_EXPECT_BAD=#{ENV.fetch('MORPH_EXPECT_BAD', nil)}"
 
       # Print summary table
       puts "\nScraping Summary:"
       summary_format = "%-20s %6s %6s %s"
 
-      puts summary_format % %w[Authority OK Bad Exception]
-      puts summary_format % ['-' * 20, '-' * 6, '-' * 6, '-' * 50]
+      puts format(summary_format, 'Authority', 'OK', 'Bad', 'Exception')
+      puts format(summary_format, "-" * 20, "-" * 6, "-" * 6, "-" * 50)
 
       authorities.each do |authority|
         stats = ScraperUtils::DataQualityMonitor.stats&.fetch(authority, {}) || {}
-    
+
         ok_records = stats[:saved] || 0
         bad_records = stats[:unprocessed] || 0
-      
+
         expect_bad_prefix = expect_bad.include?(authority) ? "[EXPECT BAD] " : ""
         exception_msg = if exceptions[authority]
                           "#{exceptions[authority].class} - #{exceptions[authority].message}"
                         else
                           "-"
                         end
-        puts summary_format % [
-          authority.to_s, 
-          ok_records, 
-          bad_records,
-          "#{expect_bad_prefix}#{exception_msg}".slice(0, 70)
-        ]
+        puts format(summary_format, authority.to_s, ok_records, bad_records,
+                    "#{expect_bad_prefix}#{exception_msg}".slice(0, 70))
       end
       puts
 
@@ -120,7 +120,8 @@ module ScraperUtils
       end
 
       if unexpected_working.any?
-        errors << "WARNING: Remove #{unexpected_working.join(',')} from MORPH_EXPECT_BAD as it now works!"
+        errors <<
+          "WARNING: Remove #{unexpected_working.join(',')} from MORPH_EXPECT_BAD as it now works!"
       end
 
       # Check for authorities with unexpected errors
@@ -137,6 +138,7 @@ module ScraperUtils
         end
       end
 
+      $stdout.flush
       if errors.any?
         errors << "See earlier output for details"
         raise errors.join("\n")