scraper_utils 0.8.2 → 0.8.3

data/lib/scraper_utils/log_utils.rb

@@ -9,12 +9,12 @@ module ScraperUtils
     LOG_TABLE = "scrape_log"
     LOG_RETENTION_DAYS = 30
 
-    # Logs a message, automatically prefixing with authority name if in a fiber
+    # Logs a message, automatically prefixing with authority name if in a sub process
     #
     # @param message [String] the message to log
     # @return [void]
     def self.log(message, authority = nil)
-      authority ||= Scheduler.current_authority
+      authority ||= ENV['AUTHORITY']
       $stderr.flush
       if authority
         puts "[#{authority}] #{message}"
@@ -174,12 +174,12 @@ module ScraperUtils
 
       # Check for authorities with unexpected errors
       unexpected_errors = authorities
-                          .select { |authority| exceptions[authority] }
-                          .reject { |authority| expect_bad.include?(authority) }
+                            .select { |authority| exceptions[authority] }
+                            .reject { |authority| expect_bad.include?(authority) }
 
       if unexpected_errors.any?
         errors << "ERROR: Unexpected errors in: #{unexpected_errors.join(',')} " \
-                  "(Add to MORPH_EXPECT_BAD?)"
+          "(Add to MORPH_EXPECT_BAD?)"
         unexpected_errors.each do |authority|
           error = exceptions[authority]
           errors << "  #{authority}: #{error.class} - #{error}"

data/lib/scraper_utils/maths_utils.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require "scraperwiki"
+
+module ScraperUtils
+  # Misc Maths Utilities
+  module MathsUtils
+    # Generate a fibonacci series
+    # @param max [Integer] The max the sequence goes up to
+    # @return [Array<Integer>] The fibonacci numbers up to max
+    def self.fibonacci_series(max)
+      result = []
+      # Start with the basic Fibonacci sequence
+      last_fib, this_fib = 1, 0
+      while this_fib <= max
+        result << this_fib
+        yield this_fib if block_given?
+        last_fib, this_fib = this_fib, this_fib + last_fib
+      end
+      result
+    end
+  end
+end

data/lib/scraper_utils/mechanize_utils/agent_config.rb

@@ -10,8 +10,7 @@ module ScraperUtils
     #
     # @example Setting global defaults
     #   ScraperUtils::MechanizeUtils::AgentConfig.configure do |config|
-    #     config.default_timeout = 90
-    #     config.default_random_delay = 5
+    #     config.default_timeout = 500
     #   end
     #
     # @example Creating an instance with defaults
@@ -24,24 +23,12 @@ module ScraperUtils
     #   )
     class AgentConfig
       DEFAULT_TIMEOUT = 60
-      DEFAULT_RANDOM_DELAY = 0
-      DEFAULT_MAX_LOAD = 50.0
-      MAX_LOAD_CAP = 80.0
 
       # Class-level defaults that can be modified
       class << self
         # @return [Integer] Default timeout in seconds for agent connections
         attr_accessor :default_timeout
 
-        # @return [Boolean] Default setting for compliance with headers and robots.txt
-        attr_accessor :default_compliant_mode
-
-        # @return [Integer, nil] Default average random delay in seconds
-        attr_accessor :default_random_delay
-
-        # @return [Float, nil] Default maximum server load percentage (nil = no response delay)
-        attr_accessor :default_max_load
-
         # @return [Boolean] Default setting for SSL certificate verification
         attr_accessor :default_disable_ssl_certificate_check
 
@@ -55,9 +42,7 @@ module ScraperUtils
         # @yield [self] Yields self for configuration
         # @example
         #   AgentConfig.configure do |config|
-        #     config.default_timeout = 90
-        #     config.default_random_delay = 5
-        #     config.default_max_load = 15
+        #     config.default_timeout = 300
         #   end
         # @return [void]
         def configure
@@ -68,9 +53,6 @@ module ScraperUtils
         # @return [void]
         def reset_defaults!
           @default_timeout = ENV.fetch('MORPH_CLIENT_TIMEOUT', DEFAULT_TIMEOUT).to_i # 60
-          @default_compliant_mode = ENV.fetch('MORPH_NOT_COMPLIANT', nil).to_s.empty? # true
-          @default_random_delay = ENV.fetch('MORPH_RANDOM_DELAY', DEFAULT_RANDOM_DELAY).to_i # 0
-          @default_max_load = ENV.fetch('MORPH_MAX_LOAD', DEFAULT_MAX_LOAD).to_f # 50.0
           @default_disable_ssl_certificate_check = !ENV.fetch('MORPH_DISABLE_SSL_CHECK', nil).to_s.empty? # false
           @default_australian_proxy = !ENV.fetch('MORPH_USE_PROXY', nil).to_s.empty? # false
           @default_user_agent = ENV.fetch('MORPH_USER_AGENT', nil) # Uses Mechanize user agent
@@ -89,10 +71,6 @@ module ScraperUtils
 
       # Creates Mechanize agent configuration with sensible defaults overridable via configure
       # @param timeout [Integer, nil] Timeout for agent connections (default: 60)
-      # @param compliant_mode [Boolean, nil] Comply with headers and robots.txt (default: true)
-      # @param random_delay [Integer, nil] Average random delay in seconds (default: 3)
-      # @param max_load [Float, nil] Maximum server load percentage (nil = no delay, default: 20%)
-      #                              When compliant_mode is true, max_load is capped at 33%
       # @param disable_ssl_certificate_check [Boolean, nil] Skip SSL verification (default: false)
       # @param australian_proxy [Boolean, nil] Use proxy if available (default: false)
       # @param user_agent [String, nil] Configure Mechanize user agent
@@ -104,10 +82,6 @@ module ScraperUtils
                      australian_proxy: nil,
                      user_agent: nil)
         @timeout = timeout.nil? ? self.class.default_timeout : timeout
-        @compliant_mode = compliant_mode.nil? ? self.class.default_compliant_mode : compliant_mode
-        @random_delay = random_delay.nil? ? self.class.default_random_delay : random_delay
-        @max_load = max_load.nil? ? self.class.default_max_load : max_load
-        @max_load = [@max_load || DEFAULT_MAX_LOAD, MAX_LOAD_CAP].min if @compliant_mode
         @user_agent = user_agent.nil? ? self.class.default_user_agent : user_agent
 
         @disable_ssl_certificate_check = if disable_ssl_certificate_check.nil?
@@ -144,13 +118,9 @@ module ScraperUtils
 
         today = Date.today.strftime("%Y-%m-%d")
         @user_agent = ENV.fetch("MORPH_USER_AGENT", nil)&.sub("TODAY", today)
-        if @compliant_mode
-          version = ScraperUtils::VERSION
-          @user_agent ||= "Mozilla/5.0 (compatible; ScraperUtils/#{version} #{today}; +https://github.com/ianheggie-oaf/scraper_utils)"
-        end
+        version = ScraperUtils::VERSION
+        @user_agent ||= "Mozilla/5.0 (compatible; ScraperUtils/#{version} #{today}; +https://github.com/ianheggie-oaf/scraper_utils)"
 
-        @robots_checker = RobotsChecker.new(@user_agent) if @user_agent
-        @adaptive_delay = AdaptiveDelay.new(max_load: @max_load) if @max_load
         display_options
       end
 
@@ -164,13 +134,11 @@ module ScraperUtils
           agent.open_timeout = @timeout
           agent.read_timeout = @timeout
         end
-        if @compliant_mode
-          agent.user_agent = user_agent
-          agent.request_headers ||= {}
-          agent.request_headers["Accept"] =
-            "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
-          agent.request_headers["Upgrade-Insecure-Requests"] = "1"
-        end
+        agent.user_agent = user_agent
+        agent.request_headers ||= {}
+        agent.request_headers["Accept"] =
+          "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+        agent.request_headers["Upgrade-Insecure-Requests"] = "1"
         if @australian_proxy
           agent.agent.set_proxy(ScraperUtils.australian_proxy)
           agent.request_headers["Accept-Language"] = "en-AU,en-US;q=0.9,en;q=0.8"
@@ -191,9 +159,6 @@ module ScraperUtils
                         else
                           "australian_proxy=#{@australian_proxy.inspect}"
                         end
-        display_args << "compliant_mode" if @compliant_mode
-        display_args << "random_delay=#{@random_delay}" if @random_delay&.positive?
-        display_args << "max_load=#{@max_load}%" if @max_load
         display_args << "disable_ssl_certificate_check" if @disable_ssl_certificate_check
         display_args << "default args" if display_args.empty?
         ScraperUtils::LogUtils.log(
@@ -220,27 +185,6 @@ module ScraperUtils
               "after #{response_time} seconds"
           )
         end
-
-        if @robots_checker&.disallowed?(uri)
-          raise ScraperUtils::UnprocessableSite,
-                "URL is disallowed by robots.txt specific rules: #{uri}"
-        end
-
-        @delay_till = nil
-        @delay = @robots_checker&.crawl_delay&.round(3)
-        debug_msg = "Delaying robots.txt: crawl_delay #{@delay} seconds"
-        unless @delay&.positive?
-          delays = {
-            max_load: @adaptive_delay&.next_delay(uri, response_time)&.round(3),
-            random: (@random_range ? (rand(@random_range) ** 2).round(3) : nil)
-          }
-          @delay = [delays[:max_load], delays[:random]].compact.sum
-          debug_msg = "Delaying #{@delay} seconds, sum of: #{delays.inspect}"
-        end
-        if @delay&.positive?
-          @delay_till = Time.now + @delay
-          ScraperUtils::LogUtils.log(debug_msg) if ScraperUtils::DebugUtils.basic?
-        end
         response
       end
 

data/lib/scraper_utils/mechanize_utils.rb

@@ -3,9 +3,7 @@
 require "mechanize"
 require "ipaddr"
 
-require_relative "mechanize_utils/adaptive_delay"
 require_relative "mechanize_utils/agent_config"
-require_relative "mechanize_utils/robots_checker"
 
 module ScraperUtils
   # Utilities for configuring and using Mechanize for web scraping

data/lib/scraper_utils/spec_support.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "scraperwiki"
+require "cgi"
 
 module ScraperUtils
   # Methods to support specs
@@ -35,6 +36,41 @@ module ScraperUtils
 
     AUSTRALIAN_POSTCODES = /\b\d{4}\b/.freeze
 
+    def self.fetch_url_with_redirects(url)
+      agent = Mechanize.new
+      # FIXME - Allow injection of a check to agree to terms if needed to set a cookie and reget the url
+      agent.get(url)
+    end
+
+    def self.authority_label(results, prefix: '', suffix: '')
+      return nil if results.nil?
+
+      authority_labels = results.map { |record| record['authority_label']}.compact.uniq
+      return nil if authority_labels.empty?
+
+      raise "Expected one authority_label, not #{authority_labels.inspect}" if authority_labels.size > 1
+      "#{prefix}#{authority_labels.first}#{suffix}"
+    end
+
+    # Validates enough addresses are geocodable
+    # @param results [Array<Hash>] The results from scraping an authority
+    # @param percentage [Integer] The min percentage of addresses expected to be geocodable (default:50)
+    # @param variation [Integer] The variation allowed in addition to percentage (default:3)
+    # @raise RuntimeError if insufficient addresses are geocodable
+    def self.validate_addresses_are_geocodable!(results, percentage: 50, variation: 3)
+      return nil if results.empty?
+
+      geocodable = results
+                     .map { |record| record["address"] }
+                     .uniq
+                     .count { |text| ScraperUtils::SpecSupport.geocodable? text }
+      puts "Found #{geocodable} out of #{results.count} unique geocodable addresses " \
+             "(#{(100.0 * geocodable / results.count).round(1)}%)"
+      expected = [((percentage.to_f / 100.0) * results.count - variation), 1].max
+      raise "Expected at least #{expected} (#{percentage}% - #{variation}) geocodable addresses, got #{geocodable}" unless geocodable >= expected
+      geocodable
+    end
+
     # Check if an address is likely to be geocodable by analyzing its format.
     # This is a bit stricter than needed - typically assert >= 75% match
     # @param address [String] The address to check
@@ -43,7 +79,7 @@ module ScraperUtils
       return false if address.nil? || address.empty?
       check_address = ignore_case ? address.upcase : address
 
-      # Basic structure check - must have a street name, suburb, state and postcode
+      # Basic structure check - must have a street type or unit/lot, uppercase suburb or postcode, state
       has_state = AUSTRALIAN_STATES.any? { |state| check_address.end_with?(" #{state}") || check_address.include?(" #{state} ") }
       has_postcode = address.match?(AUSTRALIAN_POSTCODES)
 
@@ -52,20 +88,20 @@ module ScraperUtils
 
       has_unit_or_lot = address.match?(/\b(Unit|Lot:?)\s+\d+/i)
 
-      has_suburb_stats = check_address.match?(/(\b[A-Z]{2,}(\s+[A-Z]+)*,?|,\s+[A-Z][A-Za-z ]+)(\s+\d{4})?\s+(#{AUSTRALIAN_STATES.join('|')})\b/)
+      uppercase_words = address.scan(/\b[A-Z]{2,}\b/)
+      has_uppercase_suburb = uppercase_words.any? { |word| !AUSTRALIAN_STATES.include?(word) }
 
       if ENV["DEBUG"]
         missing = []
         unless has_street_type || has_unit_or_lot
           missing << "street type / unit / lot"
         end
+        missing << "postcode/Uppercase suburb" unless has_postcode || has_uppercase_suburb
         missing << "state" unless has_state
-        missing << "postcode" unless has_postcode
-        missing << "suburb state" unless has_suburb_stats
         puts "  address: #{address} is not geocodable, missing #{missing.join(', ')}" if missing.any?
       end
 
-      (has_street_type || has_unit_or_lot) && has_state && has_postcode && has_suburb_stats
+      (has_street_type || has_unit_or_lot) && (has_postcode || has_uppercase_suburb) && has_state
     end
 
     PLACEHOLDERS = [
@@ -80,11 +116,158 @@ module ScraperUtils
       PLACEHOLDERS.any? { |placeholder| text.to_s.match?(placeholder) }
     end
 
+    # Validates enough descriptions are reasonable
+    # @param results [Array<Hash>] The results from scraping an authority
+    # @param percentage [Integer] The min percentage of descriptions expected to be reasonable (default:50)
+    # @param variation [Integer] The variation allowed in addition to percentage (default:3)
+    # @raise RuntimeError if insufficient descriptions are reasonable
+    def self.validate_descriptions_are_reasonable!(results, percentage: 50, variation: 3)
+      return nil if results.empty?
+
+      descriptions = results
+                       .map { |record| record["description"] }
+                       .uniq
+                       .count do |text|
+        selected = ScraperUtils::SpecSupport.reasonable_description? text
+        puts "  description: #{text} is not reasonable" if ENV["DEBUG"] && !selected
+        selected
+      end
+      puts "Found #{descriptions} out of #{results.count} unique reasonable descriptions " \
+             "(#{(100.0 * descriptions / results.count).round(1)}%)"
+      expected = [(percentage.to_f / 100.0) * results.count - variation, 1].max
+      raise "Expected at least #{expected} (#{percentage}% - #{variation}) reasonable descriptions, got #{descriptions}" unless descriptions >= expected
+      descriptions
+    end
+
     # Check if this looks like a "reasonable" description
     # This is a bit stricter than needed - typically assert >= 75% match
     def self.reasonable_description?(text)
       !placeholder?(text) && text.to_s.split.size >= 3
     end
+
+    # Validates that all records use the expected global info_url and it returns 200
+    # @param results [Array<Hash>] The results from scraping an authority
+    # @param expected_url [String] The expected global info_url for this authority
+    # @raise RuntimeError if records don't use the expected URL or it doesn't return 200
+    def self.validate_uses_one_valid_info_url!(results, expected_url, bot_check_expected: false)
+      info_urls = results.map { |record| record["info_url"] }.uniq
+
+      unless info_urls.size == 1
+        raise "Expected all records to use one info_url '#{expected_url}', found: #{info_urls.size}"
+      end
+      unless info_urls.first == expected_url
+        raise "Expected all records to use global info_url '#{expected_url}', found: #{info_urls.first}"
+      end
+
+      puts "Checking the one expected info_url returns 200: #{expected_url}"
+
+      if defined?(VCR)
+        VCR.use_cassette("#{authority_label(results, suffix: '_')}one_info_url") do
+          page = fetch_url_with_redirects(expected_url)
+          validate_page_response(page, bot_check_expected)
+        end
+      else
+        page = fetch_url_with_redirects(expected_url)
+        validate_page_response(page, bot_check_expected)
+      end
+    end
+
+    # Validates that info_urls have expected details (unique URLs with content validation)
+    # @param results [Array<Hash>] The results from scraping an authority
+    # @param percentage [Integer] The min percentage of detail checks expected to pass (default:75)
+    # @param variation [Integer] The variation allowed in addition to percentage (default:3)
+    # @raise RuntimeError if insufficient detail checks pass
+    def self.validate_info_urls_have_expected_details!(results, percentage: 75, variation: 3, bot_check_expected: false)
+      if defined?(VCR)
+        VCR.use_cassette("#{authority_label(results, suffix: '_')}info_url_details") do
+          check_info_url_details(results, percentage, variation, bot_check_expected)
+        end
+      else
+        check_info_url_details(results, percentage, variation, bot_check_expected)
+      end
+    end
+
+    # Check if the page response indicates bot protection
+    # @param page [Mechanize::Page] The page response to check
+    # @return [Boolean] True if bot protection is detected
+    def self.bot_protection_detected?(page)
+      return true if %w[403 429].include?(page.code)
+
+      return false unless page.body
+
+      body_lower = page.body.downcase
+
+      # Check for common bot protection indicators
+      bot_indicators = [
+        'recaptcha',
+        'cloudflare',
+        'are you human',
+        'bot detection',
+        'security check',
+        'verify you are human',
+        'access denied',
+        'blocked',
+        'captcha'
+      ]
+
+      bot_indicators.any? { |indicator| body_lower.include?(indicator) }
+    end
+
+    # Validate page response, accounting for bot protection
+    # @param page [Mechanize::Page] The page response to validate
+    # @param bot_check_expected [Boolean] Whether bot protection is acceptable
+    # @raise RuntimeError if page response is invalid and bot protection not expected
+    def self.validate_page_response(page, bot_check_expected)
+      if bot_check_expected && bot_protection_detected?(page)
+        puts "  Bot protection detected - accepting as valid response"
+        return
+      end
+
+      raise "Expected 200 response from the one expected info_url, got #{page.code}" unless page.code == "200"
+    end
+
+    private
+
+    def self.check_info_url_details(results, percentage, variation, bot_check_expected)
+      count = 0
+      failed = 0
+      fib_indices = ScraperUtils::MathsUtils.fibonacci_series(results.size - 1).uniq
+
+      fib_indices.each do |index|
+        record = results[index]
+        info_url = record["info_url"]
+        puts "Checking info_url[#{index}]: #{info_url} has the expected reference, address and description..."
+
+        page = fetch_url_with_redirects(info_url)
+
+        if bot_check_expected && bot_protection_detected?(page)
+          puts "  Bot protection detected - skipping detailed validation"
+          next
+        end
+
+        raise "Expected 200 response, got #{page.code}" unless page.code == "200"
+
+        page_body = page.body.dup.force_encoding("UTF-8").gsub(/\s\s+/, " ")
+
+        %w[council_reference address description].each do |attribute|
+          count += 1
+          expected = CGI.escapeHTML(record[attribute]).gsub(/\s\s+/, " ")
+          expected2 = expected.gsub(/(\S+)\s+(\S+)\z/, '\2 \1') # Handle Lismore post-code/state swap
+
+          next if page_body.include?(expected) || page_body.include?(expected2)
+
+          failed += 1
+          puts "  Missing: #{expected}"
+          puts "    IN: #{page_body}" if ENV['DEBUG']
+
+          min_required = [((percentage.to_f / 100.0) * count - variation), 1].max
+          passed = count - failed
+          raise "Too many failures: #{passed}/#{count} passed (min required: #{min_required})" if passed < min_required
+        end
+      end
+
+      puts "#{(100.0 * (count - failed) / count).round(1)}% detail checks passed (#{failed}/#{count} failed)!" if count > 0
+    end
+
   end
 end
-

data/lib/scraper_utils/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ScraperUtils
-  VERSION = "0.8.2"
+  VERSION = "0.8.3"
 end

data/lib/scraper_utils.rb

@@ -4,18 +4,14 @@ require "scraper_utils/version"
 
 # Public Apis (responsible for requiring their own dependencies)
 require "scraper_utils/authority_utils"
-require "scraper_utils/cycle_utils"
 require "scraper_utils/data_quality_monitor"
-require "scraper_utils/date_range_utils"
 require "scraper_utils/db_utils"
 require "scraper_utils/debug_utils"
 require "scraper_utils/log_utils"
-require "scraper_utils/randomize_utils"
-require "scraper_utils/scheduler"
+require "scraper_utils/maths_utils"
 require "scraper_utils/spec_support"
 
 # Mechanize utilities
-require "scraper_utils/mechanize_actions"
 require "scraper_utils/mechanize_utils"
 
 # Utilities for planningalerts scrapers

data/scraper_utils.gemspec

@@ -41,5 +41,6 @@ Gem::Specification.new do |spec|
   spec.add_dependency "mechanize"
   spec.add_dependency "nokogiri"
   spec.add_dependency "sqlite3"
+  # Do NOT add - it depends on a non-default branch spec.add_dependency "scraperwiki"
   spec.metadata["rubygems_mfa_required"] = "true"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scraper_utils
 version: !ruby/object:Gem::Version
-  version: 0.8.2
+  version: 0.8.3
 platform: ruby
 authors:
 - Ian Heggie
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-05-06 00:00:00.000000000 Z
+date: 2025-07-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -56,7 +56,8 @@ description: Utilities to help make planningalerts scrapers, especially multi au
   scrapers, easier to develop, run and debug.
 email:
 - ian@heggie.biz
-executables: []
+executables:
+- validate_scraper_data
 extensions: []
 extra_rdoc_files: []
 files:
@@ -79,36 +80,22 @@ files:
 - bin/setup
 - docs/debugging.md
 - docs/enhancing_specs.md
-- docs/example_scrape_with_fibers.rb
+- docs/example_custom_Rakefile
+- docs/example_dot_scraper_validation.yml
 - docs/example_scraper.rb
-- docs/fibers_and_threads.md
 - docs/getting_started.md
-- docs/interleaving_requests.md
 - docs/mechanize_utilities.md
-- docs/parallel_requests.md
-- docs/randomizing_requests.md
-- docs/reducing_server_load.md
+- docs/testing_custom_scrapers.md
+- exe/validate_scraper_data
 - lib/scraper_utils.rb
 - lib/scraper_utils/authority_utils.rb
-- lib/scraper_utils/cycle_utils.rb
 - lib/scraper_utils/data_quality_monitor.rb
-- lib/scraper_utils/date_range_utils.rb
 - lib/scraper_utils/db_utils.rb
 - lib/scraper_utils/debug_utils.rb
 - lib/scraper_utils/log_utils.rb
-- lib/scraper_utils/mechanize_actions.rb
+- lib/scraper_utils/maths_utils.rb
 - lib/scraper_utils/mechanize_utils.rb
-- lib/scraper_utils/mechanize_utils/adaptive_delay.rb
 - lib/scraper_utils/mechanize_utils/agent_config.rb
-- lib/scraper_utils/mechanize_utils/robots_checker.rb
-- lib/scraper_utils/randomize_utils.rb
-- lib/scraper_utils/scheduler.rb
-- lib/scraper_utils/scheduler/constants.rb
-- lib/scraper_utils/scheduler/operation_registry.rb
-- lib/scraper_utils/scheduler/operation_worker.rb
-- lib/scraper_utils/scheduler/process_request.rb
-- lib/scraper_utils/scheduler/thread_request.rb
-- lib/scraper_utils/scheduler/thread_response.rb
 - lib/scraper_utils/spec_support.rb
 - lib/scraper_utils/version.rb
 - scraper_utils.gemspec
@@ -119,7 +106,7 @@ metadata:
   allowed_push_host: https://rubygems.org
   homepage_uri: https://github.com/ianheggie-oaf/scraper_utils
   source_code_uri: https://github.com/ianheggie-oaf/scraper_utils
-  documentation_uri: https://rubydoc.info/gems/scraper_utils/0.8.2
+  documentation_uri: https://rubydoc.info/gems/scraper_utils/0.8.3
   changelog_uri: https://github.com/ianheggie-oaf/scraper_utils/blob/main/CHANGELOG.md
   rubygems_mfa_required: 'true'
 post_install_message:
@@ -137,7 +124,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.19
+rubygems_version: 3.4.10
 signing_key:
 specification_version: 4
 summary: planningalerts scraper utilities

data/docs/example_scrape_with_fibers.rb

@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-
-# Example scrape method updated to use ScraperUtils::FibreScheduler
-
-def scrape(authorities, attempt)
-  ScraperUtils::Scheduler.reset!
-  exceptions = {}
-  authorities.each do |authority_label|
-    ScraperUtils::Scheduler.register_operation(authority_label) do
-      ScraperUtils::LogUtils.log(
-        "Collecting feed data for #{authority_label}, attempt: #{attempt}..."
-      )
-      ScraperUtils::DataQualityMonitor.start_authority(authority_label)
-      YourScraper.scrape(authority_label) do |record|
-        record["authority_label"] = authority_label.to_s
-        ScraperUtils::DbUtils.save_record(record)
-      rescue ScraperUtils::UnprocessableRecord => e
-        ScraperUtils::DataQualityMonitor.log_unprocessable_record(e, record)
-        exceptions[authority_label] = e
-        # Continues processing other records
-      end
-    rescue StandardError => e
-      warn "#{authority_label}: ERROR: #{e}"
-      warn e.backtrace || "No backtrace available"
-      exceptions[authority_label] = e
-    end
-    # end of register_operation block
-  end
-  ScraperUtils::Scheduler.run_operations
-  exceptions
-end