scraper_utils 0.13.1 → 0.14.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3bce8cc5a624f9904ebf8bb35ccb5c5c6c831e28ed56f88d3baf3b8d19fbbd13
-  data.tar.gz: 0a481566e846a4274796b0542fb64a805f486065ed08045724cea7bc3d46710d
+  metadata.gz: 03b44a667992331d6e36bb6eca68afc286205846d7be06263694fed52b5e2d30
+  data.tar.gz: 9f0dd276223f1b22dd688453e1769199cbda34efa5141d58e546a8ddcb85c795
 SHA512:
-  metadata.gz: 231c167ffe232daacbc862b8c3dd2c0c71be6b8fc2ff061f4f36d88f2e2185a454eb0aa79653c7a99a2ed65c9857d961059456f8403af8c1ed39623cc8e2db6a
-  data.tar.gz: f287f85cdd4cc11cf17c3e5d34d5493e2809f255f3a3544bc881e756f3379c897dd70dbba5ebf16b30837bb8612f42f704872e06c6bec1cad87845606fce6231
+  metadata.gz: b42e0be0f9e42d9a83588cf7dcbb98ec079d01262340d2e6fef8ac7201c3d80faa645351631f60f767186721a58580f4f1e5e09c130a3a32aebb4f301dbfbdfc
+  data.tar.gz: e3cec3345d0af13026259600a54e417efd0c36394f1bc22ecac1a25573551a3a2e51482b060ad1b72ed7ba4850d55bf9f8032321d1b8c1ae6eab581244e92410

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## 0.14.1 - 2026-03-04
+
+* Can pass `known_suburbs: ['Suburb', ...]` to `ScraperUtils::SpecSupport.validate_addresses_are_geocodable!` and
+  `ScraperUtils::SpecSupport.geocodable?` to validate addresses that don't have postcodes nor capitalised suburb names
+* Can pass ignore_case: true to relax the requirement for either postcode or uppercase suburb when you don't want to
+  pass Known suburb.
+* Move Throttling to HostThrottler
+
 ## 0.13.1 - 2026.02-21
 
 * Added PaValidation that validates based

data/lib/scraper_utils/host_throttler.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module ScraperUtils
+  # Tracks per-host next-allowed-request time so that time spent parsing
+  # and saving records counts toward the crawl delay rather than being
+  # added on top of it.
+  #
+  # Usage:
+  #   throttler = HostThrottler.new(crawl_delay: 1.0, max_load: 50.0)
+  #   throttler.before_request(hostname)   # sleep until ready
+  #   # ... make request ...
+  #   throttler.after_request(hostname)    # record timing, schedule next slot
+  #   throttler.after_request(hostname, overloaded: true)  # double delay + 5s
+  class HostThrottler
+    MAX_DELAY = 120.0
+
+    # @param crawl_delay [Float] minimum seconds between requests per host
+    # @param max_load [Float] target server load percentage (10..100);
+    #   50 means response_time == pause_time
+    def initialize(crawl_delay: 0.0, max_load: nil)
+      @crawl_delay = crawl_delay.to_f
+      # Clamp between 10 (delay 9x response) and 100 (no extra delay)
+      @max_load = max_load ? max_load.to_f.clamp(10.0, 100.0) : nil
+      @next_request_at = {}   # hostname => Time
+      @request_started_at = {} # hostname => Time
+    end
+
+    # Sleep until this host's throttle window has elapsed.
+    # Records when the request actually started.
+    # @param hostname [String]
+    # @return [void]
+    def before_request(hostname)
+      target = @next_request_at[hostname]
+      if target
+        remaining = target - Time.now
+        sleep(remaining) if remaining > 0
+      end
+      @request_started_at[hostname] = Time.now
+    end
+
+    # Calculate and store the next allowed request time for this host.
+    # @param hostname [String]
+    # @param overloaded [Boolean] true when the server signalled overload
+    #   (HTTP 429/500/503); doubles the normal delay and adds 5 seconds.
+    # @return [void]
+    def after_request(hostname, overloaded: false)
+      started = @request_started_at[hostname] || Time.now
+      response_time = Time.now - started
+
+      delay = @crawl_delay
+      if @max_load
+        delay += (100.0 - @max_load) * response_time / @max_load
+      end
+
+      if overloaded
+        delay = delay + response_time * 2 + 5.0
+      end
+
+      delay = delay.round(3).clamp(0.0, MAX_DELAY)
+      @next_request_at[hostname] = Time.now + delay
+
+      if DebugUtils.basic?
+        msg = "HostThrottler: #{hostname} response=#{response_time.round(3)}s"
+        msg += " OVERLOADED" if overloaded
+        msg += ", Will delay #{delay}s before next request"
+        LogUtils.log(msg)
+      end
+    end
+
+    # Duck-type check for HTTP overload errors across Mechanize, HTTParty, etc.
+    # @param error [Exception]
+    # @return [Boolean]
+    def self.overload_error?(error)
+      code = if error.respond_to?(:response) && error.response.respond_to?(:code)
+               error.response.code.to_i          # HTTParty style
+             elsif error.respond_to?(:response_code)
+               error.response_code.to_i          # Mechanize style
+             end
+      [429, 500, 503].include?(code)
+    end
+  end
+end

data/lib/scraper_utils/mechanize_utils/agent_config.rb

@@ -2,6 +2,7 @@
 
 require "mechanize"
 require "ipaddr"
+require_relative "../host_throttler"
 
 module ScraperUtils
   module MechanizeUtils
@@ -76,8 +77,7 @@ module ScraperUtils
       attr_reader :user_agent
 
       # Give access for testing
-
-      attr_reader :max_load, :crawl_delay
+      attr_reader :max_load, :crawl_delay, :throttler
 
       # Creates Mechanize agent configuration with sensible defaults overridable via configure
       # @param timeout [Integer, nil] Timeout for agent connections (default: 60)
@@ -107,6 +107,7 @@ module ScraperUtils
         @crawl_delay = crawl_delay.nil? ? self.class.default_crawl_delay : crawl_delay.to_f
         # Clamp between 10 (delay 9 x response) and 100 (no delay)
         @max_load = (max_load.nil? ? self.class.default_max_load : max_load).to_f.clamp(10.0, 100.0)
+        @throttler = HostThrottler.new(crawl_delay: @crawl_delay, max_load: @max_load)
 
         # Validate proxy URL format if proxy will be used
         @australian_proxy &&= !ScraperUtils.australian_proxy.to_s.empty?
@@ -155,6 +156,7 @@ module ScraperUtils
 
         agent.pre_connect_hooks << method(:pre_connect_hook)
         agent.post_connect_hooks << method(:post_connect_hook)
+        agent.error_hooks << method(:error_hook) if agent.respond_to?(:error_hooks)
       end
 
       private
@@ -175,38 +177,41 @@ module ScraperUtils
       end
 
       def pre_connect_hook(_agent, request)
-        @connection_started_at = Time.now
-        return unless DebugUtils.verbose?
-
-        ScraperUtils::LogUtils.log(
-          "Pre Connect request: #{request.inspect} at #{@connection_started_at}"
-        )
+        hostname = request.respond_to?(:uri) ? request.uri.host : 'unknown'
+        @throttler.before_request(hostname)
+        if DebugUtils.verbose?
+          ScraperUtils::LogUtils.log(
+            "Pre Connect request: #{request.inspect}"
+          )
+        end
       end
 
       def post_connect_hook(_agent, uri, response, _body)
         raise ArgumentError, "URI must be present in post-connect hook" unless uri
 
-        response_time = Time.now - @connection_started_at
-
-        response_delay = @crawl_delay || 0.0
-        if @crawl_delay ||@max_load
-          response_delay += response_time
-          if @max_load && @max_load >= 1
-            response_delay += (100.0 - @max_load) * response_time / @max_load
-          end
-          response_delay = response_delay.round(3)
-        end
+        status = response.respond_to?(:code) ? response.code.to_i : nil
+        overloaded = [429, 500, 503].include?(status)
+        @throttler.after_request(uri.host, overloaded: overloaded)
 
         if DebugUtils.basic?
           ScraperUtils::LogUtils.log(
-            "Post Connect uri: #{uri.inspect}, response: #{response.inspect} " \
-              "after #{response_time} seconds#{response_delay > 0.0 ? ", pausing for #{response_delay} seconds" : ""}"
+            "Post Connect uri: #{uri.inspect}, response: #{response.inspect}"
           )
         end
-        sleep(response_delay) if response_delay > 0.0
         response
       end
 
+      def error_hook(_agent, error)
+        # Best-effort: record the error against whatever host we can find
+        # Mechanize errors often carry the URI in the message; fall back to 'unknown'
+        hostname = if error.respond_to?(:uri)
+                     error.uri.host
+                   else
+                     'unknown'
+                   end
+        @throttler.after_request(hostname, overloaded: HostThrottler.overload_error?(error))
+      end
+
       def verify_proxy_works(agent)
         $stderr.flush
         $stdout.flush

data/lib/scraper_utils/misc_utils.rb

@@ -1,23 +1,36 @@
 # frozen_string_literal: true
 
+require_relative "host_throttler"
+
 module ScraperUtils
   # Misc Standalone Utilities
   module MiscUtils
-    MAX_PAUSE = 120.0
+    THROTTLE_HOSTNAME = "block"
 
     class << self
-      attr_accessor :pause_duration
-
-      # Throttle block to be nice to servers we are scraping
-      def throttle_block(extra_delay: 0.5)
-        if @pause_duration&.positive?
-          puts "Pausing #{@pause_duration}s" if ScraperUtils::DebugUtils.trace?
-          sleep(@pause_duration)
+      # Throttle block to be nice to servers we are scraping.
+      # Time spent inside the block (parsing, saving) counts toward the delay.
+      def throttle_block
+        throttler.before_request(THROTTLE_HOSTNAME)
+        begin
+          result = yield
+          throttler.after_request(THROTTLE_HOSTNAME)
+          result
+        rescue StandardError => e
+          throttler.after_request(THROTTLE_HOSTNAME, overloaded: HostThrottler.overload_error?(e))
+          raise
         end
-        start_time = Time.now.to_f
-        result = yield
-        @pause_duration = (Time.now.to_f - start_time + extra_delay).round(3).clamp(0.0, MAX_PAUSE)
-        result
+      end
+
+      # Reset the internal throttler (useful in tests)
+      def reset_throttler!
+        @throttler = nil
+      end
+
+      private
+
+      def throttler
+        @throttler ||= HostThrottler.new
       end
     end
   end

data/lib/scraper_utils/spec_support.rb

@@ -95,14 +95,16 @@ module ScraperUtils
     # @param results [Array<Hash>] The results from scraping an authority
     # @param percentage [Integer] The min percentage of addresses expected to be geocodable (default:50)
     # @param variation [Integer] The variation allowed in addition to percentage (default:3)
+    # @param ignore_case [Boolean] Ignores case which relaxes suburb check
+    # @param known_suburbs [Array<String>] Known suburbs to detect in address when there is no postcode and no uppercase suburb
     # @raise RuntimeError if insufficient addresses are geocodable
-    def self.validate_addresses_are_geocodable!(results, percentage: 50, variation: 3)
+    def self.validate_addresses_are_geocodable!(results, percentage: 50, variation: 3, ignore_case: false, known_suburbs: [])
       return nil if results.empty?
 
       geocodable = results
                      .map { |record| record["address"] }
                      .uniq
-                     .count { |text| ScraperUtils::SpecSupport.geocodable? text }
+                     .count { |text| ScraperUtils::SpecSupport.geocodable? text, known_suburbs: known_suburbs, ignore_case: ignore_case }
       puts "Found #{geocodable} out of #{results.count} unique geocodable addresses " \
              "(#{(100.0 * geocodable / results.count).round(1)}%)"
       expected = [((percentage.to_f / 100.0) * results.count - variation), 1].max
@@ -115,8 +117,10 @@ module ScraperUtils
     # Check if an address is likely to be geocodable by analyzing its format.
     # This is a bit stricter than needed - typically assert >= 75% match
     # @param address [String] The address to check
+    # @param ignore_case [Boolean] Ignores case which relaxes suburb check
+    # @param known_suburbs [Array<String>] Known suburbs to detect in address when there is no postcode and no uppercase suburb
     # @return [Boolean] True if the address appears to be geocodable.
-    def self.geocodable?(address, ignore_case: false)
+    def self.geocodable?(address, ignore_case: false, known_suburbs: [])
       return false if address.nil? || address.empty?
       check_address = ignore_case ? address.upcase : address
 
@@ -129,16 +133,17 @@ module ScraperUtils
 
       uppercase_words = address.scan(/\b[A-Z]{2,}\b/)
       has_uppercase_suburb = uppercase_words.any? { |word| !AUSTRALIAN_STATES.include?(word) }
+      has_known_suburb = known_suburbs.any? { |suburb| address.include?(suburb) }
 
       if ENV["DEBUG"]
         missing = []
         missing << "street type" unless has_street_type
-        missing << "postcode/Uppercase suburb" unless has_postcode || has_uppercase_suburb
+        missing << "postcode/Uppercase suburb/Known suburb" unless has_postcode || has_uppercase_suburb || has_known_suburb
         missing << "state" unless has_state
         puts "  address: #{address} is not geocodable, missing #{missing.join(', ')}" if missing.any?
       end
 
-      has_street_type && (has_postcode || has_uppercase_suburb) && has_state
+      has_street_type && (has_postcode || has_uppercase_suburb || has_known_suburb) && has_state
     end
 
     PLACEHOLDERS = [

data/lib/scraper_utils/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ScraperUtils
-  VERSION = "0.13.1"
+  VERSION = "0.14.1"
 end

data/lib/scraper_utils.rb

@@ -5,12 +5,13 @@ require "scraper_utils/version"
 # Public Apis (responsible for requiring their own dependencies)
 require "scraper_utils/authority_utils"
 require "scraper_utils/data_quality_monitor"
-require "scraper_utils/pa_validation"
 require "scraper_utils/db_utils"
 require "scraper_utils/debug_utils"
+require "scraper_utils/host_throttler"
 require "scraper_utils/log_utils"
 require "scraper_utils/maths_utils"
 require "scraper_utils/misc_utils"
+require "scraper_utils/pa_validation"
 require "scraper_utils/spec_support"
 
 # Mechanize utilities

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scraper_utils
 version: !ruby/object:Gem::Version
-  version: 0.13.1
+  version: 0.14.1
 platform: ruby
 authors:
 - Ian Heggie
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-02-21 00:00:00.000000000 Z
+date: 2026-03-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -97,6 +97,7 @@ files:
 - lib/scraper_utils/data_quality_monitor.rb
 - lib/scraper_utils/db_utils.rb
 - lib/scraper_utils/debug_utils.rb
+- lib/scraper_utils/host_throttler.rb
 - lib/scraper_utils/log_utils.rb
 - lib/scraper_utils/maths_utils.rb
 - lib/scraper_utils/mechanize_utils.rb
@@ -113,7 +114,7 @@ metadata:
   allowed_push_host: https://rubygems.org
   homepage_uri: https://github.com/ianheggie-oaf/scraper_utils
   source_code_uri: https://github.com/ianheggie-oaf/scraper_utils
-  documentation_uri: https://rubydoc.info/gems/scraper_utils/0.13.1
+  documentation_uri: https://rubydoc.info/gems/scraper_utils/0.14.1
   changelog_uri: https://github.com/ianheggie-oaf/scraper_utils/blob/main/CHANGELOG.md
   rubygems_mfa_required: 'true'
 post_install_message: