scraper_utils 0.9.2 → 0.10.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e2d77d081c52024dbba8afdfd03f939af5d3d44d82e5491a109e4dd3e853f225
-  data.tar.gz: 67df28089a8bf743ef895c3dafdbdf9461d2a15b68273e6b0bc0cfef0f9288d5
+  metadata.gz: ffc82e602be6fd69f2086d9abb9c5003e248f96f3d427e31208153b134696eb6
+  data.tar.gz: f60ba37d659d07f55f8c04a02a32c229f083e60e99deddb5507b58390ddaeacd
 SHA512:
-  metadata.gz: 1fd29f183695351bbfa82c22fa938ac00c6278a13baf8a5dcdde89f469b8d266edcd5473393082c1972e6db70df27dbe0831fc39de078c12acb62c04787c5d8e
-  data.tar.gz: e1fa93082d439e16d27884d83da6520be20756745ec843a1eb6aa6a2b747254bdcf2089462c4164d313a67c9bc052f295da335e9fa78e6c6d71f322610d8ad2e
+  metadata.gz: 6ae035de4c7cdc76e6bad9bf6fe8053b835b85689d106172ae6916a4fc67bb712796242443ee710a0c96475607ffd49f93069746f6fec79b583c9a19ef75f68c
+  data.tar.gz: 94e0a9ad963d5557161781a26233d792aa18ec63248aee3b79fcf9541f00fd9016e671d050571b53f2ae1d16e5f8e33d42f977e555e71af6e9565a4b2f5831c4

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## 0.10.1 - 2026-01-27
+
+* Added  `ScraperUtils::DbUtils.cleanup_old_records` to Clean up records older than 30 days and approx once a month
+  vacuum the DB
+* Pauses the request time plus 0.5 seconds
+* Removed reference to random delay – it's either not required or it's not enough to make a difference
+
 ## 0.9.2 - 2026-01-27
 
 * Removed Emoticons as they are four byte UTF-8 and some databases are configured to only store 3 byte UTF-8

data/README.md

@@ -23,6 +23,8 @@ see {file:docs/getting_started.md Getting Started guide}
 
 - Configure Mechanize agents with sensible defaults
 - Supports extra actions required to get to results page
+- Plays nice with external servers by pausing (crawl_delay + response_time) between requests by default,
+  backing off when servers are slow
 - {file:docs/mechanize_utilities.md Learn more about Mechanize utilities}
 
 ### Parallel Processing
@@ -77,4 +79,4 @@ on [ianheggie-oaf/scraper_utils | GitHub](https://github.com/ianheggie-oaf/scrap
 
 ## License
 
-The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/docs/example_scraper.rb

@@ -74,6 +74,7 @@ class Scraper
       )
     end
 
+    ScraperUtils::DbUtils.cleanup_old_records 
     # Report on results, raising errors for unexpected conditions
     ScraperUtils::LogUtils.report_on_results(authorities, exceptions)
   end

data/docs/mechanize_utilities.md

@@ -16,10 +16,18 @@ agent = ScraperUtils::MechanizeUtils.mechanize_agent(**options)
 
 Add `client_options` to your AUTHORITIES configuration and move any of the following settings into it:
 
-* `timeout: Integer` - Timeout for agent connections in case the server is slower than normal
+#### Connection Settings
+
+* `timeout: Integer` - Timeout for agent connections in case the server is slower than normal (default: 60 seconds)
 * `australian_proxy: true` - Use the proxy url in the `MORPH_AUSTRALIAN_PROXY` env variable if the site is geo-locked
 * `disable_ssl_certificate_check: true` - Disabled SSL verification for old / incorrect certificates
 
+#### Delay Settings (Respectful Scraping)
+
+* `crawl_delay: Float` - Minimum delay between requests in seconds (default: 0.5)
+* `max_load: Float` - Maximum server load as a percentage. E.g., `50` pauses as long as the response took, making total
+  time 50% response + 50% pause (default: 50.0)
+
 Then adjust your code to accept `client_options` and pass then through to:
 `ScraperUtils::MechanizeUtils.mechanize_agent(client_options || {})`
 to receive a `Mechanize::Agent` configured accordingly.
@@ -42,11 +50,13 @@ ScraperUtils::MechanizeUtils::AgentConfig.configure do |config|
 end
 ```
 
-For full details, see the [MechanizeUtils class documentation](https://rubydoc.info/gems/scraper_utils/ScraperUtils/MechanizeUtils).
+For full details, see
+the [MechanizeUtils class documentation](https://rubydoc.info/gems/scraper_utils/ScraperUtils/MechanizeUtils).
 
 ## MechanizeActions
 
-The `ScraperUtils::MechanizeActions` class provides a convenient way to execute a series of actions (like clicking links, filling forms) on a Mechanize page.
+The `ScraperUtils::MechanizeActions` class provides a convenient way to execute a series of actions (like clicking
+links, filling forms) on a Mechanize page.
 
 ### Action Format
 
@@ -86,4 +96,5 @@ actions = [
 ]
 ```
 
-For full details, see the [MechanizeActions class documentation](https://rubydoc.info/gems/scraper_utils/ScraperUtils/MechanizeActions).
+For full details, see
+the [MechanizeActions class documentation](https://rubydoc.info/gems/scraper_utils/ScraperUtils/MechanizeActions).

data/lib/scraper_utils/db_utils.rb

@@ -56,5 +56,29 @@ module ScraperUtils
         ScraperUtils::DataQualityMonitor.log_saved_record(record)
       end
     end
+
+    # Clean up records older than 30 days and approx once a month vacuum the DB
+    def self.cleanup_old_records
+      cutoff_date = (Date.today - 30).to_s
+      vacuum_cutoff_date = (Date.today - 35).to_s
+
+      stats = ScraperWiki.sqliteexecute(
+        "SELECT COUNT(*) as count, MIN(date_scraped) as oldest FROM data WHERE date_scraped < ?",
+        [cutoff_date]
+      ).first
+
+      deleted_count = stats["count"]
+      oldest_date = stats["oldest"]
+
+      return unless deleted_count.positive? || ENV["VACUUM"]
+
+      LogUtils.log "Deleting #{deleted_count} applications scraped between #{oldest_date} and #{cutoff_date}"
+      ScraperWiki.sqliteexecute("DELETE FROM data WHERE date_scraped < ?", [cutoff_date])
+
+      return unless rand < 0.03 || (oldest_date && oldest_date < vacuum_cutoff_date) || ENV["VACUUM"]
+
+      LogUtils.log "  Running VACUUM to reclaim space..."
+      ScraperWiki.sqliteexecute("VACUUM")
+    end
   end
 end

data/lib/scraper_utils/mechanize_utils/agent_config.rb

@@ -19,10 +19,11 @@ module ScraperUtils
     # @example Overriding specific settings
     #   config = ScraperUtils::MechanizeUtils::AgentConfig.new(
     #     timeout: 120,
-    #     random_delay: 10
     #   )
     class AgentConfig
       DEFAULT_TIMEOUT = 60
+      DEFAULT_CRAWL_DELAY = 0.5
+      DEFAULT_MAX_LOAD = 50.0
 
       # Class-level defaults that can be modified
       class << self
@@ -38,6 +39,13 @@ module ScraperUtils
         # @return [String, nil] Default Mechanize user agent
         attr_accessor :default_user_agent
 
+        # @return [Float, nil] Default Crawl delay between requests in seconds
+        attr_accessor :default_crawl_delay
+
+        # @return [Float, nil] Default Max load presented to an external server as a percentage
+        # 50 will result in a pause the same length as the response (ie 50% of total time will be the response, 50% pausing)
+        attr_accessor :default_max_load
+
         # Configure default settings for all AgentConfig instances
         # @yield [self] Yields self for configuration
         # @example
@@ -56,6 +64,8 @@ module ScraperUtils
           @default_disable_ssl_certificate_check = !ENV.fetch('MORPH_DISABLE_SSL_CHECK', nil).to_s.empty? # false
           @default_australian_proxy = !ENV.fetch('MORPH_USE_PROXY', nil).to_s.empty? # false
           @default_user_agent = ENV.fetch('MORPH_USER_AGENT', nil) # Uses Mechanize user agent
+          @default_crawl_delay = ENV.fetch('MORPH_CLIENT_CRAWL_DELAY', DEFAULT_CRAWL_DELAY)
+          @default_max_load = ENV.fetch('MORPH_MAX_LOAD', DEFAULT_MAX_LOAD)
         end
       end
 
@@ -67,7 +77,7 @@ module ScraperUtils
 
       # Give access for testing
 
-      attr_reader :max_load, :random_range
+      attr_reader :max_load, :crawl_delay
 
       # Creates Mechanize agent configuration with sensible defaults overridable via configure
       # @param timeout [Integer, nil] Timeout for agent connections (default: 60)
@@ -76,8 +86,8 @@ module ScraperUtils
       # @param user_agent [String, nil] Configure Mechanize user agent
       def initialize(timeout: nil,
                      compliant_mode: nil,
-                     random_delay: nil,
                      max_load: nil,
+                     crawl_delay: nil,
                      disable_ssl_certificate_check: nil,
                      australian_proxy: nil,
                      user_agent: nil)
@@ -94,6 +104,9 @@ module ScraperUtils
                             else
                               australian_proxy
                             end
+        @crawl_delay = crawl_delay.nil? ? self.class.default_crawl_delay : crawl_delay.to_f
+        # Clamp between 10 (delay 9 x response) and 100 (no delay)
+        @max_load = (max_load.nil? ? self.class.default_max_load : max_load).to_f.clamp(10.0, 100.0)
 
         # Validate proxy URL format if proxy will be used
         @australian_proxy &&= !ScraperUtils.australian_proxy.to_s.empty?
@@ -111,11 +124,6 @@ module ScraperUtils
           end
         end
 
-        if @random_delay&.positive?
-          min_random = Math.sqrt(@random_delay * 3.0 / 13.0)
-          @random_range = min_random.round(3)..(3 * min_random).round(3)
-        end
-
         today = Date.today.strftime("%Y-%m-%d")
         @user_agent = ENV.fetch("MORPH_USER_AGENT", nil)&.sub("TODAY", today)
         version = ScraperUtils::VERSION
@@ -179,12 +187,23 @@ module ScraperUtils
         raise ArgumentError, "URI must be present in post-connect hook" unless uri
 
         response_time = Time.now - @connection_started_at
+
+        response_delay = @crawl_delay || 0.0
+        if @crawl_delay ||@max_load
+          response_delay += response_time
+          if @max_load && @max_load >= 1
+            response_delay += (100.0 - @max_load) * response_time / @max_load
+          end
+          response_delay = response_delay.round(3)
+        end
+
         if DebugUtils.basic?
           ScraperUtils::LogUtils.log(
             "Post Connect uri: #{uri.inspect}, response: #{response.inspect} " \
-              "after #{response_time} seconds"
+              "after #{response_time} seconds#{response_delay > 0.0 ? ", pausing for #{response_delay} seconds" : ""}"
           )
         end
+        sleep(response_delay) if response_delay > 0.0
         response
       end
 

data/lib/scraper_utils/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ScraperUtils
-  VERSION = "0.9.2"
+  VERSION = "0.10.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scraper_utils
 version: !ruby/object:Gem::Version
-  version: 0.9.2
+  version: 0.10.1
 platform: ruby
 authors:
 - Ian Heggie
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-01-26 00:00:00.000000000 Z
+date: 2026-02-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -110,7 +110,7 @@ metadata:
   allowed_push_host: https://rubygems.org
   homepage_uri: https://github.com/ianheggie-oaf/scraper_utils
   source_code_uri: https://github.com/ianheggie-oaf/scraper_utils
-  documentation_uri: https://rubydoc.info/gems/scraper_utils/0.9.2
+  documentation_uri: https://rubydoc.info/gems/scraper_utils/0.10.1
   changelog_uri: https://github.com/ianheggie-oaf/scraper_utils/blob/main/CHANGELOG.md
   rubygems_mfa_required: 'true'
 post_install_message: