scraper_utils 0.3.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a2082c406ad96266f644fc1dfc588046aa43494e9e79b8fec38fe59252c09f06
-  data.tar.gz: 5100907cdcc8c55ddd59b25cf393d212f681b573ef874c5a6c65f748a8c852ec
+  metadata.gz: 6cd5b7c0fb03a762cec4ddd35ec354d0921273e7c9ee5c8c7f332b14339f3182
+  data.tar.gz: afb80849b5c078eb8f7bf7de74bb537f8df692bcb59070f90bde29a4c68b9231
 SHA512:
-  metadata.gz: feabea23ac5b14f6b642769db303bcab954b22cfd2d95d7694af51afd661c73f1ff70a0c17b54557a730fe811275c126c0fdd4473e2ae6d2b83cfc495aa52bc6
-  data.tar.gz: d91154c0dbccfb4271fd4830a82316676c1ad5665773c30d212bcc1ab6178a09d4fee97b2b2a32480c32b8ee69da08fc9a78575460d49f8233fb91b74bf7df66
+  metadata.gz: d6e0e27e93d1b7528e96d9bda350510f5513854778972f7276683acae538b2b70e8478c5f7db2d4eb5bc0b4ebfb28c502e5885896c67f0090e8d4b85648df6de
+  data.tar.gz: e3ec0ca5d77bff719b32bbe91f6efa43fceed75c92bcc8450304d1f64be6f73f5372626b84a4541d3ed4920b3c6af889400d677fcb0c5a20fd0224fa2ec75613

data/CHANGELOG.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## 0.4.1 - 2025-03-04
+
+* Document `ScraperUtils::CycleUtils.pick(values)`
+
+## 0.4.0 - 2025-03-04
+
+* Add Cycle Utils as an alternative to Date range utils
+* Update README.md with changed defaults
+
+## 0.3.0 - 2025-03-04
+
+* Add date range utils
+* Flush $stdout and $stderr when logging to sync exception output and logging lines
+* Break out example code from README.md into docs dir
 
 ## 0.2.1 - 2025-02-28
 
@@ -12,3 +26,5 @@ Added FiberScheduler, enabled complient mode with delays by default and simplifi
 ## 0.1.0 - 2025-02-23
 
 First release for development
+
+

data/README.md

@@ -33,8 +33,7 @@ Even without specific configuration, our scrapers will, by default:
   In the default "compliant mode" this defaults to a max load of 20% and is capped at 33%.
 
 - **Add randomized delays**: We add random delays between requests to further reduce our impact on servers, which should
-  bring us
-- down to the load of a single industrious person.
+  bring us down to the load of a single industrious person.
 
 Extra utilities provided for scrapers to further reduce your server load:
 
@@ -44,6 +43,9 @@ Extra utilities provided for scrapers to further reduce your server load:
   checking the recent 4 days each day and reducing down to checking each 3 days by the end of the 33-day mark. This
   replaces the simplistic check of the last 30 days each day.
 
+- Alternative **Cycle Utilities** - a convenience class to cycle through short and longer search ranges to reduce server
+  load.
+
 Our goal is to access public planning information without negatively impacting your services.
 
 Installation
@@ -123,15 +125,14 @@ The agent returned is configured using Mechanize hooks to implement the desired
 By default, the Mechanize agent is configured with the following settings.
 As you can see, the defaults can be changed using env variables.
 
-Note - compliant mode forces max_load to be set to a value no greater than 33.
-PLEASE don't use our user agent string with a max_load higher than 33!
+Note - compliant mode forces max_load to be set to a value no greater than 50.
 
 ```ruby
 ScraperUtils::MechanizeUtils::AgentConfig.configure do |config|
   config.default_timeout = ENV.fetch('MORPH_TIMEOUT', 60).to_i # 60
   config.default_compliant_mode = ENV.fetch('MORPH_NOT_COMPLIANT', nil).to_s.empty? # true
-  config.default_random_delay = ENV.fetch('MORPH_RANDOM_DELAY', 15).to_i # 15
-  config.default_max_load = ENV.fetch('MORPH_MAX_LOAD', 20.0).to_f # 20
+  config.default_random_delay = ENV.fetch('MORPH_RANDOM_DELAY', 5).to_i # 5
+  config.default_max_load = ENV.fetch('MORPH_MAX_LOAD', 33.3).to_f # 33.3
   config.default_disable_ssl_certificate_check = !ENV.fetch('MORPH_DISABLE_SSL_CHECK', nil).to_s.empty? # false
   config.default_australian_proxy = !ENV.fetch('MORPH_USE_PROXY', nil).to_s.empty? # false
   config.default_user_agent = ENV.fetch('MORPH_USER_AGENT', nil) # Uses Mechanize user agent
@@ -268,13 +269,40 @@ Typical server load reductions:
 
 See the class documentation for customizing defaults and passing options.
 
+### Other possibilities
+
+If the site uses tags like 'L28', 'L14' and 'L7' for the last 28, 14 and 7 days, an alternative solution
+is to cycle through ['L28', 'L7', 'L14', 'L7'] which would drop the load by 50% and be less Bot like.
+
+Cycle Utils
+-----------
+Simple utility for cycling through options based on Julian day number:
+
+```ruby
+# Toggle between main and alternate behaviour
+alternate = ScraperUtils::CycleUtils.position(2).even?
+
+# OR cycle through a list of values day by day:
+period = ScraperUtils::CycleUtils.pick(['L28', 'L7', 'L14', 'L7'])
+
+# Use with any cycle size
+pos = ScraperUtils::CycleUtils.position(7) # 0-6 cycle
+
+# Test with specific date
+pos = ScraperUtils::CycleUtils.position(3, date: Date.new(2024, 1, 5))
+
+# Override for testing
+# CYCLE_POSITION=2 bundle exec ruby scraper.rb
+```
+
 Randomizing Requests
 --------------------
 
 Pass a `Collection` or `Array` to `ScraperUtils::RandomizeUtils.randomize_order` to randomize it in production, but
 receive in as is when testing.
 
-Use this with the list of records scraped from an index to randomise any requests for further information to be less Bot like.
+Use this with the list of records scraped from an index to randomise any requests for further information to be less Bot
+like.
 
 ### Spec setup
 

data/lib/scraper_utils/cycle_utils.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module ScraperUtils
+  # Provides utilities for cycling through a range of options day by day
+  module CycleUtils
+    # Returns position in cycle from zero onwards
+    # @param cycle [Integer] Cycle size (2 onwards)
+    # @param date [Date, nil] Optional date to use instead of today
+    # @return [Integer] position in cycle progressing from zero to cycle-1 and then repeating day by day
+    # Can override using CYCLE_POSITION ENV variable
+    def self.position(cycle, date: nil)
+      day = ENV.fetch('CYCLE_POSITION', (date || Date.today).jd).to_i
+      day % cycle
+    end
+
+    # Returns one value per day, cycling through all possible values in order
+    # @param values [Array] Values to cycle through
+    # @param date [Date, nil] Optional date to use instead of today to calculate position
+    # @return value from array
+    # Can override using CYCLE_POSITION ENV variable
+    def self.pick(values, date: nil)
+      values[position(values.size, date: date)]
+    end
+  end
+end

data/lib/scraper_utils/mechanize_utils/agent_config.rb

@@ -69,8 +69,8 @@ module ScraperUtils
         def reset_defaults!
           @default_timeout = ENV.fetch('MORPH_TIMEOUT', DEFAULT_TIMEOUT).to_i # 60
           @default_compliant_mode = ENV.fetch('MORPH_NOT_COMPLIANT', nil).to_s.empty? # true
-          @default_random_delay = ENV.fetch('MORPH_RANDOM_DELAY', DEFAULT_RANDOM_DELAY).to_i # 33
-          @default_max_load = ENV.fetch('MORPH_MAX_LOAD', DEFAULT_MAX_LOAD).to_f # 20
+          @default_random_delay = ENV.fetch('MORPH_RANDOM_DELAY', DEFAULT_RANDOM_DELAY).to_i # 5
+          @default_max_load = ENV.fetch('MORPH_MAX_LOAD', DEFAULT_MAX_LOAD).to_f # 33.3
           @default_disable_ssl_certificate_check = !ENV.fetch('MORPH_DISABLE_SSL_CHECK', nil).to_s.empty? # false
           @default_australian_proxy = !ENV.fetch('MORPH_USE_PROXY', nil).to_s.empty? # false
           @default_user_agent = ENV.fetch('MORPH_USER_AGENT', nil) # Uses Mechanize user agent

data/lib/scraper_utils/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ScraperUtils
-  VERSION = "0.3.0"
+  VERSION = "0.4.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scraper_utils
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.1
 platform: ruby
 authors:
 - Ian Heggie
@@ -79,6 +79,7 @@ files:
 - lib/scraper_utils.rb
 - lib/scraper_utils/adaptive_delay.rb
 - lib/scraper_utils/authority_utils.rb
+- lib/scraper_utils/cycle_utils.rb
 - lib/scraper_utils/data_quality_monitor.rb
 - lib/scraper_utils/date_range_utils.rb
 - lib/scraper_utils/db_utils.rb