scraper_utils 0.2.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b4291b6994419c04851935fe4aa4e047eb4069cab3fecf451bf65f8e91acb48d
-  data.tar.gz: 2e3a657ce230f9c6bc9defe042cf7babb9e52e2130d32f0ec8312571f5dcb26a
+  metadata.gz: 9443f89de2518d12830ddcd3d4d5e246c77a35b34e6549031a2f61a454c9c96c
+  data.tar.gz: 693aeaeb896c9779135f3c703f86aa436a3d9b10715448cc18c96c590ddf3723
 SHA512:
-  metadata.gz: 51c29aea77f43a8c7de8e874a2601b4e0b9e9c36ae512180f10dcd182b2ecc899cc08944face686e0f993b02338975672d9eaf06d8a0185ee222cfc263993244
-  data.tar.gz: ec63009a4f10677a8e9500b5ca15c68b432081fa995d5bee2eda2b2cff88cdb7090595be7ad64a0dfeacf09decbc292313b47f9f8af795a3b95c718c77f59339
+  metadata.gz: 38d23ede40ee6313c0c8098b34d1258829c2842e2b43fe39fd156503d0c387fc59929fec740205ffc232cafdfe095932a62f937b3aded2ab7b5990dc889615d7
+  data.tar.gz: 7e9b7da5194265fdd77a8136202bf11a3293a2dda0b2f5d00ffaf7503872079233889b80d81ab2267eccbd9c570b56e8778b9e4fce28cee9e3a2657788f02334

data/.gitignore

@@ -9,6 +9,9 @@
 /test/tmp/
 /test/version_tmp/
 
+# Ignore log files
+/log/
+
 # Temp files
 ,*
 *.bak

data/.rubocop.yml

@@ -1,3 +1,7 @@
+plugins:
+  - rubocop-rake
+  - rubocop-rspec
+
 AllCops:
   NewCops: enable
 

data/CHANGELOG.md

@@ -1,5 +1,26 @@
 # Changelog
 
+## 0.4.0 - 2025-03-04
+
+* Add Cycle Utils as an alternative to Date range utils
+* Update README.md with changed defaults
+
+## 0.3.0 - 2025-03-04
+
+* Add date range utils
+* Flush $stdout and $stderr when logging to sync exception output and logging lines
+* Break out example code from README.md into docs dir
+
+## 0.2.1 - 2025-02-28
+
+Fixed broken v0.2.0
+
+## 0.2.0 - 2025-02-28
+
+Added FiberScheduler, enabled complient mode with delays by default and simplified usage removing third retry without proxy
+
 ## 0.1.0 - 2025-02-23
 
-`First release for development`
+First release for development
+
+

data/Gemfile

@@ -22,12 +22,15 @@ gem "sqlite3", platform && (platform == :heroku16 ? "~> 1.4.0" : "~> 1.6.3")
 gem "scraperwiki", git: "https://github.com/openaustralia/scraperwiki-ruby.git",
                    branch: "morph_defaults"
 
-# development and test test gems
+# development and test gems
 gem "rake", platform && (platform == :heroku16 ? "~> 12.3.3" : "~> 13.0")
 gem "rspec", platform && (platform == :heroku16 ? "~> 3.9.0" : "~> 3.12")
-gem "rubocop", platform && (platform == :heroku16 ? "~> 0.80.0" : "~> 1.57")
+gem "rubocop", platform && (platform == :heroku16 ? "~> 1.28.2" : "~> 1.73")
+gem "rubocop-rake", platform && (platform == :heroku16 ? "~> 0.6.0" : "~> 0.7")
+gem "rubocop-rspec", platform && (platform == :heroku16 ? "~> 2.10.0" : "~> 3.5")
 gem "simplecov", platform && (platform == :heroku16 ? "~> 0.18.0" : "~> 0.22.0")
 gem "simplecov-console"
+gem "terminal-table"
 gem "webmock", platform && (platform == :heroku16 ? "~> 3.14.0" : "~> 3.19.0")
 
 gemspec

data/README.md

@@ -13,11 +13,12 @@ our scraper accessing your systems, here's what you should know:
 
 ### How to Control Our Behavior
 
-Our scraper utilities respect the standard server **robots.txt** control mechanisms (by default). To control our access:
+Our scraper utilities respect the standard server **robots.txt** control mechanisms (by default).
+To control our access:
 
 - Add a section for our user agent: `User-agent: ScraperUtils` (default)
-- Set a crawl delay: `Crawl-delay: 5`
-- If needed specify disallowed paths: `Disallow: /private/`
+- Set a crawl delay, eg: `Crawl-delay: 20`
+- If needed specify disallowed paths*: `Disallow: /private/`
 
 ### Built-in Politeness Features
 
@@ -26,13 +27,24 @@ Even without specific configuration, our scrapers will, by default:
 - **Identify themselves**: Our user agent clearly indicates who we are and provides a link to the project repository:
   `Mozilla/5.0 (compatible; ScraperUtils/0.2.0 2025-02-22; +https://github.com/ianheggie-oaf/scraper_utils)`
 
-- **Limit server load**: We introduce delays to avoid undue load on your server's by default based on your response
-  time.
-  The slower your server is running, the longer the delay we add between requests to help you.
-  In the default "compliant mode" this defaults to 20% and custom settings are capped at 33% maximum.
+- **Limit server load**: We slow down our requests so we should never be a significant load to your server, let alone
+  overload it.
+  The slower your server is running, the longer the delay we add between requests to help.
+  In the default "compliant mode" this defaults to a max load of 20% and is capped at 33%.
 
-- **Add randomized delays**: We add random delays between requests to avoid creating regular traffic patterns that might
-  impact server performance (enabled by default).
+- **Add randomized delays**: We add random delays between requests to further reduce our impact on servers, which should
+  bring us down to the load of a single industrious person.
+
+Extra utilities provided for scrapers to further reduce your server load:
+
+- **Interleave requests**: This spreads out the requests to your server rather than focusing on one scraper at a time.
+
+- **Intelligent Date Range selection**: This reduces server load by over 60% by a smarter choice of date range searches,
+  checking the recent 4 days each day and reducing down to checking each 3 days by the end of the 33-day mark. This
+  replaces the simplistic check of the last 30 days each day.
+
+- Alternative **Cycle Utilities** - a convenience class to cycle through short and longer search ranges to reduce server
+  load.
 
 Our goal is to access public planning information without negatively impacting your services.
 
@@ -50,10 +62,6 @@ And then execute:
 
     $ bundle
 
-Or install it yourself for testing:
-
-    $ gem install scraper_utils
-
 Usage
 -----
 
@@ -101,12 +109,12 @@ export DEBUG=1
 Add `client_options` to your AUTHORITIES configuration and move any of the following settings into it:
 
 * `timeout: Integer` - Timeout for agent connections in case the server is slower than normal
-* `australian_proxy: true` - Use the MORPH_AUSTRALIAN_PROXY as proxy url if the site is geo-locked
+* `australian_proxy: true` - Use the proxy url in the `MORPH_AUSTRALIAN_PROXY` env variable if the site is geo-locked
 * `disable_ssl_certificate_check: true` - Disabled SSL verification for old / incorrect certificates
 
 See the documentation on `ScraperUtils::MechanizeUtils::AgentConfig` for more options
 
-Then adjust your code to accept client_options and pass then through to:
+Then adjust your code to accept `client_options` and pass then through to:
 `ScraperUtils::MechanizeUtils.mechanize_agent(client_options || {})`
 to receive a `Mechanize::Agent` configured accordingly.
 
@@ -115,130 +123,47 @@ The agent returned is configured using Mechanize hooks to implement the desired
 ### Default Configuration
 
 By default, the Mechanize agent is configured with the following settings.
+As you can see, the defaults can be changed using env variables.
+
+Note - compliant mode forces max_load to be set to a value no greater than 50.
 
 ```ruby
 ScraperUtils::MechanizeUtils::AgentConfig.configure do |config|
-  config.default_timeout = 60
-  config.default_compliant_mode = true
-  config.default_random_delay = 3
-  config.default_max_load = 20 # percentage
-  config.default_disable_ssl_certificate_check = false
-  config.default_australian_proxy = false
+  config.default_timeout = ENV.fetch('MORPH_TIMEOUT', 60).to_i # 60
+  config.default_compliant_mode = ENV.fetch('MORPH_NOT_COMPLIANT', nil).to_s.empty? # true
+  config.default_random_delay = ENV.fetch('MORPH_RANDOM_DELAY', 5).to_i # 5
+  config.default_max_load = ENV.fetch('MORPH_MAX_LOAD', 33.3).to_f # 33.3
+  config.default_disable_ssl_certificate_check = !ENV.fetch('MORPH_DISABLE_SSL_CHECK', nil).to_s.empty? # false
+  config.default_australian_proxy = !ENV.fetch('MORPH_USE_PROXY', nil).to_s.empty? # false
+  config.default_user_agent = ENV.fetch('MORPH_USER_AGENT', nil) # Uses Mechanize user agent
 end
 ```
 
 You can modify these global defaults before creating any Mechanize agents. These settings will be used for all Mechanize
 agents created by `ScraperUtils::MechanizeUtils.mechanize_agent` unless overridden by passing parameters to that method.
 
-### Example updated `scraper.rb` file
-
-Update your `scraper.rb` as per the following example for basic utilities:
+To speed up testing, set the following in `spec_helper.rb`:
 
 ```ruby
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-$LOAD_PATH << "./lib"
-
-require "scraper_utils"
-require "technology_one_scraper"
-
-# Main Scraper class
-class Scraper
-  AUTHORITIES = YourScraper::AUTHORITIES
-
-  # ADD: attempt argument
-  def scrape(authorities, attempt)
-    exceptions = {}
-    # ADD: Report attempt number
-    authorities.each do |authority_label|
-      puts "\nCollecting feed data for #{authority_label}, attempt: #{attempt}..."
-
-      begin
-        # REPLACE:
-        # YourScraper.scrape(authority_label) do |record|
-        #   record["authority_label"] = authority_label.to_s
-        #   YourScraper.log(record)
-        #   ScraperWiki.save_sqlite(%w[authority_label council_reference], record)
-        # end
-        # WITH:
-        ScraperUtils::DataQualityMonitor.start_authority(authority_label)
-        YourScraper.scrape(authority_label) do |record|
-          begin
-            record["authority_label"] = authority_label.to_s
-            ScraperUtils::DbUtils.save_record(record)
-          rescue ScraperUtils::UnprocessableRecord => e
-            ScraperUtils::DataQualityMonitor.log_unprocessable_record(e, record)
-            exceptions[authority_label] = e
-          end
-        end
-        # END OF REPLACE
-      end
-    rescue StandardError => e
-      warn "#{authority_label}: ERROR: #{e}"
-      warn e.backtrace
-      exceptions[authority_label] = e
-    end
-
-    exceptions
-  end
-
-  def self.selected_authorities
-    ScraperUtils::AuthorityUtils.selected_authorities(AUTHORITIES.keys)
-  end
-
-  def self.run(authorities)
-    puts "Scraping authorities: #{authorities.join(', ')}"
-    start_time = Time.now
-    exceptions = scrape(authorities, 1)
-    # Set start_time and attempt to the call above and log run below
-    ScraperUtils::LogUtils.log_scraping_run(
-      start_time,
-      1,
-      authorities,
-      exceptions
-    )
-
-    unless exceptions.empty?
-      puts "\n***************************************************"
-      puts "Now retrying authorities which earlier had failures"
-      puts exceptions.keys.join(", ").to_s
-      puts "***************************************************"
-
-      start_time = Time.now
-      exceptions = scrape(exceptions.keys, 2)
-      # Set start_time and attempt to the call above and log run below
-      ScraperUtils::LogUtils.log_scraping_run(
-        start_time,
-        2,
-        authorities,
-        exceptions
-      )
-    end
-
-    # Report on results, raising errors for unexpected conditions
-    ScraperUtils::LogUtils.report_on_results(authorities, exceptions)
-  end
+ScraperUtils::MechanizeUtils::AgentConfig.configure do |config|
+  config.default_random_delay = nil
+  config.default_max_load = 33
 end
+```
 
-if __FILE__ == $PROGRAM_NAME
-  # Default to list of authorities we can't or won't fix in code, explain why
-  # wagga: url redirects and then reports Application error
+### Example updated `scraper.rb` file
 
-  ENV["MORPH_EXPECT_BAD"] ||= "wagga"
-  Scraper.run(Scraper.selected_authorities)
-end
-```
+Update your `scraper.rb` as per the [example scraper](docs/example_scraper.rb).
 
 Your code should raise ScraperUtils::UnprocessableRecord when there is a problem with the data presented on a page for a
 record.
 Then just before you would normally yield a record for saving, rescue that exception and:
 
-* Call ScraperUtils::DataQualityMonitor.log_unprocessable_record(e, record)
+* Call `ScraperUtils::DataQualityMonitor.log_unprocessable_record(e, record)`
 * NOT yield the record for saving
 
 In your code update where create a mechanize agent (often `YourScraper.scrape_period`) and the `AUTHORITIES` hash
-to move mechanize_agent options (like `australian_proxy` and `timeout`) to a hash under a new key: `client_options`.
+to move Mechanize agent options (like `australian_proxy` and `timeout`) to a hash under a new key: `client_options`.
 For example:
 
 ```ruby
@@ -297,44 +222,97 @@ The `ScraperUtils::FiberScheduler` provides a lightweight utility that:
 * thus optimizing the total scraper run time
 * allows you to increase the random delay for authorities without undue effect on total run time
 * For the curious, it uses [ruby fibers](https://ruby-doc.org/core-2.5.8/Fiber.html) rather than threads as that is
-  simpler to get right and debug!
+  a simpler system and thus easier to get right, understand and debug!
+* Cycles around the authorities when compliant_mode, max_load and random_delay are disabled
 
-To enable change the scrape method in the example above to;
+To enable change the scrape method to be like [example scrape method using fibers](docs/example_scrape_with_fibers.rb)
 
-```ruby
+And use `ScraperUtils::FiberScheduler.log` instead of `puts` when logging within the authority processing code.
+This will prefix the output lines with the authority name, which is needed since the system will interleave the work and
+thus the output.
+
+This uses `ScraperUtils::RandomizeUtils` as described below. Remember to add the recommended line to
+`spec/spec_heper.rb`.
+
+Intelligent Date Range Selection
+--------------------------------                                                                                                                                                                                   
+
+To further reduce server load and speed up scrapers, we provide an intelligent date range selection mechanism
+that can reduce server requests by 60% without significantly impacting delay in picking up changes.
 
-def scrape(authorities, attempt)
-  ScraperUtils::FiberScheduler.reset!
-  exceptions = {}
-  authorities.each do |authority_label|
-    ScraperUtils::FiberScheduler.register_operation(authority_label) do
-      ScraperUtils::FiberScheduler.log "Collecting feed data for #{authority_label}, attempt: #{attempt}..."
-      begin
-        ScraperUtils::DataQualityMonitor.start_authority(authority_label)
-        YourScraper.scrape(authority_label) do |record|
-          begin
-            record["authority_label"] = authority_label.to_s
-            ScraperUtils::DbUtils.save_record(record)
-          rescue ScraperUtils::UnprocessableRecord => e
-            ScraperUtils::DataQualityMonitor.log_unprocessable_record(e, record)
-            exceptions[authority_label] = e
-          end
-        end
-      rescue StandardError => e
-        warn "#{authority_label}: ERROR: #{e}"
-        warn e.backtrace
-        exceptions[authority_label] = e
-      end
-    end # end of register_operation block
+The `ScraperUtils::DateRangeUtils#calculate_date_ranges` method provides a smart approach to searching historical
+records:
+
+- Always checks the most recent 4 days daily (configurable)
+- Progressively reduces search frequency for older records
+- Uses a Fibonacci-like progression to create natural, efficient search intervals
+- Configurable `max_period` (default is 3 days)
+- merges adjacent search ranges and handles the changeover in search frequency by extending some searches
+
+Example usage in your scraper:
+
+ ```ruby                                                                                                                                                                                                            
+date_ranges = ScraperUtils::DateRangeUtils.new.calculate_date_ranges
+date_ranges.each do |from_date, to_date, _debugging_comment|
+  # Adjust your normal search code to use for this date range                                                                                                                                                                             
+  your_search_records(from_date: from_date, to_date: to_date) do |record|
+    # process as normal
   end
-  ScraperUtils::FiberScheduler.run_all
-  exceptions
 end
 ```
 
-And use `ScraperUtils::FiberScheduler.log` instead of `puts` when logging within the authority processing code.
-This will prefix the output lines with the authority name, which is needed since the system will interleave the work and
-thus the output.
+Typical server load reductions:
+
+* Max period 2 days : ~42% of the 33 days selected
+* Max period 3 days : ~37% of the 33 days selected (default)
+* Max period 5 days : ~35% (or ~31% when days = 45)
+
+See the class documentation for customizing defaults and passing options.
+
+### Other possibilities
+
+If the site uses tags like 'L28', 'L14' and 'L7' for the last 28, 14 and 7 days, an alternative solution
+is to cycle through ['L28', 'L7', 'L14', 'L7'] which would drop the load by 50% and be less Bot like.
+
+Cycle Utils
+-----------
+Simple utility for cycling through options based on Julian day number:
+
+```ruby
+# Toggle between main and alternate behaviour
+alternate = ScraperUtils::CycleUtils.position(2).even?
+
+# Use with any cycle size
+pos = ScraperUtils::CycleUtils.position(7) # 0-6 cycle
+
+# Test with specific date
+pos = ScraperUtils::CycleUtils.position(3, date: Date.new(2024, 1, 5))
+
+# Override for testing
+# CYCLE_POSITION=2 bundle exec ruby scraper.rb
+```
+
+Randomizing Requests
+--------------------
+
+Pass a `Collection` or `Array` to `ScraperUtils::RandomizeUtils.randomize_order` to randomize it in production, but
+receive in as is when testing.
+
+Use this with the list of records scraped from an index to randomise any requests for further information to be less Bot
+like.
+
+### Spec setup
+
+You should enforce sequential mode when testing by adding the following code to `spec/spec_helper.rb` :
+
+```
+ScraperUtils::RandomizeUtils.sequential = true
+```
+
+Note:
+
+* You can also force sequential mode by setting the env variable `MORPH_PROCESS_SEQUENTIALLY` to `1` (any non blank)
+* testing using VCR requires sequential mode
 
 Development
 -----------
@@ -356,7 +334,7 @@ NOTE: You need to use ruby 3.2.2 instead of 2.5.8 to release to OTP protected ac
 Contributing
 ------------
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/ianheggie-oaf/scraper_utils
+Bug reports and pull requests with working tests are welcome on [GitHub](https://github.com/ianheggie-oaf/scraper_utils)
 
 CHANGELOG.md is maintained by the author aiming to follow https://github.com/vweevers/common-changelog
 
@@ -364,3 +342,4 @@ License
 -------
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+

data/docs/example_scrape_with_fibers.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+# Example scrape method updated to use ScraperUtils::FibreScheduler
+
+def scrape(authorities, attempt)
+  ScraperUtils::FiberScheduler.reset!
+  exceptions = {}
+  authorities.each do |authority_label|
+    ScraperUtils::FiberScheduler.register_operation(authority_label) do
+      ScraperUtils::FiberScheduler.log(
+        "Collecting feed data for #{authority_label}, attempt: #{attempt}..."
+      )
+      ScraperUtils::DataQualityMonitor.start_authority(authority_label)
+      YourScraper.scrape(authority_label) do |record|
+        record["authority_label"] = authority_label.to_s
+        ScraperUtils::DbUtils.save_record(record)
+      rescue ScraperUtils::UnprocessableRecord => e
+        ScraperUtils::DataQualityMonitor.log_unprocessable_record(e, record)
+        exceptions[authority_label] = e
+        # Continues processing other records
+      end
+    rescue StandardError => e
+      warn "#{authority_label}: ERROR: #{e}"
+      warn e.backtrace || "No backtrace available"
+      exceptions[authority_label] = e
+    end
+    # end of register_operation block
+  end
+  ScraperUtils::FiberScheduler.run_all
+  exceptions
+end

data/docs/example_scraper.rb

@@ -0,0 +1,93 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+$LOAD_PATH << "./lib"
+
+require "scraper_utils"
+require "technology_one_scraper"
+
+# Main Scraper class
+class Scraper
+  AUTHORITIES = YourScraper::AUTHORITIES
+
+  # ADD: attempt argument
+  def scrape(authorities, attempt)
+    exceptions = {}
+    # ADD: Report attempt number
+    authorities.each do |authority_label|
+      puts "\nCollecting feed data for #{authority_label}, attempt: #{attempt}..."
+
+      begin
+        # REPLACE:
+        # YourScraper.scrape(authority_label) do |record|
+        #   record["authority_label"] = authority_label.to_s
+        #   YourScraper.log(record)
+        #   ScraperWiki.save_sqlite(%w[authority_label council_reference], record)
+        # end
+        # WITH:
+        ScraperUtils::DataQualityMonitor.start_authority(authority_label)
+        YourScraper.scrape(authority_label) do |record|
+          begin
+            record["authority_label"] = authority_label.to_s
+            ScraperUtils::DbUtils.save_record(record)
+          rescue ScraperUtils::UnprocessableRecord => e
+            ScraperUtils::DataQualityMonitor.log_unprocessable_record(e, record)
+            exceptions[authority_label] = e
+          end
+        end
+        # END OF REPLACE
+      end
+    rescue StandardError => e
+      warn "#{authority_label}: ERROR: #{e}"
+      warn e.backtrace
+      exceptions[authority_label] = e
+    end
+
+    exceptions
+  end
+
+  def self.selected_authorities
+    ScraperUtils::AuthorityUtils.selected_authorities(AUTHORITIES.keys)
+  end
+
+  def self.run(authorities)
+    puts "Scraping authorities: #{authorities.join(', ')}"
+    start_time = Time.now
+    exceptions = scrape(authorities, 1)
+    # Set start_time and attempt to the call above and log run below
+    ScraperUtils::LogUtils.log_scraping_run(
+      start_time,
+      1,
+      authorities,
+      exceptions
+    )
+
+    unless exceptions.empty?
+      puts "\n***************************************************"
+      puts "Now retrying authorities which earlier had failures"
+      puts exceptions.keys.join(", ").to_s
+      puts "***************************************************"
+
+      start_time = Time.now
+      exceptions = scrape(exceptions.keys, 2)
+      # Set start_time and attempt to the call above and log run below
+      ScraperUtils::LogUtils.log_scraping_run(
+        start_time,
+        2,
+        authorities,
+        exceptions
+      )
+    end
+
+    # Report on results, raising errors for unexpected conditions
+    ScraperUtils::LogUtils.report_on_results(authorities, exceptions)
+  end
+end
+
+if __FILE__ == $PROGRAM_NAME
+  # Default to list of authorities we can't or won't fix in code, explain why
+  # wagga: url redirects and then reports Application error
+
+  ENV["MORPH_EXPECT_BAD"] ||= "wagga"
+  Scraper.run(Scraper.selected_authorities)
+end