scraper_utils 0.5.0 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 63a24c24b497494b79c4d7e12f04a1bd2555068f37f50389f3906c0033817d7e
-  data.tar.gz: 6d6b96112dc3e2f9dc5a54de6318a544c240c0e3d5246ab4178c07346d0de7dc
+  metadata.gz: 28f415290516d19f6ffc95a7d349a6ed269de987a0ffe45ed512ff29bfc82902
+  data.tar.gz: 9d337d1145754bf2375f4d2e18d89da7c21231d7b9b279bc51340f92781caa35
 SHA512:
-  metadata.gz: eda8d10d996d51b7ef1d2610e21da31390c10dd29f4daa70bd5d9c3c8dc6eb9bed651803ccd6a59f53b03dae4fcd1ea016802e693f8828f4a13b92e07a0b046e
-  data.tar.gz: eba2704a99c6599a2789ec573fa335d7939a63d0c27b06886d6e905cd785e2095d7d0307e7aa1195a1209e022340fa5d027a72ccca61a350590058e998355d5d
+  metadata.gz: 8a3050451f512b2f77cf9cd806fc1602d6502b247f24248d93cfc12dea47bf5d7f02bd9be5453c0da7ee7b8acc0e3ee32cd375b52e04f0f366c2174ea7320bd9
+  data.tar.gz: 17befcb8b9305536385ddf6772aeee038df34bd42ce197c4951836bb396e1db9d3dabece2a543ece6f60e331ca244ef1f660f1b111af17d79703a2a783801183

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 0.5.1 - 2025-03-05
+
+* Remove duplicated example code in docs
+
 ## 0.5.0 - 2025-03-05
 
 * Add action processing utility

data/docs/example_scraper.rb

@@ -4,7 +4,7 @@
 $LOAD_PATH << "./lib"
 
 require "scraper_utils"
-require "technology_one_scraper"
+require "your_scraper"
 
 # Main Scraper class
 class Scraper
@@ -17,26 +17,18 @@ class Scraper
     authorities.each do |authority_label|
       puts "\nCollecting feed data for #{authority_label}, attempt: #{attempt}..."
 
-      begin
-        # REPLACE:
-        # YourScraper.scrape(authority_label) do |record|
-        #   record["authority_label"] = authority_label.to_s
-        #   YourScraper.log(record)
-        #   ScraperWiki.save_sqlite(%w[authority_label council_reference], record)
-        # end
-        # WITH:
-        ScraperUtils::DataQualityMonitor.start_authority(authority_label)
-        YourScraper.scrape(authority_label) do |record|
-          begin
-            record["authority_label"] = authority_label.to_s
-            ScraperUtils::DbUtils.save_record(record)
-          rescue ScraperUtils::UnprocessableRecord => e
-            ScraperUtils::DataQualityMonitor.log_unprocessable_record(e, record)
-            exceptions[authority_label] = e
-          end
+      # REPLACE section with:
+      ScraperUtils::DataQualityMonitor.start_authority(authority_label)
+      YourScraper.scrape(authority_label) do |record|
+        begin
+          record["authority_label"] = authority_label.to_s
+          ScraperUtils::DbUtils.save_record(record)
+        rescue ScraperUtils::UnprocessableRecord => e
+          ScraperUtils::DataQualityMonitor.log_unprocessable_record(e, record)
+          exceptions[authority_label] = e
         end
-        # END OF REPLACE
       end
+      # END OF REPLACE
     rescue StandardError => e
       warn "#{authority_label}: ERROR: #{e}"
       warn e.backtrace
@@ -86,8 +78,9 @@ end
 
 if __FILE__ == $PROGRAM_NAME
   # Default to list of authorities we can't or won't fix in code, explain why
-  # wagga: url redirects and then reports Application error
+  # some: url-for-issue Summary Reason
+  # councils : url-for-issue Summary Reason
 
-  ENV["MORPH_EXPECT_BAD"] ||= "wagga"
+  ENV["MORPH_EXPECT_BAD"] ||= "some,councils"
   Scraper.run(Scraper.selected_authorities)
 end

data/docs/getting_started.md

@@ -54,90 +54,19 @@ export DEBUG=1 # for basic, or 2 for verbose or 3 for tracing nearly everything
 
 ## Example Scraper Implementation
 
-Update your `scraper.rb` as follows:
-
-```ruby
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-$LOAD_PATH << "./lib"
-
-require "scraper_utils"
-require "your_scraper"
-
-# Main Scraper class
-class Scraper
-  AUTHORITIES = YourScraper::AUTHORITIES
-
-  def scrape(authorities, attempt)
-    exceptions = {}
-    authorities.each do |authority_label|
-      puts "\nCollecting feed data for #{authority_label}, attempt: #{attempt}..."
-
-      begin
-        ScraperUtils::DataQualityMonitor.start_authority(authority_label)
-        YourScraper.scrape(authority_label) do |record|
-          begin
-            record["authority_label"] = authority_label.to_s
-            ScraperUtils::DbUtils.save_record(record)
-          rescue ScraperUtils::UnprocessableRecord => e
-            ScraperUtils::DataQualityMonitor.log_unprocessable_record(e, record)
-            exceptions[authority_label] = e
-          end
-        end
-      rescue StandardError => e
-        warn "#{authority_label}: ERROR: #{e}"
-        warn e.backtrace
-        exceptions[authority_label] = e
-      end
-    end
-
-    exceptions
-  end
-
-  def self.selected_authorities
-    ScraperUtils::AuthorityUtils.selected_authorities(AUTHORITIES.keys)
-  end
-
-  def self.run(authorities)
-    puts "Scraping authorities: #{authorities.join(', ')}"
-    start_time = Time.now
-    exceptions = new.scrape(authorities, 1)
-    ScraperUtils::LogUtils.log_scraping_run(
-      start_time,
-      1,
-      authorities,
-      exceptions
-    )
-
-    unless exceptions.empty?
-      puts "\n***************************************************"
-      puts "Now retrying authorities which earlier had failures"
-      puts exceptions.keys.join(", ").to_s
-      puts "***************************************************"
-
-      start_time = Time.now
-      exceptions = new.scrape(exceptions.keys, 2)
-      ScraperUtils::LogUtils.log_scraping_run(
-        start_time,
-        2,
-        authorities,
-        exceptions
-      )
-    end
-
-    ScraperUtils::LogUtils.report_on_results(authorities, exceptions)
-  end
-end
-
-if __FILE__ == $PROGRAM_NAME
-  ENV["MORPH_EXPECT_BAD"] ||= "wagga"
-  Scraper.run(Scraper.selected_authorities)
-end
-```
+Update your `scraper.rb` as per [example scraper](example_scraper.rb)
 
 For more advanced implementations, see the [Interleaving Requests documentation](interleaving_requests.md).
 
+## Logging Tables
+
+The following logging tables are created for use in monitoring failure patterns and debugging issues.
+Records are automaticaly cleared after 30 days.
+
+The `ScraperUtils::LogUtils.log_scraping_run` call also logs the information to the `scrape_log` table.
+
+The `ScraperUtils::LogUtils.save_summary_record` call also logs the information to the `scrape_summary` table.
+
 ## Next Steps
 
 - [Reducing Server Load](reducing_server_load.md)

data/docs/interleaving_requests.md

@@ -11,37 +11,8 @@ The `ScraperUtils::FiberScheduler` provides a lightweight utility that:
 
 ## Implementation
 
-To enable fiber scheduling, change your scrape method to follow this pattern:
-
-```ruby
-def scrape(authorities, attempt)
-  ScraperUtils::FiberScheduler.reset!
-  exceptions = {}
-  authorities.each do |authority_label|
-    ScraperUtils::FiberScheduler.register_operation(authority_label) do
-      ScraperUtils::FiberScheduler.log(
-        "Collecting feed data for #{authority_label}, attempt: #{attempt}..."
-      )
-      ScraperUtils::DataQualityMonitor.start_authority(authority_label)
-      YourScraper.scrape(authority_label) do |record|
-        record["authority_label"] = authority_label.to_s
-        ScraperUtils::DbUtils.save_record(record)
-      rescue ScraperUtils::UnprocessableRecord => e
-        ScraperUtils::DataQualityMonitor.log_unprocessable_record(e, record)
-        exceptions[authority_label] = e
-        # Continues processing other records
-      end
-    rescue StandardError => e
-      warn "#{authority_label}: ERROR: #{e}"
-      warn e.backtrace || "No backtrace available"
-      exceptions[authority_label] = e
-    end
-    # end of register_operation block
-  end
-  ScraperUtils::FiberScheduler.run_all
-  exceptions
-end
-```
+To enable fiber scheduling, change your scrape method as per
+[example scrape with fibers](example_scrape_with_fibers.rb)
 
 ## Logging with FiberScheduler
 

data/lib/scraper_utils/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ScraperUtils
-  VERSION = "0.5.0"
+  VERSION = "0.5.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scraper_utils
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.1
 platform: ruby
 authors:
 - Ian Heggie
@@ -106,7 +106,7 @@ metadata:
   allowed_push_host: https://rubygems.org
   homepage_uri: https://github.com/ianheggie-oaf/scraper_utils
   source_code_uri: https://github.com/ianheggie-oaf/scraper_utils
-  documentation_uri: https://rubydoc.info/gems/scraper_utils/0.5.0
+  documentation_uri: https://rubydoc.info/gems/scraper_utils/0.5.1
   changelog_uri: https://github.com/ianheggie-oaf/scraper_utils/blob/main/CHANGELOG.md
   rubygems_mfa_required: 'true'
 post_install_message: