scraper_utils 0.11.1 → 0.12.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2437b1d72d0a7d4924b4f22d906844fc15eef3448095e04365b210df368a0e80
-  data.tar.gz: 860865eff6d101a28bc4c6be564a3cbd1c23ce97c7da258d4baba038c8e4713d
+  metadata.gz: fc8aab3f24d29cc1e3fe44880d9030e7149372189a8691acd6e628e2a260c57c
+  data.tar.gz: a82ecb97878e57f5d0cf75d623c94a6ff54e7805c43a7be140e6a2da7bbcca49
 SHA512:
-  metadata.gz: 8e226c73b7530ffc0e382a438896ad0f0d9fbbc8acdc6064f4d33874be017b663fa525a1de7428cba0740849dbfae38152566ddb85bae8d71669e9ef995cefff
-  data.tar.gz: 7ff0da628481fc403ca849e750b6e880f41cdca4f83ab92476d33f0aa3b42898a23825a2b92dc1618f371c47754e07a4a98fbf70d134c39bbd0e929281abd790
+  metadata.gz: bd6d178afa8669916b70f2c6bdb56b77a8a5995d18e47c64b360a8d5d334b479645d20792d80759afde1aac6550d79f46a0504030d8a8920c8dea55fa6ad6132
+  data.tar.gz: 20a2c5f144cfc8e2106d2e4643d7f3b0cb35110a5519d63bb0d8c1e655c8958fe73115089352a71272a419168c50299c8ce985d318be6bfa9635e64c9c4fb238

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## 0.12.1 - 2026.02-18
+
+* Added override for the threshold of when to abdon scraping due to unprocessable records
+  * $MORPH_UNPROCESSABLE_BASE - default 5.01
+  * $MORPH_UNPROCESSABLE_PERCENTAGE - default 10
+* Log council_reference as well as address for unprocessable records
+
 ## 0.11.1 - 2026-02-13
 
 * Output council_reference from log_saved_record as it helps me debug some web query issues

data/docs/example_scraper.rb

@@ -74,7 +74,7 @@ class Scraper
       )
     end
 
-    ScraperUtils::DbUtils.cleanup_old_records 
+    ScraperUtils::DbUtils.cleanup_old_records
     # Report on results, raising errors for unexpected conditions
     ScraperUtils::LogUtils.report_on_results(authorities, exceptions)
   end
@@ -92,6 +92,10 @@ if __FILE__ == $PROGRAM_NAME
 
     ENV["MORPH_EXPECT_BAD"] = default_expect_bad.keys.join(',')
   end
+  # If the sites have many unusable records - raise defaults
+  # ENV['MORPH_UNPROCESSABLE_BASE'] ||= "10"
+  # ENV['MORPH_UNPROCESSABLE_PERCENTAGE'] ||= "20"
+
   Scraper.run(Scraper.selected_authorities)
 
   # Dump database for morph-cli

data/lib/scraper_utils/data_quality_monitor.rb

@@ -26,8 +26,12 @@ module ScraperUtils
       authority_label
     end
 
+    # Threshold for unprocessable records
+    # Initial base of 5.01 (override using MORPH_UNPROCESSABLE_BASE)
+    # Initial percentage of 10% (override using MORPH_UNPROCESSABLE_PERCENTAGE)
     def self.threshold(authority_label)
-      5.01 + (@stats[authority_label][:saved] * 0.1) if @stats&.fetch(authority_label, nil)
+      ENV.fetch('MORPH_UNPROCESSABLE_BASE', 5.01).to_f +
+        (@stats[authority_label][:saved].to_i * ENV.fetch('MORPH_UNPROCESSABLE_PERCENTAGE', 10.0).to_f / 100.0) if @stats&.fetch(authority_label, nil)
     end
 
     # Logs an unprocessable record and raises an exception if error threshold is exceeded
@@ -40,9 +44,12 @@ module ScraperUtils
     def self.log_unprocessable_record(exception, record)
       authority_label = extract_authority(record)
       @stats[authority_label][:unprocessed] += 1
-      ScraperUtils::LogUtils.log "Erroneous record #{authority_label} - #{record&.fetch(
-        'address', nil
-      ) || record.inspect}: #{exception}"
+      details = if record&.key?('council_reference') && record&.key?('address')
+                  "#{record['council_reference']} - #{record['address']}"
+                else
+                  record.inspect
+                end
+      ScraperUtils::LogUtils.log "Erroneous record #{details}: #{exception}"
       return unless @stats[authority_label][:unprocessed] > threshold(authority_label)
 
       raise ScraperUtils::UnprocessableSite,

data/lib/scraper_utils/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ScraperUtils
-  VERSION = "0.11.1"
+  VERSION = "0.12.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scraper_utils
 version: !ruby/object:Gem::Version
-  version: 0.11.1
+  version: 0.12.1
 platform: ruby
 authors:
 - Ian Heggie
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-02-13 00:00:00.000000000 Z
+date: 2026-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -112,7 +112,7 @@ metadata:
   allowed_push_host: https://rubygems.org
   homepage_uri: https://github.com/ianheggie-oaf/scraper_utils
   source_code_uri: https://github.com/ianheggie-oaf/scraper_utils
-  documentation_uri: https://rubydoc.info/gems/scraper_utils/0.11.1
+  documentation_uri: https://rubydoc.info/gems/scraper_utils/0.12.1
   changelog_uri: https://github.com/ianheggie-oaf/scraper_utils/blob/main/CHANGELOG.md
   rubygems_mfa_required: 'true'
 post_install_message: