scraper_utils 0.7.1 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4293294d99565b0ee4adc097e9d31619868477e6efa8255f3cd5ec02dc8b275b
-  data.tar.gz: e78a08bced34a1f55b8d6e869158887da33f1d6b9803493c7d836402d64aa4ef
+  metadata.gz: 9be052a9483d914f35555b8a4a80992be5d18737f79063161053faae9e97a849
+  data.tar.gz: b170b9f1b2b973647d8e3ce6426da425773077bdb8ef10dc551b451895326e1c
 SHA512:
-  metadata.gz: ea00598b58e69cf5d911b62148d5d38e13ef017dd5a22159839f3b4437bf28c7c2b17c4832e3869688970037048f69fb543f9645922d6b92888a5fc68d88fc33
-  data.tar.gz: 57500daacbec8602c9288bf27bd7ac5f2ff407f621f9ac2c9e114a85fbbb0c555ebebfdc62782b814616b9b46dad65bd5a665b24bb1c9e6774625469bb290485
+  metadata.gz: 210d580999d5db053f1581e7ba5926fa4d968e5e4e915e73bebb063f3106d106cb83d9fbb4645ad568e3994caab909ca27a942f10b5f965d3881eceb9a83f64a
+  data.tar.gz: 063eec87dc3e124744a73494db47e27505d7de330568ba1e3562d75bccdafe1ab054f358c6b93d3904785a001274d6aec7056f3bbc2ae4665280c605a4b100e9

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 0.8.0 - 2025-05-06
+
+* Added ScraperUtils::LogUtils.project_backtrace_line to provide the first project related backtrace line
+* Included this summarized line in ScraperUtils::LogUtils.report_on_results report
+* Allow upto 250 character error message (was max 50)
+
+## 0.7.2 - 2025-04-15
+
+* Accept postcode before state as well as after
+
 ## 0.7.1 - 2025-04-15
 
 * Accept mixed case suburb names after a comma as well as uppercase suburb names as geocachable

data/Gemfile-heroku-18

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+ruby "~> 3.2.2"
+
+gem "mechanize", "~> 2.8.5"
+gem "nokogiri", "~> 1.15.0"
+gem "sqlite3", "~> 1.6.3"
+
+# Unable to list in gemspec - Include it in your projects Gemfile when using this gem
+gem "scraperwiki", git: "https://github.com/openaustralia/scraperwiki-ruby.git",
+                   branch: "morph_defaults"
+
+# development and test gems
+gem "rake", "~> 13.0"
+gem "rspec", "~> 3.12"
+gem "rubocop", "~> 1.73"
+gem "rubocop-rake", "~> 0.7"
+gem "rubocop-rspec", "~> 3.5"
+gem "simplecov", "~> 0.22.0"
+gem "simplecov-console"
+gem "terminal-table"
+gem "webmock", "~> 3.19.0"
+gem "yard"
+

data/lib/scraper_utils/log_utils.rb

@@ -67,7 +67,7 @@ module ScraperUtils
           "records_saved" => stats[:saved] || 0,
           "unprocessable_records" => stats[:unprocessed] || 0,
           "status" => status.to_s,
-          "error_message" => exception&.message,
+          "error_message" => exception&.to_s,
           "error_class" => exception&.class&.to_s,
           "error_backtrace" => extract_meaningful_backtrace(exception)
         }
@@ -88,6 +88,38 @@ module ScraperUtils
       cleanup_old_records
     end
 
+    # Extracts the first relevant line from backtrace that's from our project
+    # (not from gems, vendor, or Ruby standard library)
+    #
+    # @param backtrace [Array<String>] The exception backtrace
+    # @param options [Hash] Options hash
+    # @option options [String] :pwd The project root directory (defaults to current working directory)
+    # @option options [Boolean] :format If true, returns formatted string with brackets
+    # @return [String, nil] The relevant backtrace line without PWD prefix, or nil if none found
+    def self.project_backtrace_line(backtrace, options = {})
+      return nil if backtrace.nil? || backtrace.empty?
+
+      # Set defaults
+      pwd = options[:pwd] || Dir.pwd
+      format = options[:format] || false
+
+      # Normalize the root directory path with a trailing slash
+      pwd = File.join(pwd, '')
+
+      backtrace.each do |line|
+        next if line.include?('/gems/') ||
+                line.include?('/vendor/') ||
+                line.include?('/ruby/')
+
+        if line.start_with?(pwd)
+          relative_path = line.sub(pwd, '')
+          return format ? " [#{relative_path}]" : relative_path
+        end
+      end
+
+      format ? "" : nil
+    end
+
     # Report on the results
     # @param authorities [Array<Symbol>] List of authorities attempted to scrape
     # @param exceptions [Hash{Symbol => Exception}] Any exceptions that occurred during scraping
@@ -117,12 +149,14 @@ module ScraperUtils
 
         expect_bad_prefix = expect_bad.include?(authority) ? "[EXPECT BAD] " : ""
         exception_msg = if exceptions[authority]
-                          "#{exceptions[authority].class} - #{exceptions[authority].message}"
+                          location = self.project_backtrace_line(exceptions[authority].backtrace, format: true)
+                          puts "LOCATION: #{location.inspect}"
+                          "#{exceptions[authority].class} - #{exceptions[authority]}#{location}"
                         else
                           "-"
                         end
         puts format(summary_format, authority.to_s, ok_records, bad_records,
-                    "#{expect_bad_prefix}#{exception_msg}".slice(0, 70))
+                    "#{expect_bad_prefix}#{exception_msg}".slice(0, 250))
       end
       puts
 
@@ -149,7 +183,7 @@ module ScraperUtils
                   "(Add to MORPH_EXPECT_BAD?)"
         unexpected_errors.each do |authority|
           error = exceptions[authority]
-          errors << "  #{authority}: #{error.class} - #{error.message}"
+          errors << "  #{authority}: #{error.class} - #{error}"
         end
       end
 
@@ -212,7 +246,7 @@ module ScraperUtils
 
       lines = []
       error.backtrace.each do |line|
-        lines << line if lines.length < 2 || !line.include?("/vendor/")
+        lines << line if lines.length < 2 || !(line.include?("/vendor/") || line.include?("/gems/") || line.include?("/ruby/"))
         break if lines.length >= 6
       end
 

data/lib/scraper_utils/mechanize_utils/agent_config.rb

@@ -127,7 +127,7 @@ module ScraperUtils
           uri = begin
                   URI.parse(ScraperUtils.australian_proxy.to_s)
                 rescue URI::InvalidURIError => e
-                  raise URI::InvalidURIError, "Invalid proxy URL format: #{e.message}"
+                  raise URI::InvalidURIError, "Invalid proxy URL format: #{e}"
                 end
           unless uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS)
             raise URI::InvalidURIError, "Proxy URL must start with http:// or https://"

data/lib/scraper_utils/mechanize_utils/robots_checker.rb

@@ -78,7 +78,7 @@ module ScraperUtils
         rescue StandardError => e
           if DebugUtils.basic?
             ScraperUtils::LogUtils.log(
-              "WARNING: Failed to fetch robots.txt for #{domain}: #{e.message}"
+              "WARNING: Failed to fetch robots.txt for #{domain}: #{e}"
             )
           end
           nil

data/lib/scraper_utils/scheduler/thread_response.rb

@@ -51,7 +51,7 @@ module ScraperUtils
       # @return [String] Readable representation
       def inspect
         status = success? ? "success" : "FAILED"
-        error_info = success? ? "" : " - #{error.class}: #{error.message}"
+        error_info = success? ? "" : " - #{error.class}: #{error}"
         "#<#{self.class} authority=#{authority} #{status}#{error_info} time=#{time_taken}>"
       end
     end

data/lib/scraper_utils/spec_support.rb

@@ -11,10 +11,10 @@ module ScraperUtils
       /\bB(oulevard|lvd)\b/i,
       /\b(Circuit|Cct)\b/i,
       /\bCl(ose)?\b/i,
-      /\bC(our|r)t\b/i,
+      /\bC(our|r)?t\b/i,
       /\bCircle\b/i,
       /\bChase\b/i,
-      /\bCr(escent)?\b/i,
+      /\bCr(es(cent)?)?\b/i,
       /\bDr((ive)?|v)\b/i,
       /\bEnt(rance)?\b/i,
       /\bGr(ove)?\b/i,
@@ -33,7 +33,6 @@ module ScraperUtils
       /\bWay\b/i
     ].freeze
 
-
     AUSTRALIAN_POSTCODES = /\b\d{4}\b/.freeze
 
     # Check if an address is likely to be geocodable by analyzing its format.
@@ -53,7 +52,7 @@ module ScraperUtils
 
       has_unit_or_lot = address.match?(/\b(Unit|Lot:?)\s+\d+/i)
 
-      has_suburb_stats = check_address.match?(/(\b[A-Z]{2,}(\s+[A-Z]+)*,?|,\s+[A-Z][A-Za-z ]+)\s+(#{AUSTRALIAN_STATES.join('|')})\b/)
+      has_suburb_stats = check_address.match?(/(\b[A-Z]{2,}(\s+[A-Z]+)*,?|,\s+[A-Z][A-Za-z ]+)(\s+\d{4})?\s+(#{AUSTRALIAN_STATES.join('|')})\b/)
 
       if ENV["DEBUG"]
         missing = []

data/lib/scraper_utils/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ScraperUtils
-  VERSION = "0.7.1"
+  VERSION = "0.8.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scraper_utils
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.8.0
 platform: ruby
 authors:
 - Ian Heggie
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-04-15 00:00:00.000000000 Z
+date: 2025-05-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -68,6 +68,7 @@ files:
 - CHANGELOG.md
 - GUIDELINES.md
 - Gemfile
+- Gemfile-heroku-18
 - IMPLEMENTATION.md
 - LICENSE.txt
 - README.md
@@ -118,7 +119,7 @@ metadata:
   allowed_push_host: https://rubygems.org
   homepage_uri: https://github.com/ianheggie-oaf/scraper_utils
   source_code_uri: https://github.com/ianheggie-oaf/scraper_utils
-  documentation_uri: https://rubydoc.info/gems/scraper_utils/0.7.1
+  documentation_uri: https://rubydoc.info/gems/scraper_utils/0.8.0
   changelog_uri: https://github.com/ianheggie-oaf/scraper_utils/blob/main/CHANGELOG.md
   rubygems_mfa_required: 'true'
 post_install_message:
@@ -136,7 +137,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
+rubygems_version: 3.4.19
 signing_key:
 specification_version: 4
 summary: planningalerts scraper utilities