scraper_utils 0.7.2 → 0.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ba66a28129ee09ab76cb0937d195ff68aa5058d1c73805235de4898384fe495d
-  data.tar.gz: 76c20b7ce9bd581e59fda41b8801bca8a26909d58ddcc74c46e7755e6038970a
+  metadata.gz: e83cc7c9eb0f93f1dd1b84e9528815cc9e81564dfbcacd7afdd32ec2acb60965
+  data.tar.gz: d1be3bb53939f60470b1ff247ac61c31a2946619306076a48c390e42431d2c1e
 SHA512:
-  metadata.gz: 5ae13c2c5e4b8bb1c30c2c8a10dd30b42b349dfc7c416fc65bec504e27d7b5c9dad84b6ef195c52788412f238a6740e4fb8fc400e315a42aeb7ee57f8ada9a25
-  data.tar.gz: bf3d40831ee8667f663b442f92e76943db04eaa995095e09f4c3736e80919d26c924aa17d733a533ec32b278496c2aedd8c368eb40988e1fd9619c5febfb1567
+  metadata.gz: 4a094c22431c7b4dd34512a8eca31e36f6a41555b436be222f615ada2914fe54d38e0e69e223689cbd6c299b8a523c4085007fae02bcef2f2524fa6c2dcbef89
+  data.tar.gz: cd86b1d0d217dfb8808db194c5d14d13a06753a7009267279396e2366c6251506ca36e0c95be856ca5afbc8b096857e19802b43badd3f326d32247346badb1f2

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 0.8.1 - 2025-05-06
+
+* Removed debugging output accidentally left in
+
+## 0.8.0 - 2025-05-06
+
+* Added ScraperUtils::LogUtils.project_backtrace_line to provide the first project related backtrace line
+* Included this summarized line in ScraperUtils::LogUtils.report_on_results report
+* Allow upto 250 character error message (was max 50)
+
 ## 0.7.2 - 2025-04-15
 
 * Accept postcode before state as well as after

data/Gemfile-heroku-18

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+ruby "~> 3.2.2"
+
+gem "mechanize", "~> 2.8.5"
+gem "nokogiri", "~> 1.15.0"
+gem "sqlite3", "~> 1.6.3"
+
+# Unable to list in gemspec - Include it in your projects Gemfile when using this gem
+gem "scraperwiki", git: "https://github.com/openaustralia/scraperwiki-ruby.git",
+                   branch: "morph_defaults"
+
+# development and test gems
+gem "rake", "~> 13.0"
+gem "rspec", "~> 3.12"
+gem "rubocop", "~> 1.73"
+gem "rubocop-rake", "~> 0.7"
+gem "rubocop-rspec", "~> 3.5"
+gem "simplecov", "~> 0.22.0"
+gem "simplecov-console"
+gem "terminal-table"
+gem "webmock", "~> 3.19.0"
+gem "yard"
+

data/lib/scraper_utils/log_utils.rb

@@ -67,7 +67,7 @@ module ScraperUtils
           "records_saved" => stats[:saved] || 0,
           "unprocessable_records" => stats[:unprocessed] || 0,
           "status" => status.to_s,
-          "error_message" => exception&.message,
+          "error_message" => exception&.to_s,
           "error_class" => exception&.class&.to_s,
           "error_backtrace" => extract_meaningful_backtrace(exception)
         }
@@ -88,6 +88,38 @@ module ScraperUtils
       cleanup_old_records
     end
 
+    # Extracts the first relevant line from backtrace that's from our project
+    # (not from gems, vendor, or Ruby standard library)
+    #
+    # @param backtrace [Array<String>] The exception backtrace
+    # @param options [Hash] Options hash
+    # @option options [String] :pwd The project root directory (defaults to current working directory)
+    # @option options [Boolean] :format If true, returns formatted string with brackets
+    # @return [String, nil] The relevant backtrace line without PWD prefix, or nil if none found
+    def self.project_backtrace_line(backtrace, options = {})
+      return nil if backtrace.nil? || backtrace.empty?
+
+      # Set defaults
+      pwd = options[:pwd] || Dir.pwd
+      format = options[:format] || false
+
+      # Normalize the root directory path with a trailing slash
+      pwd = File.join(pwd, '')
+
+      backtrace.each do |line|
+        next if line.include?('/gems/') ||
+                line.include?('/vendor/') ||
+                line.include?('/ruby/')
+
+        if line.start_with?(pwd)
+          relative_path = line.sub(pwd, '')
+          return format ? " [#{relative_path}]" : relative_path
+        end
+      end
+
+      format ? "" : nil
+    end
+
     # Report on the results
     # @param authorities [Array<Symbol>] List of authorities attempted to scrape
     # @param exceptions [Hash{Symbol => Exception}] Any exceptions that occurred during scraping
@@ -117,12 +149,13 @@ module ScraperUtils
 
         expect_bad_prefix = expect_bad.include?(authority) ? "[EXPECT BAD] " : ""
         exception_msg = if exceptions[authority]
-                          "#{exceptions[authority].class} - #{exceptions[authority].message}"
+                          location = self.project_backtrace_line(exceptions[authority].backtrace, format: true)
+                          "#{exceptions[authority].class} - #{exceptions[authority]}#{location}"
                         else
                           "-"
                         end
         puts format(summary_format, authority.to_s, ok_records, bad_records,
-                    "#{expect_bad_prefix}#{exception_msg}".slice(0, 70))
+                    "#{expect_bad_prefix}#{exception_msg}".slice(0, 250))
       end
       puts
 
@@ -149,7 +182,7 @@ module ScraperUtils
                   "(Add to MORPH_EXPECT_BAD?)"
         unexpected_errors.each do |authority|
           error = exceptions[authority]
-          errors << "  #{authority}: #{error.class} - #{error.message}"
+          errors << "  #{authority}: #{error.class} - #{error}"
         end
       end
 
@@ -212,7 +245,7 @@ module ScraperUtils
 
       lines = []
       error.backtrace.each do |line|
-        lines << line if lines.length < 2 || !line.include?("/vendor/")
+        lines << line if lines.length < 2 || !(line.include?("/vendor/") || line.include?("/gems/") || line.include?("/ruby/"))
         break if lines.length >= 6
       end
 

data/lib/scraper_utils/mechanize_utils/agent_config.rb

@@ -127,7 +127,7 @@ module ScraperUtils
           uri = begin
                   URI.parse(ScraperUtils.australian_proxy.to_s)
                 rescue URI::InvalidURIError => e
-                  raise URI::InvalidURIError, "Invalid proxy URL format: #{e.message}"
+                  raise URI::InvalidURIError, "Invalid proxy URL format: #{e}"
                 end
           unless uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS)
             raise URI::InvalidURIError, "Proxy URL must start with http:// or https://"

data/lib/scraper_utils/mechanize_utils/robots_checker.rb

@@ -78,7 +78,7 @@ module ScraperUtils
         rescue StandardError => e
           if DebugUtils.basic?
             ScraperUtils::LogUtils.log(
-              "WARNING: Failed to fetch robots.txt for #{domain}: #{e.message}"
+              "WARNING: Failed to fetch robots.txt for #{domain}: #{e}"
             )
           end
           nil

data/lib/scraper_utils/scheduler/thread_response.rb

@@ -51,7 +51,7 @@ module ScraperUtils
       # @return [String] Readable representation
       def inspect
         status = success? ? "success" : "FAILED"
-        error_info = success? ? "" : " - #{error.class}: #{error.message}"
+        error_info = success? ? "" : " - #{error.class}: #{error}"
         "#<#{self.class} authority=#{authority} #{status}#{error_info} time=#{time_taken}>"
       end
     end

data/lib/scraper_utils/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ScraperUtils
-  VERSION = "0.7.2"
+  VERSION = "0.8.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scraper_utils
 version: !ruby/object:Gem::Version
-  version: 0.7.2
+  version: 0.8.1
 platform: ruby
 authors:
 - Ian Heggie
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-04-15 00:00:00.000000000 Z
+date: 2025-05-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -68,6 +68,7 @@ files:
 - CHANGELOG.md
 - GUIDELINES.md
 - Gemfile
+- Gemfile-heroku-18
 - IMPLEMENTATION.md
 - LICENSE.txt
 - README.md
@@ -118,7 +119,7 @@ metadata:
   allowed_push_host: https://rubygems.org
   homepage_uri: https://github.com/ianheggie-oaf/scraper_utils
   source_code_uri: https://github.com/ianheggie-oaf/scraper_utils
-  documentation_uri: https://rubydoc.info/gems/scraper_utils/0.7.2
+  documentation_uri: https://rubydoc.info/gems/scraper_utils/0.8.1
   changelog_uri: https://github.com/ianheggie-oaf/scraper_utils/blob/main/CHANGELOG.md
   rubygems_mfa_required: 'true'
 post_install_message:
@@ -136,7 +137,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
+rubygems_version: 3.4.19
 signing_key:
 specification_version: 4
 summary: planningalerts scraper utilities