sitediff 1.2.0 → 1.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2e37a67bb4f0d7b93f252940d4ee0e4e3184dc27a435626f995b70bad4a7fc40
-  data.tar.gz: 692d0a82b230e2dbab10fe8f6ee8591ca0128998f4c9acd6b97a37a90856c887
+  metadata.gz: dff12d889984ec88ad662c2a0f2f3e0771b2a2c6cbc8e5f0442773ab36a51e7c
+  data.tar.gz: 96541e827d456c925677821c501297b68b284828584f35e499b4b35da75f962f
 SHA512:
-  metadata.gz: 72551efe76eaa6a4a23aeacba6b25cb3bb8b4483d27299846f55ba7a7bad8e256f798de23db36f501a6739875494be808541834bfaadf587fbbd078b6fa62506
-  data.tar.gz: 977634a139f70794aa5015e42e014e542391f5cf3c07bd34bff1f7128514051c341512d607f1cb9dfb0326c2bee0c80b2c6b4b119e21897adb5bf516fa7a3140
+  metadata.gz: 3044d99f7494697d817f4ab545308987dbcaebd007f531f9113c2298f3f1550952967f92f0223a96782efed424b4c2ca97123fb0fce20a382b955086afef3386
+  data.tar.gz: 7715c7285734dad676fe95cc4fc4b6cd69411a073a4c3bca9b839ff152dfa427b0b6655159e54fc7a6e82c6ac3c0c5c9c86f312c9c7d287fc5101ec6cff0d23b

data/CHANGELOG.md

@@ -2,6 +2,11 @@
 
 Contains noteworthy changes made to SiteDiff.
 
+## Version 1.2.1
+- Fixed a bug with report exporting.
+- Prevents crawling the same site twice if the before and after urls are the same.
+- Adding a referrer to the crawler errors.
+
 ## Version 1.2.0
 - Updated requirement to Ruby 3.1.2.
 - Upgraded modules for security and compatibility.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    sitediff (1.2.0)
+    sitediff (1.2.1)
       addressable (>= 2.5.2, < 2.9.0)
       diffy (~> 3.4.0)
       minitar (~> 0.9)

data/INSTALLATION.md

@@ -36,7 +36,7 @@ avoid using `sudo` for `gem install`.
 
 ```bash
 gem install nokogiri --no-rdoc --no-ri -- --use-system-libraries=true —with-xml2-include=/usr/include/libxml2
-gem install sitediff -v '1.2.0'
+gem install sitediff -v '1.2.1'
 ```
 
 ## Docker
@@ -88,7 +88,7 @@ If possible avoid using `sudo` for `gem install`.
 
 ```bash
 gem install nokogiri --no-rdoc --no-ri -- --use-system-libraries=true —with-xml2-include=/usr/include/libxml2
-gem install sitediff -v '1.2.0'
+gem install sitediff -v '1.2.1'
 ```
 
 ## Ubuntu

data/lib/sitediff/api.rb

@@ -159,7 +159,13 @@ class SiteDiff
         max_concurrency: @config.setting(:concurrency)
       )
       @paths = {}
-      @config.roots.each do |tag, url|
+
+      ignoreAfter = @config.roots
+      if @config.roots['before'] == @config.roots['after']
+        ignoreAfter.delete('after')    
+      end
+      
+      ignoreAfter.each do |tag, url|
         Crawler.new(
           hydra,
           url,

data/lib/sitediff/crawler.rb

@@ -34,16 +34,16 @@ class SiteDiff
       @curl_opts = curl_opts
       @debug = debug
 
-      add_uri('', depth)
+      add_uri('', depth, referrer: '/')
     end
 
     # Handle a newly found relative URI
-    def add_uri(rel, depth)
+    def add_uri(rel, depth, referrer = '')
       return if @found.include? rel
 
       @found << rel
 
-      wrapper = UriWrapper.new(@base + rel, @curl_opts, debug: @debug)
+      wrapper = UriWrapper.new(@base + rel, @curl_opts, debug: @debug, referrer: referrer)
       wrapper.queue(@hydra) do |res|
         fetched_uri(rel, depth, res)
       end
@@ -90,7 +90,7 @@ class SiteDiff
       rels.each do |r|
         next if @found.include? r
 
-        add_uri(r, depth - 1)
+        add_uri(r, depth - 1, rel)
       end
     end
 

data/lib/sitediff/report.rb

@@ -147,7 +147,7 @@ class SiteDiff
       temp_path.mkpath
       report_path = temp_path + REPORT_DIR
       report_path.mkpath
-      files_path = "#{report_path}files"
+      files_path = report_path + "files"
       files_path.mkpath
       diffs_path = dir + DIFFS_DIR
 

data/lib/sitediff/uriwrapper.rb

@@ -48,12 +48,13 @@ class SiteDiff
 
     ##
     # Creates a UriWrapper.
-    def initialize(uri, curl_opts = DEFAULT_CURL_OPTS, debug: true)
+    def initialize(uri, curl_opts = DEFAULT_CURL_OPTS, debug: true, referrer: '')
       @uri = uri.respond_to?(:scheme) ? uri : Addressable::URI.parse(uri)
       # remove trailing '/'s from local URIs
       @uri.path.gsub!(%r{/*$}, '') if local?
       @curl_opts = curl_opts
       @debug = debug
+      @referrer = referrer
     end
 
     ##
@@ -136,31 +137,31 @@ class SiteDiff
           raise if @debug
 
           yield ReadResult.error(
-            "Parsing error for #{@uri}: #{e.message}"
+            "Parsing error for #{@uri}: #{e.message}  From page: #{@referrer}"
           )
         rescue StandardError => e
           raise if @debug
 
           yield ReadResult.error(
-            "Unknown parsing error for #{@uri}: #{e.message}"
+            "Unknown parsing error for #{@uri}: #{e.message}  From page: #{@referrer}"
           )
         end
       end
 
-      req.on_failure do |resp|
+      req.on_failure do |resp|           
         if resp&.status_message
           yield ReadResult.error(
-            "HTTP error when loading #{@uri} : [#{resp.response_code}] #{resp.status_message}",
+            "HTTP error when loading #{@uri} : [#{resp.response_code}] #{resp.status_message}  From page: #{@referrer}",
             resp.response_code
           )
         elsif (msg = resp.options[:return_code])
           yield ReadResult.error(
-            "Connection error when loading #{@uri} : [#{resp.options[:return_code]}] #{resp.status_message} #{msg}",
+            "Connection error when loading #{@uri} : [#{resp.options[:return_code]}] #{msg}  From page: #{@referrer}",
             resp.response_code
           )
         else
           yield ReadResult.error(
-            "Unknown error when loading #{@uri} : [#{resp.response_code}] #{resp.status_message}",
+            "Unknown error when loading #{@uri} : [#{resp.response_code}] #{resp.status_message}  From page: #{@referrer}",
             resp.response_code
           )
         end

data/sitediff.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name        = 'sitediff'
-  s.version     = '1.2.0'
+  s.version     = '1.2.1'
   s.required_ruby_version = '>= 3.1.2'
   s.summary     = 'Compare two versions of a site with ease!'
   s.description = <<DESC

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sitediff
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.2.1
 platform: ruby
 authors:
 - Alex Dergachev
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-08-29 00:00:00.000000000 Z
+date: 2022-09-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pkg-config