sitediff 1.2.0 → 1.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2e37a67bb4f0d7b93f252940d4ee0e4e3184dc27a435626f995b70bad4a7fc40
-  data.tar.gz: 692d0a82b230e2dbab10fe8f6ee8591ca0128998f4c9acd6b97a37a90856c887
+  metadata.gz: d1ee18040cb7cad571450e2e17f9144bdab9965b0c52f84199a8eecb7d046e71
+  data.tar.gz: 7d3744b782caae37bbb7fc7789e9fd3d0ae600849dbd13c9d77f60201af56792
 SHA512:
-  metadata.gz: 72551efe76eaa6a4a23aeacba6b25cb3bb8b4483d27299846f55ba7a7bad8e256f798de23db36f501a6739875494be808541834bfaadf587fbbd078b6fa62506
-  data.tar.gz: 977634a139f70794aa5015e42e014e542391f5cf3c07bd34bff1f7128514051c341512d607f1cb9dfb0326c2bee0c80b2c6b4b119e21897adb5bf516fa7a3140
+  metadata.gz: 7c3fc26f34f94fcacee9a8d87d7cdda3ea0c9a79aa40b8ff82d8a1f19b677fdda2fee147d6b91a3a9df0c4dc80541b9ae516778abc72fb4046399c18eefafb0d
+  data.tar.gz: 7f615aed415a0f313badb5a32873b611200b878587d40c5d5498d06a05088bb91531d4130c7f0f64261c6e9d33a167943b00b3023c2e9ef14c524cc2dc3d9aae

data/CHANGELOG.md

@@ -1,6 +1,24 @@
 # SiteDiff Change Log
 
 Contains noteworthy changes made to SiteDiff.
+## Version 1.2.5
+- Fix issue with whitespace in URLs.
+
+## Version 1.2.4
+- Fix issue with 'store' command.
+
+## Version 1.2.3
+- Fix issue with nil object during diff report generation.
+- Update to export documentation.
+
+## Version 1.2.2
+- Security update for Nokogiri.
+- Minor code updates.
+
+## Version 1.2.1
+- Fixed a bug with report exporting.
+- Prevents crawling the same site twice if the before and after urls are the same.
+- Adding a referrer to the crawler errors.
 
 ## Version 1.2.0
 - Updated requirement to Ruby 3.1.2.
@@ -25,4 +43,4 @@ Contains noteworthy changes made to SiteDiff.
 
 ## Prior to 1.0.0
 
-Release notes were out of date, so only tracking changes since 1.0.0 here.
+Release notes were out of date, so only tracking changes since 1.0.0 here.

data/Dockerfile

@@ -10,10 +10,10 @@ ARG DEBIAN_FRONTEND=noninteractive
 # Our build requires rake
 # Install editors: vim, nano.
 RUN apt-get update
-RUN apt-get install -y apt-utils
-RUN apt-get install -y software-properties-common
-RUN apt-get install -y make pkg-config libxml2-dev libxslt-dev
-RUN apt-get install -y vim nano git
+RUN apt-get install -y apt-utils \
+                       software-properties-common \
+                       make pkg-config libxml2-dev libxslt-dev \
+                       vim nano git
 
 # Force nokogiri gem not to compile libxml2, it takes too long
 ENV NOKOGIRI_USE_SYSTEM_LIBRARIES 1

data/Gemfile.lock

@@ -1,41 +1,39 @@
 PATH
   remote: .
   specs:
-    sitediff (1.2.0)
+    sitediff (1.2.4)
       addressable (>= 2.5.2, < 2.9.0)
       diffy (~> 3.4.0)
       minitar (~> 0.9)
-      nokogiri (>= 1.13.6)
+      nokogiri (>= 1.14.2)
       pkg-config (~> 1.4)
       rainbow (~> 3.1.1)
       thor (~> 1.2.1)
-      typhoeus (~> 1.4.0)
+      typhoeus (~> 1.4.1)
       webrick (>= 1.7)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.8.0)
-      public_suffix (>= 2.0.2, < 5.0)
+    addressable (2.8.1)
+      public_suffix (>= 2.0.2, < 6.0)
     ast (2.4.2)
     diff-lcs (1.5.0)
     diffy (3.4.2)
-    ethon (0.15.0)
-      ffi (>= 1.15.0)
+    ethon (0.10.0)
+      ffi (>= 1.3.0)
     ffi (1.15.5)
     fileutils (1.1.0)
     json (2.6.2)
-    mini_portile2 (2.8.0)
     minitar (0.9)
-    nokogiri (1.13.6)
-      mini_portile2 (~> 2.8.0)
+    nokogiri (1.14.2-arm64-darwin)
       racc (~> 1.4)
     parallel (1.22.1)
     parser (3.1.2.0)
       ast (~> 2.4.1)
-    pkg-config (1.4.7)
-    public_suffix (4.0.7)
-    racc (1.6.0)
+    pkg-config (1.5.1)
+    public_suffix (5.0.1)
+    racc (1.6.2)
     rainbow (3.1.1)
     regexp_parser (2.5.0)
     rexml (3.2.5)
@@ -66,10 +64,10 @@ GEM
       parser (>= 3.1.1.0)
     ruby-progressbar (1.11.0)
     thor (1.2.1)
-    typhoeus (1.4.0)
-      ethon (>= 0.9.0)
+    typhoeus (1.4.1)
+      ethon (= 0.10.0)
     unicode-display_width (2.2.0)
-    webrick (1.7.0)
+    webrick (1.8.1)
 
 PLATFORMS
   ruby

data/INSTALLATION.md

@@ -36,7 +36,7 @@ avoid using `sudo` for `gem install`.
 
 ```bash
 gem install nokogiri --no-rdoc --no-ri -- --use-system-libraries=true —with-xml2-include=/usr/include/libxml2
-gem install sitediff -v '1.2.0'
+gem install sitediff -v '1.2.1'
 ```
 
 ## Docker
@@ -88,7 +88,7 @@ If possible avoid using `sudo` for `gem install`.
 
 ```bash
 gem install nokogiri --no-rdoc --no-ri -- --use-system-libraries=true —with-xml2-include=/usr/include/libxml2
-gem install sitediff -v '1.2.0'
+gem install sitediff -v '1.2.1'
 ```
 
 ## Ubuntu

data/README.md

@@ -294,6 +294,13 @@ Generate a gzipped tar file containing the HTML report instead of generating
 and serving live web pages, this option overrides `--report-format`, forcing
 HTML.
 
+```
+sitediff diff --export
+sitediff diff -e
+```
+
+This will perform the diff and export the results in a gzipped tar file.
+
 ### Running inside containers
 
 If you run SiteDiff inside a container or virtual machine, the URLs in its
@@ -441,7 +448,7 @@ before comparison:
 dom_transform:
 # Remove current time block
   - type: remove
-  - selector: div#block-time
+    selector: div#block-time
 ```
 
 #### strip
@@ -458,7 +465,7 @@ To transform `<h1>  Foo and Bar\n  </h1>` to `<h1>Foo and Bar<\h1>`:
 dom_transform:
 # Strip H1 tags
   - type: strip
-  - selector: h1
+    selector: h1
 ```
 
 #### unwrap
@@ -655,7 +662,7 @@ EG:
   </div>
 </region>
 <region id="body">
-  <div class=".field-name-attribution">
+  <div class="field-name-attribution">
     <p>Lorem ipsum...
   </div>
 </region>

data/lib/sitediff/api.rb

@@ -159,7 +159,13 @@ class SiteDiff
         max_concurrency: @config.setting(:concurrency)
       )
       @paths = {}
-      @config.roots.each do |tag, url|
+
+      ignore_after = @config.roots
+      if @config.roots['before'] == @config.roots['after']
+        ignore_after.delete('after')
+      end
+
+      ignore_after.each do |tag, url|
         Crawler.new(
           hydra,
           url,
@@ -178,6 +184,10 @@ class SiteDiff
 
       # Write paths to a file.
       @paths = @paths.values.reduce(&:|).to_a.sort
+      if @paths.none? | @paths.nil?
+        return
+      end
+
       @config.paths_file_write(@paths)
 
       # Log output.
@@ -224,7 +234,7 @@ class SiteDiff
                                     @config.setting(:interval),
                                     @config.setting(:concurrency),
                                     get_curl_opts(@config.settings),
-                                    options[:debug],
+                                    debug: options[:debug],
                                     before: base)
       fetcher.run do |path, _res|
         SiteDiff.log "Visited #{path}, cached"

data/lib/sitediff/crawler.rb

@@ -34,16 +34,16 @@ class SiteDiff
       @curl_opts = curl_opts
       @debug = debug
 
-      add_uri('', depth)
+      add_uri('', depth, referrer: '/')
     end
 
     # Handle a newly found relative URI
-    def add_uri(rel, depth)
+    def add_uri(rel, depth, referrer = '')
       return if @found.include? rel
 
       @found << rel
 
-      wrapper = UriWrapper.new(@base + rel, @curl_opts, debug: @debug)
+      wrapper = UriWrapper.new(@base + rel, @curl_opts, debug: @debug, referrer:)
       wrapper.queue(@hydra) do |res|
         fetched_uri(rel, depth, res)
       end
@@ -90,12 +90,13 @@ class SiteDiff
       rels.each do |r|
         next if @found.include? r
 
-        add_uri(r, depth - 1)
+        add_uri(r, depth - 1, rel)
       end
     end
 
     # Resolve a potentially-relative link. Return nil on error.
     def resolve_link(base, rel)
+      rel = rel.strip
       base + rel
     rescue Addressable::URI::InvalidURIError
       SiteDiff.log "skipped invalid URL: '#{rel}' (at #{base})", :warning
@@ -129,6 +130,7 @@ class SiteDiff
                      u.path.start_with?(@base_uri.path)
         next unless is_sub_uri
 
+        # puts "Trying regex #{u.path}"
         is_included = @include_regex.nil? ? false : @include_regex.match(u.path)
         is_excluded = @exclude_regex.nil? ? false : @exclude_regex.match(u.path)
         if is_excluded && !is_included

data/lib/sitediff/files/report.html.erb

@@ -4,6 +4,8 @@
 
   # Pages compared.
   compared_pages = results.length
+
+  url_hash = '?' + Time.now.strftime("%s%L")
 %>
 <!DOCTYPE html>
 <html>
@@ -134,17 +136,17 @@
                 <div class="buttons">
                   <% unless relative %>
                     <% unless report['before_url_report'] === false %>
-                      <a href="<%= result.url(:before, before_url_report || before, cache) %>" class="button-before" target="_blank">Before</a>
+                      <a href="<%= result.url(:before, before_url_report || before, cache) + url_hash %>" class="button-before" target="_blank">Before</a>
                     <% end %>
                     <% unless report['after_url_report'] === false %>
-                      <a href="<%= result.url(:after, after_url_report || after, cache) %>" class="button-after" target="_blank">After</a>
+                      <a href="<%= result.url(:after, after_url_report || after, cache) + url_hash %>" class="button-after" target="_blank">After</a>
                     <% end %>
                     <% unless report['before_url_report'] === false || report['after_url_report'] === false %>
-                      <a href="/sidebyside<%= result.path %>" class="button-both">Both</a>
+                      <a href="/sidebyside<%= result.path  + url_hash %>" class="button-both">Both</a>
                     <% end %>
                   <% end %>
                   <% unless result.diff_url.nil? %>
-                    <a href="<%= result.diff_url(relative: relative) %>" class="button button-diff">View diff</a>
+                    <a href="<%= result.diff_url(relative: relative) + url_hash %>" class="button button-diff">View diff</a>
                   <% end %>
                 </div>
               </td>

data/lib/sitediff/presets/drupal.yaml

@@ -29,7 +29,7 @@ sanitization:
     pattern: '(src="[^"]*/misc/\w+\.js)?v=\d+\.\d+"'
     substitute: '\1'
   - title: Strip domain names from absolute URLs
-    pattern: 'http:\/\/[a-zA-Z0-9.:-]+'
+    pattern: 'https?:\/\/[a-zA-Z0-9.:-]+'
     substitute: '__domain__'
   - title: Strip form build ID
     selector: input
@@ -61,3 +61,7 @@ sanitization:
     selector: script
     pattern: 'js_[-\w]{40,43}\\?\.js'
     substitute: 'js__ID__.js'
+dom_transform:
+  # Strip Drupal.settings (>8.0)
+  - type: remove
+    selector: 'script[data-drupal-selector="drupal-settings-json"]'

data/lib/sitediff/report.rb

@@ -147,8 +147,8 @@ class SiteDiff
       temp_path.mkpath
       report_path = temp_path + REPORT_DIR
       report_path.mkpath
-      files_path = "#{report_path}files"
-      files_path.mkpath
+      files_path = "#{report_path}/files"
+      FileUtils.mkpath(files_path)
       diffs_path = dir + DIFFS_DIR
 
       # Move files to place.

data/lib/sitediff/sanitize.rb

@@ -220,10 +220,10 @@ class SiteDiff
 
     # Force this object to be a document, so we can apply a stylesheet
     def self.to_document(obj)
-      if Nokogiri::XML::Document == obj.class || Nokogiri::HTML::Document == obj.class
+      if obj.instance_of?(Nokogiri::XML::Document) || obj.instance_of?(Nokogiri::HTML::Document)
         obj
       # node or fragment
-      elsif Nokogiri::XML::Node == obj.class || Nokogiri::HTML::DocumentFragment == obj.class
+      elsif obj.instance_of?(Nokogiri::XML::Node) || obj.instance_of?(Nokogiri::HTML::DocumentFragment)
         domify(obj.to_s, force_doc: true)
       else
         to_document(domify(obj, force_doc: false))

data/lib/sitediff/uriwrapper.rb

@@ -48,12 +48,13 @@ class SiteDiff
 
     ##
     # Creates a UriWrapper.
-    def initialize(uri, curl_opts = DEFAULT_CURL_OPTS, debug: true)
+    def initialize(uri, curl_opts = DEFAULT_CURL_OPTS, debug: true, referrer: '')
       @uri = uri.respond_to?(:scheme) ? uri : Addressable::URI.parse(uri)
       # remove trailing '/'s from local URIs
       @uri.path.gsub!(%r{/*$}, '') if local?
       @curl_opts = curl_opts
       @debug = debug
+      @referrer = referrer
     end
 
     ##
@@ -118,6 +119,12 @@ class SiteDiff
       # Allow basic auth
       params[:userpwd] = "#{@uri.user}: #{@uri.password}" if @uri.user
 
+      # params['verbose'] = true
+      # params['ssl_verifypeer'] = false
+      # params['ssl_verifyhost'] = 0
+      # params['followlocation'] = true
+      # puts to_s
+
       req = Typhoeus::Request.new(to_s, params)
 
       req.on_success do |resp|
@@ -136,13 +143,13 @@ class SiteDiff
           raise if @debug
 
           yield ReadResult.error(
-            "Parsing error for #{@uri}: #{e.message}"
+            "Parsing error for #{@uri}: #{e.message}  From page: #{@referrer}"
           )
         rescue StandardError => e
           raise if @debug
 
           yield ReadResult.error(
-            "Unknown parsing error for #{@uri}: #{e.message}"
+            "Unknown parsing error for #{@uri}: #{e.message}  From page: #{@referrer}"
           )
         end
       end
@@ -150,17 +157,17 @@ class SiteDiff
       req.on_failure do |resp|
         if resp&.status_message
           yield ReadResult.error(
-            "HTTP error when loading #{@uri} : [#{resp.response_code}] #{resp.status_message}",
+            "HTTP error when loading #{@uri} : [#{resp.response_code}] #{resp.status_message}  From: #{@referrer}",
             resp.response_code
           )
         elsif (msg = resp.options[:return_code])
           yield ReadResult.error(
-            "Connection error when loading #{@uri} : [#{resp.options[:return_code]}] #{resp.status_message} #{msg}",
+            "Connection error when loading #{@uri} : [#{resp.options[:return_code]}] #{msg}  From: #{@referrer}",
             resp.response_code
           )
         else
           yield ReadResult.error(
-            "Unknown error when loading #{@uri} : [#{resp.response_code}] #{resp.status_message}",
+            "Unknown error when loading #{@uri} : [#{resp.response_code}] #{resp.status_message} From: #{@referrer}",
             resp.response_code
           )
         end

data/lib/sitediff.rb

@@ -137,7 +137,7 @@ class SiteDiff
       rescue StandardError => e
         raise if @debug
 
-        Result.new(path, nil, nil, nil, nil, "Sanitization error: #{e}")
+        diff = Result.new(path, nil, nil, nil, nil, "Sanitization error: #{e.message}")
       end
     end
     @results[path] = diff

data/package-lock.json

@@ -588,9 +588,9 @@
       "dev": true
     },
     "minimatch": {
-      "version": "3.0.4",
-      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
-      "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
       "dev": true,
       "requires": {
         "brace-expansion": "^1.1.7"

data/sitediff.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name        = 'sitediff'
-  s.version     = '1.2.0'
+  s.version     = '1.2.4'
   s.required_ruby_version = '>= 3.1.2'
   s.summary     = 'Compare two versions of a site with ease!'
   s.description = <<DESC
@@ -35,14 +35,14 @@ DESC
 
   s.add_dependency 'minitar', '~> 0.9'
   s.add_dependency 'thor', '~> 1.2.1'
-  s.add_dependency 'typhoeus', '~> 1.4.0'
+  s.add_dependency 'typhoeus', '~> 1.4.1'
 
   # A bug in rubygems can break rainbow 2.2
   # https://github.com/bundler/bundler/issues/5357
   s.add_dependency 'rainbow', '~> 3.1.1'
 
   # Nokogiri 1.7 is not supported on Ruby 2.0.
-  s.add_dependency 'nokogiri', '>= 1.13.6'
+  s.add_dependency 'nokogiri', '>= 1.14.2'
 
   # Diffy and addressable have a max version for Ruby 1.9.
   s.add_dependency 'addressable', '>= 2.5.2', '< 2.9.0'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sitediff
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.2.4
 platform: ruby
 authors:
 - Alex Dergachev
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-08-29 00:00:00.000000000 Z
+date: 2023-05-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pkg-config
@@ -60,14 +60,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.4.0
+        version: 1.4.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.4.0
+        version: 1.4.1
 - !ruby/object:Gem::Dependency
   name: rainbow
   requirement: !ruby/object:Gem::Requirement
@@ -88,14 +88,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.13.6
+        version: 1.14.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.13.6
+        version: 1.14.2
 - !ruby/object:Gem::Dependency
   name: addressable
   requirement: !ruby/object:Gem::Requirement
@@ -229,7 +229,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.4.8
 signing_key:
 specification_version: 4
 summary: Compare two versions of a site with ease!