sitediff 1.2.1 → 1.2.5a

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dff12d889984ec88ad662c2a0f2f3e0771b2a2c6cbc8e5f0442773ab36a51e7c
-  data.tar.gz: 96541e827d456c925677821c501297b68b284828584f35e499b4b35da75f962f
+  metadata.gz: 5dbab719d51bbf4922bbf90d806fab06801a8c29d1e0697655121a16634f84d7
+  data.tar.gz: cc42c00cd4d36393723bc28cb204851bdf86535cee30519d1cd9367c6f6e06b9
 SHA512:
-  metadata.gz: 3044d99f7494697d817f4ab545308987dbcaebd007f531f9113c2298f3f1550952967f92f0223a96782efed424b4c2ca97123fb0fce20a382b955086afef3386
-  data.tar.gz: 7715c7285734dad676fe95cc4fc4b6cd69411a073a4c3bca9b839ff152dfa427b0b6655159e54fc7a6e82c6ac3c0c5c9c86f312c9c7d287fc5101ec6cff0d23b
+  metadata.gz: ed9dfaad56b3a52761a6df4fee893d871931632cb32bc526574d0988ac5599ff09b352c785ed35f8a8717844438b65ef432f9c6a4258c756b3518a12c3b3f6da
+  data.tar.gz: 011c5c381479cbd7fc95b8a82de66708f503b649f6b840b32d1cb06b4e50471bb92eb34ee426f5b879863a949fe339a5d3f9e317f9286401b6666fa12de7d6ee

data/CHANGELOG.md

@@ -1,6 +1,22 @@
 # SiteDiff Change Log
 
 Contains noteworthy changes made to SiteDiff.
+## Version 1.2.5
+- Fix issue with whitespace in URLs.
+- Updates for Drupal preset for Drupal 8, 9, 10.
+- Bump nokogiri from 1.14.2 to 1.14.3
+- Fix basic auth derived from URL syntax
+
+## Version 1.2.4
+- Fix issue with 'store' command.
+
+## Version 1.2.3
+- Fix issue with nil object during diff report generation.
+- Update to export documentation.
+
+## Version 1.2.2
+- Security update for Nokogiri.
+- Minor code updates.
 
 ## Version 1.2.1
 - Fixed a bug with report exporting.
@@ -30,4 +46,4 @@ Contains noteworthy changes made to SiteDiff.
 
 ## Prior to 1.0.0
 
-Release notes were out of date, so only tracking changes since 1.0.0 here.
+Release notes were out of date, so only tracking changes since 1.0.0 here.

data/Dockerfile

@@ -10,10 +10,10 @@ ARG DEBIAN_FRONTEND=noninteractive
 # Our build requires rake
 # Install editors: vim, nano.
 RUN apt-get update
-RUN apt-get install -y apt-utils
-RUN apt-get install -y software-properties-common
-RUN apt-get install -y make pkg-config libxml2-dev libxslt-dev
-RUN apt-get install -y vim nano git
+RUN apt-get install -y apt-utils \
+                       software-properties-common \
+                       make pkg-config libxml2-dev libxslt-dev \
+                       vim nano git
 
 # Force nokogiri gem not to compile libxml2, it takes too long
 ENV NOKOGIRI_USE_SYSTEM_LIBRARIES 1

data/Gemfile.lock

@@ -1,41 +1,39 @@
 PATH
   remote: .
   specs:
-    sitediff (1.2.1)
+    sitediff (1.2.5a)
       addressable (>= 2.5.2, < 2.9.0)
       diffy (~> 3.4.0)
       minitar (~> 0.9)
-      nokogiri (>= 1.13.6)
+      nokogiri (>= 1.14.2)
       pkg-config (~> 1.4)
       rainbow (~> 3.1.1)
       thor (~> 1.2.1)
-      typhoeus (~> 1.4.0)
+      typhoeus (>= 1.1.1)
       webrick (>= 1.7)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.8.0)
-      public_suffix (>= 2.0.2, < 5.0)
+    addressable (2.8.1)
+      public_suffix (>= 2.0.2, < 6.0)
     ast (2.4.2)
     diff-lcs (1.5.0)
     diffy (3.4.2)
-    ethon (0.15.0)
+    ethon (0.16.0)
       ffi (>= 1.15.0)
     ffi (1.15.5)
     fileutils (1.1.0)
     json (2.6.2)
-    mini_portile2 (2.8.0)
     minitar (0.9)
-    nokogiri (1.13.6)
-      mini_portile2 (~> 2.8.0)
+    nokogiri (1.14.2-arm64-darwin)
       racc (~> 1.4)
     parallel (1.22.1)
     parser (3.1.2.0)
       ast (~> 2.4.1)
-    pkg-config (1.4.7)
-    public_suffix (4.0.7)
-    racc (1.6.0)
+    pkg-config (1.5.1)
+    public_suffix (5.0.1)
+    racc (1.6.2)
     rainbow (3.1.1)
     regexp_parser (2.5.0)
     rexml (3.2.5)
@@ -66,13 +64,14 @@ GEM
       parser (>= 3.1.1.0)
     ruby-progressbar (1.11.0)
     thor (1.2.1)
-    typhoeus (1.4.0)
-      ethon (>= 0.9.0)
+    typhoeus (1.4.1)
+      ethon (= 0.16.0)
     unicode-display_width (2.2.0)
-    webrick (1.7.0)
+    webrick (1.8.1)
 
 PLATFORMS
   ruby
+  x86_64-linux
 
 DEPENDENCIES
   fileutils (= 1.1.0)

data/README.md

@@ -294,6 +294,13 @@ Generate a gzipped tar file containing the HTML report instead of generating
 and serving live web pages, this option overrides `--report-format`, forcing
 HTML.
 
+```
+sitediff diff --export
+sitediff diff -e
+```
+
+This will perform the diff and export the results in a gzipped tar file.
+
 ### Running inside containers
 
 If you run SiteDiff inside a container or virtual machine, the URLs in its
@@ -441,7 +448,7 @@ before comparison:
 dom_transform:
 # Remove current time block
   - type: remove
-  - selector: div#block-time
+    selector: div#block-time
 ```
 
 #### strip
@@ -458,7 +465,7 @@ To transform `<h1>  Foo and Bar\n  </h1>` to `<h1>Foo and Bar<\h1>`:
 dom_transform:
 # Strip H1 tags
   - type: strip
-  - selector: h1
+    selector: h1
 ```
 
 #### unwrap
@@ -655,7 +662,7 @@ EG:
   </div>
 </region>
 <region id="body">
-  <div class=".field-name-attribution">
+  <div class="field-name-attribution">
     <p>Lorem ipsum...
   </div>
 </region>

data/lib/sitediff/api.rb

@@ -160,12 +160,12 @@ class SiteDiff
       )
       @paths = {}
 
-      ignoreAfter = @config.roots
+      ignore_after = @config.roots
       if @config.roots['before'] == @config.roots['after']
-        ignoreAfter.delete('after')    
+        ignore_after.delete('after')
       end
-      
-      ignoreAfter.each do |tag, url|
+
+      ignore_after.each do |tag, url|
         Crawler.new(
           hydra,
           url,
@@ -184,6 +184,10 @@ class SiteDiff
 
       # Write paths to a file.
       @paths = @paths.values.reduce(&:|).to_a.sort
+      if @paths.none? | @paths.nil?
+        return
+      end
+
       @config.paths_file_write(@paths)
 
       # Log output.
@@ -230,7 +234,7 @@ class SiteDiff
                                     @config.setting(:interval),
                                     @config.setting(:concurrency),
                                     get_curl_opts(@config.settings),
-                                    options[:debug],
+                                    debug: options[:debug],
                                     before: base)
       fetcher.run do |path, _res|
         SiteDiff.log "Visited #{path}, cached"

data/lib/sitediff/crawler.rb

@@ -43,7 +43,7 @@ class SiteDiff
 
       @found << rel
 
-      wrapper = UriWrapper.new(@base + rel, @curl_opts, debug: @debug, referrer: referrer)
+      wrapper = UriWrapper.new(@base + rel, @curl_opts, debug: @debug, referrer:)
       wrapper.queue(@hydra) do |res|
         fetched_uri(rel, depth, res)
       end
@@ -96,6 +96,7 @@ class SiteDiff
 
     # Resolve a potentially-relative link. Return nil on error.
     def resolve_link(base, rel)
+      rel = rel.strip
       base + rel
     rescue Addressable::URI::InvalidURIError
       SiteDiff.log "skipped invalid URL: '#{rel}' (at #{base})", :warning
@@ -129,6 +130,7 @@ class SiteDiff
                      u.path.start_with?(@base_uri.path)
         next unless is_sub_uri
 
+        # puts "Trying regex #{u.path}"
         is_included = @include_regex.nil? ? false : @include_regex.match(u.path)
         is_excluded = @exclude_regex.nil? ? false : @exclude_regex.match(u.path)
         if is_excluded && !is_included

data/lib/sitediff/files/report.html.erb

@@ -4,6 +4,8 @@
 
   # Pages compared.
   compared_pages = results.length
+
+  url_hash = '?' + Time.now.strftime("%s%L")
 %>
 <!DOCTYPE html>
 <html>
@@ -134,17 +136,17 @@
                 <div class="buttons">
                   <% unless relative %>
                     <% unless report['before_url_report'] === false %>
-                      <a href="<%= result.url(:before, before_url_report || before, cache) %>" class="button-before" target="_blank">Before</a>
+                      <a href="<%= result.url(:before, before_url_report || before, cache) + url_hash %>" class="button-before" target="_blank">Before</a>
                     <% end %>
                     <% unless report['after_url_report'] === false %>
-                      <a href="<%= result.url(:after, after_url_report || after, cache) %>" class="button-after" target="_blank">After</a>
+                      <a href="<%= result.url(:after, after_url_report || after, cache) + url_hash %>" class="button-after" target="_blank">After</a>
                     <% end %>
                     <% unless report['before_url_report'] === false || report['after_url_report'] === false %>
-                      <a href="/sidebyside<%= result.path %>" class="button-both">Both</a>
+                      <a href="/sidebyside<%= result.path  + url_hash %>" class="button-both">Both</a>
                     <% end %>
                   <% end %>
                   <% unless result.diff_url.nil? %>
-                    <a href="<%= result.diff_url(relative: relative) %>" class="button button-diff">View diff</a>
+                    <a href="<%= result.diff_url(relative: relative) + url_hash %>" class="button button-diff">View diff</a>
                   <% end %>
                 </div>
               </td>

data/lib/sitediff/presets/drupal.yaml

@@ -29,7 +29,7 @@ sanitization:
     pattern: '(src="[^"]*/misc/\w+\.js)?v=\d+\.\d+"'
     substitute: '\1'
   - title: Strip domain names from absolute URLs
-    pattern: 'http:\/\/[a-zA-Z0-9.:-]+'
+    pattern: 'https?:\/\/[a-zA-Z0-9.:-]+'
     substitute: '__domain__'
   - title: Strip form build ID
     selector: input
@@ -61,3 +61,27 @@ sanitization:
     selector: script
     pattern: 'js_[-\w]{40,43}\\?\.js'
     substitute: 'js__ID__.js'
+  - pattern: ' data-contextual-token="[^\"]*"'
+    selector: 'div'
+    substitute: ''
+  - pattern: ' data-drupal-selector="[^\"]*"'
+    selector: 'div'
+    substitute: ''
+  - pattern: ' title="[^\"]*"'
+    selector: 'input'
+    substitute: ''
+  - pattern: 'js-view-dom-id-[a-zA-Z0-9]+'
+    selector: 'div'
+    substitute: 'js-view-dom-id-__HASH__'
+  - pattern: 'value="[^\"]*"'
+    selector: 'input'
+    substitute: 'value="HASH"'
+  - pattern: 'itok=[A-Za-z0-9\-_]+'
+    substitute: ''
+  - pattern: 'hash=[^"]+"'
+    selector: 'iframe'
+    substitute: 'hash=HASH'
+dom_transform:
+  # Strip Drupal.settings (>8.0)
+  - type: remove
+    selector: 'script[data-drupal-selector="drupal-settings-json"]'

data/lib/sitediff/report.rb

@@ -147,8 +147,8 @@ class SiteDiff
       temp_path.mkpath
       report_path = temp_path + REPORT_DIR
       report_path.mkpath
-      files_path = report_path + "files"
-      files_path.mkpath
+      files_path = "#{report_path}/files"
+      FileUtils.mkpath(files_path)
       diffs_path = dir + DIFFS_DIR
 
       # Move files to place.

data/lib/sitediff/sanitize.rb

@@ -220,10 +220,10 @@ class SiteDiff
 
     # Force this object to be a document, so we can apply a stylesheet
     def self.to_document(obj)
-      if Nokogiri::XML::Document == obj.class || Nokogiri::HTML::Document == obj.class
+      if obj.instance_of?(Nokogiri::XML::Document) || obj.instance_of?(Nokogiri::HTML::Document)
         obj
       # node or fragment
-      elsif Nokogiri::XML::Node == obj.class || Nokogiri::HTML::DocumentFragment == obj.class
+      elsif obj.instance_of?(Nokogiri::XML::Node) || obj.instance_of?(Nokogiri::HTML::DocumentFragment)
         domify(obj.to_s, force_doc: true)
       else
         to_document(domify(obj, force_doc: false))

data/lib/sitediff/uriwrapper.rb

@@ -117,7 +117,13 @@ class SiteDiff
     def typhoeus_request
       params = @curl_opts.dup
       # Allow basic auth
-      params[:userpwd] = "#{@uri.user}: #{@uri.password}" if @uri.user
+      params[:userpwd] = "#{@uri.user}:#{@uri.password}" if @uri.user
+
+      # params['verbose'] = true
+      # params['ssl_verifypeer'] = false
+      # params['ssl_verifyhost'] = 0
+      # params['followlocation'] = true
+      # puts to_s
 
       req = Typhoeus::Request.new(to_s, params)
 
@@ -148,20 +154,20 @@ class SiteDiff
         end
       end
 
-      req.on_failure do |resp|           
+      req.on_failure do |resp|
         if resp&.status_message
           yield ReadResult.error(
-            "HTTP error when loading #{@uri} : [#{resp.response_code}] #{resp.status_message}  From page: #{@referrer}",
+            "HTTP error when loading #{@uri} : [#{resp.response_code}] #{resp.status_message}  From: #{@referrer}",
             resp.response_code
           )
         elsif (msg = resp.options[:return_code])
           yield ReadResult.error(
-            "Connection error when loading #{@uri} : [#{resp.options[:return_code]}] #{msg}  From page: #{@referrer}",
+            "Connection error when loading #{@uri} : [#{resp.options[:return_code]}] #{msg}  From: #{@referrer}",
             resp.response_code
           )
         else
           yield ReadResult.error(
-            "Unknown error when loading #{@uri} : [#{resp.response_code}] #{resp.status_message}  From page: #{@referrer}",
+            "Unknown error when loading #{@uri} : [#{resp.response_code}] #{resp.status_message} From: #{@referrer}",
             resp.response_code
           )
         end

data/lib/sitediff.rb

@@ -137,7 +137,7 @@ class SiteDiff
       rescue StandardError => e
         raise if @debug
 
-        Result.new(path, nil, nil, nil, nil, "Sanitization error: #{e}")
+        diff = Result.new(path, nil, nil, nil, nil, "Sanitization error: #{e.message}")
       end
     end
     @results[path] = diff

data/package-lock.json

@@ -588,9 +588,9 @@
       "dev": true
     },
     "minimatch": {
-      "version": "3.0.4",
-      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
-      "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
       "dev": true,
       "requires": {
         "brace-expansion": "^1.1.7"

data/sitediff.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name        = 'sitediff'
-  s.version     = '1.2.1'
+  s.version     = '1.2.5a'
   s.required_ruby_version = '>= 3.1.2'
   s.summary     = 'Compare two versions of a site with ease!'
   s.description = <<DESC
@@ -35,14 +35,14 @@ DESC
 
   s.add_dependency 'minitar', '~> 0.9'
   s.add_dependency 'thor', '~> 1.2.1'
-  s.add_dependency 'typhoeus', '~> 1.4.0'
+  s.add_dependency 'typhoeus', '>= 1.1.1'
 
   # A bug in rubygems can break rainbow 2.2
   # https://github.com/bundler/bundler/issues/5357
   s.add_dependency 'rainbow', '~> 3.1.1'
 
   # Nokogiri 1.7 is not supported on Ruby 2.0.
-  s.add_dependency 'nokogiri', '>= 1.13.6'
+  s.add_dependency 'nokogiri', '>= 1.14.2'
 
   # Diffy and addressable have a max version for Ruby 1.9.
   s.add_dependency 'addressable', '>= 2.5.2', '< 2.9.0'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sitediff
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.2.5a
 platform: ruby
 authors:
 - Alex Dergachev
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-09-29 00:00:00.000000000 Z
+date: 2023-05-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pkg-config
@@ -58,16 +58,16 @@ dependencies:
   name: typhoeus
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.4.0
+        version: 1.1.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.4.0
+        version: 1.1.1
 - !ruby/object:Gem::Dependency
   name: rainbow
   requirement: !ruby/object:Gem::Requirement
@@ -88,14 +88,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.13.6
+        version: 1.14.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.13.6
+        version: 1.14.2
 - !ruby/object:Gem::Dependency
   name: addressable
   requirement: !ruby/object:Gem::Requirement
@@ -225,11 +225,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 3.1.2
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.4.13
 signing_key:
 specification_version: 4
 summary: Compare two versions of a site with ease!