sitediff 1.1.1 → 1.2.1

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+require 'rubocop/rake_task'
+
+RuboCop::RakeTask.new
+
+task default: %i[spec rubocop]

data/Thorfile

@@ -0,0 +1,135 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# TODO: Determine the utility of this file.
+
+LIB_DIR = File.join(File.dirname(__FILE__), 'lib')
+$LOAD_PATH << LIB_DIR
+require 'sitediff/webserver'
+require 'sitediff/webserver/resultserver'
+
+# Thor Base class.
+class Base < Thor
+  method_options local: true
+  # Adds the option to all Base subclasses.
+  # method_options() takes different arguments than option().
+  def initialize(*args)
+    super(*args)
+    @local = options['local']
+  end
+
+  # gives us run()
+  include Thor::Actions
+
+  # Thor, by default, exits with 0 no matter what!
+  def self.exit_on_failure?
+    true
+  end
+
+  protected
+
+  def executable(gem)
+    gem = './bin/sitediff' if (gem == 'sitediff') && @local
+    "#{'bundle exec' if @local} #{gem}"
+  end
+end
+
+# Thor for Docker.
+class Docker < Base
+  IMAGE = 'evolvingweb/sitediff'
+
+  desc 'build', 'Build a docker image for sitediff'
+  # Make a build image for docker.
+  def build
+    run "docker build -t #{IMAGE} . "
+  end
+
+  desc 'run', 'Run a rake task (or a login shell if none given) inside docker'
+  # NOTE: We can't override run() (which is reserved by Thor). Luckily, Thor only
+  # checks for the first N necessary characters to match a command with a
+  # method. Cf. Thor::normalize_command_name()
+  def run_(task = 'bash')
+    docker_opts = ['-t', "-v #{File.dirname(__FILE__)}:/sitediff"]
+    finish_exec(task, docker_opts)
+  end
+
+  desc 'compose', 'Run a task inside docker without volume mounting (not supported with compose)'
+  # Run a task inside docker without volume mounting.
+  def compose(task = 'bash')
+    docker_opts = ['-t']
+    finish_exec(task, docker_opts)
+  end
+
+  no_commands do
+    # Finished exec
+    def finish_exec(task, docker_opts)
+      if task == 'bash'
+        cmd = 'bash'
+        docker_opts << '-i'
+      else
+        # pass down the local flag to docker command
+        cmd = "#{executable('thor')} #{task} #{@local ? '--local' : '--no-local'}"
+      end
+      puts "docker run #{docker_opts.join(' ')} #{IMAGE} #{cmd}"
+      run "docker run #{docker_opts.join(' ')} #{IMAGE} #{cmd}"
+    end
+  end
+end
+
+# Thor for Spec.
+class Spec < Base
+  desc 'unit', 'run RSpec unit tests'
+  # Run RSpec unit tests.
+  def unit
+    puts "#{executable('rspec')} spec/unit"
+    run "#{executable('rspec')} spec/unit"
+  end
+
+  desc 'fixture', 'run RSpec integration tests'
+  # Run RSpec integration tests.
+  def fixture
+    puts "#{executable('rspec')} spec/unit"
+    run "#{executable('rspec')} spec/fixtures"
+  end
+
+  desc 'all', 'runs both unit and fixture tests', hide: true
+  # hidden task to lump together multiple tasks
+  def all
+    unit
+    fixture
+  end
+  default_task :all
+end
+
+# Thor for fixtures.
+class Fixture < Base
+  desc 'local', 'Run a sitediff test case'
+  # Run a sitediff test case.
+  def local
+    run "#{executable('sitediff')} diff --cached=none spec/fixtures/cli/config.yaml"
+  end
+
+  desc 'http', 'Run a sitediff test case, using web servers'
+  # Run a sitediff test case, using web servers.
+  def http
+    cmd = "#{executable('sitediff')} diff --cached=none spec/fixtures/cli/config.yaml"
+    http_fixtures(cmd).kill
+  end
+
+  desc 'serve', 'Serve the result of the fixture test'
+  # Serve the result of the fixture test.
+  def serve
+    cmd = "#{executable('sitediff')} diff --cached=none --paths-file=spec/sites/ruby-doc.org/paths.txt spec/unit/cli/config.yaml"
+    http_fixtures(cmd)
+    SiteDiff::Webserver::ResultServer.new(nil, 'sitediff', quiet: true).wait
+  end
+
+  private
+
+  # HTTP Fixtures.
+  def http_fixtures(cmd)
+    serv = SiteDiff::Webserver::FixtureServer.new
+    run "#{cmd} --before #{serv.before} --after #{serv.after}"
+    serv
+  end
+end

data/config/sanitize_domains.example.yaml

@@ -0,0 +1,8 @@
+sanitization:
+  -
+    title:
+      'strip_domains_from_absolute_urls'
+    pattern:
+      'http:\/\/[a-zA-Z0-9.:-]+'
+    substitute:
+      '__domain__'

data/config/sitediff.example.yaml

@@ -0,0 +1,81 @@
+# Include other configuration files, merging them with this file.
+includes:
+  - extra-rules.yaml
+
+# Settings.
+#
+# If you use "sitediff init" with the right parameters, it will generate
+# this section for you.
+settings:
+  # Crawl 2 levels deep.
+  depth: 2
+  # Wait for 250ms between requests.
+  interval: 250
+  # Make only 1 request at a time - no simultaneous requests.
+  # Concurrency has to be one when an interval is set.
+  concurrency: 1
+  # Don't follow links to PDF files.
+  exclude: '.*\.pdf'
+  # Curl options, if any.
+  curl_opts:
+    max_recv_speed_large: 10000
+
+# Rules under this element apply only to the 'before' site.
+before:
+  # URL of the 'before' version of the site.
+  url: http://localhost/old
+
+  # Sanitizations and DOM transformations, just like the general ones
+  # demonstrated above, but applied only to the 'before' site.
+  dom_transform:
+    - title: Example
+      type: remove
+      selector: div.updates-required
+
+# Rules under this element apply only to the 'after' site.
+after:
+  # URL of the 'after' version of the site.
+  url: http://localhost/new
+
+# The root element to compare.
+#
+# Usually, sitediff compares the HTML of the entire page. If you'd rather
+# check just a subset of the page, specify a selector here. For example, the
+# line below causes only the body to be compared, ignoring the HTML head.
+selector: 'body'
+
+# General regular expression rules, applied to both versions of the site.
+sanitization:
+  # Normalize input tags containg random tokens.
+  - title: Remove form-build-id
+    pattern: '<input type="hidden" name="form_build_id" value="form-[a-zA-Z0-9_-]+" *\/?>'
+    substitute: '<input type="hidden" name="form_build_id" value="__form_build_id__">'
+
+  # Replace meta property="twitter:*" with meta name="twitter:*".
+  - title: Meta 'property' changed to 'name'
+    pattern: 'property="twitter:'
+    substitute: 'name="twitter:'
+    # 'selector' limits this rule to only within the selected elements.
+    selector: meta
+    # 'path' limits this rule to only certain pages.
+    path: /user
+
+# General DOM transforms, applied to both versions of the site.
+dom_transform:
+  # Remove article elements, replacing them with their content
+  - title: Unwrap article elements
+    type: unwrap
+    selector: article
+
+  # Remove classes from divs
+  - title: Remove classes bar and baz from divs
+    type: remove_class
+    selector: div
+    class:
+      - class-bar
+      - class-baz
+
+  # Remove a div ID.
+  - title: Remove block containing current time.
+    type: remove
+    selector: div#block-time

data/docker-compose.test.yml

@@ -0,0 +1,3 @@
+sut:
+  build: .
+  command: bundle exec rspec

data/lib/sitediff/api.rb

@@ -111,8 +111,8 @@ class SiteDiff
       sitediff = SiteDiff.new(
         @config,
         cache,
-        options[:verbose],
-        options[:debug]
+        verbose: options[:verbose],
+        debug: options[:debug]
       )
       num_failing = sitediff.run
       exit_code = num_failing.positive? ? 2 : 0
@@ -159,7 +159,13 @@ class SiteDiff
         max_concurrency: @config.setting(:concurrency)
       )
       @paths = {}
-      @config.roots.each do |tag, url|
+
+      ignoreAfter = @config.roots
+      if @config.roots['before'] == @config.roots['after']
+        ignoreAfter.delete('after')    
+      end
+      
+      ignoreAfter.each do |tag, url|
         Crawler.new(
           hydra,
           url,
@@ -168,7 +174,7 @@ class SiteDiff
           @config.setting(:exclude),
           @config.setting(:depth),
           @config.curl_opts,
-          @debug
+          debug: @debug
         ) do |info|
           SiteDiff.log "Visited #{info.uri}, cached."
           after_crawl(tag, info)
@@ -254,12 +260,23 @@ class SiteDiff
       result = info.read_result
 
       # Write result to applicable cache.
-      @cache.set(tag, path, result)
-      # If single-site, cache "after" as "before".
-      @cache.set(:before, path, result) unless @config.roots[:before]
+      # @cache.set(tag, path, result)
+      @cache.set(:before, path, result) if tag == 'before'
+      @cache.set(:after, path, result) if tag == 'after'
 
       # TODO: Restore application of rules.
       # @rules.handle_page(tag, res.content, info.document) if @rules && !res.error
     end
+
+    def get_curl_opts(options)
+      # We do want string keys here
+      bool_hash = { 'true' => true, 'false' => false }
+      curl_opts = UriWrapper::DEFAULT_CURL_OPTS
+                  .clone
+                  .merge(options['curl_options'] || {})
+                  .merge(options['curl_opts'] || {})
+      curl_opts.each { |k, v| curl_opts[k] = bool_hash.fetch(v, v) }
+      curl_opts
+    end
   end
 end

data/lib/sitediff/cache.rb

@@ -71,7 +71,8 @@ class SiteDiff
         rescue Errno::EEXIST
           curdir = filepath
           curdir = curdir.parent until curdir.exist?
-          tempname = curdir.dirname + (curdir.basename.to_s + '.temporary')
+          tempname = "#{curdir.dirname}/#{curdir.basename}.temporary"
+          # tempname = curdir.dirname + (curdir.basename.to_s + '.temporary')
           # May cause problems if action is not atomic!
           # Move existing file to dir/index.html first
           # Not robust! Should generate an UUID or something.
@@ -82,10 +83,11 @@ class SiteDiff
           filepath.dirname.mkpath
           # Should only happen in strange situations such as when the path
           # is foo/index.html/bar (i.e., index.html is a directory)
-          if (curdir + 'index.html').exist?
+          if File.exist?("#{curdir}/index.html")
             SiteDiff.log "Overwriting file #{tempname}", :warning
           end
-          tempname.rename(curdir + 'index.html')
+          File.rename(tempname, "#{curdir}/index.html")
+          # tempname.rename(curdir + 'index.html')
         end
       end
       File.open(filename, 'w') { |file| file.write(Marshal.dump(result)) }

data/lib/sitediff/cli.rb

@@ -44,8 +44,8 @@ class SiteDiff
       output = []
       output.push("Sitediff CLI #{gemspec.version}")
       if options[:verbose]
-        output.push('Website: ' + gemspec.homepage)
-        output.push('GitHub: ' + gemspec.metadata['source_code_uri'])
+        output.push("Website: #{gemspec.homepage}")
+        output.push("GitHub: #{gemspec.metadata['source_code_uri']}")
       end
       puts output.join("\n")
     end
@@ -199,11 +199,12 @@ class SiteDiff
         .merge(
           {
             after_url: urls.pop,
-            before_url: urls.pop, # may be nil
+            before_url: urls.pop,
             directory: get_dir(options['directory']),
             curl_opts: get_curl_opts(options)
           }
         )
+
       Api.init(api_options)
     end
 

data/lib/sitediff/config/creator.rb

@@ -73,22 +73,22 @@ class SiteDiff
       # Create a gitignore if we seem to be in git.
       def make_gitignore(dir)
         # Check if we're in git
-        unless dir.realpath.to_enum(:ascend).any? { |d| d.+('.git').exist? }
+        unless dir.realpath.to_enum(:ascend).any? { |d| Dir.exist?("#{d}/.git") }
           return
         end
 
-        dir.+('.gitignore').open('w') do |f|
-          f.puts <<-GITIGNORE.gsub(/^\s+/, '')
-            # Directories.
-            diffs
-            snapshot
-
-            # Files.
-            settings.yaml
-            paths.txt
-            failures.txt
-          GITIGNORE
-        end
+        f = File.open("#{dir}/.gitignore", 'w')
+        f.puts <<-GITIGNORE.gsub(/^\s+/, '')
+          # Directories.
+          diffs
+          snapshot
+
+          # Files.
+          settings.yaml
+          paths.txt
+          failures.txt
+        GITIGNORE
+        f.close
       end
 
       ##

data/lib/sitediff/config/preset.rb

@@ -12,7 +12,7 @@ class SiteDiff
       # Directory in which presets live.
       #
       # TODO: Move this outside "lib".
-      DIRECTORY = (Pathname.new(__dir__).dirname + 'presets').freeze
+      DIRECTORY = "#{Pathname.new(__dir__).dirname}/presets".freeze
 
       ##
       # Reads preset rules.
@@ -27,7 +27,7 @@ class SiteDiff
 
         # Load and cache preset config.
         if @cache[name].nil?
-          exist? name, true
+          exist? name, exception: true
           @cache[name] = Config.load_conf file(name)
         end
 
@@ -43,7 +43,7 @@ class SiteDiff
         # Load and cache preset names.
         if @all.nil?
           @all = []
-          pattern = DIRECTORY + '*.yaml'
+          pattern = "#{DIRECTORY}/*.yaml"
           Dir.glob(pattern) do |file|
             @all << File.basename(file, '.yaml')
           end
@@ -54,8 +54,8 @@ class SiteDiff
 
       ##
       # Checks whether a preset exists.
-      def self.exist?(name, exception = false)
-        result = File.exist? file(name)
+      def self.exist?(name, exception: false)
+        result = File.exist?(file(name))
 
         # Raise an exception, if required.
         if exception && !result
@@ -68,7 +68,7 @@ class SiteDiff
       ##
       # Returns the path to a preset file.
       def self.file(name)
-        DIRECTORY + "#{name}.yaml"
+        DIRECTORY + "/#{name}.yaml"
       end
     end
   end

data/lib/sitediff/config.rb

@@ -107,7 +107,7 @@ class SiteDiff
           conf[pos][key] += conf[key] if conf[key]
         end
         tools[:scalar].each { |key| conf[pos][key] ||= conf[key] }
-        conf[pos]['url'] ||= conf[pos + '_url']
+        conf[pos]['url'] ||= conf["pos#{_url}"] if defined?(_url)
         conf[pos]['curl_opts'] = conf['curl_opts']
       end
 
@@ -260,8 +260,8 @@ class SiteDiff
     end
 
     # Get "before" site configuration.
-    def before(apply_preset = false)
-      section :before, apply_preset
+    def before(apply_preset: false)
+      section(:before, with_preset: apply_preset)
     end
 
     # Get "before" site URL.
@@ -271,8 +271,8 @@ class SiteDiff
     end
 
     # Get "after" site configuration.
-    def after(apply_preset = false)
-      section :after, apply_preset
+    def after(apply_preset: false)
+      section(:after, with_preset: apply_preset)
     end
 
     # Get "after" site URL.
@@ -431,7 +431,7 @@ class SiteDiff
       end
 
       # Validate preset.
-      Preset.exist? setting(:preset), true if setting(:preset)
+      Preset.exist? setting(:preset), exception: true if setting(:preset)
     end
 
     ##
@@ -459,7 +459,7 @@ class SiteDiff
         @return_value = string_param == '' ? nil : Regexp.new(string_param)
       rescue SiteDiffException => e
         @return_value = nil
-        SiteDiff.log 'Invalid RegExp: ' + string_param, :error
+        SiteDiff.log "Invalid RegExp: #{string_param}", :error
         SiteDiff.log e.message, :error
         # TODO: Use SiteDiff.log type :debug
         # SiteDiff.log e.backtrace, :error if options[:verbose]
@@ -491,7 +491,7 @@ class SiteDiff
     #
     # @return [Hash|Nil]
     #   Section data or Nil.
-    def section(name, with_preset = false)
+    def section(name, with_preset: false)
       name = name.to_s if name.is_a? Symbol
 
       # Validate section.
@@ -531,7 +531,7 @@ class SiteDiff
     def self.load_raw_yaml(file)
       # TODO: Only show this in verbose mode.
       SiteDiff.log "Reading config file: #{Pathname.new(file).expand_path}"
-      conf = YAML.load_file(file) || {}
+      conf = YAML.load_file(file, permitted_classes: [Regexp]) || {}
 
       unless conf.is_a? Hash
         raise InvalidConfig, "Invalid configuration file: '#{file}'"

data/lib/sitediff/crawler.rb

@@ -21,7 +21,7 @@ class SiteDiff
                    exclude_regex,
                    depth = DEFAULT_DEPTH,
                    curl_opts = UriWrapper::DEFAULT_CURL_OPTS,
-                   debug = true,
+                   debug: true,
                    &block)
       @hydra = hydra
       @base_uri = Addressable::URI.parse(base)
@@ -34,16 +34,16 @@ class SiteDiff
       @curl_opts = curl_opts
       @debug = debug
 
-      add_uri('', depth)
+      add_uri('', depth, referrer: '/')
     end
 
     # Handle a newly found relative URI
-    def add_uri(rel, depth)
+    def add_uri(rel, depth, referrer = '')
       return if @found.include? rel
 
       @found << rel
 
-      wrapper = UriWrapper.new(@base + rel, @curl_opts, @debug)
+      wrapper = UriWrapper.new(@base + rel, @curl_opts, debug: @debug, referrer: referrer)
       wrapper.queue(@hydra) do |res|
         fetched_uri(rel, depth, res)
       end
@@ -90,7 +90,7 @@ class SiteDiff
       rels.each do |r|
         next if @found.include? r
 
-        add_uri(r, depth - 1)
+        add_uri(r, depth - 1, rel)
       end
     end
 
@@ -104,6 +104,16 @@ class SiteDiff
 
     # Make a link relative to @base_uri
     def relativize_link(uri)
+      # fullPath = uri.path
+      # if uri.query
+      #   fullPath += "?" + uri.query
+      # end
+      #
+      # if uri.fragment
+      #   fullPath += "#" + uri.fragment
+      # end
+      # fullPath.gsub(@base_uri.path, "")
+      #
       uri.path.slice(@base_uri.path.length, uri.path.length)
     end
 

data/lib/sitediff/diff.rb

@@ -68,7 +68,7 @@ class SiteDiff
 
     ##
     # Generates diff output for a single result.
-    def generate_diff_output(result, relative = false)
+    def generate_diff_output(result, relative: false)
       erb_path = File.join(SiteDiff::FILES_DIR, 'diff.html.erb')
       ERB.new(File.read(erb_path)).result(binding)
     end

data/lib/sitediff/fetch.rb

@@ -15,7 +15,7 @@ class SiteDiff
                    interval,
                    concurrency = 3,
                    curl_opts = nil,
-                   debug = true,
+                   debug: true,
                    **tags)
       @cache = cache
       @interval = interval
@@ -51,7 +51,7 @@ class SiteDiff
           results[tag] = UriWrapper::ReadResult.error('Not cached')
           process_results(path, results)
         else
-          uri = UriWrapper.new(base + path, @curl_opts, @debug)
+          uri = UriWrapper.new(base + path, @curl_opts, debug: @debug)
           uri.queue(@hydra) do |resl|
             # Insert delay to limit fetching rate
             if @interval != 0

data/lib/sitediff/files/report.html.erb

@@ -144,7 +144,7 @@
                     <% end %>
                   <% end %>
                   <% unless result.diff_url.nil? %>
-                    <a href="<%= result.diff_url(relative) %>" class="button button-diff">View diff</a>
+                    <a href="<%= result.diff_url(relative: relative) %>" class="button button-diff">View diff</a>
                   <% end %>
                 </div>
               </td>

data/lib/sitediff/presets/drupal.yaml

@@ -0,0 +1,63 @@
+sanitization:
+  - title: Strip Drupal.settings
+    selector: script
+    pattern: '^(<script>)?jQuery.extend\(Drupal.settings.*$'
+  - title: Strip IE CSS/JS cache IDs
+    pattern: '("[^"]*ie\d?\.(js|css))\?[a-z0-9]{6}"'
+    substitute: '\1'
+  - title: Strip form build ID
+    selector: input
+    pattern: 'name="form_build_id" value="form-[-\w]{40,43}"'
+    substitute: 'name="form_build_id" value="form-DRUPAL_FORM_BUILD_ID"'
+  - title: Strip view DOM ID
+    pattern: '(class="view .*) view-dom-id-[a-f0-9]{32}"'
+    substitute: '\1 view-dom-id-DRUPAL_VIEW_DOM_ID"'
+  - title: Strip CSS aggregation filenames
+    selector: link[rel=stylesheet]
+    pattern: '(href="[^"]*/files/css/css_)[-\w]{40,43}\.css"'
+    substitute: '\1DRUPAL_AGGREGATED_CSS.css"'
+  - title: Strip JS aggregation filenames
+    selector: script
+    pattern: '(src="[^"]*/files/js/js_)[-\w]{40,43}\.js"'
+    substitute: '\1DRUPAL_AGGREGATED_JS.js"'
+  - title: Strip CSS/JS cache IDs
+    selector: style, script
+    pattern: '("[^"]*\.(js|css))\?[a-z0-9]{6}"'
+    substitute: '\1'
+  - title: Strip Drupal JS version tags
+    selector: script
+    pattern: '(src="[^"]*/misc/\w+\.js)?v=\d+\.\d+"'
+    substitute: '\1'
+  - title: Strip domain names from absolute URLs
+    pattern: 'http:\/\/[a-zA-Z0-9.:-]+'
+    substitute: '__domain__'
+  - title: Strip form build ID
+    selector: input
+    pattern: 'autocomplete="off" data-drupal-selector="form-[-\w]{40,43}"'
+    substitute: 'autocomplete="off" data-drupal-selector="form-DRUPAL_FORM_BUILD_ID"'
+  - title: Strip form build ID 2
+    selector: input
+    pattern: 'name="form_build_id" value="form-[-\w]{40,43}"'
+    substitute: 'name="form_build_id" value="form-DRUPAL_FORM_BUILD_ID"'
+  - title: Strip Drupal CSS link queries
+    selector: link
+    pattern: '\.css\?(\w*)'
+    substitute: '\.css'
+  - title: Strip Drupal JS link queries
+    selector: script
+    pattern: '\.js\?(\w*)'
+    substitute: '\.js'
+  - title: Strip Drupal View-DOM ID
+    pattern: 'view-dom-id-\w*'
+    substitute: 'view-dom-id-_ID_'
+  - title: Strip Drupal View-DOM ID 2
+    pattern: '(views?_dom_id"?:"?)\w*'
+    substitute: '\1_ID_'
+  - title: Ignore Drupal CSS file names
+    selector: link
+    pattern: 'css_[-\w]{40,43}(\\|%5C)?\.css'
+    substitute: 'css__ID__.css'
+  - title: Ignore Drupal JS file names
+    selector: script
+    pattern: 'js_[-\w]{40,43}\\?\.js'
+    substitute: 'js__ID__.js'