sitediff 0.0.6 → 1.2.0

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+require 'rubocop/rake_task'
+
+RuboCop::RakeTask.new
+
+task default: %i[spec rubocop]

data/Thorfile

@@ -0,0 +1,135 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# TODO: Determine the utility of this file.
+
+LIB_DIR = File.join(File.dirname(__FILE__), 'lib')
+$LOAD_PATH << LIB_DIR
+require 'sitediff/webserver'
+require 'sitediff/webserver/resultserver'
+
+# Thor Base class.
+class Base < Thor
+  method_options local: true
+  # Adds the option to all Base subclasses.
+  # method_options() takes different arguments than option().
+  def initialize(*args)
+    super(*args)
+    @local = options['local']
+  end
+
+  # gives us run()
+  include Thor::Actions
+
+  # Thor, by default, exits with 0 no matter what!
+  def self.exit_on_failure?
+    true
+  end
+
+  protected
+
+  def executable(gem)
+    gem = './bin/sitediff' if (gem == 'sitediff') && @local
+    "#{'bundle exec' if @local} #{gem}"
+  end
+end
+
+# Thor for Docker.
+class Docker < Base
+  IMAGE = 'evolvingweb/sitediff'
+
+  desc 'build', 'Build a docker image for sitediff'
+  # Make a build image for docker.
+  def build
+    run "docker build -t #{IMAGE} . "
+  end
+
+  desc 'run', 'Run a rake task (or a login shell if none given) inside docker'
+  # NOTE: We can't override run() (which is reserved by Thor). Luckily, Thor only
+  # checks for the first N necessary characters to match a command with a
+  # method. Cf. Thor::normalize_command_name()
+  def run_(task = 'bash')
+    docker_opts = ['-t', "-v #{File.dirname(__FILE__)}:/sitediff"]
+    finish_exec(task, docker_opts)
+  end
+
+  desc 'compose', 'Run a task inside docker without volume mounting (not supported with compose)'
+  # Run a task inside docker without volume mounting.
+  def compose(task = 'bash')
+    docker_opts = ['-t']
+    finish_exec(task, docker_opts)
+  end
+
+  no_commands do
+    # Finished exec
+    def finish_exec(task, docker_opts)
+      if task == 'bash'
+        cmd = 'bash'
+        docker_opts << '-i'
+      else
+        # pass down the local flag to docker command
+        cmd = "#{executable('thor')} #{task} #{@local ? '--local' : '--no-local'}"
+      end
+      puts "docker run #{docker_opts.join(' ')} #{IMAGE} #{cmd}"
+      run "docker run #{docker_opts.join(' ')} #{IMAGE} #{cmd}"
+    end
+  end
+end
+
+# Thor for Spec.
+class Spec < Base
+  desc 'unit', 'run RSpec unit tests'
+  # Run RSpec unit tests.
+  def unit
+    puts "#{executable('rspec')} spec/unit"
+    run "#{executable('rspec')} spec/unit"
+  end
+
+  desc 'fixture', 'run RSpec integration tests'
+  # Run RSpec integration tests.
+  def fixture
+    puts "#{executable('rspec')} spec/unit"
+    run "#{executable('rspec')} spec/fixtures"
+  end
+
+  desc 'all', 'runs both unit and fixture tests', hide: true
+  # hidden task to lump together multiple tasks
+  def all
+    unit
+    fixture
+  end
+  default_task :all
+end
+
+# Thor for fixtures.
+class Fixture < Base
+  desc 'local', 'Run a sitediff test case'
+  # Run a sitediff test case.
+  def local
+    run "#{executable('sitediff')} diff --cached=none spec/fixtures/cli/config.yaml"
+  end
+
+  desc 'http', 'Run a sitediff test case, using web servers'
+  # Run a sitediff test case, using web servers.
+  def http
+    cmd = "#{executable('sitediff')} diff --cached=none spec/fixtures/cli/config.yaml"
+    http_fixtures(cmd).kill
+  end
+
+  desc 'serve', 'Serve the result of the fixture test'
+  # Serve the result of the fixture test.
+  def serve
+    cmd = "#{executable('sitediff')} diff --cached=none --paths-file=spec/sites/ruby-doc.org/paths.txt spec/unit/cli/config.yaml"
+    http_fixtures(cmd)
+    SiteDiff::Webserver::ResultServer.new(nil, 'sitediff', quiet: true).wait
+  end
+
+  private
+
+  # HTTP Fixtures.
+  def http_fixtures(cmd)
+    serv = SiteDiff::Webserver::FixtureServer.new
+    run "#{cmd} --before #{serv.before} --after #{serv.after}"
+    serv
+  end
+end

data/bin/sitediff

@@ -2,8 +2,15 @@
 # frozen_string_literal: true
 
 # when run as gem, $0 is /usr/local/bin/sitediff not this file
-$LOAD_PATH.unshift File.expand_path('../lib', __dir__) if $PROGRAM_NAME == __FILE__
+if $PROGRAM_NAME == __FILE__
+  $LOAD_PATH.unshift File.expand_path('../lib', __dir__)
+end
 
 require 'sitediff/cli'
 
-SiteDiff::Cli.start
+begin
+  SiteDiff::Cli.start
+rescue Interrupt
+  puts("\n")
+  SiteDiff.log('Stopping. Interrupted by user.')
+end

data/config/sanitize_domains.example.yaml

@@ -0,0 +1,8 @@
+sanitization:
+  -
+    title:
+      'strip_domains_from_absolute_urls'
+    pattern:
+      'http:\/\/[a-zA-Z0-9.:-]+'
+    substitute:
+      '__domain__'

data/config/sitediff.example.yaml

@@ -0,0 +1,81 @@
+# Include other configuration files, merging them with this file.
+includes:
+  - extra-rules.yaml
+
+# Settings.
+#
+# If you use "sitediff init" with the right parameters, it will generate
+# this section for you.
+settings:
+  # Crawl 2 levels deep.
+  depth: 2
+  # Wait for 250ms between requests.
+  interval: 250
+  # Make only 1 request at a time - no simultaneous requests.
+  # Concurrency has to be one when an interval is set.
+  concurrency: 1
+  # Don't follow links to PDF files.
+  exclude: '.*\.pdf'
+  # Curl options, if any.
+  curl_opts:
+    max_recv_speed_large: 10000
+
+# Rules under this element apply only to the 'before' site.
+before:
+  # URL of the 'before' version of the site.
+  url: http://localhost/old
+
+  # Sanitizations and DOM transformations, just like the general ones
+  # demonstrated above, but applied only to the 'before' site.
+  dom_transform:
+    - title: Example
+      type: remove
+      selector: div.updates-required
+
+# Rules under this element apply only to the 'after' site.
+after:
+  # URL of the 'after' version of the site.
+  url: http://localhost/new
+
+# The root element to compare.
+#
+# Usually, sitediff compares the HTML of the entire page. If you'd rather
+# check just a subset of the page, specify a selector here. For example, the
+# line below causes only the body to be compared, ignoring the HTML head.
+selector: 'body'
+
+# General regular expression rules, applied to both versions of the site.
+sanitization:
+  # Normalize input tags containg random tokens.
+  - title: Remove form-build-id
+    pattern: '<input type="hidden" name="form_build_id" value="form-[a-zA-Z0-9_-]+" *\/?>'
+    substitute: '<input type="hidden" name="form_build_id" value="__form_build_id__">'
+
+  # Replace meta property="twitter:*" with meta name="twitter:*".
+  - title: Meta 'property' changed to 'name'
+    pattern: 'property="twitter:'
+    substitute: 'name="twitter:'
+    # 'selector' limits this rule to only within the selected elements.
+    selector: meta
+    # 'path' limits this rule to only certain pages.
+    path: /user
+
+# General DOM transforms, applied to both versions of the site.
+dom_transform:
+  # Remove article elements, replacing them with their content
+  - title: Unwrap article elements
+    type: unwrap
+    selector: article
+
+  # Remove classes from divs
+  - title: Remove classes bar and baz from divs
+    type: remove_class
+    selector: div
+    class:
+      - class-bar
+      - class-baz
+
+  # Remove a div ID.
+  - title: Remove block containing current time.
+    type: remove
+    selector: div#block-time

data/docker-compose.test.yml

@@ -0,0 +1,3 @@
+sut:
+  build: .
+  command: bundle exec rspec

data/lib/sitediff/api.rb

@@ -0,0 +1,276 @@
+# frozen_string_literal: true
+
+require 'sitediff'
+require 'sitediff/cache'
+require 'sitediff/config'
+require 'sitediff/config/creator'
+require 'sitediff/config/preset'
+require 'sitediff/fetch'
+require 'sitediff/webserver/resultserver'
+
+class SiteDiff
+  ##
+  # Sitediff API interface.
+  class Api
+    ##
+    # Initializes new Api object.
+    def initialize(directory, config_file = nil)
+      @dir = get_dir(directory)
+      @config = SiteDiff::Config.new(config_file, @dir)
+    end
+
+    ##
+    # Intialize a SiteDiff project.
+    #
+    # Calling:
+    #     SiteDiff::Api.init(
+    #       depth: 3,
+    #       directory: 'sitediff',
+    #       concurrency: 3,
+    #       interval: 0,
+    #       include: nil,
+    #       exclude: '*.pdf',
+    #       preset: 'drupal',
+    #       curl_opts: {timeout: 60},
+    #       crawl: false
+    #     )
+    def self.init(options)
+      # Prepare a config object and write it to the file system.
+      creator = SiteDiff::Config::Creator.new(options[:debug], options[:before_url], options[:after_url])
+      include_regex = Config.create_regexp(options[:include])
+      exclude_regex = Config.create_regexp(options[:exclude])
+      creator.create(
+        depth: options[:depth],
+        directory: options[:directory],
+        concurrency: options[:concurrency],
+        interval: options[:interval],
+        include: include_regex,
+        exclude: exclude_regex,
+        preset: options[:preset],
+        curl_opts: options[:curl_opts]
+      )
+      SiteDiff.log "Created #{creator.config_file.expand_path}", :success
+
+      # TODO: implement crawl ^^^
+      # Discover paths, if enabled.
+      # if options[:crawl]
+      #   crawl(creator.config_file)
+      #   SiteDiff.log 'You can now run "sitediff diff".', :success
+      # else
+      #   SiteDiff.log 'Run "sitediff crawl" to discover paths. You should then be able to run "sitediff diff".', :info
+      # end
+    end
+
+    ##
+    # Diff the `before` and `after`.
+    #
+    # Calling:
+    #     Api.diff(
+    #       paths: options['paths'],
+    #       paths_file: options['paths-file'],
+    #       ignore_whitespace: options['ignore-whitespace'],
+    #       export: options['export'],
+    #       before: options['before'],
+    #       after: options['after'],
+    #       cached: options['cached'],
+    #       verbose: options['verbose'],
+    #       report_format: options['report-format'],
+    #       before_report: options['before-report'],
+    #       after_report: options['after-report'],
+    #       cli_mode: false
+    #     )
+    def diff(options)
+      @config.ignore_whitespace = options[:ignore_whitespace]
+      @config.export = options[:export]
+      # Apply "paths" override, if any.
+      if options[:paths]
+        @config.paths = options[:paths]
+      else
+        paths_file = options[:paths_file]
+        paths_file ||= File.join(@dir, Config::DEFAULT_PATHS_FILENAME)
+        paths_file = File.expand_path(paths_file)
+
+        paths_count = @config.paths_file_read(paths_file)
+        SiteDiff.log "Read #{paths_count} paths from: #{paths_file}"
+      end
+
+      # TODO: Why do we allow before and after override during diff?
+      @config.before['url'] = options[:before] if options[:before]
+      @config.after['url'] = options[:after] if options[:after]
+
+      # Prepare cache.
+      cache = SiteDiff::Cache.new(
+        create: options[:cached] != 'none',
+        directory: @dir
+      )
+      cache.read_tags << :before if %w[before all].include?(options[:cached])
+      cache.read_tags << :after if %w[after all].include?(options[:cached])
+      cache.write_tags << :before << :after
+
+      # Run sitediff.
+      sitediff = SiteDiff.new(
+        @config,
+        cache,
+        verbose: options[:verbose],
+        debug: options[:debug]
+      )
+      num_failing = sitediff.run
+      exit_code = num_failing.positive? ? 2 : 0
+
+      # Generate HTML report.
+      if options[:report_format] == 'html' || @config.export
+        sitediff.report.generate_html(
+          @dir,
+          options[:before_report],
+          options[:after_report]
+        )
+      end
+
+      # Generate JSON report.
+      if options[:report_format] == 'json' && @config.export == false
+        sitediff.report.generate_json @dir
+      end
+
+      SiteDiff.log 'Run "sitediff serve" to see a report.' unless options[:export]
+    rescue Config::InvalidConfig => e
+      SiteDiff.log "Invalid configuration: #{e.message}", :error
+      SiteDiff.log e.backtrace, :error if options[:verbose]
+    rescue Config::ConfigNotFound => e
+      SiteDiff.log "Invalid configuration: #{e.message}", :error
+      SiteDiff.log e.backtrace, :error if options[:verbose]
+    else # no exception was raised
+      # Thor::Error  --> exit(1), guaranteed by exit_on_failure?
+      # Failing diff --> exit(2), populated above
+      exit(exit_code) if options[:cli_mode]
+    end
+
+    ##
+    # Crawl the `before` site to determine `paths`.
+    def crawl
+      # Prepare cache.
+      @cache = SiteDiff::Cache.new(
+        create: true,
+        directory: @dir
+      )
+      @cache.write_tags << :before << :after
+
+      # Crawl with Hydra to discover paths.
+      hydra = Typhoeus::Hydra.new(
+        max_concurrency: @config.setting(:concurrency)
+      )
+      @paths = {}
+      @config.roots.each do |tag, url|
+        Crawler.new(
+          hydra,
+          url,
+          @config.setting(:interval),
+          @config.setting(:include),
+          @config.setting(:exclude),
+          @config.setting(:depth),
+          @config.curl_opts,
+          debug: @debug
+        ) do |info|
+          SiteDiff.log "Visited #{info.uri}, cached."
+          after_crawl(tag, info)
+        end
+      end
+      hydra.run
+
+      # Write paths to a file.
+      @paths = @paths.values.reduce(&:|).to_a.sort
+      @config.paths_file_write(@paths)
+
+      # Log output.
+      file = Pathname.new(@dir) + Config::DEFAULT_PATHS_FILENAME
+      SiteDiff.log ''
+      SiteDiff.log "#{@paths.length} page(s) found."
+      SiteDiff.log "Created #{file.expand_path}.", :success, 'done'
+    end
+
+    ##
+    # Serves SiteDiff report for accessing in the browser.
+    #
+    # Calling:
+    #     api.serve(browse: true, port: 13080)
+    def serve(options)
+      @cache = Cache.new(directory: @dir)
+      @cache.read_tags << :before << :after
+
+      SiteDiff::Webserver::ResultServer.new(
+        options[:port],
+        @dir,
+        browse: options[:browse],
+        cache: @cache,
+        config: @config
+      ).wait
+    rescue SiteDiffException => e
+      SiteDiff.log e.message, :error
+      SiteDiff.log e.backtrace, :error if options[:verbose]
+    end
+
+    ##
+    #
+    def store(options)
+      # TODO: Figure out how to remove this config.validate call.
+      @config.validate(need_before: false)
+      @config.paths_file_read
+
+      @cache = SiteDiff::Cache.new(directory: @dir, create: true)
+      @cache.write_tags << :before
+
+      base = options[:url] || @config.after['url']
+      fetcher = SiteDiff::Fetch.new(@cache,
+                                    @config.paths,
+                                    @config.setting(:interval),
+                                    @config.setting(:concurrency),
+                                    get_curl_opts(@config.settings),
+                                    options[:debug],
+                                    before: base)
+      fetcher.run do |path, _res|
+        SiteDiff.log "Visited #{path}, cached"
+      end
+    end
+
+    private
+
+    ##
+    # Ensures that the given directory exists.
+    def get_dir(directory)
+      # Create the dir. Must go before cache initialization!
+      @dir = Pathname.new(directory || '.')
+      @dir.mkpath unless @dir.directory?
+      @dir.to_s
+    end
+
+    ##
+    # Processes a crawled path.
+    def after_crawl(tag, info)
+      path = UriWrapper.canonicalize(info.relative)
+
+      # Register the path.
+      @paths[tag] = [] unless @paths[tag]
+      @paths[tag] << path
+
+      result = info.read_result
+
+      # Write result to applicable cache.
+      # @cache.set(tag, path, result)
+      @cache.set(:before, path, result) if tag == 'before'
+      @cache.set(:after, path, result) if tag == 'after'
+
+      # TODO: Restore application of rules.
+      # @rules.handle_page(tag, res.content, info.document) if @rules && !res.error
+    end
+
+    def get_curl_opts(options)
+      # We do want string keys here
+      bool_hash = { 'true' => true, 'false' => false }
+      curl_opts = UriWrapper::DEFAULT_CURL_OPTS
+                  .clone
+                  .merge(options['curl_options'] || {})
+                  .merge(options['curl_opts'] || {})
+      curl_opts.each { |k, v| curl_opts[k] = bool_hash.fetch(v, v) }
+      curl_opts
+    end
+  end
+end

data/lib/sitediff/cache.rb

@@ -4,28 +4,45 @@ require 'set'
 require 'fileutils'
 
 class SiteDiff
+  # SiteDiff Cache Handler.
   class Cache
+    TIMESTAMP_FILE = 'timestamp'
+
     attr_accessor :read_tags, :write_tags
 
+    ##
+    # Creates a Cache object.
     def initialize(opts = {})
       @create = opts[:create]
 
-      # Read and Write tags are sets that can contain :before and :after
-      # They indicate whether we should use the cache for reading or writing
+      # Read and Write tags are sets that can contain :before and :after.
+      # They indicate whether we should use the cache for reading or writing.
       @read_tags = Set.new
       @write_tags = Set.new
+      @timestamp_flag = { before: false, after: false }
+
+      # The directory used by the cache for storage.
       @dir = opts[:directory] || '.'
     end
 
+    ##
     # Is a tag cached?
+    # TODO: Rename it to is_cached? as it makes more sense.
     def tag?(tag)
       File.directory?(File.join(@dir, 'snapshot', tag.to_s))
     end
 
+    ##
+    # Get data from cache.
     def get(tag, path)
       return nil unless @read_tags.include? tag
 
-      filename = File.join(@dir, 'snapshot', tag.to_s, *path.split(File::SEPARATOR))
+      filename = File.join(
+        @dir,
+        'snapshot',
+        tag.to_s,
+        *path.split(File::SEPARATOR)
+      )
 
       filename = File.join(filename, 'index.html') if File.directory?(filename)
       return nil unless File.file? filename
@@ -33,10 +50,18 @@ class SiteDiff
       Marshal.load(File.read(filename))
     end
 
+    ##
+    # Set data to cache.
     def set(tag, path, result)
       return unless @write_tags.include? tag
 
-      filename = File.join(@dir, 'snapshot', tag.to_s, *path.split(File::SEPARATOR))
+      save_timestamp(tag)
+      filename = File.join(
+        @dir,
+        'snapshot',
+        tag.to_s,
+        *path.split(File::SEPARATOR)
+      )
 
       filename = File.join(filename, 'index.html') if File.directory?(filename)
       filepath = Pathname.new(filename)
@@ -46,32 +71,56 @@ class SiteDiff
         rescue Errno::EEXIST
           curdir = filepath
           curdir = curdir.parent until curdir.exist?
-          tempname = curdir.dirname + (curdir.basename.to_s + '.temporary')
+          tempname = "#{curdir.dirname}/#{curdir.basename}.temporary"
+          # tempname = curdir.dirname + (curdir.basename.to_s + '.temporary')
           # May cause problems if action is not atomic!
           # Move existing file to dir/index.html first
           # Not robust! Should generate an UUID or something.
-          SiteDiff.log "Overwriting file #{tempname}", :warn if File.exist?(tempname)
+          if File.exist?(tempname)
+            SiteDiff.log "Overwriting file #{tempname}", :warning
+          end
           curdir.rename(tempname)
           filepath.dirname.mkpath
           # Should only happen in strange situations such as when the path
           # is foo/index.html/bar (i.e., index.html is a directory)
-          SiteDiff.log "Overwriting file #{tempname}", :warn if (curdir + 'index.html').exist?
-          tempname.rename(curdir + 'index.html')
+          if File.exist?("#{curdir}/index.html")
+            SiteDiff.log "Overwriting file #{tempname}", :warning
+          end
+          File.rename(tempname, "#{curdir}/index.html")
+          # tempname.rename(curdir + 'index.html')
         end
       end
       File.open(filename, 'w') { |file| file.write(Marshal.dump(result)) }
     end
 
+    ##
+    # TODO: Document this or remove it if unused.
     def key(tag, path)
       # Ensure encoding stays the same!
       Marshal.dump([tag, path.encode('UTF-8')])
     end
 
+    ##
+    # Ensures that a directory exists.
     def get_dir(directory)
       # Create the dir. Must go before cache initialization!
       @dir = Pathname.new(directory || '.')
       @dir.mkpath unless @dir.directory?
       @dir.to_s
     end
+
+    private
+
+    def save_timestamp(tag)
+      # run once
+      return if @timestamp_flag[tag]
+
+      @timestamp_flag[tag] = true
+      cache_dir = File.join(@dir, 'snapshot', tag.to_s)
+      if File.exist? cache_dir
+        file = File.join(cache_dir, TIMESTAMP_FILE)
+        FileUtils.touch(file)
+      end
+    end
   end
 end