sitediff 0.0.6 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 1377b6bafe658b4a8a8f50ef0f54e577e99f1a87
-  data.tar.gz: 9a80e20a89b7f2f60506bbcccdc2b9f7037320f8
+SHA256:
+  metadata.gz: 1b7854497b5e81f48d810acec8106cbc66e33492d046e032e5516f76db26f142
+  data.tar.gz: a9349a79953237dd017600d49d38b8e734afc561f0ce09a1f8732e0e933530c9
 SHA512:
-  metadata.gz: 7fe3ce1b2e7bc1762d5e8f4a1bfd4ab9280963732d033d4b403087f71a4d6caa394669eadb9d82064ad963dd62918f95cae7e0b0495dd92979f105be1bfe6f5e
-  data.tar.gz: fb98c439544172ae40c0ba347272ec1287a1dc9042ab238e4abd8f720d52307ffa8a4b3fec7a70df68c7fdf845b324fe69c99693c5bbbc369f9e2c22fbe8c404
+  metadata.gz: 0e91f665f3c59b8a65f16c6942ec49b9cc9ee7fd12b0a777eadb844a0b9819ab1fd9485495bf2c757ca7342a6198dcccb5ae546c4ddf2682f234d015b64309b2
+  data.tar.gz: 64b7980bbbade8710b6069af19a67083678c2bd5fa99674df3360c1c6a3ddf8a15de7c5be4e8349ec298fc1c0dc27535b816089cd4f8852b8c8633861d72a178

data/bin/sitediff

@@ -2,8 +2,15 @@
 # frozen_string_literal: true
 
 # when run as gem, $0 is /usr/local/bin/sitediff not this file
-$LOAD_PATH.unshift File.expand_path('../lib', __dir__) if $PROGRAM_NAME == __FILE__
+if $PROGRAM_NAME == __FILE__
+  $LOAD_PATH.unshift File.expand_path('../lib', __dir__)
+end
 
 require 'sitediff/cli'
 
-SiteDiff::Cli.start
+begin
+  SiteDiff::Cli.start
+rescue Interrupt
+  puts("\n")
+  SiteDiff.log('Stopping. Interrupted by user.')
+end

data/lib/sitediff.rb

@@ -2,63 +2,85 @@
 # frozen_string_literal: true
 
 require 'sitediff/config'
+require 'sitediff/diff'
 require 'sitediff/fetch'
 require 'sitediff/result'
+require 'sitediff/report'
 require 'pathname'
 require 'rainbow'
+require 'rubygems'
 require 'yaml'
 
+# SiteDiff Object.
 class SiteDiff
-  # path to misc. static files (e.g. erb, css files)
+  attr_reader :config, :results
+
+  # SiteDiff installation directory.
+  ROOT_DIR = File.dirname(File.dirname(__FILE__))
+
+  # Path to misc files. Ex: *.erb, *.css.
   FILES_DIR = File.join(File.dirname(__FILE__), 'sitediff', 'files')
 
-  # subdirectory containing all failing diffs
-  DIFFS_DIR = 'diffs'
-
-  # files in output
-  FAILURES_FILE = 'failures.txt'
-  REPORT_FILE = 'report.html'
-  SETTINGS_FILE = 'settings.yaml'
-
-  # label will be colorized and str will not be.
-  # type dictates the color: can be :success, :error, or :failure
-  def self.log(str, type = :info, label = nil)
-    label = label ? "[sitediff] #{label}" : '[sitediff]'
-    bg = fg = nil
-    case type
-    when :info
-      bg = fg = nil
-    when :diff_success
-      bg = :green
+  # Logs a message.
+  #
+  # Label will be colorized and message will not.
+  # Type dictates the color: can be :success, :error, or :failure.
+  #
+  # TODO: Only print :debug messages in debug mode.
+  def self.log(message, type = :info, label = nil)
+    # Prepare label.
+    label ||= type unless type == :info
+    label = label.to_s
+    unless label.empty?
+      # Colorize label.
       fg = :black
-    when :diff_failure
-      bg = :red
-    when :warn
-      bg = :yellow
-      fg = :black
-    when :error
-      bg = :red
+      bg = :blue
+
+      case type
+      when :info
+        bg = :cyan
+      when :success
+        bg = :green
+      when :error
+        bg = :red
+      when :warning
+        bg = :yellow
+      end
+
+      label = '[' + label.to_s + ']'
+      label = Rainbow(label)
+      label = label.bg(bg) if bg
+      label = label.fg(fg) if fg
+
+      # Add a space after the label.
+      label += ' '
     end
-    label = Rainbow(label)
-    label = label.bg(bg) if bg
-    label = label.fg(fg) if fg
-    puts label + ' ' + str
+
+    puts label + message
   end
 
-  attr_reader :config, :results
+  ##
+  # Returns the "before" site's URL.
+  #
+  # TODO: Remove in favor of config.before_url.
   def before
     @config.before['url']
   end
 
+  ##
+  # Returns the "after" site's URL.
+  #
+  # TODO: Remove in favor of config.after_url.
   def after
     @config.after['url']
   end
 
-  def initialize(config, cache, concurrency, interval, verbose = true, debug = false)
+  # Initialize SiteDiff.
+  def initialize(config, cache, verbose = true, debug = false)
     @cache = cache
     @verbose = verbose
     @debug = debug
-    @interval = interval
+
     # Check for single-site mode
     validate_opts = {}
     if !config.before['url'] && @cache.tag?(:before)
@@ -69,37 +91,48 @@ class SiteDiff
       validate_opts[:need_before] = false
     end
     config.validate(validate_opts)
-
-    @concurrency = concurrency
+    # Configure diff.
+    Diff.diff_config(config)
     @config = config
   end
 
-  # Sanitize HTML
+  # Sanitize HTML.
   def sanitize(path, read_results)
     %i[before after].map do |tag|
       html = read_results[tag].content
+      # TODO: See why encoding is empty while running tests.
+      #
+      # The presence of an "encoding" value used to be used to determine
+      # if the sanitizer would be called. However, encoding turns up blank
+      # during rspec tests for some reason.
       encoding = read_results[tag].encoding
-      if encoding
-        config = @config.send(tag)
-        Sanitizer.new(html, config, path: path).sanitize
+      if encoding || html.length.positive?
+        section = @config.send(tag, true)
+        Sanitizer.new(html, section, path: path).sanitize
       else
         html
       end
     end
   end
 
-  # Process a set of read results
+  ##
+  # Process a set of read results.
+  #
+  # This is the callback that processes items fetched by the Fetcher.
   def process_results(path, read_results)
-    if (error = (read_results[:before].error || read_results[:after].error))
+    error = (read_results[:before].error || read_results[:after].error)
+    if error
       diff = Result.new(path, nil, nil, nil, nil, error)
     else
       begin
-        diff = Result.new(path,
-                          *sanitize(path, read_results),
-                          read_results[:before].encoding,
-                          read_results[:after].encoding,
-                          nil)
-      rescue => e
+        diff = Result.new(
+          path,
+          *sanitize(path, read_results),
+          read_results[:before].encoding,
+          read_results[:after].encoding,
+          nil
+        )
+      rescue StandardError => e
         raise if @debug
 
         Result.new(path, nil, nil, nil, nil, "Sanitization error: #{e}")
@@ -114,60 +147,72 @@ class SiteDiff
     end
   end
 
-  # Perform the comparison, populate @results and return the number of failing
-  # paths (paths with non-zero diff).
-  def run(curl_opts = {}, debug = true)
+  ##
+  # Compute diff as per config.
+  #
+  # @return [Integer]
+  #   Number of paths which have diffs.
+  def run
     # Map of path -> Result object, populated by process_results
     @results = {}
     @ordered = @config.paths.dup
 
     unless @cache.read_tags.empty?
-      SiteDiff.log('Using sites from cache: ' +
-        @cache.read_tags.sort.join(', '))
+      SiteDiff.log('Using sites from cache: ' + @cache.read_tags.sort.join(', '))
     end
 
     # TODO: Fix this after config merge refactor!
     # Not quite right. We are not passing @config.before or @config.after
     # so passing this instead but @config.after['curl_opts'] is ignored.
+    curl_opts = @config.setting :curl_opts
     config_curl_opts = @config.before['curl_opts']
     curl_opts = config_curl_opts.clone.merge(curl_opts) if config_curl_opts
-    fetcher = Fetch.new(@cache, @config.paths, @interval, @concurrency, curl_opts, debug,
-                        before: before, after: after)
+    fetcher = Fetch.new(
+      @cache,
+      @config.paths,
+      @config.setting(:interval),
+      @config.setting(:concurrency),
+      curl_opts,
+      @debug,
+      before: @config.before_url,
+      after: @config.after_url
+    )
+
+    # Run the Fetcher with "process results" as a callback.
     fetcher.run(&method(:process_results))
 
     # Order by original path order
-    @results = @config.paths.map { |p| @results[p] }
+    @results = @config.paths.map { |path| @results[path] }
     results.map { |r| r unless r.success? }.compact.length
   end
 
-  # Dump results to disk
-  def dump(dir, report_before, report_after)
-    report_before ||= before
-    report_after ||= after
-    dir = Pathname.new(dir)
-    dir.mkpath unless dir.directory?
-
-    # store diffs of each failing case, first wipe out existing diffs
-    diff_dir = dir + DIFFS_DIR
-    diff_dir.rmtree if diff_dir.exist?
-    results.each { |r| r.dump(dir) if r.status == Result::STATUS_FAILURE }
-    SiteDiff.log "All diff files were dumped inside #{dir.expand_path}"
-
-    # store failing paths
-    failures = dir + FAILURES_FILE
-    SiteDiff.log "Writing failures to #{failures.expand_path}"
-    failures.open('w') do |f|
-      results.each { |r| f.puts r.path unless r.success? }
+  ##
+  # Get a reporter object to help with report generation.
+  def report
+    if @results.nil?
+      raise SiteDiffException(
+        'No results detected. Run SiteDiff.run before SiteDiff.report.'
+      )
     end
 
-    # create report of results
-    report = Diff.generate_html_report(results, report_before, report_after,
-                                       @cache)
-    dir.+(REPORT_FILE).open('w') { |f| f.write(report) }
+    Report.new(@config, @cache, @results)
+  end
+
+  ##
+  # Get SiteDiff gemspec.
+  def self.gemspec
+    file = ROOT_DIR + '/sitediff.gemspec'
+    Gem::Specification.load(file)
+  end
 
-    # serve some settings
-    settings = { 'before' => report_before, 'after' => report_after,
-                 'cached' => %w[before after] }
-    dir.+(SETTINGS_FILE).open('w') { |f| YAML.dump(settings, f) }
+  ##
+  # Ensures that a directory exists and returns a Pathname for it.
+  #
+  # @param [String] dir
+  #   path/to/directory
+  def self.ensure_dir(dir)
+    dir = Pathname.new(dir) unless dir.is_a? Pathname
+    dir.mkpath unless dir.directory?
+    dir
   end
 end

data/lib/sitediff/cache.rb

@@ -4,28 +4,42 @@ require 'set'
 require 'fileutils'
 
 class SiteDiff
+  # SiteDiff Cache Handler.
   class Cache
     attr_accessor :read_tags, :write_tags
 
+    ##
+    # Creates a Cache object.
     def initialize(opts = {})
       @create = opts[:create]
 
-      # Read and Write tags are sets that can contain :before and :after
-      # They indicate whether we should use the cache for reading or writing
+      # Read and Write tags are sets that can contain :before and :after.
+      # They indicate whether we should use the cache for reading or writing.
       @read_tags = Set.new
       @write_tags = Set.new
+
+      # The directory used by the cache for storage.
       @dir = opts[:directory] || '.'
     end
 
+    ##
     # Is a tag cached?
+    # TODO: Rename it to is_cached? as it makes more sense.
     def tag?(tag)
       File.directory?(File.join(@dir, 'snapshot', tag.to_s))
     end
 
+    ##
+    # Get data from cache.
     def get(tag, path)
       return nil unless @read_tags.include? tag
 
-      filename = File.join(@dir, 'snapshot', tag.to_s, *path.split(File::SEPARATOR))
+      filename = File.join(
+        @dir,
+        'snapshot',
+        tag.to_s,
+        *path.split(File::SEPARATOR)
+      )
 
       filename = File.join(filename, 'index.html') if File.directory?(filename)
       return nil unless File.file? filename
@@ -33,10 +47,17 @@ class SiteDiff
       Marshal.load(File.read(filename))
     end
 
+    ##
+    # Set data to cache.
     def set(tag, path, result)
       return unless @write_tags.include? tag
 
-      filename = File.join(@dir, 'snapshot', tag.to_s, *path.split(File::SEPARATOR))
+      filename = File.join(
+        @dir,
+        'snapshot',
+        tag.to_s,
+        *path.split(File::SEPARATOR)
+      )
 
       filename = File.join(filename, 'index.html') if File.directory?(filename)
       filepath = Pathname.new(filename)
@@ -50,23 +71,31 @@ class SiteDiff
           # May cause problems if action is not atomic!
           # Move existing file to dir/index.html first
           # Not robust! Should generate an UUID or something.
-          SiteDiff.log "Overwriting file #{tempname}", :warn if File.exist?(tempname)
+          if File.exist?(tempname)
+            SiteDiff.log "Overwriting file #{tempname}", :warning
+          end
           curdir.rename(tempname)
           filepath.dirname.mkpath
           # Should only happen in strange situations such as when the path
           # is foo/index.html/bar (i.e., index.html is a directory)
-          SiteDiff.log "Overwriting file #{tempname}", :warn if (curdir + 'index.html').exist?
+          if (curdir + 'index.html').exist?
+            SiteDiff.log "Overwriting file #{tempname}", :warning
+          end
           tempname.rename(curdir + 'index.html')
         end
       end
       File.open(filename, 'w') { |file| file.write(Marshal.dump(result)) }
     end
 
+    ##
+    # TODO: Document this or remove it if unused.
     def key(tag, path)
       # Ensure encoding stays the same!
       Marshal.dump([tag, path.encode('UTF-8')])
     end
 
+    ##
+    # Ensures that a directory exists.
     def get_dir(directory)
       # Create the dir. Must go before cache initialization!
       @dir = Pathname.new(directory || '.')

data/lib/sitediff/cli.rb

@@ -5,32 +5,32 @@ require 'sitediff'
 require 'sitediff/cache'
 require 'sitediff/config'
 require 'sitediff/config/creator'
+require 'sitediff/config/preset'
 require 'sitediff/fetch'
 require 'sitediff/webserver/resultserver'
 
 class SiteDiff
+  # SiteDiff CLI.
+  # TODO: Use config.defaults to feed default values for sitediff.yaml params?
   class Cli < Thor
     class_option 'directory',
                  type: :string,
                  aliases: '-C',
                  default: 'sitediff',
                  desc: 'Configuration directory'
-    class_option :curl_options,
-                 type: :hash,
-                 default: {},
-                 desc: 'Options to be passed to curl'
-    class_option :insecure,
+    class_option :debug,
                  type: :boolean,
+                 aliases: '-d',
                  default: false,
-                 desc: 'Ignore many HTTPS/SSL errors'
-    class_option :debug,
+                 desc: 'Stop on certain errors and produce error trace backs.'
+    class_option 'verbose',
                  type: :boolean,
+                 aliases: '-v',
                  default: false,
-                 desc: 'Debug mode. Stop on certain errors and produce a traceback.'
-    class_option :interval,
-                 type: :numeric,
-                 default: 0,
-                 desc: 'Crawling delay - interval in milliseconds'
+                 desc: 'Show verbose output in terminal'
+
+    # Command aliases.
+    map recrawl: :crawl
 
     # Thor, by default, exits with 0 no matter what!
     def self.exit_on_failure?
@@ -42,6 +42,20 @@ class SiteDiff
       true
     end
 
+    desc 'version', 'Show version information'
+    ##
+    # Show version information.
+    def version
+      gemspec = SiteDiff.gemspec
+      output = []
+      output.push("Sitediff CLI #{gemspec.version}")
+      if options[:verbose]
+        output.push('Website: ' + gemspec.homepage)
+        output.push('GitHub: ' + gemspec.metadata['source_code_uri'])
+      end
+      puts output.join("\n")
+    end
+
     option 'paths-file',
            type: :string,
            desc: 'Paths are read (one at a line) from PATHS: ' \
@@ -53,79 +67,118 @@ class SiteDiff
            desc: 'Specific path or paths to fetch'
     option 'before',
            type: :string,
-           desc: 'URL used to fetch the before HTML. Acts as a prefix to specified paths',
+           desc: 'URL to the "before" site, prefixed to all paths.',
            aliases: '--before-url'
     option 'after',
            type: :string,
-           desc: 'URL used to fetch the after HTML. Acts as a prefix to specified paths.',
+           desc: 'URL to the "after" site, prefixed to all paths.',
            aliases: '--after-url'
+    option 'report-format',
+           type: :string,
+           enum: %w[html json],
+           default: 'html',
+           desc: 'The format in which a report should be generated.'
+    # TODO: Deprecate the parameters before-report / after-report?
     option 'before-report',
            type: :string,
-           desc: 'Before URL to use for reporting purposes. Useful if port forwarding.',
+           desc: 'URL to use in reports. Useful if port forwarding.',
            aliases: '--before-url-report'
     option 'after-report',
            type: :string,
-           desc: 'After URL to use for reporting purposes. Useful if port forwarding.',
+           desc: 'URL to use in reports. Useful if port forwarding.',
            aliases: '--after-url-report'
     option 'cached',
            type: :string,
            enum: %w[none all before after],
            default: 'before',
            desc: 'Use the cached version of these sites, if available.'
-    option 'verbose',
+    option 'ignore-whitespace',
            type: :boolean,
-           aliases: '-v',
            default: false,
-           desc: 'Show differences between versions for each page in terminal'
-    option :concurrency,
-           type: :numeric,
-           default: 3,
-           desc: 'Max number of concurrent connections made'
-    desc 'diff [OPTIONS] [CONFIGFILES]', 'Perform systematic diff on given URLs'
-    def diff(*config_files)
-      @interval = options['interval']
-      check_interval(@interval)
+           aliases: '-w',
+           desc: 'Ignore changes in whitespace.'
+    option 'export',
+           type: :boolean,
+           default: false,
+           aliases: '-e',
+           desc: 'Export report to files. This option forces HTML format.'
+    desc 'diff [OPTIONS] [CONFIG-FILE]',
+         'Compute diffs on configured URLs.'
+    ##
+    # Computes diffs.
+    def diff(config_file = nil)
       @dir = get_dir(options['directory'])
-      config = SiteDiff::Config.new(config_files, @dir)
-
-      # override config based on options
-      paths = options['paths']
-      if (paths_file = options['paths-file'])
-        if paths
-          SiteDiff.log "Can't have both --paths-file and --paths", :error
-          exit(-1)
-        end
+      config = SiteDiff::Config.new(config_file, @dir)
 
-        paths_file = Pathname.new(paths_file).expand_path
-        unless File.exist? paths_file
-          raise Config::InvalidConfig,
-                "Paths file '#{paths_file}' not found!"
-        end
-        SiteDiff.log "Reading paths from: #{paths_file}"
-        config.paths = File.readlines(paths_file)
+      # Determine "paths" override based on options.
+      if options['paths'] && options['paths-file']
+        SiteDiff.log "Can't specify both --paths-file and --paths.", :error
+        exit(-1)
       end
-      config.paths = paths if paths
 
+      # Ignore whitespace option.
+      config.ignore_whitespace = options['ignore-whitespace'] if options['ignore-whitespace']
+
+      # Export report option.
+      config.export = options['export']
+
+      # Apply "paths" override, if any.
+      config.paths = options['paths'] if options['paths']
+
+      # Determine and apply "paths-file", if "paths" is not specified.
+      unless options['paths']
+        paths_file = options['paths-file']
+        paths_file ||= File.join(@dir, Config::DEFAULT_PATHS_FILENAME)
+        paths_file = File.expand_path(paths_file)
+
+        paths_count = config.paths_file_read(paths_file)
+        SiteDiff.log "Read #{paths_count} paths from: #{paths_file}"
+      end
+
+      # TODO: Why do we allow before and after override during diff?
       config.before['url'] = options['before'] if options['before']
       config.after['url'] = options['after'] if options['after']
 
-      # Setup cache
-      cache = SiteDiff::Cache.new(create: options['cached'] != 'none',
-                                  directory: @dir)
+      # Prepare cache.
+      cache = SiteDiff::Cache.new(
+        create: options['cached'] != 'none',
+        directory: @dir
+      )
       cache.read_tags << :before if %w[before all].include?(options['cached'])
       cache.read_tags << :after if %w[after all].include?(options['cached'])
       cache.write_tags << :before << :after
 
-      sitediff = SiteDiff.new(config, cache, options[:concurrency], @interval,
-                              options['verbose'], options[:debug])
-      num_failing = sitediff.run(get_curl_opts(options), options[:debug])
-      exit_code = num_failing > 0 ? 2 : 0
+      # Run sitediff.
+      sitediff = SiteDiff.new(
+        config,
+        cache,
+        options['verbose'],
+        options[:debug]
+      )
+      num_failing = sitediff.run
+      exit_code = num_failing.positive? ? 2 : 0
+
+      # Generate HTML report.
+      if options['report-format'] == 'html' || config.export
+        sitediff.report.generate_html(
+          @dir,
+          options['before-report'],
+          options['after-report']
+        )
+      end
 
-      sitediff.dump(@dir, options['before-report'],
-                    options['after-report'])
+      # Generate JSON report.
+      if options['report-format'] == 'json' && config.export == false
+        sitediff.report.generate_json @dir
+      end
+
+      SiteDiff.log 'Run "sitediff serve" to see a report.' unless options['export']
     rescue Config::InvalidConfig => e
       SiteDiff.log "Invalid configuration: #{e.message}", :error
-      SiteDiff.log "at #{e.backtrace}", :error
+      SiteDiff.log e.backtrace, :error if options[:verbose]
+    rescue Config::ConfigNotFound => e
+      SiteDiff.log "Invalid configuration: #{e.message}", :error
+      SiteDiff.log e.backtrace, :error if options[:verbose]
     else # no exception was raised
       # Thor::Error  --> exit(1), guaranteed by exit_on_failure?
       # Failing diff --> exit(2), populated above
@@ -140,11 +193,14 @@ class SiteDiff
            type: :boolean,
            default: true,
            desc: 'Whether to open the served content in your browser'
-    desc 'serve [OPTIONS]', 'Serve the sitediff output directory over HTTP'
-    def serve(*config_files)
-      config = SiteDiff::Config.new(config_files, options['directory'])
-      # Could check non-empty config here but currently errors are already raised.
+    desc 'serve [OPTIONS] [CONFIG-FILE]',
+         'Serve SiteDiff report directory over HTTP.'
+    ##
+    # Serves SiteDiff report for accessing in the browser.
+    def serve(config_file = nil)
       @dir = get_dir(options['directory'])
+      config = SiteDiff::Config.new(config_file, @dir)
+
       cache = Cache.new(directory: @dir)
       cache.read_tags << :before << :after
 
@@ -157,85 +213,102 @@ class SiteDiff
       ).wait
     rescue SiteDiffException => e
       SiteDiff.log e.message, :error
-      SiteDiff.log e.backtrace, :error
+      SiteDiff.log e.backtrace, :error if options[:verbose]
     end
 
     option :depth,
            type: :numeric,
-           default: 3,
+           default: Config::DEFAULT_CONFIG['settings']['depth'],
            desc: 'How deeply to crawl the given site'
-    option :rules,
+    option :crawl,
+           type: :boolean,
+           default: true,
+           desc: 'Run "sitediff crawl" to discover paths.'
+    option :preset,
            type: :string,
-           enum: %w[yes no disabled],
-           default: 'disabled',
-           desc: 'Whether rules for the site should be auto-created'
+           enum: Config::Preset.all,
+           desc: 'Framework-specific presets to apply.'
     option :concurrency,
            type: :numeric,
-           default: 3,
-           desc: 'Max number of concurrent connections made'
+           default: Config::DEFAULT_CONFIG['settings']['concurrency'],
+           desc: 'Max number of concurrent connections made.'
+    option :interval,
+           type: :numeric,
+           default: Config::DEFAULT_CONFIG['settings']['interval'],
+           desc: 'Crawling delay - interval in milliseconds.'
     option :whitelist,
            type: :string,
-           default: '',
-           desc: 'Optional whitelist for crawling'
+           default: Config::DEFAULT_CONFIG['settings']['whitelist'],
+           desc: 'Optional whitelist for crawling.'
     option :blacklist,
            type: :string,
-           default: '',
-           desc: 'Optional blacklist for crawling'
-    desc 'init URL [URL]', 'Create a sitediff configuration'
+           default: Config::DEFAULT_CONFIG['settings']['blacklist'],
+           desc: 'Optional blacklist for crawling.'
+    # TODO: Remove this option. Always ignore SSL errors.
+    option :insecure,
+           type: :boolean,
+           default: false,
+           desc: 'Ignore many HTTPS/SSL errors'
+    option :curl_options,
+           type: :hash,
+           default: {},
+           desc: 'Options to be passed to curl'
+    desc 'init URL [URL]', 'Create a sitediff configuration.'
+    ##
+    # Initializes a sitediff (yaml) configuration file.
     def init(*urls)
       unless (1..2).cover? urls.size
         SiteDiff.log 'sitediff init requires one or two URLs', :error
         exit(2)
       end
 
-      @interval = options['interval']
-      check_interval(@interval)
+      # Prepare a config object and write it to the file system.
       @dir = get_dir(options['directory'])
-      curl_opts = get_curl_opts(options)
-      @whitelist = create_regexp(options['whitelist'])
-      @blacklist = create_regexp(options['blacklist'])
-      creator = SiteDiff::Config::Creator.new(options[:concurrency],
-                                              options['interval'],
-                                              @whitelist,
-                                              @blacklist,
-                                              curl_opts,
-                                              options[:debug],
-                                              *urls)
+      creator = SiteDiff::Config::Creator.new(options[:debug], *urls)
       creator.create(
         depth: options[:depth],
         directory: @dir,
-        rules: options[:rules] != 'no',
-        rules_disabled: (options[:rules] == 'disabled')
-      ) do |_tag, info|
-        SiteDiff.log "Visited #{info.uri}, cached"
-      end
-
+        concurrency: options[:concurrency],
+        interval: options[:interval],
+        whitelist: Config.create_regexp(options['whitelist']),
+        blacklist: Config.create_regexp(options['blacklist']),
+        preset: options[:preset],
+        curl_opts: get_curl_opts(options)
+      )
       SiteDiff.log "Created #{creator.config_file.expand_path}", :success
-      SiteDiff.log "You can now run 'sitediff diff'", :success
+
+      # Discover paths, if enabled.
+      if options[:crawl]
+        crawl(creator.config_file)
+        SiteDiff.log 'You can now run "sitediff diff".', :success
+      else
+        SiteDiff.log 'Run "sitediff crawl" to discover paths. You should then be able to run "sitediff diff".', :info
+      end
     end
 
     option :url,
            type: :string,
            desc: 'A custom base URL to fetch from'
-    option :concurrency,
-           type: :numeric,
-           default: 3,
-           desc: 'Max number of concurrent connections made'
-    desc 'store [CONFIGFILES]',
-         'Cache the current contents of a site for later comparison'
-    def store(*config_files)
+    desc 'store [CONFIG-FILE]',
+         'Cache the current contents of a site for later comparison.'
+    ##
+    # Caches the current version of the site.
+    def store(config_file = nil)
       @dir = get_dir(options['directory'])
-      config = SiteDiff::Config.new(config_files, @dir)
+      config = SiteDiff::Config.new(config_file, @dir)
+      # TODO: Figure out how to remove this config.validate call.
       config.validate(need_before: false)
+      config.paths_file_read
+
       cache = SiteDiff::Cache.new(directory: @dir, create: true)
       cache.write_tags << :before
 
       base = options[:url] || config.after['url']
       fetcher = SiteDiff::Fetch.new(cache,
                                     config.paths,
-                                    options[:interval],
-                                    options[:concurrency],
-                                    get_curl_opts(options),
+                                    config.setting(:interval),
+                                    config.setting(:concurrency),
+                                    get_curl_opts(config.settings),
                                     options[:debug],
                                     before: base)
       fetcher.run do |path, _res|
@@ -243,11 +316,70 @@ class SiteDiff
       end
     end
 
+    desc 'crawl [CONFIG-FILE]',
+         'Crawl the "before" site to discover paths.'
+    ##
+    # Crawls the "before" site to determine "paths".
+    #
+    # TODO: Move actual crawling to sitediff.crawl(config).
+    # TODO: Switch to paths = sitediff.crawl().
+    def crawl(config_file = nil)
+      # Prepare configuration.
+      @dir = get_dir(options['directory'])
+      @config = SiteDiff::Config.new(config_file, @dir)
+
+      # Prepare cache.
+      @cache = SiteDiff::Cache.new(
+        create: options['cached'] != 'none',
+        directory: @dir
+      )
+      @cache.write_tags << :before << :after
+
+      # Crawl with Hydra to discover paths.
+      hydra = Typhoeus::Hydra.new(
+        max_concurrency: @config.setting(:concurrency)
+      )
+      @paths = {}
+      @config.roots.each do |tag, url|
+        Crawler.new(
+          hydra,
+          url,
+          @config.setting(:interval),
+          @config.setting(:whitelist),
+          @config.setting(:blacklist),
+          @config.setting(:depth),
+          get_curl_opts(@config.settings),
+          @debug
+        ) do |info|
+          SiteDiff.log "Visited #{info.uri}, cached."
+          after_crawl(tag, info)
+        end
+      end
+      hydra.run
+
+      # Write paths to a file.
+      @paths = @paths.values.reduce(&:|).to_a.sort
+      @config.paths_file_write(@paths)
+
+      # Log output.
+      file = Pathname.new(@dir) + Config::DEFAULT_PATHS_FILENAME
+      SiteDiff.log ''
+      SiteDiff.log "#{@paths.length} page(s) found."
+      SiteDiff.log "Created #{file.expand_path}.", :success, 'done'
+    end
+
     no_commands do
+      # Generates CURL options.
+      #
+      # TODO: This should be in the config class instead.
+      # TODO: Make all requests insecure and avoid custom curl-opts.
       def get_curl_opts(options)
         # We do want string keys here
         bool_hash = { 'true' => true, 'false' => false }
-        curl_opts = UriWrapper::DEFAULT_CURL_OPTS.clone.merge(options[:curl_options])
+        curl_opts = UriWrapper::DEFAULT_CURL_OPTS
+                    .clone
+                    .merge(options['curl_options'] || {})
+                    .merge(options['curl_opts'] || {})
         curl_opts.each { |k, v| curl_opts[k] = bool_hash.fetch(v, v) }
         if options[:insecure]
           curl_opts[:ssl_verifypeer] = false
@@ -256,13 +388,8 @@ class SiteDiff
         curl_opts
       end
 
-      def check_interval(interval)
-        if interval != 0 && options[:concurrency] != 1
-          SiteDiff.log '--concurrency must be set to 1 in order to enable the interval feature'
-          exit(2)
-        end
-      end
-
+      ##
+      # Ensures that the given directory exists.
       def get_dir(directory)
         # Create the dir. Must go before cache initialization!
         @dir = Pathname.new(directory || '.')
@@ -270,16 +397,24 @@ class SiteDiff
         @dir.to_s
       end
 
-      def create_regexp(string_param)
-        begin
-          @return_value = string_param == '' ? nil : Regexp.new(string_param)
-        rescue SiteDiffException => e
-          @return_value = nil
-          SiteDiff.log 'whitelist and blacklist parameters must be valid regular expressions', :error
-          SiteDiff.log e.message, :error
-          SiteDiff.log e.backtrace, :error
-        end
-        return @return_value
+      ##
+      # Processes a crawled path.
+      def after_crawl(tag, info)
+        path = UriWrapper.canonicalize(info.relative)
+
+        # Register the path.
+        @paths[tag] = [] unless @paths[tag]
+        @paths[tag] << path
+
+        result = info.read_result
+
+        # Write result to applicable cache.
+        @cache.set(tag, path, result)
+        # If single-site, cache "after" as "before".
+        @cache.set(:before, path, result) unless @config.roots[:before]
+
+        # TODO: Restore application of rules.
+        # @rules.handle_page(tag, res.content, info.document) if @rules && !res.error
       end
     end
   end