sitediff 0.0.6 → 1.2.0

data/lib/sitediff/cli.rb

@@ -2,35 +2,29 @@
 
 require 'thor'
 require 'sitediff'
-require 'sitediff/cache'
-require 'sitediff/config'
-require 'sitediff/config/creator'
-require 'sitediff/fetch'
-require 'sitediff/webserver/resultserver'
+require 'sitediff/api'
 
 class SiteDiff
+  # SiteDiff CLI.
   class Cli < Thor
     class_option 'directory',
                  type: :string,
                  aliases: '-C',
                  default: 'sitediff',
                  desc: 'Configuration directory'
-    class_option :curl_options,
-                 type: :hash,
-                 default: {},
-                 desc: 'Options to be passed to curl'
-    class_option :insecure,
+    class_option :debug,
                  type: :boolean,
+                 aliases: '-d',
                  default: false,
-                 desc: 'Ignore many HTTPS/SSL errors'
-    class_option :debug,
+                 desc: 'Stop on certain errors and produce error trace backs.'
+    class_option 'verbose',
                  type: :boolean,
+                 aliases: '-v',
                  default: false,
-                 desc: 'Debug mode. Stop on certain errors and produce a traceback.'
-    class_option :interval,
-                 type: :numeric,
-                 default: 0,
-                 desc: 'Crawling delay - interval in milliseconds'
+                 desc: 'Show verbose output in terminal'
+
+    # Command aliases.
+    map recrawl: :crawl
 
     # Thor, by default, exits with 0 no matter what!
     def self.exit_on_failure?
@@ -42,6 +36,20 @@ class SiteDiff
       true
     end
 
+    desc 'version', 'Show version information'
+    ##
+    # Show version information.
+    def version
+      gemspec = SiteDiff.gemspec
+      output = []
+      output.push("Sitediff CLI #{gemspec.version}")
+      if options[:verbose]
+        output.push("Website: #{gemspec.homepage}")
+        output.push("GitHub: #{gemspec.metadata['source_code_uri']}")
+      end
+      puts output.join("\n")
+    end
+
     option 'paths-file',
            type: :string,
            desc: 'Paths are read (one at a line) from PATHS: ' \
@@ -53,83 +61,70 @@ class SiteDiff
            desc: 'Specific path or paths to fetch'
     option 'before',
            type: :string,
-           desc: 'URL used to fetch the before HTML. Acts as a prefix to specified paths',
+           desc: 'URL to the "before" site, prefixed to all paths.',
            aliases: '--before-url'
     option 'after',
            type: :string,
-           desc: 'URL used to fetch the after HTML. Acts as a prefix to specified paths.',
+           desc: 'URL to the "after" site, prefixed to all paths.',
            aliases: '--after-url'
+    option 'report-format',
+           type: :string,
+           enum: %w[html json],
+           default: 'html',
+           desc: 'The format in which a report should be generated.'
     option 'before-report',
            type: :string,
-           desc: 'Before URL to use for reporting purposes. Useful if port forwarding.',
+           desc: 'URL to use in reports. Useful if port forwarding.',
            aliases: '--before-url-report'
     option 'after-report',
            type: :string,
-           desc: 'After URL to use for reporting purposes. Useful if port forwarding.',
+           desc: 'URL to use in reports. Useful if port forwarding.',
            aliases: '--after-url-report'
     option 'cached',
            type: :string,
            enum: %w[none all before after],
            default: 'before',
            desc: 'Use the cached version of these sites, if available.'
-    option 'verbose',
+    option 'ignore-whitespace',
            type: :boolean,
-           aliases: '-v',
            default: false,
-           desc: 'Show differences between versions for each page in terminal'
-    option :concurrency,
-           type: :numeric,
-           default: 3,
-           desc: 'Max number of concurrent connections made'
-    desc 'diff [OPTIONS] [CONFIGFILES]', 'Perform systematic diff on given URLs'
-    def diff(*config_files)
-      @interval = options['interval']
-      check_interval(@interval)
-      @dir = get_dir(options['directory'])
-      config = SiteDiff::Config.new(config_files, @dir)
-
-      # override config based on options
-      paths = options['paths']
-      if (paths_file = options['paths-file'])
-        if paths
-          SiteDiff.log "Can't have both --paths-file and --paths", :error
-          exit(-1)
-        end
-
-        paths_file = Pathname.new(paths_file).expand_path
-        unless File.exist? paths_file
-          raise Config::InvalidConfig,
-                "Paths file '#{paths_file}' not found!"
-        end
-        SiteDiff.log "Reading paths from: #{paths_file}"
-        config.paths = File.readlines(paths_file)
+           aliases: '-w',
+           desc: 'Ignore changes in whitespace.'
+    option 'export',
+           type: :boolean,
+           default: false,
+           aliases: '-e',
+           desc: 'Export report to files. This option forces HTML format.'
+    desc 'diff [OPTIONS] [CONFIG-FILE]',
+         'Compute diffs on configured URLs.'
+    ##
+    # Computes diffs.
+    def diff(config_file = nil)
+      # Determine "paths" override based on options.
+      if options['paths'] && options['paths-file']
+        SiteDiff.log "Can't specify both --paths-file and --paths.", :error
+        exit(-1)
       end
-      config.paths = paths if paths
-
-      config.before['url'] = options['before'] if options['before']
-      config.after['url'] = options['after'] if options['after']
 
-      # Setup cache
-      cache = SiteDiff::Cache.new(create: options['cached'] != 'none',
-                                  directory: @dir)
-      cache.read_tags << :before if %w[before all].include?(options['cached'])
-      cache.read_tags << :after if %w[after all].include?(options['cached'])
-      cache.write_tags << :before << :after
-
-      sitediff = SiteDiff.new(config, cache, options[:concurrency], @interval,
-                              options['verbose'], options[:debug])
-      num_failing = sitediff.run(get_curl_opts(options), options[:debug])
-      exit_code = num_failing > 0 ? 2 : 0
-
-      sitediff.dump(@dir, options['before-report'],
-                    options['after-report'])
-    rescue Config::InvalidConfig => e
-      SiteDiff.log "Invalid configuration: #{e.message}", :error
-      SiteDiff.log "at #{e.backtrace}", :error
-    else # no exception was raised
-      # Thor::Error  --> exit(1), guaranteed by exit_on_failure?
-      # Failing diff --> exit(2), populated above
-      exit(exit_code)
+      api = Api.new(options['directory'], config_file)
+      api_options =
+        clean_keys(
+          options,
+          :paths,
+          :paths_file,
+          :ignore_whitespace,
+          :export,
+          :before,
+          :after,
+          :cached,
+          :verbose,
+          :debug,
+          :report_format,
+          :before_report,
+          :after_report
+        )
+      api_options[:cli_mode] = true
+      api.diff(api_options)
     end
 
     option :port,
@@ -140,129 +135,119 @@ class SiteDiff
            type: :boolean,
            default: true,
            desc: 'Whether to open the served content in your browser'
-    desc 'serve [OPTIONS]', 'Serve the sitediff output directory over HTTP'
-    def serve(*config_files)
-      config = SiteDiff::Config.new(config_files, options['directory'])
-      # Could check non-empty config here but currently errors are already raised.
-      @dir = get_dir(options['directory'])
-      cache = Cache.new(directory: @dir)
-      cache.read_tags << :before << :after
-
-      SiteDiff::Webserver::ResultServer.new(
-        options[:port],
-        options['directory'],
-        browse: options[:browse],
-        cache: cache,
-        config: config
-      ).wait
-    rescue SiteDiffException => e
-      SiteDiff.log e.message, :error
-      SiteDiff.log e.backtrace, :error
+    desc 'serve [OPTIONS] [CONFIG-FILE]',
+         'Serve SiteDiff report directory over HTTP.'
+    ##
+    # Serves SiteDiff report for accessing in the browser.
+    def serve(config_file = nil)
+      api = Api.new(options['directory'], config_file)
+      api_options = clean_keys(options, :browse, :port)
+      api.serve(api_options)
     end
 
     option :depth,
            type: :numeric,
-           default: 3,
+           default: Config::DEFAULT_CONFIG['settings']['depth'],
            desc: 'How deeply to crawl the given site'
-    option :rules,
+    option :crawl,
+           type: :boolean,
+           default: true,
+           desc: 'Run "sitediff crawl" to discover paths.'
+    option :preset,
            type: :string,
-           enum: %w[yes no disabled],
-           default: 'disabled',
-           desc: 'Whether rules for the site should be auto-created'
+           enum: Config::Preset.all,
+           desc: 'Framework-specific presets to apply.'
     option :concurrency,
            type: :numeric,
-           default: 3,
-           desc: 'Max number of concurrent connections made'
-    option :whitelist,
+           default: Config::DEFAULT_CONFIG['settings']['concurrency'],
+           desc: 'Max number of concurrent connections made.'
+    option :interval,
+           type: :numeric,
+           default: Config::DEFAULT_CONFIG['settings']['interval'],
+           desc: 'Crawling delay - interval in milliseconds.'
+    option :include,
            type: :string,
-           default: '',
-           desc: 'Optional whitelist for crawling'
-    option :blacklist,
+           default: Config::DEFAULT_CONFIG['settings']['include'],
+           desc: 'Optional URL include regex for crawling.'
+    option :exclude,
            type: :string,
-           default: '',
-           desc: 'Optional blacklist for crawling'
-    desc 'init URL [URL]', 'Create a sitediff configuration'
+           default: Config::DEFAULT_CONFIG['settings']['exclude'],
+           desc: 'Optional URL exclude regex for crawling.'
+    option :curl_options,
+           type: :hash,
+           default: {},
+           desc: 'Options to be passed to curl'
+    desc 'init URL [URL]', 'Create a sitediff configuration.'
+    ##
+    # Initializes a sitediff (yaml) configuration file.
     def init(*urls)
       unless (1..2).cover? urls.size
         SiteDiff.log 'sitediff init requires one or two URLs', :error
         exit(2)
       end
+      api_options =
+        clean_keys(
+          options,
+          :depth,
+          :concurrency,
+          :interval,
+          :include,
+          :exclude,
+          :preset,
+          :crawl
+        )
+        .merge(
+          {
+            after_url: urls.pop,
+            before_url: urls.pop,
+            directory: get_dir(options['directory']),
+            curl_opts: get_curl_opts(options)
+          }
+        )
 
-      @interval = options['interval']
-      check_interval(@interval)
-      @dir = get_dir(options['directory'])
-      curl_opts = get_curl_opts(options)
-      @whitelist = create_regexp(options['whitelist'])
-      @blacklist = create_regexp(options['blacklist'])
-      creator = SiteDiff::Config::Creator.new(options[:concurrency],
-                                              options['interval'],
-                                              @whitelist,
-                                              @blacklist,
-                                              curl_opts,
-                                              options[:debug],
-                                              *urls)
-      creator.create(
-        depth: options[:depth],
-        directory: @dir,
-        rules: options[:rules] != 'no',
-        rules_disabled: (options[:rules] == 'disabled')
-      ) do |_tag, info|
-        SiteDiff.log "Visited #{info.uri}, cached"
-      end
-
-      SiteDiff.log "Created #{creator.config_file.expand_path}", :success
-      SiteDiff.log "You can now run 'sitediff diff'", :success
+      Api.init(api_options)
     end
 
     option :url,
            type: :string,
            desc: 'A custom base URL to fetch from'
-    option :concurrency,
-           type: :numeric,
-           default: 3,
-           desc: 'Max number of concurrent connections made'
-    desc 'store [CONFIGFILES]',
-         'Cache the current contents of a site for later comparison'
-    def store(*config_files)
-      @dir = get_dir(options['directory'])
-      config = SiteDiff::Config.new(config_files, @dir)
-      config.validate(need_before: false)
-      cache = SiteDiff::Cache.new(directory: @dir, create: true)
-      cache.write_tags << :before
+    desc 'store [CONFIG-FILE]',
+         'Cache the current contents of a site for later comparison.'
+    ##
+    # Caches the current version of the site.
+    def store(config_file = nil)
+      api = Api.new(options['directory'], config_file)
+      api_options = clean_keys(options, :url, :debug)
+      api.store(api_options)
+    end
 
-      base = options[:url] || config.after['url']
-      fetcher = SiteDiff::Fetch.new(cache,
-                                    config.paths,
-                                    options[:interval],
-                                    options[:concurrency],
-                                    get_curl_opts(options),
-                                    options[:debug],
-                                    before: base)
-      fetcher.run do |path, _res|
-        SiteDiff.log "Visited #{path}, cached"
-      end
+    desc 'crawl [CONFIG-FILE]',
+         'Crawl the "before" site to discover paths.'
+    ##
+    # Crawls the "before" site to determine "paths".
+    #
+    def crawl(config_file = nil)
+      api = Api.new(options['directory'], config_file)
+      api.crawl
     end
 
     no_commands do
+      # Generates CURL options.
+      #
+      # TODO: Possibly move to API class.
       def get_curl_opts(options)
         # We do want string keys here
         bool_hash = { 'true' => true, 'false' => false }
-        curl_opts = UriWrapper::DEFAULT_CURL_OPTS.clone.merge(options[:curl_options])
+        curl_opts = UriWrapper::DEFAULT_CURL_OPTS
+                    .clone
+                    .merge(options['curl_options'] || {})
+                    .merge(options['curl_opts'] || {})
         curl_opts.each { |k, v| curl_opts[k] = bool_hash.fetch(v, v) }
-        if options[:insecure]
-          curl_opts[:ssl_verifypeer] = false
-          curl_opts[:ssl_verifyhost] = 0
-        end
         curl_opts
       end
 
-      def check_interval(interval)
-        if interval != 0 && options[:concurrency] != 1
-          SiteDiff.log '--concurrency must be set to 1 in order to enable the interval feature'
-          exit(2)
-        end
-      end
-
+      ##
+      # Ensures that the given directory exists.
       def get_dir(directory)
         # Create the dir. Must go before cache initialization!
         @dir = Pathname.new(directory || '.')
@@ -270,16 +255,11 @@ class SiteDiff
         @dir.to_s
       end
 
-      def create_regexp(string_param)
-        begin
-          @return_value = string_param == '' ? nil : Regexp.new(string_param)
-        rescue SiteDiffException => e
-          @return_value = nil
-          SiteDiff.log 'whitelist and blacklist parameters must be valid regular expressions', :error
-          SiteDiff.log e.message, :error
-          SiteDiff.log e.backtrace, :error
-        end
-        return @return_value
+      ##
+      # Clean keys - return a subset of a hash with keys as symbols.
+      def clean_keys(hash, *keys)
+        new_hash = hash.transform_keys { |k| k.tr('-', '_').to_sym }
+        new_hash.slice(*keys)
       end
     end
   end

data/lib/sitediff/config/creator.rb

@@ -3,129 +3,113 @@
 require 'sitediff/cache'
 require 'sitediff/config'
 require 'sitediff/crawler'
-require 'sitediff/rules'
 require 'pathname'
 require 'typhoeus'
 require 'yaml'
 
 class SiteDiff
   class Config
+    ##
+    # SiteDiff Config Creator Object.
     class Creator
-      def initialize(concurrency, interval, whitelist, blacklist, curl_opts, debug, *urls)
-        @concurrency = concurrency
-        @interval = interval
-        @whitelist = whitelist
-        @blacklist = blacklist
-        @after = urls.pop
-        @before = urls.pop # May be nil
-        @curl_opts = curl_opts
+      ##
+      # Creates a Creator object.
+      def initialize(debug, before, after)
+        @config = nil
+        @before = before
+        @after = after
         @debug = debug
       end
 
+      ##
+      # Determine if we're dealing with one or two URLs.
       def roots
-        @roots = begin
-          r = { after: @after }
-          r[:before] = @before if @before
-          r
-        end
+        @roots = { 'after' => @after }
+        @roots['before'] = @before || @after
+        @roots
       end
 
-      # Build a config structure, return it
-      def create(opts, &block)
+      ##
+      # Build a config structure, return it.
+      def create(options)
         @config = {}
-        @callback = block
-        @dir = Pathname.new(opts[:directory])
 
-        # Handle other options
-        @depth = opts[:depth]
-        @rules = Rules.new(@config, opts[:rules_disabled]) if opts[:rules]
+        # @callback = block
+
+        @dir = Pathname.new(options[:directory])
 
         # Setup instance vars
         @paths = Hash.new { |h, k| h[k] = Set.new }
         @cache = Cache.new(directory: @dir.to_s, create: true)
         @cache.write_tags << :before << :after
 
-        build_config
+        build_config options
         write_config
       end
 
-      def build_config
+      ##
+      # Build and populate the config object which is being created.
+      #
+      # @param [String] options
+      #   One or more options.
+      def build_config(options)
+        options = Config.stringify_keys options
+
+        # Build config for "before" and "after".
         %w[before after].each do |tag|
-          next unless (u = roots[tag.to_sym])
+          next unless (url = roots[tag])
 
-          @config[tag] = { 'url' => u }
+          @config[tag] = { 'url' => url }
         end
 
-        crawl(@depth)
-        @rules&.add_config
-
-        @config['paths'] = @paths.values.reduce(&:|).to_a.sort
-      end
-
-      def crawl(depth = nil)
-        hydra = Typhoeus::Hydra.new(max_concurrency: @concurrency)
-        roots.each do |tag, u|
-          Crawler.new(hydra, u, @interval, @whitelist, @blacklist, depth, @curl_opts, @debug) do |info|
-            crawled_path(tag, info)
-          end
+        # Build other settings.
+        @config['settings'] = {}
+        Config::ALLOWED_SETTINGS_KEYS.each do |key|
+          @config['settings'][key] = options[key]
         end
-        hydra.run
       end
 
-      # Deduplicate paths with slashes at the end
-      def canonicalize(_tag, path)
-        def altered_paths(path)
-          yield path + '/'
-          yield path.sub(%r{/$}, '')
-        end
-
-        path.empty? ? '/' : path
-      end
-
-      def crawled_path(tag, info)
-        path, dup = canonicalize(tag, info.relative)
-        return if dup
-
-        res = info.read_result
-
-        @callback[tag, info]
-        @paths[tag] << path
-        @cache.set(tag, path, res)
-
-        # If single-site, cache after as before!
-        @cache.set(:before, path, res) unless roots[:before]
-
-        # This is used to populate the list of rules we guess are
-        # applicable to the current site.
-        @rules.handle_page(tag, res.content, info.document) if @rules && !res.error
-      end
-
-      # Create a gitignore if we seem to be in git
+      ##
+      # Create a gitignore if we seem to be in git.
       def make_gitignore(dir)
         # Check if we're in git
-        return unless dir.realpath.to_enum(:ascend).any? { |d| d.+('.git').exist? }
-
-        dir.+('.gitignore').open('w') do |f|
-          f.puts <<-GITIGNORE.gsub(/^\s+/, '')
-            output
-            cache.db
-            cache.db.db
-          GITIGNORE
+        unless dir.realpath.to_enum(:ascend).any? { |d| Dir.exist?("#{d}/.git") }
+          return
         end
+
+        f = File.open("#{dir}/.gitignore", 'w')
+        f.puts <<-GITIGNORE.gsub(/^\s+/, '')
+          # Directories.
+          diffs
+          snapshot
+
+          # Files.
+          settings.yaml
+          paths.txt
+          failures.txt
+        GITIGNORE
+        f.close
       end
 
+      ##
+      # Returns the name of the config directory.
       def directory
         @dir
       end
 
+      ##
+      # Returns the name of the config file.
       def config_file
         @dir + Config::DEFAULT_FILENAME
       end
 
-      # Turn a config structure into a config file
+      ##
+      # Writes the built config into the config file.
+      # TODO: Exclude default params before writing.
       def write_config
         make_gitignore(@dir)
-        config_file.open('w') { |f| f.puts @config.to_yaml }
+        data = Config.remove_defaults(@config)
+        config_file.open('w') { |f| f.puts data.to_yaml }
       end
     end
   end