sitediff 0.0.2 → 1.1.1

data/lib/sitediff/cache.rb

@@ -1,61 +1,124 @@
+# frozen_string_literal: true
+
 require 'set'
+require 'fileutils'
 
 class SiteDiff
-class Cache
-  DEFAULT_FILENAME = 'cache.db'
+  # SiteDiff Cache Handler.
+  class Cache
+    TIMESTAMP_FILE = 'timestamp'
 
-  attr_accessor :read_tags, :write_tags
+    attr_accessor :read_tags, :write_tags
 
-  def initialize(opts = {})
-    @file = opts[:file] || DEFAULT_FILENAME
-    @create = opts[:create]
-    @read_tags = Set.new
-    @write_tags = Set.new
-  end
+    ##
+    # Creates a Cache object.
+    def initialize(opts = {})
+      @create = opts[:create]
 
-  def close; @dbm.close if defined? @dbm; end
+      # Read and Write tags are sets that can contain :before and :after.
+      # They indicate whether we should use the cache for reading or writing.
+      @read_tags = Set.new
+      @write_tags = Set.new
+      @timestamp_flag = { before: false, after: false }
 
-  # Is a tag cached?
-  def tag?(tag)
-    open
-    @dbm[tag.to_s]
-  end
+      # The directory used by the cache for storage.
+      @dir = opts[:directory] || '.'
+    end
 
-  def get(tag, path)
-    return nil unless @read_tags.include? tag
-    open or return nil
-    val = @dbm[key(tag, path)]
-    return val && Marshal.load(val)
-  end
+    ##
+    # Is a tag cached?
+    # TODO: Rename it to is_cached? as it makes more sense.
+    def tag?(tag)
+      File.directory?(File.join(@dir, 'snapshot', tag.to_s))
+    end
 
-  def set(tag, path, result)
-    return unless @write_tags.include? tag
-    open or return
-    @dbm[tag.to_s] = 'TRUE'
-    @dbm[key(tag, path)] = Marshal.dump(result)
-  end
+    ##
+    # Get data from cache.
+    def get(tag, path)
+      return nil unless @read_tags.include? tag
 
-private
-  def key(tag, path)
-    # Ensure encoding stays the same!
-    Marshal.dump([tag, path.encode('UTF-8')])
-  end
+      filename = File.join(
+        @dir,
+        'snapshot',
+        tag.to_s,
+        *path.split(File::SEPARATOR)
+      )
+
+      filename = File.join(filename, 'index.html') if File.directory?(filename)
+      return nil unless File.file? filename
+
+      Marshal.load(File.read(filename))
+    end
 
-  # Ensure the DB is open
-  def open
-    # DBM adds an extra .db, ugh
-    return false unless @create || File.exist?(@file) ||
-      File.exist?(@file + '.db')
-    return true if defined? @dbm
-
-    begin
-      require 'gdbm'
-      @dbm = GDBM.new(@file)
-    rescue LoadError
-      require 'dbm'
-      @dbm = DBM.new(@file)
+    ##
+    # Set data to cache.
+    def set(tag, path, result)
+      return unless @write_tags.include? tag
+
+      save_timestamp(tag)
+      filename = File.join(
+        @dir,
+        'snapshot',
+        tag.to_s,
+        *path.split(File::SEPARATOR)
+      )
+
+      filename = File.join(filename, 'index.html') if File.directory?(filename)
+      filepath = Pathname.new(filename)
+      unless filepath.dirname.directory?
+        begin
+          filepath.dirname.mkpath
+        rescue Errno::EEXIST
+          curdir = filepath
+          curdir = curdir.parent until curdir.exist?
+          tempname = curdir.dirname + (curdir.basename.to_s + '.temporary')
+          # May cause problems if action is not atomic!
+          # Move existing file to dir/index.html first
+          # Not robust! Should generate an UUID or something.
+          if File.exist?(tempname)
+            SiteDiff.log "Overwriting file #{tempname}", :warning
+          end
+          curdir.rename(tempname)
+          filepath.dirname.mkpath
+          # Should only happen in strange situations such as when the path
+          # is foo/index.html/bar (i.e., index.html is a directory)
+          if (curdir + 'index.html').exist?
+            SiteDiff.log "Overwriting file #{tempname}", :warning
+          end
+          tempname.rename(curdir + 'index.html')
+        end
+      end
+      File.open(filename, 'w') { |file| file.write(Marshal.dump(result)) }
+    end
+
+    ##
+    # TODO: Document this or remove it if unused.
+    def key(tag, path)
+      # Ensure encoding stays the same!
+      Marshal.dump([tag, path.encode('UTF-8')])
+    end
+
+    ##
+    # Ensures that a directory exists.
+    def get_dir(directory)
+      # Create the dir. Must go before cache initialization!
+      @dir = Pathname.new(directory || '.')
+      @dir.mkpath unless @dir.directory?
+      @dir.to_s
+    end
+
+    private
+
+    def save_timestamp(tag)
+      # run once
+      return if @timestamp_flag[tag]
+
+      @timestamp_flag[tag] = true
+      cache_dir = File.join(@dir, 'snapshot', tag.to_s)
+      if File.exist? cache_dir
+        file = File.join(cache_dir, TIMESTAMP_FILE)
+        FileUtils.touch(file)
+      end
     end
-    return true
   end
 end
-end

data/lib/sitediff/cli.rb

@@ -1,17 +1,30 @@
+# frozen_string_literal: true
+
 require 'thor'
 require 'sitediff'
-require 'sitediff/cache'
-require 'sitediff/config'
-require 'sitediff/config/creator'
-require 'sitediff/fetch'
-require 'sitediff/webserver/resultserver'
+require 'sitediff/api'
 
 class SiteDiff
+  # SiteDiff CLI.
   class Cli < Thor
     class_option 'directory',
-      :type => :string,
-      :aliases => '-C',
-      :desc => "Go to a given directory before running."
+                 type: :string,
+                 aliases: '-C',
+                 default: 'sitediff',
+                 desc: 'Configuration directory'
+    class_option :debug,
+                 type: :boolean,
+                 aliases: '-d',
+                 default: false,
+                 desc: 'Stop on certain errors and produce error trace backs.'
+    class_option 'verbose',
+                 type: :boolean,
+                 aliases: '-v',
+                 default: false,
+                 desc: 'Show verbose output in terminal'
+
+    # Command aliases.
+    map recrawl: :crawl
 
     # Thor, by default, exits with 0 no matter what!
     def self.exit_on_failure?
@@ -19,192 +32,233 @@ class SiteDiff
     end
 
     # Thor, by default, does not raise an error for use of unknown options.
-    def self.check_unknown_options?(config)
+    def self.check_unknown_options?(_config)
       true
     end
 
-    option 'dump-dir',
-      :type => :string,
-      :default => File.join('.', 'output'),
-      :desc => "Location to write the output to."
+    desc 'version', 'Show version information'
+    ##
+    # Show version information.
+    def version
+      gemspec = SiteDiff.gemspec
+      output = []
+      output.push("Sitediff CLI #{gemspec.version}")
+      if options[:verbose]
+        output.push('Website: ' + gemspec.homepage)
+        output.push('GitHub: ' + gemspec.metadata['source_code_uri'])
+      end
+      puts output.join("\n")
+    end
+
     option 'paths-file',
-      :type => :string,
-      :desc => 'Paths are read (one at a line) from PATHS: ' +
-               'useful for iterating over sanitization rules',
-      :aliases => '--paths-from-file'
+           type: :string,
+           desc: 'Paths are read (one at a line) from PATHS: ' \
+                    'useful for iterating over sanitization rules',
+           aliases: '--paths-from-file'
     option 'paths',
-      :type => :array,
-      :aliases => '-p',
-      :desc => "Fetch only these specific paths"
+           type: :array,
+           aliases: '-p',
+           desc: 'Specific path or paths to fetch'
     option 'before',
-      :type => :string,
-      :desc => "URL used to fetch the before HTML. Acts as a prefix to specified paths",
-      :aliases => '--before-url'
+           type: :string,
+           desc: 'URL to the "before" site, prefixed to all paths.',
+           aliases: '--before-url'
     option 'after',
-      :type => :string,
-      :desc => "URL used to fetch the after HTML. Acts as a prefix to specified paths.",
-      :aliases => '--after-url'
+           type: :string,
+           desc: 'URL to the "after" site, prefixed to all paths.',
+           aliases: '--after-url'
+    option 'report-format',
+           type: :string,
+           enum: %w[html json],
+           default: 'html',
+           desc: 'The format in which a report should be generated.'
     option 'before-report',
-      :type => :string,
-      :desc => "Before URL to use for reporting purposes. Useful if port forwarding.",
-      :aliases => '--before-url-report'
+           type: :string,
+           desc: 'URL to use in reports. Useful if port forwarding.',
+           aliases: '--before-url-report'
     option 'after-report',
-      :type => :string,
-      :desc => "After URL to use for reporting purposes. Useful if port forwarding.",
-      :aliases => '--after-url-report'
+           type: :string,
+           desc: 'URL to use in reports. Useful if port forwarding.',
+           aliases: '--after-url-report'
     option 'cached',
-      :type => :string,
-      :enum => %w[none all before after],
-      :default => 'before',
-      :desc => "Use the cached version of these sites, if available."
-    option 'quiet',
-      :type => :boolean,
-      :aliases => '-q',
-      :default => false,
-      :desc => "Show the difference between versions for each page"
-    desc "diff [OPTIONS] [CONFIGFILES]", "Perform systematic diff on given URLs"
-    def diff(*config_files)
-      config = chdir(config_files)
-
-      # override config based on options
-      paths = options['paths']
-      if paths_file = options['paths-file']
-        if paths then
-          SiteDiff::log "Can't have both --paths-file and --paths", :error
-          exit -1
-        end
-
-        paths_file = Pathname.new(paths_file).expand_path
-        unless File.exists? paths_file
-          raise Config::InvalidConfig,
-            "Paths file '#{paths_file}' not found!"
-        end
-        SiteDiff::log "Reading paths from: #{paths_file}"
-        config.paths = File.readlines(paths_file)
+           type: :string,
+           enum: %w[none all before after],
+           default: 'before',
+           desc: 'Use the cached version of these sites, if available.'
+    option 'ignore-whitespace',
+           type: :boolean,
+           default: false,
+           aliases: '-w',
+           desc: 'Ignore changes in whitespace.'
+    option 'export',
+           type: :boolean,
+           default: false,
+           aliases: '-e',
+           desc: 'Export report to files. This option forces HTML format.'
+    desc 'diff [OPTIONS] [CONFIG-FILE]',
+         'Compute diffs on configured URLs.'
+    ##
+    # Computes diffs.
+    def diff(config_file = nil)
+      # Determine "paths" override based on options.
+      if options['paths'] && options['paths-file']
+        SiteDiff.log "Can't specify both --paths-file and --paths.", :error
+        exit(-1)
       end
-      config.paths = paths if paths
-
-      config.before['url'] = options['before'] if options['before']
-      config.after['url'] = options['after'] if options['after']
-
-      # Setup cache
-      cache = SiteDiff::Cache.new(:create => options['cached'] != 'none')
-      cache.read_tags << :before if %w[before all].include?(options['cached'])
-      cache.read_tags << :after if %w[after all].include?(options['cached'])
-      cache.write_tags << :before << :after
-
-      sitediff = SiteDiff.new(config, cache, !options['quiet'])
-      num_failing = sitediff.run
-      exit_code = (num_failing > 0) ? 2 : 0;
 
-      sitediff.dump(options['dump-dir'], options['before-report'],
-        options['after-report'])
-    rescue Config::InvalidConfig => e
-      SiteDiff.log "Invalid configuration: #{e.message}", :error
-    rescue SiteDiffException => e
-      SiteDiff.log e.message, :error
-    else # no exception was raised
-      # Thor::Error  --> exit(1), guaranteed by exit_on_failure?
-      # Failing diff --> exit(2), populated above
-      exit(exit_code)
+      api = Api.new(options['directory'], config_file)
+      api_options =
+        clean_keys(
+          options,
+          :paths,
+          :paths_file,
+          :ignore_whitespace,
+          :export,
+          :before,
+          :after,
+          :cached,
+          :verbose,
+          :debug,
+          :report_format,
+          :before_report,
+          :after_report
+        )
+      api_options[:cli_mode] = true
+      api.diff(api_options)
     end
 
     option :port,
-      :type => :numeric,
-      :default => SiteDiff::Webserver::DEFAULT_PORT,
-      :desc => 'The port to serve on'
-    option 'dump-dir',
-      :type => :string,
-      :default => 'output',
-      :desc => 'The directory to serve'
+           type: :numeric,
+           default: SiteDiff::Webserver::DEFAULT_PORT,
+           desc: 'The port to serve on'
     option :browse,
-      :type => :boolean,
-      :default => true,
-      :desc => "Whether to open the served content in your browser"
-    desc "serve [OPTIONS]", "Serve the sitediff output directory over HTTP"
-    def serve(*config_files)
-      config = chdir(config_files, :config => false)
-
-      cache = Cache.new
-      cache.read_tags << :before << :after
-
-      SiteDiff::Webserver::ResultServer.new(
-        options[:port],
-        options['dump-dir'],
-        :browse => options[:browse],
-        :cache => cache,
-        :config => config,
-      ).wait
+           type: :boolean,
+           default: true,
+           desc: 'Whether to open the served content in your browser'
+    desc 'serve [OPTIONS] [CONFIG-FILE]',
+         'Serve SiteDiff report directory over HTTP.'
+    ##
+    # Serves SiteDiff report for accessing in the browser.
+    def serve(config_file = nil)
+      api = Api.new(options['directory'], config_file)
+      api_options = clean_keys(options, :browse, :port)
+      api.serve(api_options)
     end
 
-    option :output,
-      :type => :string,
-      :default => 'sitediff',
-      :desc => 'Where to place the configuration',
-      :aliases => ['-o']
     option :depth,
-      :type => :numeric,
-      :default => 3,
-      :desc => 'How deeply to crawl the given site'
-    option :rules,
-      :type => :string,
-      :enum => %w[yes no disabled],
-      :default => 'disabled',
-      :desc => 'Whether rules for the site should be auto-created'
-    desc "init URL [URL]", "Create a sitediff configuration"
+           type: :numeric,
+           default: Config::DEFAULT_CONFIG['settings']['depth'],
+           desc: 'How deeply to crawl the given site'
+    option :crawl,
+           type: :boolean,
+           default: true,
+           desc: 'Run "sitediff crawl" to discover paths.'
+    option :preset,
+           type: :string,
+           enum: Config::Preset.all,
+           desc: 'Framework-specific presets to apply.'
+    option :concurrency,
+           type: :numeric,
+           default: Config::DEFAULT_CONFIG['settings']['concurrency'],
+           desc: 'Max number of concurrent connections made.'
+    option :interval,
+           type: :numeric,
+           default: Config::DEFAULT_CONFIG['settings']['interval'],
+           desc: 'Crawling delay - interval in milliseconds.'
+    option :include,
+           type: :string,
+           default: Config::DEFAULT_CONFIG['settings']['include'],
+           desc: 'Optional URL include regex for crawling.'
+    option :exclude,
+           type: :string,
+           default: Config::DEFAULT_CONFIG['settings']['exclude'],
+           desc: 'Optional URL exclude regex for crawling.'
+    option :curl_options,
+           type: :hash,
+           default: {},
+           desc: 'Options to be passed to curl'
+    desc 'init URL [URL]', 'Create a sitediff configuration.'
+    ##
+    # Initializes a sitediff (yaml) configuration file.
     def init(*urls)
-      unless (1..2).include? urls.size
-        SiteDiff.log "sitediff init requires one or two URLs", :error
-        exit 2
+      unless (1..2).cover? urls.size
+        SiteDiff.log 'sitediff init requires one or two URLs', :error
+        exit(2)
       end
-
-      chdir([], :search => false)
-      creator = SiteDiff::Config::Creator.new(*urls)
-      creator.create(
-        :depth => options[:depth],
-        :directory => options[:output],
-        :rules => options[:rules] != 'no',
-        :rules_disabled => (options[:rules] == 'disabled'),
-      ) do |tag, info|
-        SiteDiff.log "Visited #{info.uri}, cached"
-      end
-
-      SiteDiff.log "Created #{creator.config_file.expand_path}", :success
-      SiteDiff.log "You can now run 'sitediff diff'", :success
+      api_options =
+        clean_keys(
+          options,
+          :depth,
+          :concurrency,
+          :interval,
+          :include,
+          :exclude,
+          :preset,
+          :crawl
+        )
+        .merge(
+          {
+            after_url: urls.pop,
+            before_url: urls.pop, # may be nil
+            directory: get_dir(options['directory']),
+            curl_opts: get_curl_opts(options)
+          }
+        )
+      Api.init(api_options)
     end
 
     option :url,
-      :type => :string,
-      :desc => 'A custom base URL to fetch from'
-    desc "store [CONFIGFILES]",
-      "Cache the current contents of a site for later comparison"
-    def store(*config_files)
-      config = chdir(config_files)
-      config.validate(:need_before => false)
-
-      cache = SiteDiff::Cache.new(:create => true)
-      cache.write_tags << :before
+           type: :string,
+           desc: 'A custom base URL to fetch from'
+    desc 'store [CONFIG-FILE]',
+         'Cache the current contents of a site for later comparison.'
+    ##
+    # Caches the current version of the site.
+    def store(config_file = nil)
+      api = Api.new(options['directory'], config_file)
+      api_options = clean_keys(options, :url, :debug)
+      api.store(api_options)
+    end
 
-      base = options[:url] || config.after['url']
-      fetcher = SiteDiff::Fetch.new(cache, config.paths, :before => base)
-      fetcher.run do |path, res|
-        SiteDiff.log "Visited #{path}, cached"
-      end
+    desc 'crawl [CONFIG-FILE]',
+         'Crawl the "before" site to discover paths.'
+    ##
+    # Crawls the "before" site to determine "paths".
+    #
+    def crawl(config_file = nil)
+      api = Api.new(options['directory'], config_file)
+      api.crawl
     end
 
-  private
-    def chdir(files, opts = {})
-      opts = { :config => true, :search => true }.merge(opts)
+    no_commands do
+      # Generates CURL options.
+      #
+      # TODO: Possibly move to API class.
+      def get_curl_opts(options)
+        # We do want string keys here
+        bool_hash = { 'true' => true, 'false' => false }
+        curl_opts = UriWrapper::DEFAULT_CURL_OPTS
+                    .clone
+                    .merge(options['curl_options'] || {})
+                    .merge(options['curl_opts'] || {})
+        curl_opts.each { |k, v| curl_opts[k] = bool_hash.fetch(v, v) }
+        curl_opts
+      end
 
-      dir = options['directory']
-      Dir.chdir(dir) if dir
+      ##
+      # Ensures that the given directory exists.
+      def get_dir(directory)
+        # Create the dir. Must go before cache initialization!
+        @dir = Pathname.new(directory || '.')
+        @dir.mkpath unless @dir.directory?
+        @dir.to_s
+      end
 
-      return unless opts[:search]
-      begin
-        SiteDiff::Config.new(files, :search => !dir)
-      rescue SiteDiff::Config::ConfigNotFound => e
-        raise if opts[:config]
-        # If no config required, allow it to pass
+      ##
+      # Clean keys - return a subset of a hash with keys as symbols.
+      def clean_keys(hash, *keys)
+        new_hash = hash.transform_keys { |k| k.tr('-', '_').to_sym }
+        new_hash.slice(*keys)
       end
     end
   end