sitediff 0.0.3 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1113cf29b633160c7556d52da5e7109bb76c1362
-  data.tar.gz: '04118bf895870a85cd83cc94d3c674520e24e36c'
+  metadata.gz: 93ab2ffc296a3c9de8ea835e47f435e0193e7854
+  data.tar.gz: 5a80c5bb738912114aeb029c4ff283ff890f2ce2
 SHA512:
-  metadata.gz: 192c737e7eefa45055de28c34fc59377e93017393bd4ae212b64d67e645a26509427bec7365c5b95f803c5d269a6dfd94aceddfc8277d578b4b3ce8335a8acc9
-  data.tar.gz: e7f8e5673f4919b0fabf2af1a379b84059beae6b01c7dcded0b335af2db83c7de9b2d52e975c67095c33e84ec16bca33c5fd849bd8d6c022c669c1ad1b2ca931
+  metadata.gz: fd07a6ff9a14a8da965f4f5b09acb1c7f30be594d97f6383c3bec6545d1b5bb3fc26451a9943ed3fc3c81d261a0fa4ca6c5fcc5355d7263ae09700800af5af9d
+  data.tar.gz: bb940ede7b68b1e047dbda46d66fade88d06e0f54d9e257842a311f119efb4043939ea9b83021cdf2fd5c76ac3a23da92e7da91072cc5d4421a1717902093e17

data/bin/sitediff

@@ -1,9 +1,8 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 # when run as gem, $0 is /usr/local/bin/sitediff not this file
-if $0 == __FILE__
-  $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
-end
+$LOAD_PATH.unshift File.expand_path('../lib', __dir__) if $PROGRAM_NAME == __FILE__
 
 require 'sitediff/cli'
 

data/lib/sitediff.rb

@@ -1,4 +1,6 @@
 #!/bin/env ruby
+# frozen_string_literal: true
+
 require 'sitediff/config'
 require 'sitediff/fetch'
 require 'sitediff/result'
@@ -20,11 +22,12 @@ class SiteDiff
 
   # label will be colorized and str will not be.
   # type dictates the color: can be :success, :error, or :failure
-  def self.log(str, type=:info, label=nil)
+  def self.log(str, type = :info, label = nil)
     label = label ? "[sitediff] #{label}" : '[sitediff]'
     bg = fg = nil
     case type
     when :info
+      bg = fg = nil
     when :diff_success
       bg = :green
       fg = :black
@@ -46,47 +49,50 @@ class SiteDiff
   def before
     @config.before['url']
   end
+
   def after
     @config.after['url']
   end
 
-  def initialize(config, cache, verbose=true)
+  def initialize(config, cache, concurrency, verbose = true)
     @cache = cache
     @verbose = verbose
 
     # Check for single-site mode
     validate_opts = {}
     if !config.before['url'] && @cache.tag?(:before)
-      raise SiteDiffException,
-        "A cached 'before' is required for single-site mode" \
-        unless @cache.read_tags.include?(:before)
+      unless @cache.read_tags.include?(:before)
+        raise SiteDiffException,
+              "A cached 'before' is required for single-site mode"
+      end
       validate_opts[:need_before] = false
     end
     config.validate(validate_opts)
 
+    @concurrency = concurrency
     @config = config
   end
 
   # Sanitize HTML
   def sanitize(path, read_results)
-    [:before, :after].map do |tag|
+    %i[before after].map do |tag|
       html = read_results[tag].content
       config = @config.send(tag)
-      Sanitizer.new(html, config, :path => path).sanitize
+      Sanitizer.new(html, config, path: path).sanitize
     end
   end
 
   # Process a set of read results
   def process_results(path, read_results)
-    if error = read_results[:before].error || read_results[:after].error
-      diff = Result.new(path, nil, nil, error)
-    else
-      diff = Result.new(path, *sanitize(path, read_results), nil)
-    end
+    diff = if (error = (read_results[:before].error || read_results[:after].error))
+             Result.new(path, nil, nil, error)
+           else
+             Result.new(path, *sanitize(path, read_results), nil)
+           end
     @results[path] = diff
 
     # Print results in order!
-    while next_diff = @results[@ordered.first]
+    while (next_diff = @results[@ordered.first])
       next_diff.log(@verbose)
       @ordered.shift
     end
@@ -94,23 +100,28 @@ class SiteDiff
 
   # Perform the comparison, populate @results and return the number of failing
   # paths (paths with non-zero diff).
-  def run
+  def run(curl_opts = {})
     # Map of path -> Result object, populated by process_results
     @results = {}
     @ordered = @config.paths.dup
 
     unless @cache.read_tags.empty?
-      SiteDiff.log("Using sites from cache: " +
+      SiteDiff.log('Using sites from cache: ' +
         @cache.read_tags.sort.join(', '))
     end
 
-    fetcher = Fetch.new(@cache, @config.paths,
-      :before => before, :after => after)
-    fetcher.run(&self.method(:process_results))
+    # TODO: Fix this after config merge refactor!
+    # Not quite right. We are not passing @config.before or @config.after
+    # so passing this instead but @config.after['curl_opts'] is ignored.
+    config_curl_opts = @config.before['curl_opts']
+    curl_opts = config_curl_opts.clone.merge(curl_opts) if config_curl_opts
+    fetcher = Fetch.new(@cache, @config.paths, @concurrency, curl_opts,
+                        before: before, after: after)
+    fetcher.run(&method(:process_results))
 
     # Order by original path order
     @results = @config.paths.map { |p| @results[p] }
-    return results.map{ |r| r unless r.success? }.compact.length
+    results.map { |r| r unless r.success? }.compact.length
   end
 
   # Dump results to disk
@@ -124,23 +135,23 @@ class SiteDiff
     diff_dir = dir + DIFFS_DIR
     diff_dir.rmtree if diff_dir.exist?
     results.each { |r| r.dump(dir) if r.status == Result::STATUS_FAILURE }
-    SiteDiff::log "All diff files were dumped inside #{dir.expand_path}"
+    SiteDiff.log "All diff files were dumped inside #{dir.expand_path}"
 
     # store failing paths
     failures = dir + FAILURES_FILE
-    SiteDiff::log "Writing failures to #{failures.expand_path}"
+    SiteDiff.log "Writing failures to #{failures.expand_path}"
     failures.open('w') do |f|
       results.each { |r| f.puts r.path unless r.success? }
     end
 
     # create report of results
-    report = Diff::generate_html_report(results, report_before, report_after,
-      @cache)
+    report = Diff.generate_html_report(results, report_before, report_after,
+                                       @cache)
     dir.+(REPORT_FILE).open('w') { |f| f.write(report) }
 
     # serve some settings
     settings = { 'before' => report_before, 'after' => report_after,
-       'cached' => @cache.read_tags.map { |t| t.to_s } }
+                 'cached' => %w[before after] }
     dir.+(SETTINGS_FILE).open('w') { |f| YAML.dump(settings, f) }
   end
 end

data/lib/sitediff/cache.rb

@@ -1,61 +1,67 @@
+# frozen_string_literal: true
+
 require 'set'
+require 'fileutils'
 
 class SiteDiff
-class Cache
-  DEFAULT_FILENAME = 'cache.db'
+  class Cache
+    attr_accessor :read_tags, :write_tags
 
-  attr_accessor :read_tags, :write_tags
+    def initialize(opts = {})
+      @dir = opts[:dir] || '.'
+      @create = opts[:create]
+      @read_tags = Set.new
+      @write_tags = Set.new
+    end
 
-  def initialize(opts = {})
-    @file = opts[:file] || DEFAULT_FILENAME
-    @create = opts[:create]
-    @read_tags = Set.new
-    @write_tags = Set.new
-  end
+    # Is a tag cached?
+    def tag?(tag)
+      File.directory?(File.join(@dir, 'snapshot', tag.to_s))
+    end
 
-  def close; @dbm.close if defined? @dbm; end
+    def get(tag, path)
+      return nil unless @read_tags.include? tag
 
-  # Is a tag cached?
-  def tag?(tag)
-    open
-    @dbm[tag.to_s]
-  end
+      filename = File.join(@dir, 'snapshot', tag.to_s, *path.split(File::SEPARATOR))
 
-  def get(tag, path)
-    return nil unless @read_tags.include? tag
-    open or return nil
-    val = @dbm[key(tag, path)]
-    return val && Marshal.load(val)
-  end
+      filename = File.join(filename, 'index.html') if File.directory?(filename)
+      return nil unless File.file? filename
 
-  def set(tag, path, result)
-    return unless @write_tags.include? tag
-    open or return
-    @dbm[tag.to_s] = 'TRUE'
-    @dbm[key(tag, path)] = Marshal.dump(result)
-  end
+      Marshal.load(File.read(filename))
+    end
 
-private
-  def key(tag, path)
-    # Ensure encoding stays the same!
-    Marshal.dump([tag, path.encode('UTF-8')])
-  end
+    def set(tag, path, result)
+      return unless @write_tags.include? tag
+
+      filename = File.join(@dir, 'snapshot', tag.to_s, *path.split(File::SEPARATOR))
 
-  # Ensure the DB is open
-  def open
-    # DBM adds an extra .db, ugh
-    return false unless @create || File.exist?(@file) ||
-      File.exist?(@file + '.db')
-    return true if defined? @dbm
-
-    begin
-      require 'gdbm'
-      @dbm = GDBM.new(@file)
-    rescue LoadError
-      require 'dbm'
-      @dbm = DBM.new(@file)
+      filename = File.join(filename, 'index.html') if File.directory?(filename)
+      filepath = Pathname.new(filename)
+      unless filepath.dirname.directory?
+        begin
+          filepath.dirname.mkpath
+        rescue Errno::EEXIST
+          curdir = filepath
+          curdir = curdir.parent until curdir.exist?
+          tempname = curdir.dirname + (curdir.basename.to_s + '.temporary')
+          # May cause problems if action is not atomic!
+          # Move existing file to dir/index.html first
+          # Not robust! Should generate an UUID or something.
+          SiteDiff.log "Overwriting file #{tempname}", :warn if File.exist?(tempname)
+          curdir.rename(tempname)
+          filepath.dirname.mkpath
+          # Should only happen in strange situations such as when the path
+          # is foo/index.html/bar (i.e., index.html is a directory)
+          SiteDiff.log "Overwriting file #{tempname}", :warn if (curdir + 'index.html').exist?
+          tempname.rename(curdir + 'index.html')
+        end
+      end
+      File.open(filename, 'w') { |file| file.write(Marshal.dump(result)) }
+    end
+
+    def key(tag, path)
+      # Ensure encoding stays the same!
+      Marshal.dump([tag, path.encode('UTF-8')])
     end
-    return true
   end
 end
-end

data/lib/sitediff/cli.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'thor'
 require 'sitediff'
 require 'sitediff/cache'
@@ -9,9 +11,18 @@ require 'sitediff/webserver/resultserver'
 class SiteDiff
   class Cli < Thor
     class_option 'directory',
-      :type => :string,
-      :aliases => '-C',
-      :desc => "Go to a given directory before running."
+                 type: :string,
+                 aliases: '-C',
+                 default: 'sitediff',
+                 desc: 'Configuration directory'
+    class_option :curl_options,
+                 type: :hash,
+                 default: {},
+                 desc: 'Options to be passed to curl'
+    class_option :insecure,
+                 type: :boolean,
+                 default: false,
+                 desc: 'Ignore many HTTPS/SSL errors'
 
     # Thor, by default, exits with 0 no matter what!
     def self.exit_on_failure?
@@ -19,67 +30,67 @@ class SiteDiff
     end
 
     # Thor, by default, does not raise an error for use of unknown options.
-    def self.check_unknown_options?(config)
+    def self.check_unknown_options?(_config)
       true
     end
 
-    option 'dump-dir',
-      :type => :string,
-      :default => File.join('.', 'output'),
-      :desc => "Location to write the output to."
     option 'paths-file',
-      :type => :string,
-      :desc => 'Paths are read (one at a line) from PATHS: ' +
-               'useful for iterating over sanitization rules',
-      :aliases => '--paths-from-file'
+           type: :string,
+           desc: 'Paths are read (one at a line) from PATHS: ' \
+                    'useful for iterating over sanitization rules',
+           aliases: '--paths-from-file'
     option 'paths',
-      :type => :array,
-      :aliases => '-p',
-      :desc => "Specific path or paths to fetch"
+           type: :array,
+           aliases: '-p',
+           desc: 'Specific path or paths to fetch'
     option 'before',
-      :type => :string,
-      :desc => "URL used to fetch the before HTML. Acts as a prefix to specified paths",
-      :aliases => '--before-url'
+           type: :string,
+           desc: 'URL used to fetch the before HTML. Acts as a prefix to specified paths',
+           aliases: '--before-url'
     option 'after',
-      :type => :string,
-      :desc => "URL used to fetch the after HTML. Acts as a prefix to specified paths.",
-      :aliases => '--after-url'
+           type: :string,
+           desc: 'URL used to fetch the after HTML. Acts as a prefix to specified paths.',
+           aliases: '--after-url'
     option 'before-report',
-      :type => :string,
-      :desc => "Before URL to use for reporting purposes. Useful if port forwarding.",
-      :aliases => '--before-url-report'
+           type: :string,
+           desc: 'Before URL to use for reporting purposes. Useful if port forwarding.',
+           aliases: '--before-url-report'
     option 'after-report',
-      :type => :string,
-      :desc => "After URL to use for reporting purposes. Useful if port forwarding.",
-      :aliases => '--after-url-report'
+           type: :string,
+           desc: 'After URL to use for reporting purposes. Useful if port forwarding.',
+           aliases: '--after-url-report'
     option 'cached',
-      :type => :string,
-      :enum => %w[none all before after],
-      :default => 'before',
-      :desc => "Use the cached version of these sites, if available."
-    option 'quiet',
-      :type => :boolean,
-      :aliases => '-q',
-      :default => false,
-      :desc => "Do not show differences between versions for each page"
-    desc "diff [OPTIONS] [CONFIGFILES]", "Perform systematic diff on given URLs"
+           type: :string,
+           enum: %w[none all before after],
+           default: 'before',
+           desc: 'Use the cached version of these sites, if available.'
+    option 'verbose',
+           type: :boolean,
+           aliases: '-v',
+           default: false,
+           desc: 'Show differences between versions for each page in terminal'
+    option :concurrency,
+           type: :numeric,
+           default: 3,
+           desc: 'Max number of concurrent connections made'
+    desc 'diff [OPTIONS] [CONFIGFILES]', 'Perform systematic diff on given URLs'
     def diff(*config_files)
-      config = chdir(config_files)
+      config = SiteDiff::Config.new(config_files, options[:directory])
 
       # override config based on options
       paths = options['paths']
-      if paths_file = options['paths-file']
-        if paths then
-          SiteDiff::log "Can't have both --paths-file and --paths", :error
-          exit -1
+      if (paths_file = options['paths-file'])
+        if paths
+          SiteDiff.log "Can't have both --paths-file and --paths", :error
+          exit(-1)
         end
 
         paths_file = Pathname.new(paths_file).expand_path
-        unless File.exists? paths_file
+        unless File.exist? paths_file
           raise Config::InvalidConfig,
-            "Paths file '#{paths_file}' not found!"
+                "Paths file '#{paths_file}' not found!"
         end
-        SiteDiff::log "Reading paths from: #{paths_file}"
+        SiteDiff.log "Reading paths from: #{paths_file}"
         config.paths = File.readlines(paths_file)
       end
       config.paths = paths if paths
@@ -88,21 +99,21 @@ class SiteDiff
       config.after['url'] = options['after'] if options['after']
 
       # Setup cache
-      cache = SiteDiff::Cache.new(:create => options['cached'] != 'none')
+      cache = SiteDiff::Cache.new(create: options['cached'] != 'none',
+                                  dir: options['directory'])
       cache.read_tags << :before if %w[before all].include?(options['cached'])
       cache.read_tags << :after if %w[after all].include?(options['cached'])
       cache.write_tags << :before << :after
 
-      sitediff = SiteDiff.new(config, cache, !options['quiet'])
-      num_failing = sitediff.run
-      exit_code = (num_failing > 0) ? 2 : 0;
+      sitediff = SiteDiff.new(config, cache, options[:concurrency],
+                              options['verbose'])
+      num_failing = sitediff.run(get_curl_opts(options))
+      exit_code = num_failing > 0 ? 2 : 0
 
-      sitediff.dump(options['dump-dir'], options['before-report'],
-        options['after-report'])
+      sitediff.dump(options['directory'], options['before-report'],
+                    options['after-report'])
     rescue Config::InvalidConfig => e
       SiteDiff.log "Invalid configuration: #{e.message}", :error
-    rescue SiteDiffException => e
-      SiteDiff.log e.message, :error
     else # no exception was raised
       # Thor::Error  --> exit(1), guaranteed by exit_on_failure?
       # Failing diff --> exit(2), populated above
@@ -110,62 +121,61 @@ class SiteDiff
     end
 
     option :port,
-      :type => :numeric,
-      :default => SiteDiff::Webserver::DEFAULT_PORT,
-      :desc => 'The port to serve on'
-    option 'dump-dir',
-      :type => :string,
-      :default => 'output',
-      :desc => 'The directory to serve'
+           type: :numeric,
+           default: SiteDiff::Webserver::DEFAULT_PORT,
+           desc: 'The port to serve on'
     option :browse,
-      :type => :boolean,
-      :default => true,
-      :desc => "Whether to open the served content in your browser"
-    desc "serve [OPTIONS]", "Serve the sitediff output directory over HTTP"
+           type: :boolean,
+           default: true,
+           desc: 'Whether to open the served content in your browser'
+    desc 'serve [OPTIONS]', 'Serve the sitediff output directory over HTTP'
     def serve(*config_files)
-      config = chdir(config_files, :config => false)
+      config = SiteDiff::Config.new(config_files, options['directory'])
+      # Could check non-empty config here but currently errors are already raised.
 
-      cache = Cache.new
+      cache = Cache.new(dir: options['directory'])
       cache.read_tags << :before << :after
 
       SiteDiff::Webserver::ResultServer.new(
         options[:port],
-        options['dump-dir'],
-        :browse => options[:browse],
-        :cache => cache,
-        :config => config,
+        options['directory'],
+        browse: options[:browse],
+        cache: cache,
+        config: config
       ).wait
+    rescue SiteDiffException => e
+      SiteDiff.log e.message, :error
     end
 
-    option :output,
-      :type => :string,
-      :default => 'sitediff',
-      :desc => 'Directory in which to place the configuration',
-      :aliases => ['-o']
     option :depth,
-      :type => :numeric,
-      :default => 3,
-      :desc => 'How deeply to crawl the given site'
+           type: :numeric,
+           default: 3,
+           desc: 'How deeply to crawl the given site'
     option :rules,
-      :type => :string,
-      :enum => %w[yes no disabled],
-      :default => 'disabled',
-      :desc => 'Whether rules for the site should be auto-created'
-    desc "init URL [URL]", "Create a sitediff configuration"
+           type: :string,
+           enum: %w[yes no disabled],
+           default: 'disabled',
+           desc: 'Whether rules for the site should be auto-created'
+    option :concurrency,
+           type: :numeric,
+           default: 3,
+           desc: 'Max number of concurrent connections made'
+    desc 'init URL [URL]', 'Create a sitediff configuration'
     def init(*urls)
-      unless (1..2).include? urls.size
-        SiteDiff.log "sitediff init requires one or two URLs", :error
+      unless (1..2).cover? urls.size
+        SiteDiff.log 'sitediff init requires one or two URLs', :error
         exit 2
       end
 
-      chdir([], :search => false)
-      creator = SiteDiff::Config::Creator.new(*urls)
+      curl_opts = get_curl_opts(options)
+
+      creator = SiteDiff::Config::Creator.new(options[:concurrency], curl_opts, *urls)
       creator.create(
-        :depth => options[:depth],
-        :directory => options[:output],
-        :rules => options[:rules] != 'no',
-        :rules_disabled => (options[:rules] == 'disabled'),
-      ) do |tag, info|
+        depth: options[:depth],
+        directory: options[:directory],
+        rules: options[:rules] != 'no',
+        rules_disabled: (options[:rules] == 'disabled')
+      ) do |_tag, info|
         SiteDiff.log "Visited #{info.uri}, cached"
       end
 
@@ -174,37 +184,40 @@ class SiteDiff
     end
 
     option :url,
-      :type => :string,
-      :desc => 'A custom base URL to fetch from'
-    desc "store [CONFIGFILES]",
-      "Cache the current contents of a site for later comparison"
+           type: :string,
+           desc: 'A custom base URL to fetch from'
+    option :concurrency,
+           type: :numeric,
+           default: 3,
+           desc: 'Max number of concurrent connections made'
+    desc 'store [CONFIGFILES]',
+         'Cache the current contents of a site for later comparison'
     def store(*config_files)
-      config = chdir(config_files)
-      config.validate(:need_before => false)
+      config = SiteDiff::Config.new(config_files, options['directory'])
+      config.validate(need_before: false)
 
-      cache = SiteDiff::Cache.new(:create => true)
+      cache = SiteDiff::Cache.new(create: true)
       cache.write_tags << :before
 
       base = options[:url] || config.after['url']
-      fetcher = SiteDiff::Fetch.new(cache, config.paths, :before => base)
-      fetcher.run do |path, res|
+      fetcher = SiteDiff::Fetch.new(cache, config.paths, options['concurrency'],
+                                    before: base)
+      fetcher.run do |path, _res|
         SiteDiff.log "Visited #{path}, cached"
       end
     end
 
-  private
-    def chdir(files, opts = {})
-      opts = { :config => true, :search => true }.merge(opts)
-
-      dir = options['directory']
-      Dir.chdir(dir) if dir
-
-      return unless opts[:search]
-      begin
-        SiteDiff::Config.new(files, :search => !dir)
-      rescue SiteDiff::Config::ConfigNotFound => e
-        raise if opts[:config]
-        # If no config required, allow it to pass
+    no_commands do
+      def get_curl_opts(options)
+        # We do want string keys here
+        bool_hash = { 'true' => true, 'false' => false }
+        curl_opts = UriWrapper::DEFAULT_CURL_OPTS.clone.merge(options[:curl_options])
+        curl_opts.each { |k, v| curl_opts[k] = bool_hash.fetch(v, v) }
+        if options[:insecure]
+          curl_opts[:ssl_verifypeer] = false
+          curl_opts[:ssl_verifyhost] = 0
+        end
+        curl_opts
       end
     end
   end