sitediff 0.0.3 → 0.0.5

data/lib/sitediff/config.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'sitediff/exception'
 require 'sitediff/sanitize'
 require 'pathname'
@@ -9,7 +11,7 @@ class SiteDiff
 
     # keys allowed in configuration files
     CONF_KEYS = Sanitizer::TOOLS.values.flatten(1) +
-                %w[paths before after before_url after_url includes]
+                %w[paths before after before_url after_url includes curl_opts]
 
     class InvalidConfig < SiteDiffException; end
     class ConfigNotFound < SiteDiffException; end
@@ -41,13 +43,14 @@ class SiteDiff
           conf[pos][key] ||= []
           conf[pos][key] += conf[key] if conf[key]
         end
-        tools[:scalar].each {|key| conf[pos][key] ||= conf[key]}
+        tools[:scalar].each { |key| conf[pos][key] ||= conf[key] }
         conf[pos]['url'] ||= conf[pos + '_url']
+        conf[pos]['curl_opts'] = conf['curl_opts']
       end
       # normalize paths
-      conf['paths'] = Config::normalize_paths(conf['paths'])
+      conf['paths'] = Config.normalize_paths(conf['paths'])
 
-      conf.select {|k,v| %w[before after paths].include? k}
+      conf.select { |k, _v| %w[before after paths curl_opts].include? k }
     end
 
     # Merges two normalized Hashes according to the following rules:
@@ -72,56 +75,33 @@ class SiteDiff
           next
         end
         result[pos] = first[pos].merge!(second[pos]) do |key, a, b|
-          if Sanitizer::TOOLS[:array].include? key # rule 2a
-            result[pos][key] = (a || []) + (b|| [])
-          else
-            result[pos][key] = a || b # rule 2b
-          end
+          result[pos][key] = if Sanitizer::TOOLS[:array].include? key # rule 2a
+                               (a || []) + (b || [])
+                             else
+                               a || b # rule 2b
+                             end
         end
       end
       result
     end
 
-    # Search for a config file. If found, change to the containing directory,
-    # and return an array of config files found.
-    def self.search
-      subdirs = %w[. sitediff]
-      root_indicators = %w[.git .svn]
-
-      Pathname.pwd.ascend do |dir|
-        subdirs.each do |subdir|
-          d = dir + subdir + DEFAULT_FILENAME
-          if d.exist?
-            Dir.chdir(dir.+(subdir).to_s)
-            return [DEFAULT_FILENAME]
-          end
-        end
-
-        root_indicators.each { |r| return [] if dir.+(r).exist? }
-      end
-
-      return []
-    end
-
-    def initialize(files, opts = {})
-      @config = {'paths' => [], 'before' => {}, 'after' => {} }
-
-      files = Config.search if files.empty? && opts[:search]
-      files = [DEFAULT_FILENAME] if files.empty? &&
-        File.exists?(DEFAULT_FILENAME)
-      raise ConfigNotFound, "No configuration file found." if files.empty?
+    def initialize(files, dir)
+      @config = { 'paths' => [], 'before' => {}, 'after' => {} }
 
+      files = [File.join(dir, DEFAULT_FILENAME)] if files.empty?
       files.each do |file|
-        raise InvalidConfig,
-          "Missing config file %s." % File.expand_path(file) \
-          unless File.exist?(file)
-        @config = Config::merge(@config, Config::load_conf(file))
+        unless File.exist?(file)
+          raise InvalidConfig,
+                format('Missing config file %s.', File.expand_path(file))
+        end
+        @config = Config.merge(@config, Config.load_conf(file))
       end
     end
 
     def before
       @config['before']
     end
+
     def after
       @config['after']
     end
@@ -129,61 +109,57 @@ class SiteDiff
     def paths
       @config['paths']
     end
+
     def paths=(paths)
-      @config['paths'] = Config::normalize_paths(paths)
+      @config['paths'] = Config.normalize_paths(paths)
     end
 
     # Checks if the configuration is usable for diff-ing.
     def validate(opts = {})
-      opts = { :need_before => true }.merge(opts)
+      opts = { need_before: true }.merge(opts)
 
       raise InvalidConfig, "Undefined 'before' base URL." if \
         opts[:need_before] && !before['url']
       raise InvalidConfig, "Undefined 'after' base URL." unless after['url']
-      raise InvalidConfig, "Undefined 'paths'." unless (paths and !paths.empty?)
+      raise InvalidConfig, "Undefined 'paths'." unless paths && !paths.empty?
     end
 
     private
 
     def self.normalize_paths(paths)
       paths ||= []
-      return paths.map { |p| (p[0] == '/' ? p : "/#{p}").chomp }
+      paths.map { |p| (p[0] == '/' ? p : "/#{p}").chomp }
     end
 
     # reads a YAML file and raises an InvalidConfig if the file is not valid.
     def self.load_raw_yaml(file)
-      SiteDiff::log "Reading config file: #{Pathname.new(file).expand_path}"
+      SiteDiff.log "Reading config file: #{Pathname.new(file).expand_path}"
       conf = YAML.load_file(file) || {}
-      unless conf.is_a? Hash
-        raise InvalidConfig, "Invalid configuration file: '#{file}'"
-      end
-      conf.each do |k,v|
-        unless CONF_KEYS.include? k
-          raise InvalidConfig, "Unknown configuration key (#{file}): '#{k}'"
-        end
+      raise InvalidConfig, "Invalid configuration file: '#{file}'" unless conf.is_a? Hash
+
+      conf.each_key do |k, _v|
+        raise InvalidConfig, "Unknown configuration key (#{file}): '#{k}'" unless CONF_KEYS.include? k
       end
       conf
     end
 
     # loads a single YAML configuration file, merges all its 'included' files
     # and returns a normalized Hash.
-    def self.load_conf(file, visited=[])
+    def self.load_conf(file, visited = [])
       # don't get fooled by a/../a/ or symlinks
       file = File.realpath(file)
-      if visited.include? file
-        raise InvalidConfig, "Circular dependency: #{file}"
-      end
+      raise InvalidConfig, "Circular dependency: #{file}" if visited.include? file
 
       conf = load_raw_yaml(file) # not normalized yet
       visited << file
 
       # normalize and merge includes
       includes = conf['includes'] || []
-      conf = Config::normalize(conf)
+      conf = Config.normalize(conf)
       includes.each do |dep|
         # include paths are relative to the including file.
         dep = File.join(File.dirname(file), dep)
-        conf = Config::merge(conf, load_conf(dep, visited))
+        conf = Config.merge(conf, load_conf(dep, visited))
       end
       conf
     end

data/lib/sitediff/config/creator.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'sitediff/cache'
 require 'sitediff/config'
 require 'sitediff/crawler'
@@ -7,116 +9,119 @@ require 'typhoeus'
 require 'yaml'
 
 class SiteDiff
-class Config
-class Creator
-  def initialize(*urls, &block)
-    @after = urls.pop
-    @before = urls.pop # May be nil
-  end
+  class Config
+    class Creator
+      def initialize(concurrency, curl_opts, *urls)
+        @concurrency = concurrency
+        @after = urls.pop
+        @before = urls.pop # May be nil
+        @curl_opts = curl_opts
+      end
 
-  def roots
-    @roots = begin
-      r = { :after => @after }
-      r[:before] = @before if @before
-      r
-    end
-  end
+      def roots
+        @roots = begin
+          r = { after: @after }
+          r[:before] = @before if @before
+          r
+        end
+      end
 
-  # Build a config structure, return it
-  def create(opts, &block)
-    @config = {}
-    @callback = block
+      # Build a config structure, return it
+      def create(opts, &block)
+        @config = {}
+        @callback = block
 
-    # Handle options
-    @dir = Pathname.new(opts[:directory])
-    @depth = opts[:depth]
-    @rules = Rules.new(@config, opts[:rules_disabled]) if opts[:rules]
+        # Handle options
+        @dir = Pathname.new(opts[:directory])
+        @depth = opts[:depth]
+        @rules = Rules.new(@config, opts[:rules_disabled]) if opts[:rules]
 
-    # Create the dir. Must go before cache initialization!
-    @dir.mkpath unless @dir.directory?
+        # Create the dir. Must go before cache initialization!
+        @dir.mkpath unless @dir.directory?
 
-    # Setup instance vars
-    @paths = Hash.new { |h,k| h[k] = Set.new }
-    @cache = Cache.new(:file => @dir.+(Cache::DEFAULT_FILENAME).to_s,
-      :create => true)
-    @cache.write_tags << :before << :after
+        # Setup instance vars
+        @paths = Hash.new { |h, k| h[k] = Set.new }
+        @cache = Cache.new(dir: @dir.to_s, create: true)
+        @cache.write_tags << :before << :after
 
-    build_config
-    write_config
-  end
+        build_config
+        write_config
+      end
 
-  def build_config
-    %w[before after].each do |tag|
-      next unless u = roots[tag.to_sym]
-      @config[tag] = {'url' => u}
-    end
+      def build_config
+        %w[before after].each do |tag|
+          next unless (u = roots[tag.to_sym])
 
-    crawl(@depth)
-    @cache.close
-    @rules.add_config if @rules
+          @config[tag] = { 'url' => u }
+        end
 
-    @config['paths'] = @paths.values.reduce(&:|).to_a.sort
-  end
+        crawl(@depth)
+        @rules&.add_config
 
-  def crawl(depth = nil)
-    hydra = Typhoeus::Hydra.new(max_concurrency: 10)
-    roots.each do |tag, u|
-      Crawler.new(hydra, u, depth) do |info|
-        crawled_path(tag, info)
+        @config['paths'] = @paths.values.reduce(&:|).to_a.sort
       end
-    end
-    hydra.run
-  end
 
-  # Deduplicate paths with slashes at the end
-  def canonicalize(tag, path)
-    def altered_paths(path)
-      yield path + '/'
-      yield path.sub(%r[/$], '')
-    end
+      def crawl(depth = nil)
+        hydra = Typhoeus::Hydra.new(max_concurrency: @concurrency)
+        roots.each do |tag, u|
+          Crawler.new(hydra, u, depth, @curl_opts) do |info|
+            crawled_path(tag, info)
+          end
+        end
+        hydra.run
+      end
 
-    return path.empty? ? '/' : path
-  end
+      # Deduplicate paths with slashes at the end
+      def canonicalize(_tag, path)
+        def altered_paths(path)
+          yield path + '/'
+          yield path.sub(%r{/$}, '')
+        end
 
-  def crawled_path(tag, info)
-    path, dup = canonicalize(tag, info.relative)
-    return if dup
+        path.empty? ? '/' : path
+      end
 
-    res = info.read_result
+      def crawled_path(tag, info)
+        path, dup = canonicalize(tag, info.relative)
+        return if dup
 
-    @callback[tag, info]
-    @paths[tag] << path
-    @cache.set(tag, path, res)
+        res = info.read_result
 
-    # If single-site, cache after as before!
-    @cache.set(:before, path, res) unless roots[:before]
+        @callback[tag, info]
+        @paths[tag] << path
+        @cache.set(tag, path, res)
 
-    @rules.handle_page(tag, res.content, info.document) if @rules && !res.error
-  end
+        # If single-site, cache after as before!
+        @cache.set(:before, path, res) unless roots[:before]
 
-  # Create a gitignore if we seem to be in git
-  def make_gitignore(dir)
-    # Check if we're in git
-    return unless dir.realpath.to_enum(:ascend).any? { |d| d.+('.git').exist? }
-
-    dir.+('.gitignore').open('w') do |f|
-      f.puts <<-EOF.gsub(/^\s+/, '')
-        output
-        cache.db
-        cache.db.db
-      EOF
-    end
-  end
+        # This is used to populate the list of rules we guess are
+        # applicable to the current site.
+        @rules.handle_page(tag, res.content, info.document) if @rules && !res.error
+      end
 
-  def config_file
-    @dir + Config::DEFAULT_FILENAME
-  end
+      # Create a gitignore if we seem to be in git
+      def make_gitignore(dir)
+        # Check if we're in git
+        return unless dir.realpath.to_enum(:ascend).any? { |d| d.+('.git').exist? }
+
+        dir.+('.gitignore').open('w') do |f|
+          f.puts <<-GITIGNORE.gsub(/^\s+/, '')
+            output
+            cache.db
+            cache.db.db
+          GITIGNORE
+        end
+      end
+
+      def config_file
+        @dir + Config::DEFAULT_FILENAME
+      end
 
-  # Turn a config structure into a config file
-  def write_config
-    make_gitignore(@dir)
-    config_file.open('w') { |f| f.puts @config.to_yaml }
+      # Turn a config structure into a config file
+      def write_config
+        make_gitignore(@dir)
+        config_file.open('w') { |f| f.puts @config.to_yaml }
+      end
+    end
   end
 end
-end
-end

data/lib/sitediff/crawler.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'sitediff'
 require 'sitediff/uriwrapper'
 require 'addressable/uri'
@@ -6,90 +8,99 @@ require 'ostruct'
 require 'set'
 
 class SiteDiff
-class Crawler
-  class Info < OpenStruct; end
-
-  DEFAULT_DEPTH = 3
+  class Crawler
+    class Info < OpenStruct; end
+
+    DEFAULT_DEPTH = 3
+
+    # Create a crawler with a base URL
+    def initialize(hydra, base, depth = DEFAULT_DEPTH,
+                   curl_opts = UriWrapper::DEFAULT_CURL_OPTS, &block)
+      @hydra = hydra
+      @base_uri = Addressable::URI.parse(base)
+      @base = base
+      @found = Set.new
+      @callback = block
+      @curl_opts = curl_opts
+
+      add_uri('', depth)
+    end
 
-  # Create a crawler with a base URL
-  def initialize(hydra, base, depth = DEFAULT_DEPTH, &block)
-    @hydra = hydra
-    @base_uri = Addressable::URI.parse(base)
-    @base = base
-    @found = Set.new
-    @callback = block
+    # Handle a newly found relative URI
+    def add_uri(rel, depth)
+      return if @found.include? rel
 
-    add_uri('', depth)
-  end
+      @found << rel
 
-  # Handle a newly found relative URI
-  def add_uri(rel, depth)
-    return if @found.include? rel
-    @found << rel
-
-    wrapper = UriWrapper.new(@base + rel)
-    wrapper.queue(@hydra) do |res|
-      fetched_uri(rel, depth, res)
+      wrapper = UriWrapper.new(@base + rel, @curl_opts)
+      wrapper.queue(@hydra) do |res|
+        fetched_uri(rel, depth, res)
+      end
     end
-  end
 
-  # Handle the fetch of a URI
-  def fetched_uri(rel, depth, res)
-    return unless res.content # Ignore errors
-    return unless depth >= 0
-
-    base = Addressable::URI.parse(@base + rel)
-    doc = Nokogiri::HTML(res.content)
-
-    # Call the callback
-    info = Info.new(
-      :relative => rel,
-      :uri => base,
-      :read_result => res,
-      :document => doc,
-    )
-    @callback[info]
-
-    # Find links
-    links = find_links(doc)
-    uris = links.map { |l| resolve_link(base, l) }.compact
-    uris = filter_links(uris)
-
-    # Make them relative
-    rels = uris.map { |u| relativize_link(u) }
-
-    # Queue them in turn
-    rels.each do |r|
-      next if @found.include? r
-      add_uri(r, depth - 1)
+    # Handle the fetch of a URI
+    def fetched_uri(rel, depth, res)
+      if res.error
+        SiteDiff.log(res.error, :error)
+        return
+      elsif !res.content
+        SiteDiff.log('Response is missing content. Treating as an error.', :error)
+        return
+      end
+
+      base = Addressable::URI.parse(@base + rel)
+      doc = Nokogiri::HTML(res.content)
+
+      # Call the callback
+      info = Info.new(
+        relative: rel,
+        uri: base,
+        read_result: res,
+        document: doc
+      )
+      @callback[info]
+
+      return unless depth >= 1
+
+      # Find links
+      links = find_links(doc)
+      uris = links.map { |l| resolve_link(base, l) }.compact
+      uris = filter_links(uris)
+
+      # Make them relative
+      rels = uris.map { |u| relativize_link(u) }
+
+      # Queue them in turn
+      rels.each do |r|
+        next if @found.include? r
+
+        add_uri(r, depth - 1)
+      end
     end
-  end
 
-  # Resolve a potentially-relative link. Return nil on error.
-  def resolve_link(base, rel)
-    begin
-      return base + rel
+    # Resolve a potentially-relative link. Return nil on error.
+    def resolve_link(base, rel)
+      base + rel
     rescue Addressable::URI::InvalidURIError
-      SiteDiff.log "skipped invalid URL: '#{rel}'", :warn
-      return nil
+      SiteDiff.log "skipped invalid URL: '#{rel}' (at #{base})", :warn
+      nil
     end
-  end
 
-  # Make a link relative to @base_uri
-  def relativize_link(uri)
-    uri.path.slice(@base_uri.path.length, uri.path.length)
-  end
+    # Make a link relative to @base_uri
+    def relativize_link(uri)
+      uri.path.slice(@base_uri.path.length, uri.path.length)
+    end
 
-  # Return a list of string links found on a page.
-  def find_links(doc)
-    return doc.xpath('//a[@href]').map { |e| e['href'] }
-  end
+    # Return a list of string links found on a page.
+    def find_links(doc)
+      doc.xpath('//a[@href]').map { |e| e['href'] }
+    end
 
-  # Filter out links we don't want. Links passed in are absolute URIs.
-  def filter_links(uris)
-    uris.find_all do |u|
-      u.host == @base_uri.host && u.path.start_with?(@base_uri.path)
+    # Filter out links we don't want. Links passed in are absolute URIs.
+    def filter_links(uris)
+      uris.find_all do |u|
+        u.host == @base_uri.host && u.path.start_with?(@base_uri.path)
+      end
     end
   end
 end
-end