sitediff 0.0.2 → 1.1.1

data/lib/sitediff/config/creator.rb

@@ -1,122 +1,116 @@
+# frozen_string_literal: true
+
 require 'sitediff/cache'
 require 'sitediff/config'
 require 'sitediff/crawler'
-require 'sitediff/rules'
 require 'pathname'
 require 'typhoeus'
 require 'yaml'
 
 class SiteDiff
-class Config
-class Creator
-  def initialize(*urls, &block)
-    @after = urls.pop
-    @before = urls.pop # May be nil
-  end
-
-  def roots
-    @roots = begin
-      r = { :after => @after }
-      r[:before] = @before if @before
-      r
-    end
-  end
-
-  # Build a config structure, return it
-  def create(opts, &block)
-    @config = {}
-    @callback = block
-
-    # Handle options
-    @dir = Pathname.new(opts[:directory])
-    @depth = opts[:depth]
-    @rules = Rules.new(@config, opts[:rules_disabled]) if opts[:rules]
-
-    # Create the dir. Must go before cache initialization!
-    @dir.mkpath unless @dir.directory?
+  class Config
+    ##
+    # SiteDiff Config Creator Object.
+    class Creator
+      ##
+      # Creates a Creator object.
+      def initialize(debug, before, after)
+        @config = nil
+        @before = before
+        @after = after
+        @debug = debug
+      end
 
-    # Setup instance vars
-    @paths = Hash.new { |h,k| h[k] = Set.new }
-    @cache = Cache.new(:file => @dir.+(Cache::DEFAULT_FILENAME).to_s,
-      :create => true)
-    @cache.write_tags << :before << :after
+      ##
+      # Determine if we're dealing with one or two URLs.
+      def roots
+        @roots = { 'after' => @after }
+        @roots['before'] = @before || @after
+        @roots
+      end
 
-    build_config
-    write_config
-  end
+      ##
+      # Build a config structure, return it.
+      def create(options)
+        @config = {}
 
-  def build_config
-    %w[before after].each do |tag|
-      next unless u = roots[tag.to_sym]
-      @config[tag] = {'url' => u}
-    end
+        # @callback = block
 
-    crawl(@depth)
-    @cache.close
-    @rules.add_config if @rules
+        @dir = Pathname.new(options[:directory])
 
-    @config['paths'] = @paths.values.reduce(&:|).to_a.sort
-  end
+        # Setup instance vars
+        @paths = Hash.new { |h, k| h[k] = Set.new }
+        @cache = Cache.new(directory: @dir.to_s, create: true)
+        @cache.write_tags << :before << :after
 
-  def crawl(depth = nil)
-    hydra = Typhoeus::Hydra.new(max_concurrency: 10)
-    roots.each do |tag, u|
-      Crawler.new(hydra, u, depth) do |info|
-        crawled_path(tag, info)
+        build_config options
+        write_config
       end
-    end
-    hydra.run
-  end
-
-  # Deduplicate paths with slashes at the end
-  def canonicalize(tag, path)
-    def altered_paths(path)
-      yield path + '/'
-      yield path.sub(%r[/$], '')
-    end
-
-    return path.empty? ? '/' : path
-  end
-
-  def crawled_path(tag, info)
-    path, dup = canonicalize(tag, info.relative)
-    return if dup
 
-    res = info.read_result
+      ##
+      # Build and populate the config object which is being created.
+      #
+      # @param [String] options
+      #   One or more options.
+      def build_config(options)
+        options = Config.stringify_keys options
+
+        # Build config for "before" and "after".
+        %w[before after].each do |tag|
+          next unless (url = roots[tag])
+
+          @config[tag] = { 'url' => url }
+        end
+
+        # Build other settings.
+        @config['settings'] = {}
+        Config::ALLOWED_SETTINGS_KEYS.each do |key|
+          @config['settings'][key] = options[key]
+        end
+      end
 
-    @callback[tag, info]
-    @paths[tag] << path
-    @cache.set(tag, path, res)
+      ##
+      # Create a gitignore if we seem to be in git.
+      def make_gitignore(dir)
+        # Check if we're in git
+        unless dir.realpath.to_enum(:ascend).any? { |d| d.+('.git').exist? }
+          return
+        end
+
+        dir.+('.gitignore').open('w') do |f|
+          f.puts <<-GITIGNORE.gsub(/^\s+/, '')
+            # Directories.
+            diffs
+            snapshot
+
+            # Files.
+            settings.yaml
+            paths.txt
+            failures.txt
+          GITIGNORE
+        end
+      end
 
-    # If single-site, cache after as before!
-    @cache.set(:before, path, res) unless roots[:before]
+      ##
+      # Returns the name of the config directory.
+      def directory
+        @dir
+      end
 
-    @rules.handle_page(tag, res.content, info.document) if @rules && !res.error
-  end
+      ##
+      # Returns the name of the config file.
+      def config_file
+        @dir + Config::DEFAULT_FILENAME
+      end
 
-  # Create a gitignore if we seem to be in git
-  def make_gitignore(dir)
-    # Check if we're in git
-    return unless dir.realpath.to_enum(:ascend).any? { |d| d.+('.git').exist? }
-
-    dir.+('.gitignore').open('w') do |f|
-      f.puts <<-EOF.gsub(/^\s+/, '')
-        output
-        cache.db
-        cache.db.db
-      EOF
+      ##
+      # Writes the built config into the config file.
+      # TODO: Exclude default params before writing.
+      def write_config
+        make_gitignore(@dir)
+        data = Config.remove_defaults(@config)
+        config_file.open('w') { |f| f.puts data.to_yaml }
+      end
     end
   end
-
-  def config_file
-    @dir + Config::DEFAULT_FILENAME
-  end
-
-  # Turn a config structure into a config file
-  def write_config
-    make_gitignore(@dir)
-    config_file.open('w') { |f| f.puts @config.to_yaml }
-  end
-end
-end
 end

data/lib/sitediff/config/preset.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require 'pathname'
+require 'sitediff/config'
+
+class SiteDiff
+  class Config
+    ##
+    # Preset helper.
+    class Preset
+      ##
+      # Directory in which presets live.
+      #
+      # TODO: Move this outside "lib".
+      DIRECTORY = (Pathname.new(__dir__).dirname + 'presets').freeze
+
+      ##
+      # Reads preset rules.
+      #
+      # @param [String] preset
+      #   Presets
+      #
+      # @return [Hash]
+      #   A hash containing the preset's rules.
+      def self.read(name)
+        @cache = {} if @cache.nil?
+
+        # Load and cache preset config.
+        if @cache[name].nil?
+          exist? name, true
+          @cache[name] = Config.load_conf file(name)
+        end
+
+        @cache[name]
+      end
+
+      ##
+      # Get all possible rules.
+      #
+      # @return [Array]
+      #   All presets.
+      def self.all
+        # Load and cache preset names.
+        if @all.nil?
+          @all = []
+          pattern = DIRECTORY + '*.yaml'
+          Dir.glob(pattern) do |file|
+            @all << File.basename(file, '.yaml')
+          end
+        end
+
+        @all
+      end
+
+      ##
+      # Checks whether a preset exists.
+      def self.exist?(name, exception = false)
+        result = File.exist? file(name)
+
+        # Raise an exception, if required.
+        if exception && !result
+          raise Config::InvalidConfig, "Preset not found: #{name}"
+        end
+
+        result
+      end
+
+      ##
+      # Returns the path to a preset file.
+      def self.file(name)
+        DIRECTORY + "#{name}.yaml"
+      end
+    end
+  end
+end

data/lib/sitediff/crawler.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'sitediff'
 require 'sitediff/uriwrapper'
 require 'addressable/uri'
@@ -6,90 +8,124 @@ require 'ostruct'
 require 'set'
 
 class SiteDiff
-class Crawler
-  class Info < OpenStruct; end
-
-  DEFAULT_DEPTH = 3
+  # SiteDiff Crawler.
+  class Crawler
+    class Info < OpenStruct; end
+
+    DEFAULT_DEPTH = 3
+
+    # Create a crawler with a base URL
+    def initialize(hydra, base,
+                   interval,
+                   include_regex,
+                   exclude_regex,
+                   depth = DEFAULT_DEPTH,
+                   curl_opts = UriWrapper::DEFAULT_CURL_OPTS,
+                   debug = true,
+                   &block)
+      @hydra = hydra
+      @base_uri = Addressable::URI.parse(base)
+      @base = base
+      @interval = interval
+      @include_regex = include_regex
+      @exclude_regex = exclude_regex
+      @found = Set.new
+      @callback = block
+      @curl_opts = curl_opts
+      @debug = debug
+
+      add_uri('', depth)
+    end
 
-  # Create a crawler with a base URL
-  def initialize(hydra, base, depth = DEFAULT_DEPTH, &block)
-    @hydra = hydra
-    @base_uri = Addressable::URI.parse(base)
-    @base = base
-    @found = Set.new
-    @callback = block
+    # Handle a newly found relative URI
+    def add_uri(rel, depth)
+      return if @found.include? rel
 
-    add_uri('', depth)
-  end
+      @found << rel
 
-  # Handle a newly found relative URI
-  def add_uri(rel, depth)
-    return if @found.include? rel
-    @found << rel
-
-    wrapper = UriWrapper.new(@base + rel)
-    wrapper.queue(@hydra) do |res|
-      fetched_uri(rel, depth, res)
+      wrapper = UriWrapper.new(@base + rel, @curl_opts, @debug)
+      wrapper.queue(@hydra) do |res|
+        fetched_uri(rel, depth, res)
+      end
     end
-  end
 
-  # Handle the fetch of a URI
-  def fetched_uri(rel, depth, res)
-    return unless res.content # Ignore errors
-    return unless depth >= 0
-
-    base = Addressable::URI.parse(@base + rel)
-    doc = Nokogiri::HTML(res.content)
-
-    # Call the callback
-    info = Info.new(
-      :relative => rel,
-      :uri => base,
-      :read_result => res,
-      :document => doc,
-    )
-    @callback[info]
-
-    # Find links
-    links = find_links(doc)
-    uris = links.map { |l| resolve_link(base, l) }.compact
-    uris = filter_links(uris)
-
-    # Make them relative
-    rels = uris.map { |u| relativize_link(u) }
-
-    # Queue them in turn
-    rels.each do |r|
-      next if @found.include? r
-      add_uri(r, depth - 1)
+    # Handle the fetch of a URI
+    def fetched_uri(rel, depth, res)
+      if res.error
+        SiteDiff.log(res.error, :error)
+        return
+      elsif !res.content
+        SiteDiff.log('Response is missing content. Treating as an error.', :error)
+        return
+      end
+
+      base = Addressable::URI.parse(@base + rel)
+      doc = Nokogiri::HTML(res.content)
+
+      # Call the callback
+      info = Info.new(
+        relative: rel,
+        uri: base,
+        read_result: res,
+        document: doc
+      )
+      # Insert delay to limit fetching rate
+      if @interval != 0
+        SiteDiff.log("Waiting #{@interval} milliseconds.", :info)
+        sleep(@interval / 1000.0)
+      end
+      @callback[info]
+
+      return unless depth >= 1
+
+      # Find links
+      links = find_links(doc)
+      uris = links.map { |l| resolve_link(base, l) }.compact
+      uris = filter_links(uris)
+
+      # Make them relative
+      rels = uris.map { |u| relativize_link(u) }
+
+      # Queue them in turn
+      rels.each do |r|
+        next if @found.include? r
+
+        add_uri(r, depth - 1)
+      end
     end
-  end
 
-  # Resolve a potentially-relative link. Return nil on error.
-  def resolve_link(base, rel)
-    begin
-      return base + rel
+    # Resolve a potentially-relative link. Return nil on error.
+    def resolve_link(base, rel)
+      base + rel
     rescue Addressable::URI::InvalidURIError
-      SiteDiff.log "skipped invalid URL: '#{rel}'", :warn
-      return nil
+      SiteDiff.log "skipped invalid URL: '#{rel}' (at #{base})", :warning
+      nil
     end
-  end
 
-  # Make a link relative to @base_uri
-  def relativize_link(uri)
-    uri.path.slice(@base_uri.path.length, uri.path.length)
-  end
+    # Make a link relative to @base_uri
+    def relativize_link(uri)
+      uri.path.slice(@base_uri.path.length, uri.path.length)
+    end
 
-  # Return a list of string links found on a page.
-  def find_links(doc)
-    return doc.xpath('//a[@href]').map { |e| e['href'] }
-  end
+    # Return a list of string links found on a page.
+    def find_links(doc)
+      doc.xpath('//a[@href]').map { |e| e['href'] }
+    end
 
-  # Filter out links we don't want. Links passed in are absolute URIs.
-  def filter_links(uris)
-    uris.find_all do |u|
-      u.host == @base_uri.host && u.path.start_with?(@base_uri.path)
+    # Filter out links we don't want. Links passed in are absolute URIs.
+    def filter_links(uris)
+      uris.find_all do |u|
+        is_sub_uri = (u.host == @base_uri.host) &&
+                     u.path.start_with?(@base_uri.path)
+        next unless is_sub_uri
+
+        is_included = @include_regex.nil? ? false : @include_regex.match(u.path)
+        is_excluded = @exclude_regex.nil? ? false : @exclude_regex.match(u.path)
+        if is_excluded && !is_included
+          SiteDiff.log "Ignoring excluded URL #{u.path}", :info
+        end
+        is_included || !is_excluded
+      end
     end
   end
 end
-end