sitediff 0.0.1 → 1.0.0

data/lib/sitediff/config/creator.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require 'sitediff/cache'
+require 'sitediff/config'
+require 'sitediff/crawler'
+require 'pathname'
+require 'typhoeus'
+require 'yaml'
+
+class SiteDiff
+  class Config
+    ##
+    # SiteDiff Config Creator Object.
+    class Creator
+      ##
+      # Creates a Creator object.
+      def initialize(debug, *urls)
+        @config = nil
+        @after = urls.pop
+        @before = urls.pop # May be nil
+        @debug = debug
+      end
+
+      ##
+      # Determine if we're dealing with one or two URLs.
+      def roots
+        @roots = { 'after' => @after }
+        @roots['before'] = @before if @before
+        @roots
+      end
+
+      ##
+      # Build a config structure, return it.
+      def create(options, &block)
+        @config = {}
+        @callback = block
+        @dir = Pathname.new(options[:directory])
+
+        # Setup instance vars
+        @paths = Hash.new { |h, k| h[k] = Set.new }
+        @cache = Cache.new(directory: @dir.to_s, create: true)
+        @cache.write_tags << :before << :after
+
+        build_config options
+        write_config
+      end
+
+      ##
+      # Build and populate the config object which is being created.
+      #
+      # @param [String] options
+      #   One or more options.
+      def build_config(options)
+        options = Config.stringify_keys options
+
+        # Build config for "before" and "after".
+        %w[before after].each do |tag|
+          next unless (url = roots[tag])
+
+          @config[tag] = { 'url' => url }
+        end
+
+        # Build other settings.
+        @config['settings'] = {}
+        Config::ALLOWED_SETTINGS_KEYS.each do |key|
+          @config['settings'][key] = options[key]
+        end
+      end
+
+      ##
+      # Create a gitignore if we seem to be in git.
+      def make_gitignore(dir)
+        # Check if we're in git
+        unless dir.realpath.to_enum(:ascend).any? { |d| d.+('.git').exist? }
+          return
+        end
+
+        dir.+('.gitignore').open('w') do |f|
+          f.puts <<-GITIGNORE.gsub(/^\s+/, '')
+            # Directories.
+            diffs
+            snapshot
+
+            # Files.
+            settings.yaml
+            paths.txt
+            failures.txt
+          GITIGNORE
+        end
+      end
+
+      ##
+      # Returns the name of the config directory.
+      def directory
+        @dir
+      end
+
+      ##
+      # Returns the name of the config file.
+      def config_file
+        @dir + Config::DEFAULT_FILENAME
+      end
+
+      ##
+      # Writes the built config into the config file.
+      # TODO: Exclude default params before writing.
+      def write_config
+        make_gitignore(@dir)
+        data = Config.remove_defaults(@config)
+        config_file.open('w') { |f| f.puts data.to_yaml }
+      end
+    end
+  end
+end

data/lib/sitediff/config/preset.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require 'pathname'
+require 'sitediff/config'
+
+class SiteDiff
+  class Config
+    ##
+    # Preset helper.
+    class Preset
+      ##
+      # Directory in which presets live.
+      #
+      # TODO: Move this outside "lib".
+      DIRECTORY = (Pathname.new(__dir__).dirname + 'presets').freeze
+
+      ##
+      # Reads preset rules.
+      #
+      # @param [String] preset
+      #   Presets
+      #
+      # @return [Hash]
+      #   A hash containing the preset's rules.
+      def self.read(name)
+        @cache = {} if @cache.nil?
+
+        # Load and cache preset config.
+        if @cache[name].nil?
+          exist? name, true
+          @cache[name] = Config.load_conf file(name)
+        end
+
+        @cache[name]
+      end
+
+      ##
+      # Get all possible rules.
+      #
+      # @return [Array]
+      #   All presets.
+      def self.all
+        # Load and cache preset names.
+        if @all.nil?
+          @all = []
+          pattern = DIRECTORY + '*.yaml'
+          Dir.glob(pattern) do |file|
+            @all << File.basename(file, '.yaml')
+          end
+        end
+
+        @all
+      end
+
+      ##
+      # Checks whether a preset exists.
+      def self.exist?(name, exception = false)
+        result = File.exist? file(name)
+
+        # Raise an exception, if required.
+        if exception && !result
+          raise Config::InvalidConfig, "Preset not found: #{name}"
+        end
+
+        result
+      end
+
+      ##
+      # Returns the path to a preset file.
+      def self.file(name)
+        DIRECTORY + "#{name}.yaml"
+      end
+    end
+  end
+end

data/lib/sitediff/crawler.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+require 'sitediff'
+require 'sitediff/uriwrapper'
+require 'addressable/uri'
+require 'nokogiri'
+require 'ostruct'
+require 'set'
+
+class SiteDiff
+  # SiteDiff Crawler.
+  class Crawler
+    class Info < OpenStruct; end
+
+    DEFAULT_DEPTH = 3
+
+    # Create a crawler with a base URL
+    def initialize(hydra, base,
+                   interval,
+                   whitelist,
+                   blacklist,
+                   depth = DEFAULT_DEPTH,
+                   curl_opts = UriWrapper::DEFAULT_CURL_OPTS,
+                   debug = true,
+                   &block)
+      @hydra = hydra
+      @base_uri = Addressable::URI.parse(base)
+      @base = base
+      @interval = interval
+      @whitelist = whitelist
+      @blacklist = blacklist
+      @found = Set.new
+      @callback = block
+      @curl_opts = curl_opts
+      @debug = debug
+
+      add_uri('', depth)
+    end
+
+    # Handle a newly found relative URI
+    def add_uri(rel, depth)
+      return if @found.include? rel
+
+      @found << rel
+
+      wrapper = UriWrapper.new(@base + rel, @curl_opts, @debug)
+      wrapper.queue(@hydra) do |res|
+        fetched_uri(rel, depth, res)
+      end
+    end
+
+    # Handle the fetch of a URI
+    def fetched_uri(rel, depth, res)
+      if res.error
+        SiteDiff.log(res.error, :error)
+        return
+      elsif !res.content
+        SiteDiff.log('Response is missing content. Treating as an error.', :error)
+        return
+      end
+
+      base = Addressable::URI.parse(@base + rel)
+      doc = Nokogiri::HTML(res.content)
+
+      # Call the callback
+      info = Info.new(
+        relative: rel,
+        uri: base,
+        read_result: res,
+        document: doc
+      )
+      # Insert delay to limit fetching rate
+      if @interval != 0
+        SiteDiff.log("Waiting #{@interval} milliseconds.", :info)
+        sleep(@interval / 1000.0)
+      end
+      @callback[info]
+
+      return unless depth >= 1
+
+      # Find links
+      links = find_links(doc)
+      uris = links.map { |l| resolve_link(base, l) }.compact
+      uris = filter_links(uris)
+
+      # Make them relative
+      rels = uris.map { |u| relativize_link(u) }
+
+      # Queue them in turn
+      rels.each do |r|
+        next if @found.include? r
+
+        add_uri(r, depth - 1)
+      end
+    end
+
+    # Resolve a potentially-relative link. Return nil on error.
+    def resolve_link(base, rel)
+      base + rel
+    rescue Addressable::URI::InvalidURIError
+      SiteDiff.log "skipped invalid URL: '#{rel}' (at #{base})", :warning
+      nil
+    end
+
+    # Make a link relative to @base_uri
+    def relativize_link(uri)
+      uri.path.slice(@base_uri.path.length, uri.path.length)
+    end
+
+    # Return a list of string links found on a page.
+    def find_links(doc)
+      doc.xpath('//a[@href]').map { |e| e['href'] }
+    end
+
+    # Filter out links we don't want. Links passed in are absolute URIs.
+    def filter_links(uris)
+      uris.find_all do |u|
+        is_sub_uri = (u.host == @base_uri.host) &&
+                     u.path.start_with?(@base_uri.path)
+        next unless is_sub_uri
+
+        is_whitelisted = @whitelist.nil? ? false : @whitelist.match(u.path)
+        is_blacklisted = @blacklist.nil? ? false : @blacklist.match(u.path)
+        if is_blacklisted && !is_whitelisted
+          SiteDiff.log "Ignoring blacklisted URL #{u.path}", :info
+        end
+        is_whitelisted || !is_blacklisted
+      end
+    end
+  end
+end

data/lib/sitediff/diff.rb

@@ -1,37 +1,82 @@
+# frozen_string_literal: true
+
+require 'sitediff'
 require 'diffy'
 require 'erb'
 require 'rainbow'
+require 'digest'
 
 class SiteDiff
+  # SiteDiff Diff Object.
   module Diff
     module_function
 
+    ##
+    # Generates HTML diff.
     def html_diffy(before_html, after_html)
       diff = Diffy::Diff.new(before_html, after_html)
-      diff.first ?  # Is it non-empty?
-        diff.to_s(:html) : nil
+      # If the diff is non-empty, convert it to string.
+      diff.first ? diff.to_s(:html) : nil
+    end
+
+    ##
+    # Generates a description about encoding.
+    def encoding_blurb(encoding)
+      if encoding
+        "Text content returned - charset #{encoding}"
+      else
+        'Binary content returned'
+      end
+    end
+
+    ##
+    # Computes diff of binary files using MD5 hashes.
+    def binary_diffy(before, after, before_encoding, after_encoding)
+      if before_encoding || after_encoding
+        Diffy::Diff.new(encoding_blurb(before_encoding),
+                        encoding_blurb(after_encoding)).to_s(:html)
+      elsif before == after
+        nil
+      else
+        md5_before = Digest::MD5.hexdigest(before)
+        md5_after = Digest::MD5.hexdigest(after)
+        Diffy::Diff.new("Binary content returned md5: #{md5_before}",
+                        "Binary content returned md5: #{md5_after}").to_s(:html)
+      end
     end
 
+    ##
+    # Generates diff for CLI output.
     def terminal_diffy(before_html, after_html)
       args = []
       args << :color if Rainbow.enabled
-      return Diffy::Diff.new(before_html, after_html, :context => 3).
-        to_s(*args)
+      Diffy::Diff.new(before_html, after_html, context: 3)
+                 .to_s(*args)
     end
 
-    def generate_html_report(results, before, after)
-      erb_path = File.join(SiteDiff::FILES_DIR, 'html_report.html.erb')
-      report_html = ERB.new(File.read(erb_path)).result(binding)
-      return report_html
+    ##
+    # Generates an HTML report.
+    # TODO: Generate the report in SiteDif::Report instead.
+    def generate_html(results, before, after, cache, relative = false)
+      erb_path = File.join(SiteDiff::FILES_DIR, 'report.html.erb')
+      ERB.new(File.read(erb_path)).result(binding)
     end
 
-    def generate_diff_output(result)
+    ##
+    # Generates diff output for a single result.
+    def generate_diff_output(result, relative = false)
       erb_path = File.join(SiteDiff::FILES_DIR, 'diff.html.erb')
-      return ERB.new(File.read(erb_path)).result(binding)
+      ERB.new(File.read(erb_path)).result(binding)
     end
 
-    def css
-      File.read(File.join(SiteDiff::FILES_DIR, 'sitediff.css'))
+    ##
+    # Set configuration for Diffy.
+    def diff_config(config)
+      diff_options = Diffy::Diff.default_options[:diff]
+      diff_options = [diff_options] unless diff_options.is_a?(Array)
+      # ignore_whitespace option
+      diff_options.push('-w').uniq if config.ignore_whitespace
+      Diffy::Diff.default_options[:diff] = diff_options
     end
   end
 end

data/lib/sitediff/exception.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class SiteDiff
+  class SiteDiffException < RuntimeError; end
+end

data/lib/sitediff/fetch.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require 'sitediff/uriwrapper'
+require 'typhoeus'
+
+class SiteDiff
+  # SiteDiff Data Fetcher.
+  # TODO: Rename this to Fetcher.
+  class Fetch
+    # Cache is a cache object, see sitediff/cache
+    # Paths is a list of sub-paths
+    # Tags is a hash of tag names => base URLs.
+    def initialize(cache,
+                   paths,
+                   interval,
+                   concurrency = 3,
+                   curl_opts = nil,
+                   debug = true,
+                   **tags)
+      @cache = cache
+      @interval = interval
+      @paths = paths
+      @tags = tags
+      @curl_opts = curl_opts || UriWrapper::DEFAULT_CURL_OPTS
+      @concurrency = concurrency
+      @debug = debug
+    end
+
+    # Fetch all the paths, once per tag.
+    # When a path has been fetched for every tag, block will be called with the
+    # path, and a hash of tag => UriWrapper::ReadResult objects.
+    def run(&block)
+      @callback = block
+      @hydra = Typhoeus::Hydra.new(max_concurrency: @concurrency)
+      @paths.each { |path| queue_path(path) }
+      @hydra.run
+    end
+
+    private
+
+    # Queue a path for fetching
+    def queue_path(path)
+      results = {}
+
+      @tags.each do |tag, base|
+        if (res = @cache.get(tag, path))
+          results[tag] = res
+          process_results(path, results)
+        elsif !base
+          # We only have the cache, but this item isn't cached!
+          results[tag] = UriWrapper::ReadResult.error('Not cached')
+          process_results(path, results)
+        else
+          uri = UriWrapper.new(base + path, @curl_opts, @debug)
+          uri.queue(@hydra) do |resl|
+            # Insert delay to limit fetching rate
+            if @interval != 0
+              SiteDiff.log("Waiting #{@interval} milliseconds.", :info)
+              sleep(@interval / 1000.0)
+            end
+            @cache.set(tag, path, resl)
+            results[tag] = resl
+            process_results(path, results)
+          end
+        end
+      end
+    end
+
+    # Process fetch results
+    def process_results(path, results)
+      return unless results.size == @tags.size
+
+      @callback[path, results]
+    end
+  end
+end