sitediff 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 93ab2ffc296a3c9de8ea835e47f435e0193e7854
-  data.tar.gz: 5a80c5bb738912114aeb029c4ff283ff890f2ce2
+  metadata.gz: 1377b6bafe658b4a8a8f50ef0f54e577e99f1a87
+  data.tar.gz: 9a80e20a89b7f2f60506bbcccdc2b9f7037320f8
 SHA512:
-  metadata.gz: fd07a6ff9a14a8da965f4f5b09acb1c7f30be594d97f6383c3bec6545d1b5bb3fc26451a9943ed3fc3c81d261a0fa4ca6c5fcc5355d7263ae09700800af5af9d
-  data.tar.gz: bb940ede7b68b1e047dbda46d66fade88d06e0f54d9e257842a311f119efb4043939ea9b83021cdf2fd5c76ac3a23da92e7da91072cc5d4421a1717902093e17
+  metadata.gz: 7fe3ce1b2e7bc1762d5e8f4a1bfd4ab9280963732d033d4b403087f71a4d6caa394669eadb9d82064ad963dd62918f95cae7e0b0495dd92979f105be1bfe6f5e
+  data.tar.gz: fb98c439544172ae40c0ba347272ec1287a1dc9042ab238e4abd8f720d52307ffa8a4b3fec7a70df68c7fdf845b324fe69c99693c5bbbc369f9e2c22fbe8c404

data/lib/sitediff.rb

@@ -54,10 +54,11 @@ class SiteDiff
     @config.after['url']
   end
 
-  def initialize(config, cache, concurrency, verbose = true)
+  def initialize(config, cache, concurrency, interval, verbose = true, debug = false)
     @cache = cache
     @verbose = verbose
-
+    @debug = debug
+    @interval = interval
     # Check for single-site mode
     validate_opts = {}
     if !config.before['url'] && @cache.tag?(:before)
@@ -77,18 +78,33 @@ class SiteDiff
   def sanitize(path, read_results)
     %i[before after].map do |tag|
       html = read_results[tag].content
-      config = @config.send(tag)
-      Sanitizer.new(html, config, path: path).sanitize
+      encoding = read_results[tag].encoding
+      if encoding
+        config = @config.send(tag)
+        Sanitizer.new(html, config, path: path).sanitize
+      else
+        html
+      end
     end
   end
 
   # Process a set of read results
   def process_results(path, read_results)
-    diff = if (error = (read_results[:before].error || read_results[:after].error))
-             Result.new(path, nil, nil, error)
-           else
-             Result.new(path, *sanitize(path, read_results), nil)
-           end
+    if (error = (read_results[:before].error || read_results[:after].error))
+      diff = Result.new(path, nil, nil, nil, nil, error)
+    else
+      begin
+        diff = Result.new(path,
+                          *sanitize(path, read_results),
+                          read_results[:before].encoding,
+                          read_results[:after].encoding,
+                          nil)
+      rescue => e
+        raise if @debug
+
+        Result.new(path, nil, nil, nil, nil, "Sanitization error: #{e}")
+      end
+    end
     @results[path] = diff
 
     # Print results in order!
@@ -100,7 +116,7 @@ class SiteDiff
 
   # Perform the comparison, populate @results and return the number of failing
   # paths (paths with non-zero diff).
-  def run(curl_opts = {})
+  def run(curl_opts = {}, debug = true)
     # Map of path -> Result object, populated by process_results
     @results = {}
     @ordered = @config.paths.dup
@@ -115,7 +131,7 @@ class SiteDiff
     # so passing this instead but @config.after['curl_opts'] is ignored.
     config_curl_opts = @config.before['curl_opts']
     curl_opts = config_curl_opts.clone.merge(curl_opts) if config_curl_opts
-    fetcher = Fetch.new(@cache, @config.paths, @concurrency, curl_opts,
+    fetcher = Fetch.new(@cache, @config.paths, @interval, @concurrency, curl_opts, debug,
                         before: before, after: after)
     fetcher.run(&method(:process_results))
 

data/lib/sitediff/cache.rb

@@ -8,10 +8,13 @@ class SiteDiff
     attr_accessor :read_tags, :write_tags
 
     def initialize(opts = {})
-      @dir = opts[:dir] || '.'
       @create = opts[:create]
+
+      # Read and Write tags are sets that can contain :before and :after
+      # They indicate whether we should use the cache for reading or writing
       @read_tags = Set.new
       @write_tags = Set.new
+      @dir = opts[:directory] || '.'
     end
 
     # Is a tag cached?
@@ -63,5 +66,12 @@ class SiteDiff
       # Ensure encoding stays the same!
       Marshal.dump([tag, path.encode('UTF-8')])
     end
+
+    def get_dir(directory)
+      # Create the dir. Must go before cache initialization!
+      @dir = Pathname.new(directory || '.')
+      @dir.mkpath unless @dir.directory?
+      @dir.to_s
+    end
   end
 end

data/lib/sitediff/cli.rb

@@ -23,6 +23,14 @@ class SiteDiff
                  type: :boolean,
                  default: false,
                  desc: 'Ignore many HTTPS/SSL errors'
+    class_option :debug,
+                 type: :boolean,
+                 default: false,
+                 desc: 'Debug mode. Stop on certain errors and produce a traceback.'
+    class_option :interval,
+                 type: :numeric,
+                 default: 0,
+                 desc: 'Crawling delay - interval in milliseconds'
 
     # Thor, by default, exits with 0 no matter what!
     def self.exit_on_failure?
@@ -75,7 +83,10 @@ class SiteDiff
            desc: 'Max number of concurrent connections made'
     desc 'diff [OPTIONS] [CONFIGFILES]', 'Perform systematic diff on given URLs'
     def diff(*config_files)
-      config = SiteDiff::Config.new(config_files, options[:directory])
+      @interval = options['interval']
+      check_interval(@interval)
+      @dir = get_dir(options['directory'])
+      config = SiteDiff::Config.new(config_files, @dir)
 
       # override config based on options
       paths = options['paths']
@@ -100,20 +111,21 @@ class SiteDiff
 
       # Setup cache
       cache = SiteDiff::Cache.new(create: options['cached'] != 'none',
-                                  dir: options['directory'])
+                                  directory: @dir)
       cache.read_tags << :before if %w[before all].include?(options['cached'])
       cache.read_tags << :after if %w[after all].include?(options['cached'])
       cache.write_tags << :before << :after
 
-      sitediff = SiteDiff.new(config, cache, options[:concurrency],
-                              options['verbose'])
-      num_failing = sitediff.run(get_curl_opts(options))
+      sitediff = SiteDiff.new(config, cache, options[:concurrency], @interval,
+                              options['verbose'], options[:debug])
+      num_failing = sitediff.run(get_curl_opts(options), options[:debug])
       exit_code = num_failing > 0 ? 2 : 0
 
-      sitediff.dump(options['directory'], options['before-report'],
+      sitediff.dump(@dir, options['before-report'],
                     options['after-report'])
     rescue Config::InvalidConfig => e
       SiteDiff.log "Invalid configuration: #{e.message}", :error
+      SiteDiff.log "at #{e.backtrace}", :error
     else # no exception was raised
       # Thor::Error  --> exit(1), guaranteed by exit_on_failure?
       # Failing diff --> exit(2), populated above
@@ -132,8 +144,8 @@ class SiteDiff
     def serve(*config_files)
       config = SiteDiff::Config.new(config_files, options['directory'])
       # Could check non-empty config here but currently errors are already raised.
-
-      cache = Cache.new(dir: options['directory'])
+      @dir = get_dir(options['directory'])
+      cache = Cache.new(directory: @dir)
       cache.read_tags << :before << :after
 
       SiteDiff::Webserver::ResultServer.new(
@@ -145,6 +157,7 @@ class SiteDiff
       ).wait
     rescue SiteDiffException => e
       SiteDiff.log e.message, :error
+      SiteDiff.log e.backtrace, :error
     end
 
     option :depth,
@@ -160,19 +173,37 @@ class SiteDiff
            type: :numeric,
            default: 3,
            desc: 'Max number of concurrent connections made'
+    option :whitelist,
+           type: :string,
+           default: '',
+           desc: 'Optional whitelist for crawling'
+    option :blacklist,
+           type: :string,
+           default: '',
+           desc: 'Optional blacklist for crawling'
     desc 'init URL [URL]', 'Create a sitediff configuration'
     def init(*urls)
       unless (1..2).cover? urls.size
         SiteDiff.log 'sitediff init requires one or two URLs', :error
-        exit 2
+        exit(2)
       end
 
+      @interval = options['interval']
+      check_interval(@interval)
+      @dir = get_dir(options['directory'])
       curl_opts = get_curl_opts(options)
-
-      creator = SiteDiff::Config::Creator.new(options[:concurrency], curl_opts, *urls)
+      @whitelist = create_regexp(options['whitelist'])
+      @blacklist = create_regexp(options['blacklist'])
+      creator = SiteDiff::Config::Creator.new(options[:concurrency],
+                                              options['interval'],
+                                              @whitelist,
+                                              @blacklist,
+                                              curl_opts,
+                                              options[:debug],
+                                              *urls)
       creator.create(
         depth: options[:depth],
-        directory: options[:directory],
+        directory: @dir,
         rules: options[:rules] != 'no',
         rules_disabled: (options[:rules] == 'disabled')
       ) do |_tag, info|
@@ -193,14 +224,19 @@ class SiteDiff
     desc 'store [CONFIGFILES]',
          'Cache the current contents of a site for later comparison'
     def store(*config_files)
-      config = SiteDiff::Config.new(config_files, options['directory'])
+      @dir = get_dir(options['directory'])
+      config = SiteDiff::Config.new(config_files, @dir)
       config.validate(need_before: false)
-
-      cache = SiteDiff::Cache.new(create: true)
+      cache = SiteDiff::Cache.new(directory: @dir, create: true)
       cache.write_tags << :before
 
       base = options[:url] || config.after['url']
-      fetcher = SiteDiff::Fetch.new(cache, config.paths, options['concurrency'],
+      fetcher = SiteDiff::Fetch.new(cache,
+                                    config.paths,
+                                    options[:interval],
+                                    options[:concurrency],
+                                    get_curl_opts(options),
+                                    options[:debug],
                                     before: base)
       fetcher.run do |path, _res|
         SiteDiff.log "Visited #{path}, cached"
@@ -219,6 +255,32 @@ class SiteDiff
         end
         curl_opts
       end
+
+      def check_interval(interval)
+        if interval != 0 && options[:concurrency] != 1
+          SiteDiff.log '--concurrency must be set to 1 in order to enable the interval feature'
+          exit(2)
+        end
+      end
+
+      def get_dir(directory)
+        # Create the dir. Must go before cache initialization!
+        @dir = Pathname.new(directory || '.')
+        @dir.mkpath unless @dir.directory?
+        @dir.to_s
+      end
+
+      def create_regexp(string_param)
+        begin
+          @return_value = string_param == '' ? nil : Regexp.new(string_param)
+        rescue SiteDiffException => e
+          @return_value = nil
+          SiteDiff.log 'whitelist and blacklist parameters must be valid regular expressions', :error
+          SiteDiff.log e.message, :error
+          SiteDiff.log e.backtrace, :error
+        end
+        return @return_value
+      end
     end
   end
 end

data/lib/sitediff/config/creator.rb

@@ -11,11 +11,15 @@ require 'yaml'
 class SiteDiff
   class Config
     class Creator
-      def initialize(concurrency, curl_opts, *urls)
+      def initialize(concurrency, interval, whitelist, blacklist, curl_opts, debug, *urls)
         @concurrency = concurrency
+        @interval = interval
+        @whitelist = whitelist
+        @blacklist = blacklist
         @after = urls.pop
         @before = urls.pop # May be nil
         @curl_opts = curl_opts
+        @debug = debug
       end
 
       def roots
@@ -30,18 +34,15 @@ class SiteDiff
       def create(opts, &block)
         @config = {}
         @callback = block
-
-        # Handle options
         @dir = Pathname.new(opts[:directory])
+
+        # Handle other options
         @depth = opts[:depth]
         @rules = Rules.new(@config, opts[:rules_disabled]) if opts[:rules]
 
-        # Create the dir. Must go before cache initialization!
-        @dir.mkpath unless @dir.directory?
-
         # Setup instance vars
         @paths = Hash.new { |h, k| h[k] = Set.new }
-        @cache = Cache.new(dir: @dir.to_s, create: true)
+        @cache = Cache.new(directory: @dir.to_s, create: true)
         @cache.write_tags << :before << :after
 
         build_config
@@ -64,7 +65,7 @@ class SiteDiff
       def crawl(depth = nil)
         hydra = Typhoeus::Hydra.new(max_concurrency: @concurrency)
         roots.each do |tag, u|
-          Crawler.new(hydra, u, depth, @curl_opts) do |info|
+          Crawler.new(hydra, u, @interval, @whitelist, @blacklist, depth, @curl_opts, @debug) do |info|
             crawled_path(tag, info)
           end
         end
@@ -113,6 +114,10 @@ class SiteDiff
         end
       end
 
+      def directory
+        @dir
+      end
+
       def config_file
         @dir + Config::DEFAULT_FILENAME
       end

data/lib/sitediff/crawler.rb

@@ -14,14 +14,24 @@ class SiteDiff
     DEFAULT_DEPTH = 3
 
     # Create a crawler with a base URL
-    def initialize(hydra, base, depth = DEFAULT_DEPTH,
-                   curl_opts = UriWrapper::DEFAULT_CURL_OPTS, &block)
+    def initialize(hydra, base,
+                   interval,
+                   whitelist,
+                   blacklist,
+                   depth = DEFAULT_DEPTH,
+                   curl_opts = UriWrapper::DEFAULT_CURL_OPTS,
+                   debug = true,
+                   &block)
       @hydra = hydra
       @base_uri = Addressable::URI.parse(base)
       @base = base
+      @interval = interval
+      @whitelist = whitelist
+      @blacklist = blacklist
       @found = Set.new
       @callback = block
       @curl_opts = curl_opts
+      @debug = debug
 
       add_uri('', depth)
     end
@@ -32,7 +42,7 @@ class SiteDiff
 
       @found << rel
 
-      wrapper = UriWrapper.new(@base + rel, @curl_opts)
+      wrapper = UriWrapper.new(@base + rel, @curl_opts, @debug)
       wrapper.queue(@hydra) do |res|
         fetched_uri(rel, depth, res)
       end
@@ -58,6 +68,11 @@ class SiteDiff
         read_result: res,
         document: doc
       )
+      # Insert delay to limit fetching rate
+      if @interval != 0
+        SiteDiff.log("Waiting #{@interval} milliseconds.", :info)
+        sleep(@interval / 1000.0)
+      end
       @callback[info]
 
       return unless depth >= 1
@@ -99,7 +114,21 @@ class SiteDiff
     # Filter out links we don't want. Links passed in are absolute URIs.
     def filter_links(uris)
       uris.find_all do |u|
-        u.host == @base_uri.host && u.path.start_with?(@base_uri.path)
+        is_sub_uri = (u.host == @base_uri.host) && u.path.start_with?(@base_uri.path)
+        if is_sub_uri
+          is_whitelisted = @whitelist.nil? ? false : @whitelist.match(u.path)
+          is_blacklisted = @blacklist.nil? ? false : @blacklist.match(u.path)
+          if is_blacklisted && !is_whitelisted
+            SiteDiff.log "Ignoring blacklisted URL #{u.path}", :info
+          end
+          is_whitelisted || !is_blacklisted
+        end
+        # SiteDiff.log "Filtering URL #{u.path}", :info
+        # SiteDiff.log Regexp.new(@blacklist).match(u.path).inspect, :info
+        # (u.host == @base_uri.host) &&
+        # (u.path.start_with?(@base_uri.path)) &&
+        # (@whitelist == '' || Regexp.new(@whitelist).match(u.path)) &&
+        # (@blacklist == '' || !(Regexp.new(@blacklist).match(u.path)))
       end
     end
   end

data/lib/sitediff/diff.rb

@@ -4,6 +4,7 @@ require 'sitediff'
 require 'diffy'
 require 'erb'
 require 'rainbow'
+require 'digest'
 
 class SiteDiff
   module Diff
@@ -15,6 +16,28 @@ class SiteDiff
         diff.to_s(:html) : nil
     end
 
+    def encoding_blurb(encoding)
+      if encoding
+        "Text content returned - charset #{encoding}"
+      else
+        'Binary content returned'
+      end
+    end
+
+    def binary_diffy(before, after, before_encoding, after_encoding)
+      if before_encoding || after_encoding
+        Diffy::Diff.new(encoding_blurb(before_encoding),
+                        encoding_blurb(after_encoding)).to_s(:html)
+      elsif before == after
+        nil
+      else
+        md5_before = Digest::MD5.hexdigest(before)
+        md5_after = Digest::MD5.hexdigest(after)
+        Diffy::Diff.new("Binary content returned md5: #{md5_before}",
+                        "Binary content returned md5: #{md5_after}").to_s(:html)
+      end
+    end
+
     def terminal_diffy(before_html, after_html)
       args = []
       args << :color if Rainbow.enabled

data/lib/sitediff/fetch.rb

@@ -8,12 +8,15 @@ class SiteDiff
     # Cache is a cache object, see sitediff/cache
     # Paths is a list of sub-paths
     # Tags is a hash of tag names => base URLs.
-    def initialize(cache, paths, concurrency = 3, curl_opts = nil, **tags)
+    def initialize(cache, paths, interval, concurrency = 3, curl_opts = nil,
+                   debug = true, **tags)
       @cache = cache
+      @interval = interval
       @paths = paths
       @tags = tags
       @curl_opts = curl_opts || UriWrapper::DEFAULT_CURL_OPTS
       @concurrency = concurrency
+      @debug = debug
     end
 
     # Fetch all the paths, once per tag.
@@ -41,8 +44,13 @@ class SiteDiff
           results[tag] = UriWrapper::ReadResult.error('Not cached')
           process_results(path, results)
         else
-          uri = UriWrapper.new(base + path, @curl_opts)
+          uri = UriWrapper.new(base + path, @curl_opts, @debug)
           uri.queue(@hydra) do |resl|
+            # Insert delay to limit fetching rate
+            if @interval != 0
+              SiteDiff.log("Waiting #{@interval} milliseconds.", :info)
+              sleep(@interval / 1000.0)
+            end
             @cache.set(tag, path, resl)
             results[tag] = resl
             process_results(path, results)

data/lib/sitediff/result.rb

@@ -6,7 +6,7 @@ require 'digest/sha1'
 require 'fileutils'
 
 class SiteDiff
-  class Result < Struct.new(:path, :before, :after, :error, :verbose)
+  class Result < Struct.new(:path, :before, :after, :before_encoding, :after_encoding, :error, :verbose)
     STATUS_SUCCESS  = 0   # Identical before and after
     STATUS_FAILURE  = 1   # Different before and after
     STATUS_ERROR    = 2   # Couldn't fetch page
@@ -19,7 +19,11 @@ class SiteDiff
       if error
         @status = STATUS_ERROR
       else
-        @diff = Diff.html_diffy(before, after)
+        if !before_encoding || !after_encoding
+          @diff = Diff.binary_diffy(before, after, before_encoding, after_encoding)
+        else
+          @diff = Diff.html_diffy(before, after)
+        end
         @status = @diff ? STATUS_FAILURE : STATUS_SUCCESS
       end
     end

data/lib/sitediff/sanitize.rb

@@ -96,6 +96,10 @@ class SiteDiff
       selector.each { |r| r.apply(@node) }
       @html = Sanitizer.prettify(@node)
       @node = nil
+      # Prevent potential UTF-8 encoding errors by removing bytes
+      # Not the only solution. An alternative is to return the
+      # string unmodified.
+      @html = @html.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
       global.each { |r| r.apply(@html) }
     end
 
@@ -144,6 +148,10 @@ class SiteDiff
 
       # There's a lot of cruft left over,that we don't want
 
+      # Prevent potential UTF-8 encoding errors by removing invalid bytes.
+      # Not the only solution.
+      # An alternative is to return the string unmodified.
+      str = str.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
       # Remove xml declaration and <html> tags
       str.sub!(/\A<\?xml.*$\n/, '')
       str.sub!(/\A^<html>$\n/, '')

data/lib/sitediff/sanitize/regexp.rb

@@ -47,6 +47,8 @@ class SiteDiff
       def gsub!(str)
         re = ::Regexp.new(@rule['pattern'])
         sub = @rule['substitute'] || ''
+        # Expecting a mutation here. Do not reassign the variable str
+        # for the purpose of removing UTF-8 encoding errors.
         str.gsub!(re, sub)
         str
       end

data/lib/sitediff/uriwrapper.rb

@@ -18,10 +18,11 @@ class SiteDiff
 
     # This lets us treat errors or content as one object
     class ReadResult
-      attr_accessor :content, :error_code, :error
+      attr_accessor :encoding, :content, :error_code, :error
 
-      def initialize(content = nil)
+      def initialize(content = nil, encoding = 'utf-8')
         @content = content
+        @encoding = encoding
         @error = nil
         @error_code = nil
       end
@@ -34,11 +35,12 @@ class SiteDiff
       end
     end
 
-    def initialize(uri, curl_opts = DEFAULT_CURL_OPTS)
+    def initialize(uri, curl_opts = DEFAULT_CURL_OPTS, debug = true)
       @uri = uri.respond_to?(:scheme) ? uri : Addressable::URI.parse(uri)
       # remove trailing '/'s from local URIs
       @uri.path.gsub!(%r{/*$}, '') if local?
       @curl_opts = curl_opts
+      @debug = debug
     end
 
     def user
@@ -78,7 +80,7 @@ class SiteDiff
 
     # Returns the encoding of an HTTP response from headers , nil if not
     # specified.
-    def http_encoding(http_headers)
+    def charset_encoding(http_headers)
       if (content_type = http_headers['Content-Type'])
         if (md = /;\s*charset=([-\w]*)/.match(content_type))
           md[1]
@@ -101,10 +103,23 @@ class SiteDiff
         body = resp.body
         # Typhoeus does not respect HTTP headers when setting the encoding
         # resp.body; coerce if possible.
-        if (encoding = http_encoding(resp.headers))
+        if (encoding = charset_encoding(resp.headers))
           body.force_encoding(encoding)
         end
-        yield ReadResult.new(body)
+        # Should be wrapped with rescue I guess? Maybe this entire function?
+        # Should at least be an option in the Cli to disable this.
+        # "stop on first error"
+        begin
+          yield ReadResult.new(body, encoding)
+        rescue ArgumentError => e
+          raise if @debug
+
+          yield ReadResult.error("Parsing error for #{@uri}: #{e.message}")
+        rescue => e
+          raise if @debug
+
+          yield ReadResult.error("Unknown parsing error for #{@uri}: #{e.message}")
+        end
       end
 
       req.on_failure do |resp|

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sitediff
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - Alex Dergachev
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-12-14 00:00:00.000000000 Z
+date: 2019-04-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pkg-config
@@ -167,7 +167,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 2.5.2.3
 signing_key: 
 specification_version: 4
 summary: Compare two versions of a site with ease!