RubyGems - sitediff - Versions diffs - 0.0.6 → 1.2.0 - Mend

sitediff 0.0.6 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (55) hide show

checksums.yaml +5 -5
data/.eslintignore +1 -0
data/.eslintrc.json +28 -0
data/.project +11 -0
data/.rubocop.yml +179 -0
data/.rubocop_todo.yml +51 -0
data/CHANGELOG.md +28 -0
data/Dockerfile +33 -0
data/Gemfile +11 -0
data/Gemfile.lock +85 -0
data/INSTALLATION.md +146 -0
data/LICENSE +339 -0
data/README.md +810 -0
data/Rakefile +12 -0
data/Thorfile +135 -0
data/bin/sitediff +9 -2
data/config/.gitkeep +0 -0
data/config/sanitize_domains.example.yaml +8 -0
data/config/sitediff.example.yaml +81 -0
data/docker-compose.test.yml +3 -0
data/lib/sitediff/api.rb +276 -0
data/lib/sitediff/cache.rb +57 -8
data/lib/sitediff/cli.rb +156 -176
data/lib/sitediff/config/creator.rb +61 -77
data/lib/sitediff/config/preset.rb +75 -0
data/lib/sitediff/config.rb +436 -31
data/lib/sitediff/crawler.rb +27 -21
data/lib/sitediff/diff.rb +32 -9
data/lib/sitediff/fetch.rb +10 -3
data/lib/sitediff/files/diff.html.erb +20 -2
data/lib/sitediff/files/jquery.min.js +2 -0
data/lib/sitediff/files/normalize.css +349 -0
data/lib/sitediff/files/report.html.erb +171 -0
data/lib/sitediff/files/sidebyside.html.erb +5 -2
data/lib/sitediff/files/sitediff.css +303 -30
data/lib/sitediff/files/sitediff.js +367 -0
data/lib/sitediff/presets/drupal.yaml +63 -0
data/lib/sitediff/report.rb +254 -0
data/lib/sitediff/result.rb +50 -20
data/lib/sitediff/sanitize/dom_transform.rb +47 -8
data/lib/sitediff/sanitize/regexp.rb +24 -3
data/lib/sitediff/sanitize.rb +81 -12
data/lib/sitediff/uriwrapper.rb +65 -23
data/lib/sitediff/webserver/resultserver.rb +30 -33
data/lib/sitediff/webserver.rb +15 -3
data/lib/sitediff.rb +130 -83
data/misc/sitediff - overview report.png +0 -0
data/misc/sitediff - page report.png +0 -0
data/package-lock.json +878 -0
data/package.json +25 -0
data/sitediff.gemspec +51 -0
metadata +91 -29
data/lib/sitediff/files/html_report.html.erb +0 -66
data/lib/sitediff/files/rules/drupal.yaml +0 -63
data/lib/sitediff/rules.rb +0 -65

data/lib/sitediff/sanitize.rb CHANGED Viewed

@@ -8,21 +8,26 @@ require 'nokogiri'
 require 'set'
 class SiteDiff
+  # SiteDiff Sanitizer.
   class Sanitizer
     class InvalidSanitization < SiteDiffException; end
     TOOLS = {
       array: %w[dom_transform sanitization],
-      scalar: %w[selector remove_spacing]
+      scalar: %w[selector remove_spacing ignore_whitespace]
     }.freeze
-    DOM_TRANSFORMS = Set.new(%w[remove unwrap_root unwrap remove_class])
+    DOM_TRANSFORMS = Set.new(%w[remove strip unwrap_root unwrap remove_class])
+    ##
+    # Creates a Sanitizer.
     def initialize(html, config, opts = {})
       @html = html
       @config = config
       @opts = opts
     end
+    ##
+    # Performs sanitization.
     def sanitize
       return '' if @html == '' # Quick return on empty input
@@ -30,7 +35,7 @@ class SiteDiff
       @html = nil
       remove_spacing
-      selector
+      regions || selector
       dom_transforms
       regexps
@@ -56,13 +61,13 @@ class SiteDiff
     def canonicalize_rule(name)
       (rules = @config[name]) || (return nil)
-      if rules[0]&.respond_to?(:[]) && rules[0]['value']
-        # Already an array
+      # Already an array? Do nothing.
+      if rules[0].respond_to?('each') && rules[0]&.fetch('value')
+      # If it is a hash, put it in an array.
       elsif rules['value']
-        # Hash, put it in an array
         rules = [rules]
+      # If it is a scalar value, put it in an array.
       else
-        # Scalar, put it in a hash
         rules = [{ 'value' => rules }]
       end
@@ -79,6 +84,13 @@ class SiteDiff
       Sanitizer.remove_node_spacing(@node) if rule['value']
     end
+    # Perform 'regions' action, don't perform 'selector' if regions exist.
+    def regions
+      return unless validate_regions
+      @node = select_regions(@node, @config['regions'], @opts[:output])
+    end
     # Perform 'selector' action, to choose a new root
     def selector
       (rule = canonicalize_rule('selector')) || return
@@ -99,7 +111,13 @@ class SiteDiff
       # Prevent potential UTF-8 encoding errors by removing bytes
       # Not the only solution. An alternative is to return the
       # string unmodified.
-      @html = @html.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
+      @html = @html.encode(
+        'UTF-8',
+        'binary',
+        invalid: :replace,
+        undef: :replace,
+        replace: ''
+      )
       global.each { |r| r.apply(@html) }
     end
@@ -124,6 +142,20 @@ class SiteDiff
       end
     end
+    # Restructure the node into regions.
+    def select_regions(node, regions, output)
+      regions = output.map do |name|
+        selector = get_named_region(regions, name)['selector']
+        region = Nokogiri::XML.fragment("<region id=\"#{name}\"></region>").at_css('region')
+        matching = node.css(selector)
+        matching.each { |m| region.add_child m }
+        region
+      end
+      node = Nokogiri::HTML.fragment('')
+      regions.each { |r| node.add_child r }
+      node
+    end
     # Get a fragment consisting of the elements matching the selector(s)
     def self.select_fragments(node, sel)
       # When we choose a new root, we always become a DocumentFragment,
@@ -151,7 +183,13 @@ class SiteDiff
       # Prevent potential UTF-8 encoding errors by removing invalid bytes.
       # Not the only solution.
       # An alternative is to return the string unmodified.
-      str = str.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
+      str = str.encode(
+        'UTF-8',
+        'binary',
+        invalid: :replace,
+        undef: :replace,
+        replace: ''
+      )
       # Remove xml declaration and <html> tags
       str.sub!(/\A<\?xml.*$\n/, '')
       str.sub!(/\A^<html>$\n/, '')
@@ -164,11 +202,15 @@ class SiteDiff
       # Remove blank lines
       str.gsub!(/^\s*$\n/, '')
+      # Remove DOS newlines
+      str.gsub!(/\x0D$/, '')
+      str.gsub!(/&#13;$/, '')
       str
     end
     # Parse HTML into a node
-    def self.domify(str, force_doc = false)
+    def self.domify(str, force_doc: false)
       if force_doc || /<!DOCTYPE/.match(str[0, 512])
         Nokogiri::HTML(str)
       else
@@ -182,10 +224,37 @@ class SiteDiff
         obj
       # node or fragment
       elsif Nokogiri::XML::Node == obj.class || Nokogiri::HTML::DocumentFragment == obj.class
-        domify(obj.to_s, true)
+        domify(obj.to_s, force_doc: true)
       else
-        to_document(domify(obj, false))
+        to_document(domify(obj, force_doc: false))
+      end
+    end
+    private
+    # Validate `regions` and `output` from config.
+    def validate_regions
+      return false unless @config['regions'].is_a?(Array)
+      return false unless @opts[:output].is_a?(Array)
+      regions = @config['regions']
+      output = @opts[:output]
+      regions.each do |region|
+        return false unless region.key?('name') && region.key?('selector')
+      end
+      # Check that each named output has an associated region.
+      output.each do |name|
+        return false unless get_named_region(regions, name)
       end
+      true
+    end
+    # Return the selector from a named region.
+    def get_named_region(regions, name)
+      regions.find { |region| region['name'] == name }
     end
   end
 end

data/lib/sitediff/uriwrapper.rb CHANGED Viewed

@@ -7,19 +7,28 @@ require 'addressable/uri'
 class SiteDiff
   class SiteDiffReadFailure < SiteDiffException; end
+  # SiteDiff URI Wrapper.
   class UriWrapper
+    # TODO: Move these CURL OPTS to Config.DEFAULT_CONFIG.
     DEFAULT_CURL_OPTS = {
-      connecttimeout: 3,     # Don't hang on servers that don't exist
-      followlocation: true,  # Follow HTTP redirects (code 301 and 302)
+      # Don't hang on servers that don't exist.
+      connecttimeout: 3,
+      # Follow HTTP redirects (code 301 and 302).
+      followlocation: true,
       headers: {
         'User-Agent' => 'Sitediff - https://github.com/evolvingweb/sitediff'
-      }
+      },
+      # always accept SSL certs
+      ssl_verifypeer: false,
+      ssl_verifyhost: 0
     }.freeze
     # This lets us treat errors or content as one object
     class ReadResult
       attr_accessor :encoding, :content, :error_code, :error
+      ##
+      # Creates a ReadResult.
       def initialize(content = nil, encoding = 'utf-8')
         @content = content
         @encoding = encoding
@@ -27,15 +36,19 @@ class SiteDiff
         @error_code = nil
       end
-      def self.error(err, code = nil)
+      ##
+      # Creates a ReadResult with an error.
+      def self.error(message, code = nil)
         res = new
         res.error_code = code
-        res.error = err
+        res.error = message
         res
       end
     end
-    def initialize(uri, curl_opts = DEFAULT_CURL_OPTS, debug = true)
+    ##
+    # Creates a UriWrapper.
+    def initialize(uri, curl_opts = DEFAULT_CURL_OPTS, debug: true)
       @uri = uri.respond_to?(:scheme) ? uri : Addressable::URI.parse(uri)
       # remove trailing '/'s from local URIs
       @uri.path.gsub!(%r{/*$}, '') if local?
@@ -43,14 +56,20 @@ class SiteDiff
       @debug = debug
     end
+    ##
+    # Returns the "user" part of the URI.
     def user
       @uri.user
     end
+    ##
+    # Returns the "password" part of the URI.
     def password
       @uri.password
     end
+    ##
+    # Converts the URI to a string.
     def to_s
       uri = @uri.dup
       uri.user = nil
@@ -58,19 +77,22 @@ class SiteDiff
       uri.to_s
     end
+    ##
     # Is this a local filesystem path?
     def local?
       @uri.scheme.nil?
     end
+    ## What does this one do?
     # FIXME: this is not used anymore
-    def +(path)
+    def +(other)
       # 'path' for SiteDiff includes (parts of) path, query, and fragment.
       sep = ''
       sep = '/' if local? || @uri.path.empty?
-      self.class.new(@uri.to_s + sep + path)
+      self.class.new(@uri.to_s + sep + other)
     end
+    ##
     # Reads a file and yields to the completion handler, see .queue()
     def read_file
       File.open(@uri.to_s, 'r:UTF-8') { |f| yield ReadResult.new(f.read) }
@@ -81,10 +103,9 @@ class SiteDiff
     # Returns the encoding of an HTTP response from headers , nil if not
     # specified.
     def charset_encoding(http_headers)
-      if (content_type = http_headers['Content-Type'])
-        if (md = /;\s*charset=([-\w]*)/.match(content_type))
-          md[1]
-        end
+      content_type = http_headers['Content-Type']
+      if (md = /;\s*charset=([-\w]*)/.match(content_type))
+        md[1]
       end
     end
@@ -95,7 +116,7 @@ class SiteDiff
     def typhoeus_request
       params = @curl_opts.dup
       # Allow basic auth
-      params[:userpwd] = @uri.user + ':' + @uri.password if @uri.user
+      params[:userpwd] = "#{@uri.user}: #{@uri.password}" if @uri.user
       req = Typhoeus::Request.new(to_s, params)
@@ -114,25 +135,34 @@ class SiteDiff
         rescue ArgumentError => e
           raise if @debug
-          yield ReadResult.error("Parsing error for #{@uri}: #{e.message}")
-        rescue => e
+          yield ReadResult.error(
+            "Parsing error for #{@uri}: #{e.message}"
+          )
+        rescue StandardError => e
           raise if @debug
-          yield ReadResult.error("Unknown parsing error for #{@uri}: #{e.message}")
+          yield ReadResult.error(
+            "Unknown parsing error for #{@uri}: #{e.message}"
+          )
         end
       end
       req.on_failure do |resp|
         if resp&.status_message
-          msg = resp.status_message
-          yield ReadResult.error("HTTP error when loading #{@uri}: #{msg}",
-                                 resp.response_code)
+          yield ReadResult.error(
+            "HTTP error when loading #{@uri} : [#{resp.response_code}] #{resp.status_message}",
+            resp.response_code
+          )
         elsif (msg = resp.options[:return_code])
-          yield ReadResult.error("Connection error when loading #{@uri}: #{msg}",
-                                 resp.response_code)
+          yield ReadResult.error(
+            "Connection error when loading #{@uri} : [#{resp.options[:return_code]}] #{resp.status_message} #{msg}",
+            resp.response_code
+          )
         else
-          yield ReadResult.error("Unknown error when loading #{@uri}: #{msg}",
-                                 resp.response_code)
+          yield ReadResult.error(
+            "Unknown error when loading #{@uri} : [#{resp.response_code}] #{resp.status_message}",
+            resp.response_code
+          )
         end
       end
@@ -152,5 +182,17 @@ class SiteDiff
         hydra.queue(typhoeus_request(&handler))
       end
     end
+    ##
+    # Canonicalize a path.
+    #
+    # @param [String] path
+    #   A base relative path. Example: /foo/bar
+    def self.canonicalize(path)
+      # Ignore trailing slashes for all paths except "/" (front page).
+      path = path.chomp('/') unless path == '/'
+      # If the path is empty, assume that it's the front page.
+      path.empty? ? '/' : path
+    end
   end
 end

data/lib/sitediff/webserver/resultserver.rb CHANGED Viewed

@@ -1,18 +1,25 @@
 # frozen_string_literal: true
 require 'sitediff'
+require 'sitediff/report'
 require 'sitediff/webserver'
 require 'erb'
 class SiteDiff
   class Webserver
+    # SiteDiff Result Server.
     class ResultServer < Webserver
       # Display a page from the cache
       class CacheServlet < WEBrick::HTTPServlet::AbstractServlet
+        ##
+        # Creates a Cache Servlet.
         def initialize(_server, cache)
+          super
           @cache = cache
         end
+        ##
+        # Performs a GET request.
         def do_GET(req, res)
           path = req.path_info
           (md = %r{^/([^/]+)(/.*)$}.match(path)) ||
@@ -29,13 +36,19 @@ class SiteDiff
         end
       end
-      # Display two pages side by side
+      ##
+      # Display two pages side by side.
       class SideBySideServlet < WEBrick::HTTPServlet::AbstractServlet
+        ##
+        # Creates a Side By Side Servlet.
         def initialize(_server, cache, settings)
+          super
           @cache = cache
           @settings = settings
         end
+        ##
+        # Generates URLs for a given path.
         def urls(path)
           %w[before after].map do |tag|
             base = @settings[tag]
@@ -44,6 +57,8 @@ class SiteDiff
           end
         end
+        ##
+        # Performs a GET request.
         def do_GET(req, res)
           path = req.path_info
           before, after = *urls(path)
@@ -54,52 +69,29 @@ class SiteDiff
         end
       end
-      # Run sitediff command from browser. Probably dangerous in general.
-      class RunServlet < WEBrick::HTTPServlet::AbstractServlet
-        def initialize(_server, dir)
-          @dir = dir
-        end
-        def do_GET(req, res)
-          path = req.path_info
-          if path != '/diff'
-            res['content-type'] = 'text/plain'
-            res.body = 'ERROR: Only /run/diff is supported by the /run API at the moment'
-            return
-          end
-          # Thor assumes only one command is called and some values like
-          # `options` are share across all SiteDiff::Cli instances so
-          # we can't just call SiteDiff::Cli.new().diff
-          # This is likely to go very wrong depending on how `sitediff serve`
-          # was actually called
-          cmd = "#{$PROGRAM_NAME} diff -C #{@dir} --cached=all"
-          system(cmd)
-          # Could also add a message to indicate success/failure
-          # But for the moment, all our files are static
-          res.set_redirect(WEBrick::HTTPStatus::Found,
-                           "/files/#{SiteDiff::REPORT_FILE}")
-        end
-      end
+      ##
+      # Creates a Result Server.
       def initialize(port, dir, opts = {})
-        unless File.exist?(File.join(dir, SiteDiff::SETTINGS_FILE))
+        unless File.exist?(File.join(dir, Report::SETTINGS_FILE))
           raise SiteDiffException,
                 "Please run 'sitediff diff' before running 'sitediff serve'"
         end
-        @settings = YAML.load_file(File.join(dir, SiteDiff::SETTINGS_FILE))
+        @settings = YAML.load_file(File.join(dir, Report::SETTINGS_FILE))
+        puts @settings
         @cache = opts[:cache]
         super(port, [dir], opts)
       end
+      ##
+      # TODO: Document what this method does.
       def server(opts)
         dir = opts.delete(:DocumentRoot)
         srv = super(opts)
         srv.mount_proc('/') do |req, res|
           if req.path == '/'
             res.set_redirect(WEBrick::HTTPStatus::Found,
-                             "/files/#{SiteDiff::REPORT_FILE}")
+                             "/files/#{Report::REPORT_FILE_HTML}")
           else
             res.set_redirect(WEBrick::HTTPStatus::TemporaryRedirect,
                              "#{@settings['after']}#{req.path}")
@@ -109,10 +101,11 @@ class SiteDiff
         srv.mount('/files', WEBrick::HTTPServlet::FileHandler, dir, true)
         srv.mount('/cache', CacheServlet, @cache)
         srv.mount('/sidebyside', SideBySideServlet, @cache, @settings)
-        srv.mount('/run', RunServlet, dir)
         srv
       end
+      ##
+      # Sets up the server.
       def setup
         super
         root = uris.first
@@ -120,6 +113,8 @@ class SiteDiff
         open_in_browser(root) if @opts[:browse]
       end
+      ##
+      # Opens a URL in a browser.
       def open_in_browser(url)
         commands = %w[xdg-open open]
         cmd = commands.find { |c| which(c) }
@@ -127,6 +122,8 @@ class SiteDiff
         cmd
       end
+      ##
+      # TODO: Document what this method does.
       def which(cmd)
         ENV['PATH'].split(File::PATH_SEPARATOR).each do |path|
           file = File.join(path, cmd)

data/lib/sitediff/webserver.rb CHANGED Viewed

@@ -3,13 +3,15 @@
 require 'webrick'
 class SiteDiff
+  # SiteDiff Web Server.
   class Webserver
-    # Simple webserver for testing purposes
+    # Simple web server for testing purposes.
     DEFAULT_PORT = 13_080
     attr_accessor :ports
-    # Serve a list of directories
+    ##
+    # Serve a list of directories.
     def initialize(start_port, dirs, opts = {})
       start_port ||= DEFAULT_PORT
       @ports = (start_port...(start_port + dirs.size)).to_a
@@ -25,14 +27,20 @@ class SiteDiff
       end
     end
+    ##
+    # Kills the server.
     def kill
       @threads.each(&:kill)
     end
+    ##
+    # Waits for the server.
     def wait
       @threads.each(&:join)
     end
+    ##
+    # Maps URIs to defined ports and returns a list of URIs.
     def uris
       ports.map { |p| "http://localhost:#{p}" }
     end
@@ -63,20 +71,24 @@ class SiteDiff
     public
+    # SiteDiff Fixture Server.
     class FixtureServer < Webserver
       PORT = DEFAULT_PORT + 1
-      BASE = 'spec/fixtures/ruby-doc.org'
+      BASE = 'spec/sites/ruby-doc.org'
       NAMES = %w[core-1.9.3 core-2.0].freeze
+      # Initialize web server.
       def initialize(port = PORT, base = BASE, names = NAMES)
         dirs = names.map { |n| File.join(base, n) }
         super(port, dirs, quiet: true)
       end
+      # Get the before site uri.
       def before
         uris.first
       end
+      # Get the after site uri.
       def after
         uris.last
       end