sitediff 0.0.2 → 1.1.1

data/lib/sitediff/sanitize/dom_transform.rb

@@ -1,92 +1,130 @@
+# frozen_string_literal: true
+
 require 'sitediff/sanitize'
 require 'nokogiri'
 
 class SiteDiff
-class Sanitizer
+  class Sanitizer
+    # Currently supported transforms:
+    #
+    #  * { :type => "unwrap_root" }
+    #  * { :type => "unwrap", :selector => "div.field-item" }
+    #  * { :type => "remove", :selector => "div.extra-stuff" }
+    #  * { :type => "remove_class", :class => 'class1' }
+    #  * { :type => "strip", :selector => 'h1' }
+    class DomTransform
+      # Supported dom_transform types.
+      TRANSFORMS = {}
 
-# Currently supported transforms:
-#
-#  * { :type => "unwrap_root" }
-#  * { :type => "unwrap", :selector => "div.field-item" }
-#  * { :type => "remove", :selector => "div.extra-stuff" }
-#  * { :type => "remove_class", :class => 'class1' }
-class DomTransform
+      ##
+      # Creates a DOM Transform.
+      def initialize(rule)
+        @rule = rule
+      end
 
-Transforms = {}
+      ##
+      # Often an array or scalar are both ok values. Turn either into an array.
+      def to_array(val)
+        [val].flatten
+      end
 
-def initialize(rule)
-  @rule = rule
-end
+      ##
+      # TODO: Document what this method does.
+      def targets(node)
+        selectors = to_array(@rule['selector'])
+        selectors.each do |sel|
+          node.css(sel).each { |n| yield n }
+        end
+      end
 
-# Often an array or scalar are both ok values. Turn either into an array.
-def to_array(v)
-  return [v].flatten
-end
+      ##
+      # Applies the transformation to a DOM node.
+      def apply(node)
+        targets(node) { |t| process(t) }
+      end
 
-def targets(node)
-  selectors = to_array(@rule['selector'])
-  selectors.each do |sel|
-    node.css(sel).each { |n| yield n }
-  end
-end
+      ##
+      # Registers a DOM Transform plugin.
+      def self.register(name)
+        TRANSFORMS[name] = self
+      end
 
-def apply(node)
-  targets(node) { |t| process(t) }
-end
+      ##
+      # Creates a DOM Transform as per rule.
+      def self.create(rule)
+        (type = rule['type']) ||
+          raise(InvalidSanitization, 'DOM transform needs a type')
+        (transform = TRANSFORMS[type]) ||
+          raise(InvalidSanitization, "No DOM transform named #{type}")
+        transform.new(rule)
+      end
 
-def self.register(name)
-  Transforms[name] = self
-end
+      ##
+      # Remove elements matching 'selector'.
+      class Remove < DomTransform
+        register 'remove'
 
-def self.create(rule)
-  type = rule['type'] or
-    raise InvalidSanitization, "DOM transform needs a type"
-  transform = Transforms[type] or
-    raise InvalidSanitization, "No DOM transform named #{type}"
-  return transform.new(rule)
-end
+        ##
+        # Processes a node.
+        def process(node)
+          node.remove
+        end
+      end
 
-# Remove elements matching 'selector'
-class Remove < DomTransform
-  register "remove"
-  def process(node)
-    node.remove
-  end
-end
+      # Squeeze whitespace from a tag matching 'selector'.
+      class Strip < DomTransform
+        register 'strip'
 
-# Unwrap elements matching 'selector'
-class Unwrap < DomTransform
-  register "unwrap"
-  def process(node)
-    node.add_next_sibling(node.children)
-    node.remove
-  end
-end
+        ##
+        # Processes a node.
+        def process(node)
+          node.content = node.content.strip
+        end
+      end
 
-# Remove classes from elements matching selector
-class RemoveClass < DomTransform
-  register "remove_class"
-  def process(node)
-    classes = to_array(@rule['class'])
+      # Unwrap elements matching 'selector'.
+      class Unwrap < DomTransform
+        register 'unwrap'
 
-    # Must call remove_class on a NodeSet!
-    ns = Nokogiri::XML::NodeSet.new(node.document, [node])
-    classes.each do |class_name|
-      ns.remove_class(class_name)
-    end
-  end
-end
+        ##
+        # Processes a node.
+        def process(node)
+          node.add_next_sibling(node.children)
+          node.remove
+        end
+      end
 
-# Unwrap the root element
-class UnwrapRoot < DomTransform
-  register "unwrap_root"
-  def apply(node)
-    node.children.size == 1 or
-      raise InvalidSanitization, "Multiple root elements in unwrap_root"
-    node.children = node.children[0].children
-  end
-end
+      ##
+      # Remove classes from elements matching selector
+      class RemoveClass < DomTransform
+        register 'remove_class'
 
-end
-end
+        ##
+        # Processes a node.
+        def process(node)
+          classes = to_array(@rule['class'])
+
+          # Must call remove_class on a NodeSet!
+          ns = Nokogiri::XML::NodeSet.new(node.document, [node])
+          classes.each do |class_name|
+            ns.remove_class(class_name)
+          end
+        end
+      end
+
+      ##
+      # Unwrap the root element.
+      class UnwrapRoot < DomTransform
+        register 'unwrap_root'
+
+        ##
+        # Applies the transformation to a DOM node.
+        def apply(node)
+          (node.children.size == 1) ||
+            raise(InvalidSanitization, 'Multiple root elements in unwrap_root')
+          node.children = node.children[0].children
+        end
+      end
+    end
+  end
 end

data/lib/sitediff/sanitize/regexp.rb

@@ -1,56 +1,82 @@
+# frozen_string_literal: true
+
 class SiteDiff
-class Sanitizer
-class Regexp
-  def initialize(rule)
-    @rule = rule
-  end
+  class Sanitizer
+    # Regular Expression Object.
+    class Regexp
+      ##
+      # Creates a RegExp object.
+      def initialize(rule)
+        @rule = rule
+      end
 
-  def selector?
-    false
-  end
+      ##
+      # Whether the RegExp has a selector.
+      def selector?
+        false
+      end
 
-  def applies?(html, node)
-    applies_to_string?(html)
-  end
+      ##
+      # Whether the RegExp applies to the given markup.
+      def applies?(html, _node)
+        applies_to_string?(html)
+      end
 
-  def apply(html)
-    gsub!(html)
-  end
+      ##
+      # Applies the RegExp to the markup.
+      def apply(html)
+        gsub!(html)
+      end
 
-  def self.create(rule)
-    rule['selector'] ? WithSelector.new(rule) : new(rule)
-  end
+      ##
+      # Creates a RegExp object as per rule.
+      def self.create(rule)
+        rule['selector'] ? WithSelector.new(rule) : new(rule)
+      end
 
-  class WithSelector < Regexp
-    def selector?
-      true
-    end
+      ##
+      # A RegExp with selector.
+      class WithSelector < Regexp
+        ##
+        # Whether the RegExp has a selector.
+        def selector?
+          true
+        end
 
-    def contexts(node)
-      sels = @rule['selector']
-      node.css(sels).each { |e| yield(e) }
-    end
+        ##
+        # TODO: Document what this method does.
+        def contexts(node)
+          selectors = @rule['selector']
+          node.css(selectors).each { |e| yield(e) }
+        end
 
-    def applies?(html, node)
-      enum_for(:contexts, node).any? { |e| applies_to_string?(e.to_html) }
-    end
+        ##
+        # Whether the RegExp applies to the given markup.
+        def applies?(_html, node)
+          enum_for(:contexts, node).any? { |e| applies_to_string?(e.to_html) }
+        end
 
-    def apply(node)
-      contexts(node) { |e| e.replace(gsub!(e.to_html)) }
-    end
-  end
+        ##
+        # Applies the RegExp to the markup.
+        def apply(node)
+          contexts(node) { |e| e.replace(gsub!(e.to_html)) }
+        end
+      end
 
-protected
-  def gsub!(str)
-    re = ::Regexp.new(@rule['pattern'])
-    sub = @rule['substitute'] || ''
-    str.gsub!(re, sub)
-    str
-  end
+      protected
 
-  def applies_to_string?(str)
-    gsub!(str.dup) != str
+      def gsub!(str)
+        re = ::Regexp.new(@rule['pattern'])
+        sub = @rule['substitute'] || ''
+        # Expecting a mutation here. Do not reassign the variable str
+        # for the purpose of removing UTF-8 encoding errors.
+        str.gsub!(re, sub)
+        str
+      end
+
+      def applies_to_string?(str)
+        gsub!(str.dup) != str
+      end
+    end
   end
 end
-end
-end

data/lib/sitediff/uriwrapper.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'sitediff/exception'
 require 'typhoeus'
 require 'addressable/uri'
@@ -5,63 +7,94 @@ require 'addressable/uri'
 class SiteDiff
   class SiteDiffReadFailure < SiteDiffException; end
 
+  # SiteDiff URI Wrapper.
   class UriWrapper
+    # TODO: Move these CURL OPTS to Config.DEFAULT_CONFIG.
+    DEFAULT_CURL_OPTS = {
+      # Don't hang on servers that don't exist.
+      connecttimeout: 3,
+      # Follow HTTP redirects (code 301 and 302).
+      followlocation: true,
+      headers: {
+        'User-Agent' => 'Sitediff - https://github.com/evolvingweb/sitediff'
+      },
+      # always accept SSL certs
+      ssl_verifypeer: false,
+      ssl_verifyhost: 0
+    }.freeze
+
     # This lets us treat errors or content as one object
     class ReadResult
-      attr_accessor :content, :error_code, :error
+      attr_accessor :encoding, :content, :error_code, :error
 
-      def initialize(content = nil)
+      ##
+      # Creates a ReadResult.
+      def initialize(content = nil, encoding = 'utf-8')
         @content = content
+        @encoding = encoding
         @error = nil
         @error_code = nil
       end
 
-      def self.error(err, code = nil)
+      ##
+      # Creates a ReadResult with an error.
+      def self.error(message, code = nil)
         res = new
         res.error_code = code
-        res.error = err
-        return res
+        res.error = message
+        res
       end
     end
 
-    def initialize(uri)
+    ##
+    # Creates a UriWrapper.
+    def initialize(uri, curl_opts = DEFAULT_CURL_OPTS, debug = true)
       @uri = uri.respond_to?(:scheme) ? uri : Addressable::URI.parse(uri)
       # remove trailing '/'s from local URIs
-      @uri.path.gsub!(/\/*$/, '') if local?
+      @uri.path.gsub!(%r{/*$}, '') if local?
+      @curl_opts = curl_opts
+      @debug = debug
     end
 
+    ##
+    # Returns the "user" part of the URI.
     def user
       @uri.user
     end
 
+    ##
+    # Returns the "password" part of the URI.
     def password
       @uri.password
     end
 
+    ##
+    # Converts the URI to a string.
     def to_s
       uri = @uri.dup
       uri.user = nil
       uri.password = nil
-      return uri.to_s
+      uri.to_s
     end
 
+    ##
     # Is this a local filesystem path?
     def local?
-      @uri.scheme == nil
+      @uri.scheme.nil?
     end
 
-    # FIXME this is not used anymore
-    def +(path)
+    ## What does this one do?
+    # FIXME: this is not used anymore
+    def +(other)
       # 'path' for SiteDiff includes (parts of) path, query, and fragment.
       sep = ''
-      if local? || @uri.path.empty?
-        sep = '/'
-      end
-      self.class.new(@uri.to_s + sep + path)
+      sep = '/' if local? || @uri.path.empty?
+      self.class.new(@uri.to_s + sep + other)
     end
 
+    ##
     # Reads a file and yields to the completion handler, see .queue()
-    def read_file(&handler)
+    def read_file
       File.open(@uri.to_s, 'r:UTF-8') { |f| yield ReadResult.new(f.read) }
     rescue Errno::ENOENT, Errno::ENOTDIR, Errno::EACCES, Errno::EISDIR => e
       yield ReadResult.error(e.message)
@@ -69,10 +102,10 @@ class SiteDiff
 
     # Returns the encoding of an HTTP response from headers , nil if not
     # specified.
-    def http_encoding(http_headers)
-      if content_type = http_headers['Content-Type']
-        if md = /;\s*charset=([-\w]*)/.match(content_type)
-          return md[1]
+    def charset_encoding(http_headers)
+      if (content_type = http_headers['Content-Type'])
+        if (md = /;\s*charset=([-\w]*)/.match(content_type))
+          md[1]
         end
       end
     end
@@ -81,33 +114,58 @@ class SiteDiff
     #
     # Completion callbacks of the request wrap the given handler which is
     # assumed to accept a single ReadResult argument.
-    def typhoeus_request(&handler)
-      params = {
-        :connecttimeout => 3,     # Don't hang on servers that don't exist
-        :followlocation => true,  # Follow HTTP redirects (code 301 and 302)
-        :headers => {
-          "User-Agent" => "Sitediff - https://github.com/evolvingweb/sitediff"
-        }
-      }
+    def typhoeus_request
+      params = @curl_opts.dup
       # Allow basic auth
       params[:userpwd] = @uri.user + ':' + @uri.password if @uri.user
 
-      req = Typhoeus::Request.new(self.to_s, params)
+      req = Typhoeus::Request.new(to_s, params)
 
       req.on_success do |resp|
         body = resp.body
         # Typhoeus does not respect HTTP headers when setting the encoding
         # resp.body; coerce if possible.
-        if encoding = http_encoding(resp.headers)
+        if (encoding = charset_encoding(resp.headers))
           body.force_encoding(encoding)
         end
-        yield ReadResult.new(body)
+        # Should be wrapped with rescue I guess? Maybe this entire function?
+        # Should at least be an option in the Cli to disable this.
+        # "stop on first error"
+        begin
+          yield ReadResult.new(body, encoding)
+        rescue ArgumentError => e
+          raise if @debug
+
+          yield ReadResult.error(
+            "Parsing error for #{@uri}: #{e.message}"
+          )
+        rescue StandardError => e
+          raise if @debug
+
+          yield ReadResult.error(
+            "Unknown parsing error for #{@uri}: #{e.message}"
+          )
+        end
       end
 
       req.on_failure do |resp|
-        msg = 'Unknown Error'
-        msg = resp.status_message if resp and resp.status_message
-        yield ReadResult.error("HTTP error #{@uri}: #{msg}", resp.response_code)
+        if resp&.status_message
+          msg = resp.status_message
+          yield ReadResult.error(
+            "HTTP error when loading #{@uri}: #{msg}",
+            resp.response_code
+          )
+        elsif (msg = resp.options[:return_code])
+          yield ReadResult.error(
+            "Connection error when loading #{@uri}: #{msg}",
+            resp.response_code
+          )
+        else
+          yield ReadResult.error(
+            "Unknown error when loading #{@uri}: #{msg}",
+            resp.response_code
+          )
+        end
       end
 
       req
@@ -126,5 +184,17 @@ class SiteDiff
         hydra.queue(typhoeus_request(&handler))
       end
     end
+
+    ##
+    # Canonicalize a path.
+    #
+    # @param [String] path
+    #   A base relative path. Example: /foo/bar
+    def self.canonicalize(path)
+      # Ignore trailing slashes for all paths except "/" (front page).
+      path = path.chomp('/') unless path == '/'
+      # If the path is empty, assume that it's the front page.
+      path.empty? ? '/' : path
+    end
   end
 end