sitediff 0.0.3 → 0.0.5

data/lib/sitediff/sanitize/dom_transform.rb

@@ -1,92 +1,91 @@
+# frozen_string_literal: true
+
 require 'sitediff/sanitize'
 require 'nokogiri'
 
 class SiteDiff
-class Sanitizer
+  class Sanitizer
+    # Currently supported transforms:
+    #
+    #  * { :type => "unwrap_root" }
+    #  * { :type => "unwrap", :selector => "div.field-item" }
+    #  * { :type => "remove", :selector => "div.extra-stuff" }
+    #  * { :type => "remove_class", :class => 'class1' }
+    class DomTransform
+      Transforms = {}
 
-# Currently supported transforms:
-#
-#  * { :type => "unwrap_root" }
-#  * { :type => "unwrap", :selector => "div.field-item" }
-#  * { :type => "remove", :selector => "div.extra-stuff" }
-#  * { :type => "remove_class", :class => 'class1' }
-class DomTransform
+      def initialize(rule)
+        @rule = rule
+      end
 
-Transforms = {}
+      # Often an array or scalar are both ok values. Turn either into an array.
+      def to_array(val)
+        [val].flatten
+      end
 
-def initialize(rule)
-  @rule = rule
-end
+      def targets(node)
+        selectors = to_array(@rule['selector'])
+        selectors.each do |sel|
+          node.css(sel).each { |n| yield n }
+        end
+      end
 
-# Often an array or scalar are both ok values. Turn either into an array.
-def to_array(v)
-  return [v].flatten
-end
-
-def targets(node)
-  selectors = to_array(@rule['selector'])
-  selectors.each do |sel|
-    node.css(sel).each { |n| yield n }
-  end
-end
+      def apply(node)
+        targets(node) { |t| process(t) }
+      end
 
-def apply(node)
-  targets(node) { |t| process(t) }
-end
+      def self.register(name)
+        Transforms[name] = self
+      end
 
-def self.register(name)
-  Transforms[name] = self
-end
+      def self.create(rule)
+        (type = rule['type']) ||
+          raise(InvalidSanitization, 'DOM transform needs a type')
+        (transform = Transforms[type]) ||
+          raise(InvalidSanitization, "No DOM transform named #{type}")
+        transform.new(rule)
+      end
 
-def self.create(rule)
-  type = rule['type'] or
-    raise InvalidSanitization, "DOM transform needs a type"
-  transform = Transforms[type] or
-    raise InvalidSanitization, "No DOM transform named #{type}"
-  return transform.new(rule)
-end
+      # Remove elements matching 'selector'
+      class Remove < DomTransform
+        register 'remove'
+        def process(node)
+          node.remove
+        end
+      end
 
-# Remove elements matching 'selector'
-class Remove < DomTransform
-  register "remove"
-  def process(node)
-    node.remove
-  end
-end
+      # Unwrap elements matching 'selector'
+      class Unwrap < DomTransform
+        register 'unwrap'
+        def process(node)
+          node.add_next_sibling(node.children)
+          node.remove
+        end
+      end
 
-# Unwrap elements matching 'selector'
-class Unwrap < DomTransform
-  register "unwrap"
-  def process(node)
-    node.add_next_sibling(node.children)
-    node.remove
-  end
-end
+      # Remove classes from elements matching selector
+      class RemoveClass < DomTransform
+        register 'remove_class'
+        def process(node)
+          classes = to_array(@rule['class'])
 
-# Remove classes from elements matching selector
-class RemoveClass < DomTransform
-  register "remove_class"
-  def process(node)
-    classes = to_array(@rule['class'])
+          # Must call remove_class on a NodeSet!
+          ns = Nokogiri::XML::NodeSet.new(node.document, [node])
+          classes.each do |class_name|
+            ns.remove_class(class_name)
+          end
+        end
+      end
 
-    # Must call remove_class on a NodeSet!
-    ns = Nokogiri::XML::NodeSet.new(node.document, [node])
-    classes.each do |class_name|
-      ns.remove_class(class_name)
+      # Unwrap the root element
+      class UnwrapRoot < DomTransform
+        register 'unwrap_root'
+        def apply(node)
+          (node.children.size == 1) ||
+            raise(InvalidSanitization, 'Multiple root elements in unwrap_root')
+          node.children = node.children[0].children
+        end
+      end
     end
   end
 end
-
-# Unwrap the root element
-class UnwrapRoot < DomTransform
-  register "unwrap_root"
-  def apply(node)
-    node.children.size == 1 or
-      raise InvalidSanitization, "Multiple root elements in unwrap_root"
-    node.children = node.children[0].children
-  end
-end
-
-end
-end
-end

data/lib/sitediff/sanitize/regexp.rb

@@ -1,56 +1,59 @@
-class SiteDiff
-class Sanitizer
-class Regexp
-  def initialize(rule)
-    @rule = rule
-  end
-
-  def selector?
-    false
-  end
-
-  def applies?(html, node)
-    applies_to_string?(html)
-  end
-
-  def apply(html)
-    gsub!(html)
-  end
-
-  def self.create(rule)
-    rule['selector'] ? WithSelector.new(rule) : new(rule)
-  end
-
-  class WithSelector < Regexp
-    def selector?
-      true
-    end
+# frozen_string_literal: true
 
-    def contexts(node)
-      sels = @rule['selector']
-      node.css(sels).each { |e| yield(e) }
-    end
-
-    def applies?(html, node)
-      enum_for(:contexts, node).any? { |e| applies_to_string?(e.to_html) }
-    end
-
-    def apply(node)
-      contexts(node) { |e| e.replace(gsub!(e.to_html)) }
+class SiteDiff
+  class Sanitizer
+    class Regexp
+      def initialize(rule)
+        @rule = rule
+      end
+
+      def selector?
+        false
+      end
+
+      def applies?(html, _node)
+        applies_to_string?(html)
+      end
+
+      def apply(html)
+        gsub!(html)
+      end
+
+      def self.create(rule)
+        rule['selector'] ? WithSelector.new(rule) : new(rule)
+      end
+
+      class WithSelector < Regexp
+        def selector?
+          true
+        end
+
+        def contexts(node)
+          sels = @rule['selector']
+          node.css(sels).each { |e| yield(e) }
+        end
+
+        def applies?(_html, node)
+          enum_for(:contexts, node).any? { |e| applies_to_string?(e.to_html) }
+        end
+
+        def apply(node)
+          contexts(node) { |e| e.replace(gsub!(e.to_html)) }
+        end
+      end
+
+      protected
+
+      def gsub!(str)
+        re = ::Regexp.new(@rule['pattern'])
+        sub = @rule['substitute'] || ''
+        str.gsub!(re, sub)
+        str
+      end
+
+      def applies_to_string?(str)
+        gsub!(str.dup) != str
+      end
     end
   end
-
-protected
-  def gsub!(str)
-    re = ::Regexp.new(@rule['pattern'])
-    sub = @rule['substitute'] || ''
-    str.gsub!(re, sub)
-    str
-  end
-
-  def applies_to_string?(str)
-    gsub!(str.dup) != str
-  end
-end
-end
 end

data/lib/sitediff/uriwrapper.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'sitediff/exception'
 require 'typhoeus'
 require 'addressable/uri'
@@ -6,6 +8,14 @@ class SiteDiff
   class SiteDiffReadFailure < SiteDiffException; end
 
   class UriWrapper
+    DEFAULT_CURL_OPTS = {
+      connecttimeout: 3,     # Don't hang on servers that don't exist
+      followlocation: true,  # Follow HTTP redirects (code 301 and 302)
+      headers: {
+        'User-Agent' => 'Sitediff - https://github.com/evolvingweb/sitediff'
+      }
+    }.freeze
+
     # This lets us treat errors or content as one object
     class ReadResult
       attr_accessor :content, :error_code, :error
@@ -20,14 +30,15 @@ class SiteDiff
         res = new
         res.error_code = code
         res.error = err
-        return res
+        res
       end
     end
 
-    def initialize(uri)
+    def initialize(uri, curl_opts = DEFAULT_CURL_OPTS)
       @uri = uri.respond_to?(:scheme) ? uri : Addressable::URI.parse(uri)
       # remove trailing '/'s from local URIs
-      @uri.path.gsub!(/\/*$/, '') if local?
+      @uri.path.gsub!(%r{/*$}, '') if local?
+      @curl_opts = curl_opts
     end
 
     def user
@@ -42,26 +53,24 @@ class SiteDiff
       uri = @uri.dup
       uri.user = nil
       uri.password = nil
-      return uri.to_s
+      uri.to_s
     end
 
     # Is this a local filesystem path?
     def local?
-      @uri.scheme == nil
+      @uri.scheme.nil?
     end
 
-    # FIXME this is not used anymore
+    # FIXME: this is not used anymore
     def +(path)
       # 'path' for SiteDiff includes (parts of) path, query, and fragment.
       sep = ''
-      if local? || @uri.path.empty?
-        sep = '/'
-      end
+      sep = '/' if local? || @uri.path.empty?
       self.class.new(@uri.to_s + sep + path)
     end
 
     # Reads a file and yields to the completion handler, see .queue()
-    def read_file(&handler)
+    def read_file
       File.open(@uri.to_s, 'r:UTF-8') { |f| yield ReadResult.new(f.read) }
     rescue Errno::ENOENT, Errno::ENOTDIR, Errno::EACCES, Errno::EISDIR => e
       yield ReadResult.error(e.message)
@@ -70,9 +79,9 @@ class SiteDiff
     # Returns the encoding of an HTTP response from headers , nil if not
     # specified.
     def http_encoding(http_headers)
-      if content_type = http_headers['Content-Type']
-        if md = /;\s*charset=([-\w]*)/.match(content_type)
-          return md[1]
+      if (content_type = http_headers['Content-Type'])
+        if (md = /;\s*charset=([-\w]*)/.match(content_type))
+          md[1]
         end
       end
     end
@@ -81,33 +90,35 @@ class SiteDiff
     #
     # Completion callbacks of the request wrap the given handler which is
     # assumed to accept a single ReadResult argument.
-    def typhoeus_request(&handler)
-      params = {
-        :connecttimeout => 3,     # Don't hang on servers that don't exist
-        :followlocation => true,  # Follow HTTP redirects (code 301 and 302)
-        :headers => {
-          "User-Agent" => "Sitediff - https://github.com/evolvingweb/sitediff"
-        }
-      }
+    def typhoeus_request
+      params = @curl_opts.dup
       # Allow basic auth
       params[:userpwd] = @uri.user + ':' + @uri.password if @uri.user
 
-      req = Typhoeus::Request.new(self.to_s, params)
+      req = Typhoeus::Request.new(to_s, params)
 
       req.on_success do |resp|
         body = resp.body
         # Typhoeus does not respect HTTP headers when setting the encoding
         # resp.body; coerce if possible.
-        if encoding = http_encoding(resp.headers)
+        if (encoding = http_encoding(resp.headers))
           body.force_encoding(encoding)
         end
         yield ReadResult.new(body)
       end
 
       req.on_failure do |resp|
-        msg = 'Unknown Error'
-        msg = resp.status_message if resp and resp.status_message
-        yield ReadResult.error("HTTP error #{@uri}: #{msg}", resp.response_code)
+        if resp&.status_message
+          msg = resp.status_message
+          yield ReadResult.error("HTTP error when loading #{@uri}: #{msg}",
+                                 resp.response_code)
+        elsif (msg = resp.options[:return_code])
+          yield ReadResult.error("Connection error when loading #{@uri}: #{msg}",
+                                 resp.response_code)
+        else
+          yield ReadResult.error("Unknown error when loading #{@uri}: #{msg}",
+                                 resp.response_code)
+        end
       end
 
       req

data/lib/sitediff/webserver.rb

@@ -1,82 +1,85 @@
+# frozen_string_literal: true
+
 require 'webrick'
 
 class SiteDiff
-class Webserver
-# Simple webserver for testing purposes
-DEFAULT_PORT = 13080
-
-attr_accessor :ports
-
-# Serve a list of directories
-def initialize(start_port, dirs, opts = {})
-  start_port ||= DEFAULT_PORT
-  @ports = (start_port...(start_port + dirs.size)).to_a
-  @dirs = dirs
-  @opts = opts
-
-  setup
-  start_servers
-
-  if block_given?
-    yield self
-    kill
-  end
-end
-
-def kill
-  @threads.each { |t| t.kill }
-end
-
-def wait
-  @threads.each { |t| t.join }
-end
-
-def uris
-  ports.map { |p| "http://localhost:#{p}" }
-end
-
-protected
-def setup
-  @server_opts = {}
-  if @opts[:quiet]
-    @server_opts[:Logger] = WEBrick::Log.new(IO::NULL)
-    @server_opts[:AccessLog] = []
-  end
-end
-
-def server(opts)
-  WEBrick::HTTPServer.new(opts)
-end
-
-def start_servers
-  @threads = []
-  @dirs.each_with_index do |dir, idx|
-    @server_opts[:Port] = @ports[idx]
-    @server_opts[:DocumentRoot] = dir
-    srv = server(@server_opts)
-    @threads << Thread.new { srv.start }
+  class Webserver
+    # Simple webserver for testing purposes
+    DEFAULT_PORT = 13_080
+
+    attr_accessor :ports
+
+    # Serve a list of directories
+    def initialize(start_port, dirs, opts = {})
+      start_port ||= DEFAULT_PORT
+      @ports = (start_port...(start_port + dirs.size)).to_a
+      @dirs = dirs
+      @opts = opts
+
+      setup
+      start_servers
+
+      if block_given?
+        yield self
+        kill
+      end
+    end
+
+    def kill
+      @threads.each(&:kill)
+    end
+
+    def wait
+      @threads.each(&:join)
+    end
+
+    def uris
+      ports.map { |p| "http://localhost:#{p}" }
+    end
+
+    protected
+
+    def setup
+      @server_opts = {}
+      if @opts[:quiet]
+        @server_opts[:Logger] = WEBrick::Log.new(IO::NULL)
+        @server_opts[:AccessLog] = []
+      end
+    end
+
+    def server(opts)
+      WEBrick::HTTPServer.new(opts)
+    end
+
+    def start_servers
+      @threads = []
+      @dirs.each_with_index do |dir, idx|
+        @server_opts[:Port] = @ports[idx]
+        @server_opts[:DocumentRoot] = dir
+        srv = server(@server_opts)
+        @threads << Thread.new { srv.start }
+      end
+    end
+
+    public
+
+    class FixtureServer < Webserver
+      PORT = DEFAULT_PORT + 1
+      BASE = 'spec/fixtures/ruby-doc.org'
+      NAMES = %w[core-1.9.3 core-2.0].freeze
+
+      def initialize(port = PORT, base = BASE, names = NAMES)
+        dirs = names.map { |n| File.join(base, n) }
+        super(port, dirs, quiet: true)
+      end
+
+      def before
+        uris.first
+      end
+
+      def after
+        uris.last
+      end
+    end
   end
 end
-
-public
-
-class FixtureServer < Webserver
-  PORT = DEFAULT_PORT + 1
-  BASE = 'spec/fixtures/ruby-doc.org'
-  NAMES = %w[core-1.9.3 core-2.0]
-
-  def initialize(port = PORT, base = BASE, names = NAMES)
-    dirs = names.map { |n| File.join(base, n) }
-    super(port, dirs, :quiet => true)
-  end
-
-  def before
-    uris.first
-  end
-  def after
-    uris.last
-  end
-end
-
-end
-end