sitediff 0.0.1 → 0.0.2

data/lib/sitediff/sanitize/dom_transform.rb

@@ -0,0 +1,92 @@
+require 'sitediff/sanitize'
+require 'nokogiri'
+
+class SiteDiff
+class Sanitizer
+
+# Currently supported transforms:
+#
+#  * { :type => "unwrap_root" }
+#  * { :type => "unwrap", :selector => "div.field-item" }
+#  * { :type => "remove", :selector => "div.extra-stuff" }
+#  * { :type => "remove_class", :class => 'class1' }
+class DomTransform
+
+Transforms = {}
+
+def initialize(rule)
+  @rule = rule
+end
+
+# Often an array or scalar are both ok values. Turn either into an array.
+def to_array(v)
+  return [v].flatten
+end
+
+def targets(node)
+  selectors = to_array(@rule['selector'])
+  selectors.each do |sel|
+    node.css(sel).each { |n| yield n }
+  end
+end
+
+def apply(node)
+  targets(node) { |t| process(t) }
+end
+
+def self.register(name)
+  Transforms[name] = self
+end
+
+def self.create(rule)
+  type = rule['type'] or
+    raise InvalidSanitization, "DOM transform needs a type"
+  transform = Transforms[type] or
+    raise InvalidSanitization, "No DOM transform named #{type}"
+  return transform.new(rule)
+end
+
+# Remove elements matching 'selector'
+class Remove < DomTransform
+  register "remove"
+  def process(node)
+    node.remove
+  end
+end
+
+# Unwrap elements matching 'selector'
+class Unwrap < DomTransform
+  register "unwrap"
+  def process(node)
+    node.add_next_sibling(node.children)
+    node.remove
+  end
+end
+
+# Remove classes from elements matching selector
+class RemoveClass < DomTransform
+  register "remove_class"
+  def process(node)
+    classes = to_array(@rule['class'])
+
+    # Must call remove_class on a NodeSet!
+    ns = Nokogiri::XML::NodeSet.new(node.document, [node])
+    classes.each do |class_name|
+      ns.remove_class(class_name)
+    end
+  end
+end
+
+# Unwrap the root element
+class UnwrapRoot < DomTransform
+  register "unwrap_root"
+  def apply(node)
+    node.children.size == 1 or
+      raise InvalidSanitization, "Multiple root elements in unwrap_root"
+    node.children = node.children[0].children
+  end
+end
+
+end
+end
+end

data/lib/sitediff/sanitize/regexp.rb

@@ -0,0 +1,56 @@
+class SiteDiff
+class Sanitizer
+class Regexp
+  def initialize(rule)
+    @rule = rule
+  end
+
+  def selector?
+    false
+  end
+
+  def applies?(html, node)
+    applies_to_string?(html)
+  end
+
+  def apply(html)
+    gsub!(html)
+  end
+
+  def self.create(rule)
+    rule['selector'] ? WithSelector.new(rule) : new(rule)
+  end
+
+  class WithSelector < Regexp
+    def selector?
+      true
+    end
+
+    def contexts(node)
+      sels = @rule['selector']
+      node.css(sels).each { |e| yield(e) }
+    end
+
+    def applies?(html, node)
+      enum_for(:contexts, node).any? { |e| applies_to_string?(e.to_html) }
+    end
+
+    def apply(node)
+      contexts(node) { |e| e.replace(gsub!(e.to_html)) }
+    end
+  end
+
+protected
+  def gsub!(str)
+    re = ::Regexp.new(@rule['pattern'])
+    sub = @rule['substitute'] || ''
+    str.gsub!(re, sub)
+    str
+  end
+
+  def applies_to_string?(str)
+    gsub!(str.dup) != str
+  end
+end
+end
+end

data/lib/sitediff/uriwrapper.rb

@@ -1,19 +1,31 @@
+require 'sitediff/exception'
 require 'typhoeus'
+require 'addressable/uri'
 
 class SiteDiff
-  class SiteDiffReadFailure < Exception; end
+  class SiteDiffReadFailure < SiteDiffException; end
 
   class UriWrapper
     # This lets us treat errors or content as one object
-    class ReadResult < Struct.new(:content, :error)
-      def initialize(cont, err = nil)
-        super(cont, err)
+    class ReadResult
+      attr_accessor :content, :error_code, :error
+
+      def initialize(content = nil)
+        @content = content
+        @error = nil
+        @error_code = nil
+      end
+
+      def self.error(err, code = nil)
+        res = new
+        res.error_code = code
+        res.error = err
+        return res
       end
-      def self.error(err); new(nil, err); end
     end
 
     def initialize(uri)
-      @uri = uri.respond_to?(:scheme) ? uri : URI.parse(uri)
+      @uri = uri.respond_to?(:scheme) ? uri : Addressable::URI.parse(uri)
       # remove trailing '/'s from local URIs
       @uri.path.gsub!(/\/*$/, '') if local?
     end
@@ -95,7 +107,7 @@ class SiteDiff
       req.on_failure do |resp|
         msg = 'Unknown Error'
         msg = resp.status_message if resp and resp.status_message
-        yield ReadResult.error("HTTP error #{@uri}: #{msg}")
+        yield ReadResult.error("HTTP error #{@uri}: #{msg}", resp.response_code)
       end
 
       req

data/lib/sitediff/webserver.rb

@@ -0,0 +1,82 @@
+require 'webrick'
+
+class SiteDiff
+class Webserver
+# Simple webserver for testing purposes
+DEFAULT_PORT = 13080
+
+attr_accessor :ports
+
+# Serve a list of directories
+def initialize(start_port, dirs, opts = {})
+  start_port ||= DEFAULT_PORT
+  @ports = (start_port...(start_port + dirs.size)).to_a
+  @dirs = dirs
+  @opts = opts
+
+  setup
+  start_servers
+
+  if block_given?
+    yield self
+    kill
+  end
+end
+
+def kill
+  @threads.each { |t| t.kill }
+end
+
+def wait
+  @threads.each { |t| t.join }
+end
+
+def uris
+  ports.map { |p| "http://localhost:#{p}" }
+end
+
+protected
+def setup
+  @server_opts = {}
+  if @opts[:quiet]
+    @server_opts[:Logger] = WEBrick::Log.new(IO::NULL)
+    @server_opts[:AccessLog] = []
+  end
+end
+
+def server(opts)
+  WEBrick::HTTPServer.new(opts)
+end
+
+def start_servers
+  @threads = []
+  @dirs.each_with_index do |dir, idx|
+    @server_opts[:Port] = @ports[idx]
+    @server_opts[:DocumentRoot] = dir
+    srv = server(@server_opts)
+    @threads << Thread.new { srv.start }
+  end
+end
+
+public
+
+class FixtureServer < Webserver
+  PORT = DEFAULT_PORT + 1
+  BASE = 'spec/fixtures/ruby-doc.org'
+  NAMES = %w[core-1.9.3 core-2.0]
+
+  def initialize(port = PORT, base = BASE, names = NAMES)
+    dirs = names.map { |n| File.join(base, n) }
+    super(port, dirs, :quiet => true)
+  end
+
+  def before
+    uris.first
+  end
+  def after
+    uris.last
+  end
+end
+
+end
+end

data/lib/sitediff/webserver/resultserver.rb

@@ -0,0 +1,98 @@
+require 'sitediff'
+require 'sitediff/webserver'
+require 'erb'
+
+class SiteDiff
+class Webserver
+class ResultServer < Webserver
+  # Display a page from the cache
+  class CacheServlet < WEBrick::HTTPServlet::AbstractServlet
+    def initialize(server, cache)
+      @cache = cache
+    end
+
+    def do_GET(req, res)
+      path = req.path_info
+      md = %r[^/([^/]+)(/.*)$].match(path) or
+        raise WEBrick::HTTPStatus::NotFound
+      tag, path = *md.captures
+      r = @cache.get(tag.to_sym, path) or
+        raise WEBrick::HTTPStatus::NotFound
+
+      raise WEBrick::HTTPStatus[r.error_code] if r.error_code
+      raise WEBrick::HTTPStatus::InternalServerError, r.error if r.error
+
+      res['content-type'] = 'text/html'
+      res.body = r.content
+    end
+  end
+
+  # Display two pages side by side
+  class SideBySideServlet < WEBrick::HTTPServlet::AbstractServlet
+    def initialize(server, cache, settings)
+      @cache = cache
+      @settings = settings
+    end
+
+    def urls(path)
+      %w[before after].map do |tag|
+        base = @settings[tag]
+        base = "/cache/#{tag}" if @settings['cached'].include? tag
+        base + path
+      end
+    end
+
+    def do_GET(req, res)
+      path = req.path_info
+      before, after = *urls(path)
+
+      res['content-type'] = 'text/html'
+      erb = File.join(SiteDiff::FILES_DIR, 'sidebyside.html.erb')
+      res.body = ERB.new(File.read(erb)).result(binding)
+    end
+  end
+
+  def initialize(port, dir, opts = {})
+    @settings = YAML.load_file(File.join(dir, SiteDiff::SETTINGS_FILE))
+    @cache = opts[:cache]
+    super(port, [dir], opts)
+  end
+
+  def server(opts)
+    dir = opts.delete(:DocumentRoot)
+    srv = super(opts)
+    srv.mount_proc('/') do |req, res|
+      res.set_redirect(WEBrick::HTTPStatus::Found,
+        "/files/#{SiteDiff::REPORT_FILE}")
+    end
+
+    srv.mount('/files', WEBrick::HTTPServlet::FileHandler, dir, true)
+    srv.mount('/cache', CacheServlet, @cache)
+    srv.mount('/sidebyside', SideBySideServlet, @cache, @settings)
+    return srv
+  end
+
+  def setup
+    super
+    root = uris.first
+    puts "Serving at #{root}"
+    open_in_browser(root) if @opts[:browse]
+  end
+
+  def open_in_browser(url)
+    commands = %w[xdg-open open]
+    cmd = commands.find { |c| which(c) }
+    system(cmd, url) if cmd
+    return cmd
+  end
+
+  def which(cmd)
+    ENV['PATH'].split(File::PATH_SEPARATOR).each do |path|
+      file = File.join(path, cmd)
+      return file if File.executable?(file)
+    end
+    return nil
+  end
+end
+end
+end

metadata

@@ -1,7 +1,8 @@
 --- !ruby/object:Gem::Specification
 name: sitediff
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
+  prerelease: 
 platform: ruby
 authors:
 - Alex Dergachev
@@ -10,122 +11,166 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-21 00:00:00.000000000 Z
+date: 2015-05-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: diffy
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: typhoeus
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rainbow
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-description: |
-  SiteDiff makes it easy to see differences between two versions of a website. It accepts a set of paths to compare two versions of the site together with potential normalization/sanitization rules. From the provided paths and configuration SiteDiff generates an HTML report of all the status of HTML comparison between the given paths together with a readable diff-like HTML for each specified path containing the differences between the two versions of the site. It is useful tool for QAing re-deployments, site upgrades, etc.
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ! 'SiteDiff makes it easy to see differences between two versions of
+  a website. It accepts a set of paths to compare two versions of the site together
+  with potential normalization/sanitization rules. From the provided paths and configuration
+  SiteDiff generates an HTML report of all the status of HTML comparison between the
+  given paths together with a readable diff-like HTML for each specified path containing
+  the differences between the two versions of the site. It is useful tool for QAing
+  re-deployments, site upgrades, etc.
+
+'
 email: alex@evolvingweb.ca
 executables:
 - sitediff
 extensions: []
 extra_rdoc_files: []
 files:
+- lib/sitediff.rb
+- lib/sitediff/crawler.rb
+- lib/sitediff/sanitize/dom_transform.rb
+- lib/sitediff/sanitize/regexp.rb
+- lib/sitediff/config/creator.rb
 - lib/sitediff/cli.rb
+- lib/sitediff/fetch.rb
+- lib/sitediff/exception.rb
+- lib/sitediff/cache.rb
 - lib/sitediff/config.rb
-- lib/sitediff/diff.rb
+- lib/sitediff/uriwrapper.rb
 - lib/sitediff/result.rb
 - lib/sitediff/sanitize.rb
-- lib/sitediff/uriwrapper.rb
-- lib/sitediff/util/cache.rb
-- lib/sitediff/util/webserver.rb
-- lib/sitediff.rb
+- lib/sitediff/webserver/resultserver.rb
+- lib/sitediff/rules.rb
+- lib/sitediff/webserver.rb
+- lib/sitediff/diff.rb
 - lib/sitediff/files/diff.html.erb
+- lib/sitediff/files/sidebyside.html.erb
+- lib/sitediff/files/sitediff.css
 - lib/sitediff/files/html_report.html.erb
 - lib/sitediff/files/pretty_print.xsl
-- lib/sitediff/files/sitediff.css
+- lib/sitediff/files/rules/drupal.yaml
 - bin/sitediff
 homepage: https://github.com/evolvingweb/sitediff/
 licenses:
 - GPL-2
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: 1.9.3
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14
+rubygems_version: 1.8.23
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: Compare two versions of a site with ease!
 test_files: []