sitediff 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1dc3a624b91cd4b7ef1c926116630cd795532024
+  data.tar.gz: e49f227ae303f574b704ffe3a226f79a120ae30f
+SHA512:
+  metadata.gz: 90ca5508b834d32ac7c96aa6a94a6aa8488921e978e76890e142b1249da20bc620ddcfa237f3defc1e6928d83dd0a22583c9dded150855c320f94140e1bffdf1
+  data.tar.gz: 24bf7969b6f17c269bb407d1ff1684f6556318d0cfa7c6c92a8327ddd0d86ee4f153778affa4d1ef115e47a3b69b31b4dac5f01ed8b7f464a05fd98f9f98212b

data/bin/sitediff

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+# when run as gem, $0 is /usr/local/bin/sitediff not this file
+if $0 == __FILE__
+  $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+end
+
+require 'sitediff'
+
+SiteDiff::Cli.start

data/lib/sitediff.rb

@@ -0,0 +1,130 @@
+#!/bin/env ruby
+require 'sitediff/cli.rb'
+require 'sitediff/config.rb'
+require 'sitediff/result.rb'
+require 'sitediff/uriwrapper'
+require 'sitediff/util/cache'
+require 'typhoeus'
+require 'rainbow'
+
+class SiteDiff
+  # path to misc. static files (e.g. erb, css files)
+  FILES_DIR = File.join(File.dirname(__FILE__), 'sitediff', 'files')
+
+  # subdirectory containing all failing diffs
+  DIFFS_DIR = 'diffs'
+
+  # label will be colorized and str will not be.
+  # type dictates the color: can be :success, :error, or :failure
+  def self.log(str, type=nil, label=nil)
+    label = label ? "[sitediff] #{label}" : '[sitediff]'
+    bg = fg = nil
+    case type
+    when :success
+      bg = :green
+      fg = :black
+    when :failure
+      bg = :red
+    when :error
+      bg = :yellow
+      fg = :black
+    end
+    label = Rainbow(label)
+    label = label.bg(bg) if bg
+    label = label.fg(fg) if fg
+    puts label + ' ' + str
+  end
+
+  attr_reader :config, :results
+  def before
+    @config.before['url']
+  end
+  def after
+    @config.after['url']
+  end
+
+  def cache=(file)
+    # FIXME: Non-global cache would be nice
+    return unless file
+    if Gem::Version.new(Typhoeus::VERSION) >= Gem::Version.new('0.6.4')
+      Typhoeus::Config.cache = SiteDiff::Util::Cache.new(file)
+    else
+      # Bug, see: https://github.com/typhoeus/typhoeus/pull/296
+      SiteDiff::log("Cache unsupported on Typhoeus version < 0.6.4", :failure)
+    end
+  end
+
+  def initialize(config, cache)
+    config.validate
+    @config = config
+    self.cache = cache
+  end
+
+  # Sanitize an HTML string based on configuration for either before or after
+  def sanitize(html, pos)
+    Sanitize::sanitize(html, @config.send(pos))
+  end
+
+  # Queues fetching before and after URLs with a Typhoeus::Hydra instance
+  #
+  # Upon completion of both before and after, prints and saves the diff to
+  # @results.
+  def queue_read(hydra, path)
+    # ( :before | after ) => ReadResult object
+    reads = {}
+    [:before, :after].each do |pos|
+      uri = UriWrapper.new(send(pos) + path)
+
+      uri.queue(hydra) do |res|
+        reads[pos] = res
+        next unless reads.size == 2
+
+        # we have read both before and after; calculate diff
+        if error = reads[:before].error || reads[:after].error
+          diff = Result.new(path, nil, nil, error)
+        else
+          diff = Result.new(path, sanitize(reads[:before].content, :before),
+                            sanitize(reads[:after].content,:after), nil)
+        end
+        diff.log
+        @results[path] = diff
+      end
+    end
+  end
+
+  # Perform the comparison
+  def run
+    # Map of path -> Result object, queue_read sets callbacks to populate this
+    @results = {}
+
+    hydra = Typhoeus::Hydra.new(max_concurrency: 3)
+    @config.paths.each { |path| queue_read(hydra, path) }
+    hydra.run
+
+    # Order by original path order
+    @results = @config.paths.map { |p| @results[p] }
+  end
+
+  # Dump results to disk
+  def dump(dir, report_before, report_after, failing_paths)
+    report_before ||= before
+    report_after ||= after
+    FileUtils.mkdir_p(dir)
+
+    # store diffs of each failing case, first wipe out existing diffs
+    diff_dir = File.join(dir, DIFFS_DIR)
+    FileUtils.rm_rf(diff_dir)
+    results.each { |r| r.dump(dir) if r.status == Result::STATUS_FAILURE }
+    SiteDiff::log "All diff files were dumped inside #{dir}"
+
+    # store failing paths
+    SiteDiff::log "Writing failures to #{failing_paths}"
+    File.open(failing_paths, 'w') do |f|
+      results.each { |r| f.puts r.path unless r.success? }
+    end
+
+    # create report of results
+    report = Diff::generate_html_report(results, report_before, report_after)
+    File.open(File.join(dir, "/report.html") , 'w') { |f| f.write(report) }
+  end
+end

data/lib/sitediff/cli.rb

@@ -0,0 +1,90 @@
+require 'thor'
+require 'sitediff/diff'
+require 'sitediff/sanitize'
+require 'sitediff/util/webserver'
+require 'open-uri'
+require 'uri'
+
+class SiteDiff
+  class Cli < Thor
+    # Thor, by default, exits with 0 no matter what!
+    def self.exit_on_failure?
+      true
+    end
+
+    # Thor, by default, does not raise an error for use of unknown options.
+    def self.check_unknown_options?(config)
+      true
+    end
+
+    option 'dump-dir',
+      :type => :string,
+      :default => File.join('.', 'output'),
+      :desc => "Location to write the output to."
+    option 'paths',
+      :type => :string,
+      :desc => 'Paths are read (one at a line) from PATHS: ' +
+               'useful for iterating over sanitization rules',
+      :aliases => '--paths-from-file'
+    option 'before',
+      :type => :string,
+      :desc => "URL used to fetch the before HTML. Acts as a prefix to specified paths",
+      :aliases => '--before-url'
+    option 'after',
+      :type => :string,
+      :desc => "URL used to fetch the after HTML. Acts as a prefix to specified paths.",
+      :aliases => '--after-url'
+    option 'before-report',
+      :type => :string,
+      :desc => "Before URL to use for reporting purposes. Useful if port forwarding.",
+      :aliases => '--before-url-report'
+    option 'after-report',
+      :type => :string,
+      :desc => "After URL to use for reporting purposes. Useful if port forwarding.",
+      :aliases => '--after-url-report'
+    option 'cache',
+      :type => :string,
+      :desc => "Filename to use for caching requests.",
+      :lazy_default => 'cache.db'
+    desc "diff [OPTIONS] [CONFIGFILES]", "Perform systematic diff on given URLs"
+    def diff(*config_files)
+      config = SiteDiff::Config.new(config_files)
+
+      # override config based on options
+      if paths_file = options['paths']
+        unless File.exists? paths_file
+          raise Config::InvalidConfig,
+            "Paths file '#{paths_file}' not found!"
+        end
+        SiteDiff::log "Reading paths from: #{paths_file}"
+        config.paths = File.readlines(paths_file)
+      end
+      config.before['url'] = options['before'] if options['before']
+      config.after['url'] = options['after'] if options['after']
+
+      sitediff = SiteDiff.new(config, options['cache'])
+      sitediff.run
+
+      failing_paths = File.join(options['dump-dir'], 'failures.txt')
+      sitediff.dump(options['dump-dir'], options['before-report'],
+        options['after-report'], failing_paths)
+    rescue Config::InvalidConfig => e
+      SiteDiff.log "Invalid configuration: #{e.message}", :failure
+    end
+
+    option :port,
+      :type => :numeric,
+      :default => SiteDiff::Util::Webserver::DEFAULT_PORT,
+      :desc => 'The port to serve on'
+    option :directory,
+      :type => :string,
+      :default => 'output',
+      :desc => 'The directory to serve',
+      :aliases => '--dump-dir'
+    desc "serve [OPTIONS]", "Serve the sitediff output directory over HTTP"
+    def serve
+      SiteDiff::Util::Webserver.serve(options[:port], options[:directory],
+        :announce => true).wait
+    end
+  end
+end

data/lib/sitediff/config.rb

@@ -0,0 +1,154 @@
+require 'yaml'
+
+class SiteDiff
+  class Config
+
+    # keys allowed in configuration files
+    CONF_KEYS = Sanitize::TOOLS.values.flatten(1) +
+                %w[paths before after before_url after_url includes]
+
+    class InvalidConfig < Exception; end
+
+    # Takes a Hash and normalizes it to the following form by merging globals
+    # into before and after. A normalized config Hash looks like this:
+    #
+    #     paths:
+    #     - /about
+    #
+    #     before:
+    #       url: http://before
+    #       selector: body
+    #       dom_transform:
+    #       - type: remove
+    #         selector: script
+    #
+    #     after:
+    #       url: http://after
+    #       selector: body
+    #
+    def self.normalize(conf)
+      tools = Sanitize::TOOLS
+
+      # merge globals
+      %w[before after].each do |pos|
+        conf[pos] ||= {}
+        tools[:array].each do |key|
+          conf[pos][key] ||= []
+          conf[pos][key] += conf[key] if conf[key]
+        end
+        tools[:scalar].each {|key| conf[pos][key] ||= conf[key]}
+        conf[pos]['url'] ||= conf[pos + '_url']
+      end
+      # normalize paths
+      conf['paths'] = Config::normalize_paths(conf['paths'])
+
+      conf.select {|k,v| %w[before after paths].include? k}
+    end
+
+    # Merges two normalized Hashes according to the following rules:
+    # 1 paths are merged as arrays.
+    # 2 before and after: for each subhash H (e.g. ['before']['dom_transform']):
+    #   a)  if first[H] and second[H] are expected to be arrays, their values
+    #       are merged as such,
+    #   b)  if first[H] and second[H] are expected to be scalars, the value for
+    #       second[H] is kept if and only if first[H] is nil.
+    #
+    # For example, merge(h1, h2) results in h3:
+    #
+    # (h1) before: {selector: foo, sanitization: [pattern: foo]}
+    # (h2) before: {selector: bar, sanitization: [pattern: bar]}
+    # (h3) before: {selector: foo, sanitization: [pattern: foo, pattern: bar]}
+    def self.merge(first, second)
+      result = { 'paths' => {}, 'before' => {}, 'after' => {} }
+      result['paths'] = (first['paths'] || []) + (second['paths'] || []) # rule 1
+      %w[before after].each do |pos|
+        unless first[pos]
+          result[pos] = second[pos] || {}
+          next
+        end
+        result[pos] = first[pos].merge!(second[pos]) do |key, a, b|
+          if Sanitize::TOOLS[:array].include? key # rule 2a
+            result[pos][key] = (a || []) + (b|| [])
+          else
+            result[pos][key] = a || b # rule 2b
+          end
+        end
+      end
+      result
+    end
+
+    def initialize(files)
+      @config = {'paths' => [], 'before' => {}, 'after' => {} }
+      files.each do |file|
+        @config = Config::merge(@config, Config::load_conf(file))
+      end
+    end
+
+    def before
+      @config['before']
+    end
+    def after
+      @config['after']
+    end
+
+    def paths
+      @config['paths']
+    end
+    def paths=(paths)
+      @config['paths'] = Config::normalize_paths(paths)
+    end
+
+    # Checks if the configuration is usable for diff-ing.
+    def validate
+      raise InvalidConfig, "Undefined 'before' base URL." unless before['url']
+      raise InvalidConfig, "Undefined 'after' base URL." unless after['url']
+      raise InvalidConfig, "Undefined 'paths'." unless (paths and !paths.empty?)
+    end
+
+    private
+
+    def self.normalize_paths(paths)
+      paths ||= []
+      return paths.map { |p| (p[0] == '/' ? p : "/#{p}").chomp }
+    end
+
+    # reads a YAML file and raises an InvalidConfig if the file is not valid.
+    def self.load_raw_yaml(file)
+      SiteDiff::log "Reading config file: #{file}"
+      conf = YAML.load_file(file) || {}
+      unless conf.is_a? Hash
+        raise InvalidConfig, "Invalid configuration file: '#{file}'"
+      end
+      conf.each do |k,v|
+        unless CONF_KEYS.include? k
+          raise InvalidConfig, "Unknown configuration key (#{file}): '#{k}'"
+        end
+      end
+      conf
+    end
+
+    # loads a single YAML configuration file, merges all its 'included' files
+    # and returns a normalized Hash.
+    def self.load_conf(file, visited=[])
+      # don't get fooled by a/../a/ or symlinks
+      file = File.realpath(file)
+      if visited.include? file
+        raise InvalidConfig, "Circular dependency: #{file}"
+      end
+
+      conf = load_raw_yaml(file) # not normalized yet
+      visited << file
+
+      # normalize and merge includes
+      includes = conf['includes'] || []
+      conf = Config::normalize(conf)
+      includes.each do |dep|
+        # include paths are relative to the including file.
+        dep = File.join(File.dirname(file), dep)
+        conf = Config::merge(conf, load_conf(dep, visited))
+      end
+      conf
+    end
+
+  end
+end

data/lib/sitediff/diff.rb

@@ -0,0 +1,37 @@
+require 'diffy'
+require 'erb'
+require 'rainbow'
+
+class SiteDiff
+  module Diff
+    module_function
+
+    def html_diffy(before_html, after_html)
+      diff = Diffy::Diff.new(before_html, after_html)
+      diff.first ?  # Is it non-empty?
+        diff.to_s(:html) : nil
+    end
+
+    def terminal_diffy(before_html, after_html)
+      args = []
+      args << :color if Rainbow.enabled
+      return Diffy::Diff.new(before_html, after_html, :context => 3).
+        to_s(*args)
+    end
+
+    def generate_html_report(results, before, after)
+      erb_path = File.join(SiteDiff::FILES_DIR, 'html_report.html.erb')
+      report_html = ERB.new(File.read(erb_path)).result(binding)
+      return report_html
+    end
+
+    def generate_diff_output(result)
+      erb_path = File.join(SiteDiff::FILES_DIR, 'diff.html.erb')
+      return ERB.new(File.read(erb_path)).result(binding)
+    end
+
+    def css
+      File.read(File.join(SiteDiff::FILES_DIR, 'sitediff.css'))
+    end
+  end
+end

data/lib/sitediff/files/diff.html.erb

@@ -0,0 +1,11 @@
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <style>
+      <%= Diffy::CSS %>
+    </style>
+  </head>
+  <body>
+    <%= result.diff %>
+  </body>
+</html>

data/lib/sitediff/files/html_report.html.erb

@@ -0,0 +1,47 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <!-- important: otherwise chrome will choke on non-ascii characters -->
+    <meta charset="utf-8" />
+    <style>
+      <%= SiteDiff::Diff.css %>
+    </style>
+    <title> SiteDiff Report </title>
+  </head>
+  <body>
+    <div class="sitediff">
+      <div class="legend">
+        <strong>before</strong> (base url): <a href="<%=before%>"><%=before%></a> |
+        <strong>after </strong> (base url): <a href="<%=after%>" ><%=after %></a>
+      </div>
+      <table class="results">
+
+        <colgroup>
+          <col class="before-col">
+          <col class="after-col">
+          <col class="path-col">
+          <col class="diff-stat-col">
+        </colgroup>
+
+        <thead>
+          <tr>
+            <th> Before </th>
+            <th> After </th>
+            <th> Path </th>
+            <th> Status </th>
+          </tr>
+        </thead>
+
+        <% results.each do |result| %>
+        <tr class="<%= result.status_text %>">
+          <td class="before"><a href="<%= result.url(before) %>">[before]</a></td>
+          <td class="after"><a href="<%= result.url(after) %>">[after]</a></td>
+          <td class="path"><%= result.path %></td>
+          <td class="status"><%= result.link %></td>
+        </tr>
+        <% end %>
+
+      </table>
+    </div>
+  </body>
+</html>

data/lib/sitediff/files/pretty_print.xsl

@@ -0,0 +1,9 @@
+<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
+  <xsl:output method="xml" encoding="UTF-8" indent="yes"/>
+  <xsl:param name="indent-increment" select="'   '"/>
+  <xsl:strip-space elements="*"/>
+
+  <xsl:template match="/">
+    <xsl:copy-of select="."/>
+  </xsl:template>
+</xsl:stylesheet>

data/lib/sitediff/files/sitediff.css

@@ -0,0 +1,42 @@
+.sitediff {
+  font-family: monospace;
+  font-size: 1.2em;
+}
+.sitediff .legend {
+  width: 95%;
+  margin: 1em auto;
+  text-align: center;
+}
+.sitediff .results thead {
+  background: black;
+  color: white;
+}
+.sitediff .results td {
+  text-align: center;
+}
+.sitediff .results td.path {
+  text-align: left;
+  padding-left: 1em;
+}
+.sitediff .results {
+  padding: 1em;
+  width: 95%;
+  margin: 1em auto;
+  font-size: 1em;
+}
+.sitediff tr.error > td.status,
+.sitediff tr.error > td.path {
+  background-color: khaki;
+}
+.sitediff tr.failure > td.status,
+.sitediff tr.failure > td.path {
+  background-color: salmon;
+}
+.sitediff .before-col,
+.sitediff .after-col,
+.sitediff .diff-stat-col {
+  width: 10%;
+}
+.sitediff .path-col {
+  width: 55%;
+}

data/lib/sitediff/result.rb

@@ -0,0 +1,74 @@
+require 'fileutils'
+require 'digest/sha1'
+
+class SiteDiff
+  class Result < Struct.new(:path, :before, :after, :error)
+    STATUS_SUCCESS  = 0   # Identical before and after
+    STATUS_FAILURE  = 1   # Different before and after
+    STATUS_ERROR    = 2   # Couldn't fetch page
+    STATUS_TEXT = %w[success failure error]
+
+    attr_reader :status, :diff
+
+    def initialize(*args)
+      super
+      if error
+        @status = STATUS_ERROR
+      else
+        @diff = Diff::html_diffy(before, after)
+        @status = @diff ? STATUS_FAILURE : STATUS_SUCCESS
+      end
+    end
+
+    def success?
+      status == STATUS_SUCCESS
+    end
+
+    # Textual representation of the status
+    def status_text
+      return STATUS_TEXT[status]
+    end
+
+    # Printable URL
+    def url(prefix)
+      prefix.to_s + path
+    end
+
+    # Filename to store diff
+    def filename
+      File.join(SiteDiff::DIFFS_DIR, Digest::SHA1.hexdigest(self.path) + '.html')
+    end
+
+    # Text of the link in the HTML report
+    def link
+      case status
+      when STATUS_ERROR then error
+      when STATUS_SUCCESS then status_text
+      when STATUS_FAILURE then "<a href='#{filename}'>DIFF</a>"
+      end
+    end
+
+    # Log the result to the terminal
+    def log
+      case status
+      when STATUS_SUCCESS then
+        SiteDiff::log path, :success, 'SUCCESS'
+      when STATUS_ERROR then
+        SiteDiff::log path, :error, "ERROR (#{error})"
+      when STATUS_FAILURE then
+        SiteDiff::log path, :failure, "FAILURE"
+        puts Diff::terminal_diffy(before, after)
+      end
+    end
+
+    # Dump the result to a file
+    def dump(dir)
+      dump_path = File.join(dir, filename)
+      base = File.dirname(dump_path)
+      FileUtils::mkdir_p(base) unless File.exists?(base)
+      File.open(dump_path, 'w') do |f|
+        f.write(Diff::generate_diff_output(self))
+      end
+    end
+  end
+end

data/lib/sitediff/sanitize.rb

@@ -0,0 +1,193 @@
+require 'nokogiri'
+require 'set'
+
+class SiteDiff
+  module Sanitize
+    class InvalidSanitization < Exception; end
+
+    TOOLS = {
+      :array => %w[dom_transform sanitization],
+      :scalar => %w[selector remove_spacing],
+    }
+    DOM_TRANSFORMS = Set.new(%w[remove unwrap_root unwrap remove_class])
+
+    module_function
+
+    # Performs dom transformations.
+    #
+    # Currently supported transforms:
+    #
+    #  * { :type => "unwrap_root" }
+    #  * { :type => "unwrap", :selector => "div.field-item" }
+    #  * { :type => "remove", :selector => "div.extra-stuff" }
+    #
+    #  @arg node - Nokogiri document or Node
+    #  @arg rules - array of dom_transform rules
+    #  @return - transformed Nokogiri document node
+    def perform_dom_transforms(node, rules)
+      rules.each do |rule|
+        type = rule['type'] or
+          raise InvalidSanitization, "DOM transform needs a type"
+        DOM_TRANSFORMS.include?(type) or
+          raise InvalidSanitization, "No DOM transform named #{type}"
+
+        meth = 'transform_' + type
+
+        if sels = rule['selector']
+          sels = [sels].flatten # Either array or scalar is fine
+          # Call method for each node the selectors find
+          sels.each do |sel|
+            node.css(sel).each { |e| send(meth, rule, e) }
+          end
+        else
+          send(meth, rule, node)
+        end
+      end
+    end
+
+    def transform_remove(rule, el)
+      el.remove
+    end
+    def transform_unwrap(rule, el)
+      el.add_next_sibling(el.children)
+      el.remove
+    end
+    def transform_remove_class(rule, el)
+      # Must call remove_class on a NodeSet!
+      ns = Nokogiri::XML::NodeSet.new(el.document, [el])
+      [rule['class']].flatten.each do |class_name|
+        ns.remove_class(class_name)
+      end
+    end
+    def transform_unwrap_root(rule, node)
+      node.children.size == 1 or
+        raise InvalidSanitization, "Multiple root elements in unwrap_root"
+      node.children = node.children[0].children
+    end
+
+    def parse(str, force_doc = false, log_errors = false)
+      if force_doc || /<!DOCTYPE/.match(str[0, 512])
+        doc = Nokogiri::HTML(str)
+        doc
+      else
+        doc = Nokogiri::HTML.fragment(str)
+      end
+      if log_errors
+        doc.errors.each do |e|
+          SiteDiff::log "Error in parsing HTML document: #{e}", :error
+        end
+      end
+      doc
+    end
+
+    # Force this object to be a document, so we can apply a stylesheet
+    def to_document(obj)
+      if Nokogiri::XML::Document === obj
+        return obj
+      elsif Nokogiri::XML::Node === obj # or fragment
+        return parse(obj.to_s, true)
+
+        # This ought to work, and would be faster,
+        # but seems to segfault Nokogiri
+        # doc = Nokogiri::HTML('<html><body>')
+        # doc.at('body').children = obj.children
+        # return doc
+      else
+        return to_document(parse(obj))
+      end
+    end
+
+    # Pretty-print the HTML
+    def prettify(obj)
+      @stylesheet ||= begin
+        stylesheet_path = File.join(SiteDiff::FILES_DIR, 'pretty_print.xsl')
+        Nokogiri::XSLT(File.read(stylesheet_path))
+      end
+
+      # Pull out the html element's children
+      # The obvious way to do this is to iterate over pretty.css('html'),
+      # but that tends to segfault Nokogiri
+      str = @stylesheet.apply_to(to_document(obj))
+
+      # Remove xml declaration and <html> tags
+      str.sub!(/\A<\?xml.*$\n/, '')
+      str.sub!(/\A^<html>$\n/, '')
+      str.sub!(%r[</html>\n\Z], '')
+
+      # Remove top-level indentation
+      indent = /\A(\s*)/.match(str)[1].size
+      str.gsub!(/^\s{,#{indent}}/, '')
+
+      # Remove blank lines
+      str.gsub!(/^\s*$\n/, '')
+
+      return str
+    end
+
+    def remove_spacing(doc)
+      # remove double spacing, but only inside text nodes (eg not attributes)
+      doc.xpath('//text()').each do |node|
+        node.content = node.content.gsub(/  +/, ' ')
+      end
+    end
+
+    # Do one regexp transformation on a string
+    def substitute(str, rule)
+      #FIXME escape forward slashes, right now we are escaping them in YAML!
+      str.gsub!(/#{rule['pattern']}/, rule['substitute'] || '' )
+      str
+    end
+
+    # Do all regexp sanitization rules
+    def perform_regexps(node, rules)
+      rules ||= []
+
+      # First do rules with a selector
+      rules.each do |rule|
+        if sel = rule['selector']
+          node.css(sel).each do |e|
+            e.replace(substitute(e.to_html, rule))
+          end
+        end
+      end
+
+      # If needed, do rules without a selector. We'd rather not convert to
+      # a string unless necessary.
+      global_rules = rules.reject { |r| r['selector'] }
+      return node if global_rules.empty?
+
+      str = node.to_html # Convert to string
+      global_rules.each { |r| substitute(str, r) }
+      return str
+    end
+
+    def select_root(node, sel)
+      return node unless sel
+
+      # When we choose a new root, we always become a DocumentFragment,
+      # and lose any DOCTYPE and such.
+      ns = node.css(sel)
+      unless node.fragment?
+        node = Nokogiri::HTML.fragment('')
+      end
+      node.children = ns
+      return node
+    end
+
+    def sanitize(str, config)
+      return '' if str == ''
+
+      node = parse(str)
+
+      remove_spacing(node) if config['remove_spacing']
+      node = select_root(node, config['selector'])
+      if transform = config['dom_transform']
+        perform_dom_transforms(node, transform)
+      end
+
+      obj = perform_regexps(node, config['sanitization'])
+
+      return prettify(obj)
+    end
+  end
+end

data/lib/sitediff/uriwrapper.rb

@@ -0,0 +1,118 @@
+require 'typhoeus'
+
+class SiteDiff
+  class SiteDiffReadFailure < Exception; end
+
+  class UriWrapper
+    # This lets us treat errors or content as one object
+    class ReadResult < Struct.new(:content, :error)
+      def initialize(cont, err = nil)
+        super(cont, err)
+      end
+      def self.error(err); new(nil, err); end
+    end
+
+    def initialize(uri)
+      @uri = uri.respond_to?(:scheme) ? uri : URI.parse(uri)
+      # remove trailing '/'s from local URIs
+      @uri.path.gsub!(/\/*$/, '') if local?
+    end
+
+    def user
+      @uri.user
+    end
+
+    def password
+      @uri.password
+    end
+
+    def to_s
+      uri = @uri.dup
+      uri.user = nil
+      uri.password = nil
+      return uri.to_s
+    end
+
+    # Is this a local filesystem path?
+    def local?
+      @uri.scheme == nil
+    end
+
+    # FIXME this is not used anymore
+    def +(path)
+      # 'path' for SiteDiff includes (parts of) path, query, and fragment.
+      sep = ''
+      if local? || @uri.path.empty?
+        sep = '/'
+      end
+      self.class.new(@uri.to_s + sep + path)
+    end
+
+    # Reads a file and yields to the completion handler, see .queue()
+    def read_file(&handler)
+      File.open(@uri.to_s, 'r:UTF-8') { |f| yield ReadResult.new(f.read) }
+    rescue Errno::ENOENT, Errno::ENOTDIR, Errno::EACCES, Errno::EISDIR => e
+      yield ReadResult.error(e.message)
+    end
+
+    # Returns the encoding of an HTTP response from headers , nil if not
+    # specified.
+    def http_encoding(http_headers)
+      if content_type = http_headers['Content-Type']
+        if md = /;\s*charset=([-\w]*)/.match(content_type)
+          return md[1]
+        end
+      end
+    end
+
+    # Returns a Typhoeus::Request to fetch @uri
+    #
+    # Completion callbacks of the request wrap the given handler which is
+    # assumed to accept a single ReadResult argument.
+    def typhoeus_request(&handler)
+      params = {
+        :connecttimeout => 3,     # Don't hang on servers that don't exist
+        :followlocation => true,  # Follow HTTP redirects (code 301 and 302)
+        :headers => {
+          "User-Agent" => "Sitediff - https://github.com/evolvingweb/sitediff"
+        }
+      }
+      # Allow basic auth
+      params[:userpwd] = @uri.user + ':' + @uri.password if @uri.user
+
+      req = Typhoeus::Request.new(self.to_s, params)
+
+      req.on_success do |resp|
+        body = resp.body
+        # Typhoeus does not respect HTTP headers when setting the encoding
+        # resp.body; coerce if possible.
+        if encoding = http_encoding(resp.headers)
+          body.force_encoding(encoding)
+        end
+        yield ReadResult.new(body)
+      end
+
+      req.on_failure do |resp|
+        msg = 'Unknown Error'
+        msg = resp.status_message if resp and resp.status_message
+        yield ReadResult.error("HTTP error #{@uri}: #{msg}")
+      end
+
+      req
+    end
+
+    # Queue reading this URL, with a completion handler to run after.
+    #
+    # The handler should be callable as handler[ReadResult].
+    #
+    # This method may choose not to queue the request at all, but simply
+    # execute right away.
+    def queue(hydra, &handler)
+      if local?
+        read_file(&handler)
+      else
+        hydra.queue(typhoeus_request(&handler))
+      end
+    end
+  end
+end

data/lib/sitediff/util/cache.rb

@@ -0,0 +1,32 @@
+class SiteDiff
+  module Util
+    # A typhoeus cache, backed by DBM
+    class Cache
+      def initialize(file)
+        # Default to GDBM, if we have it, we don't want pag/dir files
+        begin
+          require 'gdbm'
+          @dbm = GDBM.new(file)
+        rescue LoadError
+          require 'dbm'
+          @dbm = DBM.new(file)
+        end
+      end
+
+      # Older Typhoeus doesn't have cache_key
+      def cache_key(req)
+        return req.cache_key if req.respond_to?(:cache_key)
+        return Marshal.dump([req.base_url, req.options])
+      end
+
+      def get(req)
+        resp = @dbm[cache_key(req)] or return nil
+        Marshal.load(resp)
+      end
+
+      def set(req, resp)
+        @dbm[cache_key(req)] = Marshal.dump(resp)
+      end
+    end
+  end
+end

data/lib/sitediff/util/webserver.rb

@@ -0,0 +1,77 @@
+require 'webrick'
+
+class SiteDiff
+  module Util
+    # Simple webserver for testing purposes
+    class Webserver
+      DEFAULT_PORT = 13080
+
+      attr_accessor :ports
+
+      # Serve a list of directories
+      def initialize(start_port, dirs, params = {})
+        start_port ||= DEFAULT_PORT
+        @ports = (start_port...(start_port + dirs.size)).to_a
+
+        if params[:announce]
+          puts "Serving at #{uris.join(", ")}"
+        end
+
+        opts = {}
+        if params[:quiet]
+          opts[:Logger] = WEBrick::Log.new(IO::NULL)
+          opts[:AccessLog] = []
+        end
+
+        @threads = []
+        dirs.each_with_index do |dir, idx|
+          opts[:Port] = @ports[idx]
+          opts[:DocumentRoot] = dir
+          server = WEBrick::HTTPServer.new(opts)
+          @threads << Thread.new { server.start }
+        end
+
+        if block_given?
+          yield self
+          kill
+        end
+      end
+
+      def kill
+        @threads.each { |t| t.kill }
+      end
+
+      def wait
+        @threads.each { |t| t.join }
+      end
+
+      def uris
+        ports.map { |p| "http://localhost:#{p}" }
+      end
+
+
+      # Helper to serve one dir
+      def self.serve(port, dir, params = {})
+        new(port, [dir], params)
+      end
+    end
+
+    class FixtureServer < Webserver
+      PORT = DEFAULT_PORT + 1
+      BASE = 'spec/fixtures/ruby-doc.org'
+      NAMES = %w[core-1.9.3 core-2.0]
+
+      def initialize(port = PORT, base = BASE, names = NAMES)
+        dirs = names.map { |n| File.join(base, n) }
+        super(port, dirs, :quiet => true)
+      end
+
+      def before
+        uris.first
+      end
+      def after
+        uris.last
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,131 @@
+--- !ruby/object:Gem::Specification
+name: sitediff
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Alex Dergachev
+- Amir Kadivar
+- Dave Vasilevsky
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-04-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: diffy
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: typhoeus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rainbow
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: |
+  SiteDiff makes it easy to see differences between two versions of a website. It accepts a set of paths to compare two versions of the site together with potential normalization/sanitization rules. From the provided paths and configuration SiteDiff generates an HTML report of all the status of HTML comparison between the given paths together with a readable diff-like HTML for each specified path containing the differences between the two versions of the site. It is useful tool for QAing re-deployments, site upgrades, etc.
+email: alex@evolvingweb.ca
+executables:
+- sitediff
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/sitediff/cli.rb
+- lib/sitediff/config.rb
+- lib/sitediff/diff.rb
+- lib/sitediff/result.rb
+- lib/sitediff/sanitize.rb
+- lib/sitediff/uriwrapper.rb
+- lib/sitediff/util/cache.rb
+- lib/sitediff/util/webserver.rb
+- lib/sitediff.rb
+- lib/sitediff/files/diff.html.erb
+- lib/sitediff/files/html_report.html.erb
+- lib/sitediff/files/pretty_print.xsl
+- lib/sitediff/files/sitediff.css
+- bin/sitediff
+homepage: https://github.com/evolvingweb/sitediff/
+licenses:
+- GPL-2
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: 1.9.3
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: Compare two versions of a site with ease!
+test_files: []