sitediff 0.0.1 → 0.0.2

data/bin/sitediff

@@ -5,6 +5,6 @@ if $0 == __FILE__
   $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
 end
 
-require 'sitediff'
+require 'sitediff/cli'
 
 SiteDiff::Cli.start

data/lib/sitediff.rb

@@ -1,11 +1,10 @@
 #!/bin/env ruby
-require 'sitediff/cli.rb'
-require 'sitediff/config.rb'
-require 'sitediff/result.rb'
-require 'sitediff/uriwrapper'
-require 'sitediff/util/cache'
-require 'typhoeus'
+require 'sitediff/config'
+require 'sitediff/fetch'
+require 'sitediff/result'
+require 'pathname'
 require 'rainbow'
+require 'yaml'
 
 class SiteDiff
   # path to misc. static files (e.g. erb, css files)
@@ -14,20 +13,28 @@ class SiteDiff
   # subdirectory containing all failing diffs
   DIFFS_DIR = 'diffs'
 
+  # files in output
+  FAILURES_FILE = 'failures.txt'
+  REPORT_FILE = 'report.html'
+  SETTINGS_FILE = 'settings.yaml'
+
   # label will be colorized and str will not be.
   # type dictates the color: can be :success, :error, or :failure
-  def self.log(str, type=nil, label=nil)
+  def self.log(str, type=:info, label=nil)
     label = label ? "[sitediff] #{label}" : '[sitediff]'
     bg = fg = nil
     case type
-    when :success
+    when :info
+    when :diff_success
       bg = :green
       fg = :black
-    when :failure
+    when :diff_failure
       bg = :red
-    when :error
+    when :warn
       bg = :yellow
       fg = :black
+    when :error
+      bg = :red
     end
     label = Rainbow(label)
     label = label.bg(bg) if bg
@@ -43,88 +50,97 @@ class SiteDiff
     @config.after['url']
   end
 
-  def cache=(file)
-    # FIXME: Non-global cache would be nice
-    return unless file
-    if Gem::Version.new(Typhoeus::VERSION) >= Gem::Version.new('0.6.4')
-      Typhoeus::Config.cache = SiteDiff::Util::Cache.new(file)
-    else
-      # Bug, see: https://github.com/typhoeus/typhoeus/pull/296
-      SiteDiff::log("Cache unsupported on Typhoeus version < 0.6.4", :failure)
+  def initialize(config, cache, verbose=true)
+    @cache = cache
+    @verbose = verbose
+
+    # Check for single-site mode
+    validate_opts = {}
+    if !config.before['url'] && @cache.tag?(:before)
+      raise SiteDiffException,
+        "A cached 'before' is required for single-site mode" \
+        unless @cache.read_tags.include?(:before)
+      validate_opts[:need_before] = false
     end
-  end
+    config.validate(validate_opts)
 
-  def initialize(config, cache)
-    config.validate
     @config = config
-    self.cache = cache
   end
 
-  # Sanitize an HTML string based on configuration for either before or after
-  def sanitize(html, pos)
-    Sanitize::sanitize(html, @config.send(pos))
+  # Sanitize HTML
+  def sanitize(path, read_results)
+    [:before, :after].map do |tag|
+      html = read_results[tag].content
+      config = @config.send(tag)
+      Sanitizer.new(html, config, :path => path).sanitize
+    end
   end
 
-  # Queues fetching before and after URLs with a Typhoeus::Hydra instance
-  #
-  # Upon completion of both before and after, prints and saves the diff to
-  # @results.
-  def queue_read(hydra, path)
-    # ( :before | after ) => ReadResult object
-    reads = {}
-    [:before, :after].each do |pos|
-      uri = UriWrapper.new(send(pos) + path)
-
-      uri.queue(hydra) do |res|
-        reads[pos] = res
-        next unless reads.size == 2
-
-        # we have read both before and after; calculate diff
-        if error = reads[:before].error || reads[:after].error
-          diff = Result.new(path, nil, nil, error)
-        else
-          diff = Result.new(path, sanitize(reads[:before].content, :before),
-                            sanitize(reads[:after].content,:after), nil)
-        end
-        diff.log
-        @results[path] = diff
-      end
+  # Process a set of read results
+  def process_results(path, read_results)
+    if error = read_results[:before].error || read_results[:after].error
+      diff = Result.new(path, nil, nil, error)
+    else
+      diff = Result.new(path, *sanitize(path, read_results), nil)
+    end
+    @results[path] = diff
+
+    # Print results in order!
+    while next_diff = @results[@ordered.first]
+      next_diff.log(@verbose)
+      @ordered.shift
     end
   end
 
-  # Perform the comparison
+  # Perform the comparison, populate @results and return the number of failing
+  # paths (paths with non-zero diff).
   def run
-    # Map of path -> Result object, queue_read sets callbacks to populate this
+    # Map of path -> Result object, populated by process_results
     @results = {}
+    @ordered = @config.paths.dup
+
+    unless @cache.read_tags.empty?
+      SiteDiff.log("Using sites from cache: " +
+        @cache.read_tags.sort.join(', '))
+    end
 
-    hydra = Typhoeus::Hydra.new(max_concurrency: 3)
-    @config.paths.each { |path| queue_read(hydra, path) }
-    hydra.run
+    fetcher = Fetch.new(@cache, @config.paths,
+      :before => before, :after => after)
+    fetcher.run(&self.method(:process_results))
 
     # Order by original path order
     @results = @config.paths.map { |p| @results[p] }
+    return results.map{ |r| r unless r.success? }.compact.length
   end
 
   # Dump results to disk
-  def dump(dir, report_before, report_after, failing_paths)
+  def dump(dir, report_before, report_after)
     report_before ||= before
     report_after ||= after
-    FileUtils.mkdir_p(dir)
+    dir = Pathname.new(dir)
+    dir.mkpath unless dir.directory?
 
     # store diffs of each failing case, first wipe out existing diffs
-    diff_dir = File.join(dir, DIFFS_DIR)
-    FileUtils.rm_rf(diff_dir)
+    diff_dir = dir + DIFFS_DIR
+    diff_dir.rmtree if diff_dir.exist?
     results.each { |r| r.dump(dir) if r.status == Result::STATUS_FAILURE }
-    SiteDiff::log "All diff files were dumped inside #{dir}"
+    SiteDiff::log "All diff files were dumped inside #{dir.expand_path}"
 
     # store failing paths
-    SiteDiff::log "Writing failures to #{failing_paths}"
-    File.open(failing_paths, 'w') do |f|
+    failures = dir + FAILURES_FILE
+    SiteDiff::log "Writing failures to #{failures.expand_path}"
+    failures.open('w') do |f|
       results.each { |r| f.puts r.path unless r.success? }
     end
 
     # create report of results
-    report = Diff::generate_html_report(results, report_before, report_after)
-    File.open(File.join(dir, "/report.html") , 'w') { |f| f.write(report) }
+    report = Diff::generate_html_report(results, report_before, report_after,
+      @cache)
+    dir.+(REPORT_FILE).open('w') { |f| f.write(report) }
+
+    # serve some settings
+    settings = { 'before' => report_before, 'after' => report_after,
+       'cached' => @cache.read_tags.map { |t| t.to_s } }
+    dir.+(SETTINGS_FILE).open('w') { |f| YAML.dump(settings, f) }
   end
 end

data/lib/sitediff/cache.rb

@@ -0,0 +1,61 @@
+require 'set'
+
+class SiteDiff
+class Cache
+  DEFAULT_FILENAME = 'cache.db'
+
+  attr_accessor :read_tags, :write_tags
+
+  def initialize(opts = {})
+    @file = opts[:file] || DEFAULT_FILENAME
+    @create = opts[:create]
+    @read_tags = Set.new
+    @write_tags = Set.new
+  end
+
+  def close; @dbm.close if defined? @dbm; end
+
+  # Is a tag cached?
+  def tag?(tag)
+    open
+    @dbm[tag.to_s]
+  end
+
+  def get(tag, path)
+    return nil unless @read_tags.include? tag
+    open or return nil
+    val = @dbm[key(tag, path)]
+    return val && Marshal.load(val)
+  end
+
+  def set(tag, path, result)
+    return unless @write_tags.include? tag
+    open or return
+    @dbm[tag.to_s] = 'TRUE'
+    @dbm[key(tag, path)] = Marshal.dump(result)
+  end
+
+private
+  def key(tag, path)
+    # Ensure encoding stays the same!
+    Marshal.dump([tag, path.encode('UTF-8')])
+  end
+
+  # Ensure the DB is open
+  def open
+    # DBM adds an extra .db, ugh
+    return false unless @create || File.exist?(@file) ||
+      File.exist?(@file + '.db')
+    return true if defined? @dbm
+
+    begin
+      require 'gdbm'
+      @dbm = GDBM.new(@file)
+    rescue LoadError
+      require 'dbm'
+      @dbm = DBM.new(@file)
+    end
+    return true
+  end
+end
+end

data/lib/sitediff/cli.rb

@@ -1,12 +1,18 @@
 require 'thor'
-require 'sitediff/diff'
-require 'sitediff/sanitize'
-require 'sitediff/util/webserver'
-require 'open-uri'
-require 'uri'
+require 'sitediff'
+require 'sitediff/cache'
+require 'sitediff/config'
+require 'sitediff/config/creator'
+require 'sitediff/fetch'
+require 'sitediff/webserver/resultserver'
 
 class SiteDiff
   class Cli < Thor
+    class_option 'directory',
+      :type => :string,
+      :aliases => '-C',
+      :desc => "Go to a given directory before running."
+
     # Thor, by default, exits with 0 no matter what!
     def self.exit_on_failure?
       true
@@ -21,11 +27,15 @@ class SiteDiff
       :type => :string,
       :default => File.join('.', 'output'),
       :desc => "Location to write the output to."
-    option 'paths',
+    option 'paths-file',
       :type => :string,
       :desc => 'Paths are read (one at a line) from PATHS: ' +
                'useful for iterating over sanitization rules',
       :aliases => '--paths-from-file'
+    option 'paths',
+      :type => :array,
+      :aliases => '-p',
+      :desc => "Fetch only these specific paths"
     option 'before',
       :type => :string,
       :desc => "URL used to fetch the before HTML. Acts as a prefix to specified paths",
@@ -42,16 +52,29 @@ class SiteDiff
       :type => :string,
       :desc => "After URL to use for reporting purposes. Useful if port forwarding.",
       :aliases => '--after-url-report'
-    option 'cache',
+    option 'cached',
       :type => :string,
-      :desc => "Filename to use for caching requests.",
-      :lazy_default => 'cache.db'
+      :enum => %w[none all before after],
+      :default => 'before',
+      :desc => "Use the cached version of these sites, if available."
+    option 'quiet',
+      :type => :boolean,
+      :aliases => '-q',
+      :default => false,
+      :desc => "Show the difference between versions for each page"
     desc "diff [OPTIONS] [CONFIGFILES]", "Perform systematic diff on given URLs"
     def diff(*config_files)
-      config = SiteDiff::Config.new(config_files)
+      config = chdir(config_files)
 
       # override config based on options
-      if paths_file = options['paths']
+      paths = options['paths']
+      if paths_file = options['paths-file']
+        if paths then
+          SiteDiff::log "Can't have both --paths-file and --paths", :error
+          exit -1
+        end
+
+        paths_file = Pathname.new(paths_file).expand_path
         unless File.exists? paths_file
           raise Config::InvalidConfig,
             "Paths file '#{paths_file}' not found!"
@@ -59,32 +82,130 @@ class SiteDiff
         SiteDiff::log "Reading paths from: #{paths_file}"
         config.paths = File.readlines(paths_file)
       end
+      config.paths = paths if paths
+
       config.before['url'] = options['before'] if options['before']
       config.after['url'] = options['after'] if options['after']
 
-      sitediff = SiteDiff.new(config, options['cache'])
-      sitediff.run
+      # Setup cache
+      cache = SiteDiff::Cache.new(:create => options['cached'] != 'none')
+      cache.read_tags << :before if %w[before all].include?(options['cached'])
+      cache.read_tags << :after if %w[after all].include?(options['cached'])
+      cache.write_tags << :before << :after
+
+      sitediff = SiteDiff.new(config, cache, !options['quiet'])
+      num_failing = sitediff.run
+      exit_code = (num_failing > 0) ? 2 : 0;
 
-      failing_paths = File.join(options['dump-dir'], 'failures.txt')
       sitediff.dump(options['dump-dir'], options['before-report'],
-        options['after-report'], failing_paths)
+        options['after-report'])
     rescue Config::InvalidConfig => e
-      SiteDiff.log "Invalid configuration: #{e.message}", :failure
+      SiteDiff.log "Invalid configuration: #{e.message}", :error
+    rescue SiteDiffException => e
+      SiteDiff.log e.message, :error
+    else # no exception was raised
+      # Thor::Error  --> exit(1), guaranteed by exit_on_failure?
+      # Failing diff --> exit(2), populated above
+      exit(exit_code)
     end
 
     option :port,
       :type => :numeric,
-      :default => SiteDiff::Util::Webserver::DEFAULT_PORT,
+      :default => SiteDiff::Webserver::DEFAULT_PORT,
       :desc => 'The port to serve on'
-    option :directory,
+    option 'dump-dir',
       :type => :string,
       :default => 'output',
-      :desc => 'The directory to serve',
-      :aliases => '--dump-dir'
+      :desc => 'The directory to serve'
+    option :browse,
+      :type => :boolean,
+      :default => true,
+      :desc => "Whether to open the served content in your browser"
     desc "serve [OPTIONS]", "Serve the sitediff output directory over HTTP"
-    def serve
-      SiteDiff::Util::Webserver.serve(options[:port], options[:directory],
-        :announce => true).wait
+    def serve(*config_files)
+      config = chdir(config_files, :config => false)
+
+      cache = Cache.new
+      cache.read_tags << :before << :after
+
+      SiteDiff::Webserver::ResultServer.new(
+        options[:port],
+        options['dump-dir'],
+        :browse => options[:browse],
+        :cache => cache,
+        :config => config,
+      ).wait
+    end
+
+    option :output,
+      :type => :string,
+      :default => 'sitediff',
+      :desc => 'Where to place the configuration',
+      :aliases => ['-o']
+    option :depth,
+      :type => :numeric,
+      :default => 3,
+      :desc => 'How deeply to crawl the given site'
+    option :rules,
+      :type => :string,
+      :enum => %w[yes no disabled],
+      :default => 'disabled',
+      :desc => 'Whether rules for the site should be auto-created'
+    desc "init URL [URL]", "Create a sitediff configuration"
+    def init(*urls)
+      unless (1..2).include? urls.size
+        SiteDiff.log "sitediff init requires one or two URLs", :error
+        exit 2
+      end
+
+      chdir([], :search => false)
+      creator = SiteDiff::Config::Creator.new(*urls)
+      creator.create(
+        :depth => options[:depth],
+        :directory => options[:output],
+        :rules => options[:rules] != 'no',
+        :rules_disabled => (options[:rules] == 'disabled'),
+      ) do |tag, info|
+        SiteDiff.log "Visited #{info.uri}, cached"
+      end
+
+      SiteDiff.log "Created #{creator.config_file.expand_path}", :success
+      SiteDiff.log "You can now run 'sitediff diff'", :success
+    end
+
+    option :url,
+      :type => :string,
+      :desc => 'A custom base URL to fetch from'
+    desc "store [CONFIGFILES]",
+      "Cache the current contents of a site for later comparison"
+    def store(*config_files)
+      config = chdir(config_files)
+      config.validate(:need_before => false)
+
+      cache = SiteDiff::Cache.new(:create => true)
+      cache.write_tags << :before
+
+      base = options[:url] || config.after['url']
+      fetcher = SiteDiff::Fetch.new(cache, config.paths, :before => base)
+      fetcher.run do |path, res|
+        SiteDiff.log "Visited #{path}, cached"
+      end
+    end
+
+  private
+    def chdir(files, opts = {})
+      opts = { :config => true, :search => true }.merge(opts)
+
+      dir = options['directory']
+      Dir.chdir(dir) if dir
+
+      return unless opts[:search]
+      begin
+        SiteDiff::Config.new(files, :search => !dir)
+      rescue SiteDiff::Config::ConfigNotFound => e
+        raise if opts[:config]
+        # If no config required, allow it to pass
+      end
     end
   end
 end