sitediff 0.0.1 → 0.0.2

data/lib/sitediff/config.rb

@@ -1,13 +1,18 @@
+require 'sitediff/exception'
+require 'sitediff/sanitize'
+require 'pathname'
 require 'yaml'
 
 class SiteDiff
   class Config
+    DEFAULT_FILENAME = 'sitediff.yaml'
 
     # keys allowed in configuration files
-    CONF_KEYS = Sanitize::TOOLS.values.flatten(1) +
+    CONF_KEYS = Sanitizer::TOOLS.values.flatten(1) +
                 %w[paths before after before_url after_url includes]
 
-    class InvalidConfig < Exception; end
+    class InvalidConfig < SiteDiffException; end
+    class ConfigNotFound < SiteDiffException; end
 
     # Takes a Hash and normalizes it to the following form by merging globals
     # into before and after. A normalized config Hash looks like this:
@@ -27,7 +32,7 @@ class SiteDiff
     #       selector: body
     #
     def self.normalize(conf)
-      tools = Sanitize::TOOLS
+      tools = Sanitizer::TOOLS
 
       # merge globals
       %w[before after].each do |pos|
@@ -67,7 +72,7 @@ class SiteDiff
           next
         end
         result[pos] = first[pos].merge!(second[pos]) do |key, a, b|
-          if Sanitize::TOOLS[:array].include? key # rule 2a
+          if Sanitizer::TOOLS[:array].include? key # rule 2a
             result[pos][key] = (a || []) + (b|| [])
           else
             result[pos][key] = a || b # rule 2b
@@ -77,9 +82,39 @@ class SiteDiff
       result
     end
 
-    def initialize(files)
+    # Search for a config file. If found, change to the containing directory,
+    # and return an array of config files found.
+    def self.search
+      subdirs = %w[. sitediff]
+      root_indicators = %w[.git .svn]
+
+      Pathname.pwd.ascend do |dir|
+        subdirs.each do |subdir|
+          d = dir + subdir + DEFAULT_FILENAME
+          if d.exist?
+            Dir.chdir(dir.+(subdir).to_s)
+            return [DEFAULT_FILENAME]
+          end
+        end
+
+        root_indicators.each { |r| return [] if dir.+(r).exist? }
+      end
+
+      return []
+    end
+
+    def initialize(files, opts = {})
       @config = {'paths' => [], 'before' => {}, 'after' => {} }
+
+      files = Config.search if files.empty? && opts[:search]
+      files = [DEFAULT_FILENAME] if files.empty? &&
+        File.exists?(DEFAULT_FILENAME)
+      raise ConfigNotFound, "No configuration file found." if files.empty?
+
       files.each do |file|
+        raise InvalidConfig,
+          "Missing config file %s." % File.expand_path(file) \
+          unless File.exist?(file)
         @config = Config::merge(@config, Config::load_conf(file))
       end
     end
@@ -99,8 +134,11 @@ class SiteDiff
     end
 
     # Checks if the configuration is usable for diff-ing.
-    def validate
-      raise InvalidConfig, "Undefined 'before' base URL." unless before['url']
+    def validate(opts = {})
+      opts = { :need_before => true }.merge(opts)
+
+      raise InvalidConfig, "Undefined 'before' base URL." if \
+        opts[:need_before] && !before['url']
       raise InvalidConfig, "Undefined 'after' base URL." unless after['url']
       raise InvalidConfig, "Undefined 'paths'." unless (paths and !paths.empty?)
     end
@@ -114,7 +152,7 @@ class SiteDiff
 
     # reads a YAML file and raises an InvalidConfig if the file is not valid.
     def self.load_raw_yaml(file)
-      SiteDiff::log "Reading config file: #{file}"
+      SiteDiff::log "Reading config file: #{Pathname.new(file).expand_path}"
       conf = YAML.load_file(file) || {}
       unless conf.is_a? Hash
         raise InvalidConfig, "Invalid configuration file: '#{file}'"
@@ -149,6 +187,5 @@ class SiteDiff
       end
       conf
     end
-
   end
 end

data/lib/sitediff/config/creator.rb

@@ -0,0 +1,122 @@
+require 'sitediff/cache'
+require 'sitediff/config'
+require 'sitediff/crawler'
+require 'sitediff/rules'
+require 'pathname'
+require 'typhoeus'
+require 'yaml'
+
+class SiteDiff
+class Config
+class Creator
+  def initialize(*urls, &block)
+    @after = urls.pop
+    @before = urls.pop # May be nil
+  end
+
+  def roots
+    @roots = begin
+      r = { :after => @after }
+      r[:before] = @before if @before
+      r
+    end
+  end
+
+  # Build a config structure, return it
+  def create(opts, &block)
+    @config = {}
+    @callback = block
+
+    # Handle options
+    @dir = Pathname.new(opts[:directory])
+    @depth = opts[:depth]
+    @rules = Rules.new(@config, opts[:rules_disabled]) if opts[:rules]
+
+    # Create the dir. Must go before cache initialization!
+    @dir.mkpath unless @dir.directory?
+
+    # Setup instance vars
+    @paths = Hash.new { |h,k| h[k] = Set.new }
+    @cache = Cache.new(:file => @dir.+(Cache::DEFAULT_FILENAME).to_s,
+      :create => true)
+    @cache.write_tags << :before << :after
+
+    build_config
+    write_config
+  end
+
+  def build_config
+    %w[before after].each do |tag|
+      next unless u = roots[tag.to_sym]
+      @config[tag] = {'url' => u}
+    end
+
+    crawl(@depth)
+    @cache.close
+    @rules.add_config if @rules
+
+    @config['paths'] = @paths.values.reduce(&:|).to_a.sort
+  end
+
+  def crawl(depth = nil)
+    hydra = Typhoeus::Hydra.new(max_concurrency: 10)
+    roots.each do |tag, u|
+      Crawler.new(hydra, u, depth) do |info|
+        crawled_path(tag, info)
+      end
+    end
+    hydra.run
+  end
+
+  # Deduplicate paths with slashes at the end
+  def canonicalize(tag, path)
+    def altered_paths(path)
+      yield path + '/'
+      yield path.sub(%r[/$], '')
+    end
+
+    return path.empty? ? '/' : path
+  end
+
+  def crawled_path(tag, info)
+    path, dup = canonicalize(tag, info.relative)
+    return if dup
+
+    res = info.read_result
+
+    @callback[tag, info]
+    @paths[tag] << path
+    @cache.set(tag, path, res)
+
+    # If single-site, cache after as before!
+    @cache.set(:before, path, res) unless roots[:before]
+
+    @rules.handle_page(tag, res.content, info.document) if @rules && !res.error
+  end
+
+  # Create a gitignore if we seem to be in git
+  def make_gitignore(dir)
+    # Check if we're in git
+    return unless dir.realpath.to_enum(:ascend).any? { |d| d.+('.git').exist? }
+
+    dir.+('.gitignore').open('w') do |f|
+      f.puts <<-EOF.gsub(/^\s+/, '')
+        output
+        cache.db
+        cache.db.db
+      EOF
+    end
+  end
+
+  def config_file
+    @dir + Config::DEFAULT_FILENAME
+  end
+
+  # Turn a config structure into a config file
+  def write_config
+    make_gitignore(@dir)
+    config_file.open('w') { |f| f.puts @config.to_yaml }
+  end
+end
+end
+end

data/lib/sitediff/crawler.rb

@@ -0,0 +1,95 @@
+require 'sitediff'
+require 'sitediff/uriwrapper'
+require 'addressable/uri'
+require 'nokogiri'
+require 'ostruct'
+require 'set'
+
+class SiteDiff
+class Crawler
+  class Info < OpenStruct; end
+
+  DEFAULT_DEPTH = 3
+
+  # Create a crawler with a base URL
+  def initialize(hydra, base, depth = DEFAULT_DEPTH, &block)
+    @hydra = hydra
+    @base_uri = Addressable::URI.parse(base)
+    @base = base
+    @found = Set.new
+    @callback = block
+
+    add_uri('', depth)
+  end
+
+  # Handle a newly found relative URI
+  def add_uri(rel, depth)
+    return if @found.include? rel
+    @found << rel
+
+    wrapper = UriWrapper.new(@base + rel)
+    wrapper.queue(@hydra) do |res|
+      fetched_uri(rel, depth, res)
+    end
+  end
+
+  # Handle the fetch of a URI
+  def fetched_uri(rel, depth, res)
+    return unless res.content # Ignore errors
+    return unless depth >= 0
+
+    base = Addressable::URI.parse(@base + rel)
+    doc = Nokogiri::HTML(res.content)
+
+    # Call the callback
+    info = Info.new(
+      :relative => rel,
+      :uri => base,
+      :read_result => res,
+      :document => doc,
+    )
+    @callback[info]
+
+    # Find links
+    links = find_links(doc)
+    uris = links.map { |l| resolve_link(base, l) }.compact
+    uris = filter_links(uris)
+
+    # Make them relative
+    rels = uris.map { |u| relativize_link(u) }
+
+    # Queue them in turn
+    rels.each do |r|
+      next if @found.include? r
+      add_uri(r, depth - 1)
+    end
+  end
+
+  # Resolve a potentially-relative link. Return nil on error.
+  def resolve_link(base, rel)
+    begin
+      return base + rel
+    rescue Addressable::URI::InvalidURIError
+      SiteDiff.log "skipped invalid URL: '#{rel}'", :warn
+      return nil
+    end
+  end
+
+  # Make a link relative to @base_uri
+  def relativize_link(uri)
+    uri.path.slice(@base_uri.path.length, uri.path.length)
+  end
+
+  # Return a list of string links found on a page.
+  def find_links(doc)
+    return doc.xpath('//a[@href]').map { |e| e['href'] }
+  end
+
+  # Filter out links we don't want. Links passed in are absolute URIs.
+  def filter_links(uris)
+    uris.find_all do |u|
+      u.host == @base_uri.host && u.path.start_with?(@base_uri.path)
+    end
+  end
+end
+end

data/lib/sitediff/diff.rb

@@ -1,3 +1,4 @@
+require 'sitediff'
 require 'diffy'
 require 'erb'
 require 'rainbow'
@@ -19,7 +20,7 @@ class SiteDiff
         to_s(*args)
     end
 
-    def generate_html_report(results, before, after)
+    def generate_html_report(results, before, after, cache)
       erb_path = File.join(SiteDiff::FILES_DIR, 'html_report.html.erb')
       report_html = ERB.new(File.read(erb_path)).result(binding)
       return report_html

data/lib/sitediff/exception.rb

@@ -0,0 +1,3 @@
+class SiteDiff
+  class SiteDiffException < Exception; end
+end

data/lib/sitediff/fetch.rb

@@ -0,0 +1,55 @@
+require 'sitediff/uriwrapper'
+require 'typhoeus'
+
+class SiteDiff
+class Fetch
+  # Cache is a cache object, see sitediff/cache
+  # Paths is a list of sub-paths
+  # Tags is a hash of tag names => base URLs.
+  def initialize(cache, paths, tags)
+    @cache = cache
+    @paths = paths
+    @tags = tags
+  end
+
+  # Fetch all the paths, once per tag.
+  # When a path has been fetched for every tag, block will be called with the
+  # path, and a hash of tag => UriWrapper::ReadResult objects.
+  def run(&block)
+    @callback = block
+    @hydra = Typhoeus::Hydra.new(max_concurrency: 3)
+    @paths.each { |path| queue_path(path) }
+    @hydra.run
+  end
+
+private
+  # Queue a path for fetching
+  def queue_path(path)
+    results = {}
+
+    @tags.each do |tag, base|
+      if res = @cache.get(tag, path)
+        results[tag] = res
+        process_results(path, results)
+      elsif !base
+        # We only have the cache, but this item isn't cached!
+        results[tag] = UriWrapper::ReadResult.error("Not cached")
+        process_results(path, results)
+      else
+        uri = UriWrapper.new(base + path)
+        uri.queue(@hydra) do |res|
+          @cache.set(tag, path, res)
+          results[tag] = res
+          process_results(path, results)
+        end
+      end
+    end
+  end
+
+  # Process fetch results
+  def process_results(path, results)
+    return unless results.size == @tags.size
+    @callback[path, results]
+  end
+end
+end

data/lib/sitediff/files/html_report.html.erb

@@ -11,14 +11,22 @@
   <body>
     <div class="sitediff">
       <div class="legend">
-        <strong>before</strong> (base url): <a href="<%=before%>"><%=before%></a> |
-        <strong>after </strong> (base url): <a href="<%=after%>" ><%=after %></a>
+        <%
+           tags = %w[before after]
+           tags.each do |tag| %>
+             <% if tags.first != tag %> | <% end %>
+             <% notes = ['base url']
+                notes << 'cached' if cache.read_tags.include?(tag.to_sym) %>
+             <strong><%= tag %></strong> (<%= notes.join(', ') %>):
+               <a href="<%= eval(tag) %>"><%= eval(tag) %></a>
+        <% end %>
       </div>
       <table class="results">
 
         <colgroup>
           <col class="before-col">
           <col class="after-col">
+          <col class="both-col">
           <col class="path-col">
           <col class="diff-stat-col">
         </colgroup>
@@ -27,6 +35,7 @@
           <tr>
             <th> Before </th>
             <th> After </th>
+            <th> Both </th>
             <th> Path </th>
             <th> Status </th>
           </tr>
@@ -34,8 +43,15 @@
 
         <% results.each do |result| %>
         <tr class="<%= result.status_text %>">
-          <td class="before"><a href="<%= result.url(before) %>">[before]</a></td>
-          <td class="after"><a href="<%= result.url(after) %>">[after]</a></td>
+          <td class="before">
+            <a href="<%= result.url(:before, before, cache) %>">[before]</a>
+          </td>
+          <td class="after">
+            <a href="<%= result.url(:after, after, cache) %>">[after]</a>
+          </td>
+          <td class="both">
+            <a href="/sidebyside<%= result.path %>">[both]</a>
+          </td>
           <td class="path"><%= result.path %></td>
           <td class="status"><%= result.link %></td>
         </tr>