sitediff 0.0.1 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 1dc3a624b91cd4b7ef1c926116630cd795532024
-  data.tar.gz: e49f227ae303f574b704ffe3a226f79a120ae30f
+SHA256:
+  metadata.gz: 1b7854497b5e81f48d810acec8106cbc66e33492d046e032e5516f76db26f142
+  data.tar.gz: a9349a79953237dd017600d49d38b8e734afc561f0ce09a1f8732e0e933530c9
 SHA512:
-  metadata.gz: 90ca5508b834d32ac7c96aa6a94a6aa8488921e978e76890e142b1249da20bc620ddcfa237f3defc1e6928d83dd0a22583c9dded150855c320f94140e1bffdf1
-  data.tar.gz: 24bf7969b6f17c269bb407d1ff1684f6556318d0cfa7c6c92a8327ddd0d86ee4f153778affa4d1ef115e47a3b69b31b4dac5f01ed8b7f464a05fd98f9f98212b
+  metadata.gz: 0e91f665f3c59b8a65f16c6942ec49b9cc9ee7fd12b0a777eadb844a0b9819ab1fd9485495bf2c757ca7342a6198dcccb5ae546c4ddf2682f234d015b64309b2
+  data.tar.gz: 64b7980bbbade8710b6069af19a67083678c2bd5fa99674df3360c1c6a3ddf8a15de7c5be4e8349ec298fc1c0dc27535b816089cd4f8852b8c8633861d72a178

data/bin/sitediff

@@ -1,10 +1,16 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 # when run as gem, $0 is /usr/local/bin/sitediff not this file
-if $0 == __FILE__
-  $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+if $PROGRAM_NAME == __FILE__
+  $LOAD_PATH.unshift File.expand_path('../lib', __dir__)
 end
 
-require 'sitediff'
+require 'sitediff/cli'
 
-SiteDiff::Cli.start
+begin
+  SiteDiff::Cli.start
+rescue Interrupt
+  puts("\n")
+  SiteDiff.log('Stopping. Interrupted by user.')
+end

data/lib/sitediff.rb

@@ -1,130 +1,218 @@
 #!/bin/env ruby
-require 'sitediff/cli.rb'
-require 'sitediff/config.rb'
-require 'sitediff/result.rb'
-require 'sitediff/uriwrapper'
-require 'sitediff/util/cache'
-require 'typhoeus'
+# frozen_string_literal: true
+
+require 'sitediff/config'
+require 'sitediff/diff'
+require 'sitediff/fetch'
+require 'sitediff/result'
+require 'sitediff/report'
+require 'pathname'
 require 'rainbow'
+require 'rubygems'
+require 'yaml'
 
+# SiteDiff Object.
 class SiteDiff
-  # path to misc. static files (e.g. erb, css files)
+  attr_reader :config, :results
+
+  # SiteDiff installation directory.
+  ROOT_DIR = File.dirname(File.dirname(__FILE__))
+
+  # Path to misc files. Ex: *.erb, *.css.
   FILES_DIR = File.join(File.dirname(__FILE__), 'sitediff', 'files')
 
-  # subdirectory containing all failing diffs
-  DIFFS_DIR = 'diffs'
-
-  # label will be colorized and str will not be.
-  # type dictates the color: can be :success, :error, or :failure
-  def self.log(str, type=nil, label=nil)
-    label = label ? "[sitediff] #{label}" : '[sitediff]'
-    bg = fg = nil
-    case type
-    when :success
-      bg = :green
-      fg = :black
-    when :failure
-      bg = :red
-    when :error
-      bg = :yellow
+  # Logs a message.
+  #
+  # Label will be colorized and message will not.
+  # Type dictates the color: can be :success, :error, or :failure.
+  #
+  # TODO: Only print :debug messages in debug mode.
+  def self.log(message, type = :info, label = nil)
+    # Prepare label.
+    label ||= type unless type == :info
+    label = label.to_s
+    unless label.empty?
+      # Colorize label.
       fg = :black
+      bg = :blue
+
+      case type
+      when :info
+        bg = :cyan
+      when :success
+        bg = :green
+      when :error
+        bg = :red
+      when :warning
+        bg = :yellow
+      end
+
+      label = '[' + label.to_s + ']'
+      label = Rainbow(label)
+      label = label.bg(bg) if bg
+      label = label.fg(fg) if fg
+
+      # Add a space after the label.
+      label += ' '
     end
-    label = Rainbow(label)
-    label = label.bg(bg) if bg
-    label = label.fg(fg) if fg
-    puts label + ' ' + str
+
+    puts label + message
   end
 
-  attr_reader :config, :results
+  ##
+  # Returns the "before" site's URL.
+  #
+  # TODO: Remove in favor of config.before_url.
   def before
     @config.before['url']
   end
+
+  ##
+  # Returns the "after" site's URL.
+  #
+  # TODO: Remove in favor of config.after_url.
   def after
     @config.after['url']
   end
 
-  def cache=(file)
-    # FIXME: Non-global cache would be nice
-    return unless file
-    if Gem::Version.new(Typhoeus::VERSION) >= Gem::Version.new('0.6.4')
-      Typhoeus::Config.cache = SiteDiff::Util::Cache.new(file)
-    else
-      # Bug, see: https://github.com/typhoeus/typhoeus/pull/296
-      SiteDiff::log("Cache unsupported on Typhoeus version < 0.6.4", :failure)
+  # Initialize SiteDiff.
+  def initialize(config, cache, verbose = true, debug = false)
+    @cache = cache
+    @verbose = verbose
+    @debug = debug
+
+    # Check for single-site mode
+    validate_opts = {}
+    if !config.before['url'] && @cache.tag?(:before)
+      unless @cache.read_tags.include?(:before)
+        raise SiteDiffException,
+              "A cached 'before' is required for single-site mode"
+      end
+      validate_opts[:need_before] = false
     end
-  end
-
-  def initialize(config, cache)
-    config.validate
+    config.validate(validate_opts)
+    # Configure diff.
+    Diff.diff_config(config)
     @config = config
-    self.cache = cache
   end
 
-  # Sanitize an HTML string based on configuration for either before or after
-  def sanitize(html, pos)
-    Sanitize::sanitize(html, @config.send(pos))
+  # Sanitize HTML.
+  def sanitize(path, read_results)
+    %i[before after].map do |tag|
+      html = read_results[tag].content
+      # TODO: See why encoding is empty while running tests.
+      #
+      # The presence of an "encoding" value used to be used to determine
+      # if the sanitizer would be called. However, encoding turns up blank
+      # during rspec tests for some reason.
+      encoding = read_results[tag].encoding
+      if encoding || html.length.positive?
+        section = @config.send(tag, true)
+        Sanitizer.new(html, section, path: path).sanitize
+      else
+        html
+      end
+    end
   end
 
-  # Queues fetching before and after URLs with a Typhoeus::Hydra instance
+  ##
+  # Process a set of read results.
   #
-  # Upon completion of both before and after, prints and saves the diff to
-  # @results.
-  def queue_read(hydra, path)
-    # ( :before | after ) => ReadResult object
-    reads = {}
-    [:before, :after].each do |pos|
-      uri = UriWrapper.new(send(pos) + path)
-
-      uri.queue(hydra) do |res|
-        reads[pos] = res
-        next unless reads.size == 2
-
-        # we have read both before and after; calculate diff
-        if error = reads[:before].error || reads[:after].error
-          diff = Result.new(path, nil, nil, error)
-        else
-          diff = Result.new(path, sanitize(reads[:before].content, :before),
-                            sanitize(reads[:after].content,:after), nil)
-        end
-        diff.log
-        @results[path] = diff
+  # This is the callback that processes items fetched by the Fetcher.
+  def process_results(path, read_results)
+    error = (read_results[:before].error || read_results[:after].error)
+    if error
+      diff = Result.new(path, nil, nil, nil, nil, error)
+    else
+      begin
+        diff = Result.new(
+          path,
+          *sanitize(path, read_results),
+          read_results[:before].encoding,
+          read_results[:after].encoding,
+          nil
+        )
+      rescue StandardError => e
+        raise if @debug
+
+        Result.new(path, nil, nil, nil, nil, "Sanitization error: #{e}")
       end
     end
+    @results[path] = diff
+
+    # Print results in order!
+    while (next_diff = @results[@ordered.first])
+      next_diff.log(@verbose)
+      @ordered.shift
+    end
   end
 
-  # Perform the comparison
+  ##
+  # Compute diff as per config.
+  #
+  # @return [Integer]
+  #   Number of paths which have diffs.
   def run
-    # Map of path -> Result object, queue_read sets callbacks to populate this
+    # Map of path -> Result object, populated by process_results
     @results = {}
+    @ordered = @config.paths.dup
 
-    hydra = Typhoeus::Hydra.new(max_concurrency: 3)
-    @config.paths.each { |path| queue_read(hydra, path) }
-    hydra.run
+    unless @cache.read_tags.empty?
+      SiteDiff.log('Using sites from cache: ' + @cache.read_tags.sort.join(', '))
+    end
+
+    # TODO: Fix this after config merge refactor!
+    # Not quite right. We are not passing @config.before or @config.after
+    # so passing this instead but @config.after['curl_opts'] is ignored.
+    curl_opts = @config.setting :curl_opts
+    config_curl_opts = @config.before['curl_opts']
+    curl_opts = config_curl_opts.clone.merge(curl_opts) if config_curl_opts
+    fetcher = Fetch.new(
+      @cache,
+      @config.paths,
+      @config.setting(:interval),
+      @config.setting(:concurrency),
+      curl_opts,
+      @debug,
+      before: @config.before_url,
+      after: @config.after_url
+    )
+
+    # Run the Fetcher with "process results" as a callback.
+    fetcher.run(&method(:process_results))
 
     # Order by original path order
-    @results = @config.paths.map { |p| @results[p] }
+    @results = @config.paths.map { |path| @results[path] }
+    results.map { |r| r unless r.success? }.compact.length
   end
 
-  # Dump results to disk
-  def dump(dir, report_before, report_after, failing_paths)
-    report_before ||= before
-    report_after ||= after
-    FileUtils.mkdir_p(dir)
-
-    # store diffs of each failing case, first wipe out existing diffs
-    diff_dir = File.join(dir, DIFFS_DIR)
-    FileUtils.rm_rf(diff_dir)
-    results.each { |r| r.dump(dir) if r.status == Result::STATUS_FAILURE }
-    SiteDiff::log "All diff files were dumped inside #{dir}"
-
-    # store failing paths
-    SiteDiff::log "Writing failures to #{failing_paths}"
-    File.open(failing_paths, 'w') do |f|
-      results.each { |r| f.puts r.path unless r.success? }
+  ##
+  # Get a reporter object to help with report generation.
+  def report
+    if @results.nil?
+      raise SiteDiffException(
+        'No results detected. Run SiteDiff.run before SiteDiff.report.'
+      )
     end
 
-    # create report of results
-    report = Diff::generate_html_report(results, report_before, report_after)
-    File.open(File.join(dir, "/report.html") , 'w') { |f| f.write(report) }
+    Report.new(@config, @cache, @results)
+  end
+
+  ##
+  # Get SiteDiff gemspec.
+  def self.gemspec
+    file = ROOT_DIR + '/sitediff.gemspec'
+    Gem::Specification.load(file)
+  end
+
+  ##
+  # Ensures that a directory exists and returns a Pathname for it.
+  #
+  # @param [String] dir
+  #   path/to/directory
+  def self.ensure_dir(dir)
+    dir = Pathname.new(dir) unless dir.is_a? Pathname
+    dir.mkpath unless dir.directory?
+    dir
   end
 end

data/lib/sitediff/cache.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+require 'set'
+require 'fileutils'
+
+class SiteDiff
+  # SiteDiff Cache Handler.
+  class Cache
+    attr_accessor :read_tags, :write_tags
+
+    ##
+    # Creates a Cache object.
+    def initialize(opts = {})
+      @create = opts[:create]
+
+      # Read and Write tags are sets that can contain :before and :after.
+      # They indicate whether we should use the cache for reading or writing.
+      @read_tags = Set.new
+      @write_tags = Set.new
+
+      # The directory used by the cache for storage.
+      @dir = opts[:directory] || '.'
+    end
+
+    ##
+    # Is a tag cached?
+    # TODO: Rename it to is_cached? as it makes more sense.
+    def tag?(tag)
+      File.directory?(File.join(@dir, 'snapshot', tag.to_s))
+    end
+
+    ##
+    # Get data from cache.
+    def get(tag, path)
+      return nil unless @read_tags.include? tag
+
+      filename = File.join(
+        @dir,
+        'snapshot',
+        tag.to_s,
+        *path.split(File::SEPARATOR)
+      )
+
+      filename = File.join(filename, 'index.html') if File.directory?(filename)
+      return nil unless File.file? filename
+
+      Marshal.load(File.read(filename))
+    end
+
+    ##
+    # Set data to cache.
+    def set(tag, path, result)
+      return unless @write_tags.include? tag
+
+      filename = File.join(
+        @dir,
+        'snapshot',
+        tag.to_s,
+        *path.split(File::SEPARATOR)
+      )
+
+      filename = File.join(filename, 'index.html') if File.directory?(filename)
+      filepath = Pathname.new(filename)
+      unless filepath.dirname.directory?
+        begin
+          filepath.dirname.mkpath
+        rescue Errno::EEXIST
+          curdir = filepath
+          curdir = curdir.parent until curdir.exist?
+          tempname = curdir.dirname + (curdir.basename.to_s + '.temporary')
+          # May cause problems if action is not atomic!
+          # Move existing file to dir/index.html first
+          # Not robust! Should generate an UUID or something.
+          if File.exist?(tempname)
+            SiteDiff.log "Overwriting file #{tempname}", :warning
+          end
+          curdir.rename(tempname)
+          filepath.dirname.mkpath
+          # Should only happen in strange situations such as when the path
+          # is foo/index.html/bar (i.e., index.html is a directory)
+          if (curdir + 'index.html').exist?
+            SiteDiff.log "Overwriting file #{tempname}", :warning
+          end
+          tempname.rename(curdir + 'index.html')
+        end
+      end
+      File.open(filename, 'w') { |file| file.write(Marshal.dump(result)) }
+    end
+
+    ##
+    # TODO: Document this or remove it if unused.
+    def key(tag, path)
+      # Ensure encoding stays the same!
+      Marshal.dump([tag, path.encode('UTF-8')])
+    end
+
+    ##
+    # Ensures that a directory exists.
+    def get_dir(directory)
+      # Create the dir. Must go before cache initialization!
+      @dir = Pathname.new(directory || '.')
+      @dir.mkpath unless @dir.directory?
+      @dir.to_s
+    end
+  end
+end