sitediff 0.0.6 → 1.0.0

data/lib/sitediff/report.rb

@@ -0,0 +1,238 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'json'
+require 'minitar'
+require 'sitediff'
+require 'sitediff/config'
+require 'zlib'
+
+class SiteDiff
+  ##
+  # SiteDiff Report Helper.
+  class Report
+    attr_reader :results, :cache
+
+    ##
+    # Directory where diffs will be generated.
+    DIFFS_DIR = 'diffs'
+
+    ##
+    # Name of file containing a list of pages with diffs.
+    FAILURES_FILE = 'failures.txt'
+
+    ##
+    # Name of file containing HTML report of diffs.
+    REPORT_FILE_HTML = 'report.html'
+
+    ##
+    # Name of file containing JSON report of diffs.
+    REPORT_FILE_JSON = 'report.json'
+
+    ##
+    # Name of file containing exported file archive.
+    REPORT_FILE_TAR = 'report.tgz'
+
+    ##
+    # Name of directory in which to build the portable report.
+    REPORT_BUILD_DIR = '_tmp_report'
+
+    ##
+    # Name of the portable report directory.
+    REPORT_DIR = 'report'
+
+    ##
+    # Path to settings used for report.
+    SETTINGS_FILE = 'settings.yaml'
+
+    ##
+    # Creates a Reporter object.
+    #
+    # @param [Config] config.
+    # @param [Cache] cache.
+    # @param [Array] results.
+    def initialize(config, cache, results)
+      @config = config
+      @cache = cache
+      @results = results
+    end
+
+    ##
+    # Generates an HTML report.
+    #
+    # @param [String] dir
+    #   The directory in which the report is to be generated.
+    def generate_html(
+      dir,
+      report_before = nil,
+      report_after = nil
+    )
+      report_before ||= @config.before_url
+      report_after ||= @config.after_url
+
+      dir = SiteDiff.ensure_dir dir
+
+      write_diffs dir
+      write_failures dir
+
+      # Prepare report.
+      report = Diff.generate_html(
+        @results,
+        report_before,
+        report_after,
+        @cache,
+        @config.export
+      )
+
+      # Write report.
+      report_file = dir + REPORT_FILE_HTML
+      report_file.unlink if report_file.file?
+      report_file.open('w') { |f| f.write(report) }
+
+      write_settings dir, report_before, report_after
+
+      if @config.export
+        package_report(dir)
+      else
+        SiteDiff.log 'Report generated to ' + report_file.expand_path.to_s
+      end
+    end
+
+    ##
+    # Generates a JSON report.
+    #
+    # @param dir
+    #   The directory in which the report is to be generated.
+    def generate_json(dir)
+      dir = SiteDiff.ensure_dir dir
+      write_diffs dir
+      write_failures dir
+
+      # Prepare report.
+      report = {
+        paths_compared: @results.length,
+        paths_diffs: 0,
+        paths: {}
+      }
+      @results.each do |item|
+        report[:paths_diffs] += 1 unless item.success?
+
+        item_report = {
+          path: item.path,
+          status: item.status,
+          message: item.error
+        }
+        report[:paths][item.path] = item_report
+      end
+      report = JSON report
+
+      # Write report.
+      report_file = dir + REPORT_FILE_JSON
+      report_file.unlink if report_file.file?
+      report_file.open('w') { |f| f.write(report) }
+
+      write_settings dir
+
+      SiteDiff.log 'Report generated to ' + report_file.expand_path.to_s
+    end
+
+    ##
+    # Package report for export.
+    def package_report(dir)
+      # Create temporaryreport directories.
+      temp_path = dir + REPORT_BUILD_DIR
+      temp_path.rmtree if temp_path.directory?
+      temp_path.mkpath
+      report_path = temp_path + REPORT_DIR
+      report_path.mkpath
+      files_path = report_path + 'files'
+      files_path.mkpath
+      diffs_path = dir + DIFFS_DIR
+
+      # Move files to place.
+      FileUtils.move(dir + REPORT_FILE_HTML, report_path)
+      FileUtils.move(diffs_path, files_path) if diffs_path.directory?
+
+      # Make tar file.
+      Dir.chdir(temp_path) do
+        Minitar.pack(
+          REPORT_DIR,
+          Zlib::GzipWriter.new(File.open(REPORT_FILE_TAR, 'wb'))
+        )
+      end
+      FileUtils.move(temp_path + REPORT_FILE_TAR, dir)
+      temp_path.rmtree
+      SiteDiff.log 'Archived report generated to ' + dir.join(REPORT_FILE_TAR).to_s
+    end
+
+    ##
+    # Creates diff files in a directory named "diffs".
+    #
+    # If "dir" is /foo/bar, then diffs will be placed in /foo/bar/diffs.
+    #
+    # @param [Pathname] dir
+    #   The directory in which a "diffs" directory is to be generated.
+    def write_diffs(dir)
+      raise Exception 'dir must be a Pathname' unless dir.is_a? Pathname
+
+      # Delete existing "diffs" dir, if exists.
+      diff_dir = dir + DIFFS_DIR
+      diff_dir.rmtree if diff_dir.exist?
+
+      # Write diffs to the diff directory.
+      @results.each { |r| r.dump(dir, @config.export) if r.status == Result::STATUS_FAILURE }
+      SiteDiff.log "All diff files written to #{diff_dir.expand_path}" unless @config.export
+    end
+
+    ##
+    # Writes paths with diffs into a file.
+    #
+    # @param [Pathname] dir
+    #   The directory in which the report is to be generated.
+    def write_failures(dir)
+      raise Exception 'dir must be a Pathname' unless dir.is_a? Pathname
+
+      failures = dir + FAILURES_FILE
+      SiteDiff.log "All failures written to #{failures.expand_path}"
+      failures.open('w') do |f|
+        @results.each { |r| f.puts r.path unless r.success? }
+      end
+    end
+
+    ##
+    # Creates report settings.yaml file.
+    #
+    # TODO: Find a way to avoid having to create this file.
+    #
+    # @param [Pathname] dir
+    #   The directory in which the report is to be generated.
+    def write_settings(dir, report_before = nil, report_after = nil)
+      raise Exception 'dir must be a Pathname' unless dir.is_a? Pathname
+
+      settings = {
+        'before' => report_before,
+        'after' => report_after,
+        'cached' => %w[before after]
+      }
+      dir.+(SETTINGS_FILE).open('w') { |f| YAML.dump(settings, f) }
+    end
+
+    ##
+    # Returns CSS for HTML report.
+    def self.css
+      output = ''
+      output += File.read(File.join(SiteDiff::FILES_DIR, 'normalize.css'))
+      output += File.read(File.join(SiteDiff::FILES_DIR, 'sitediff.css'))
+      output
+    end
+
+    ##
+    # Returns JS for HTML report.
+    def self.js
+      output = ''
+      output += File.read(File.join(SiteDiff::FILES_DIR, 'jquery.min.js'))
+      output += File.read(File.join(SiteDiff::FILES_DIR, 'sitediff.js'))
+      output
+    end
+  end
+end

data/lib/sitediff/result.rb

@@ -2,25 +2,42 @@
 
 require 'sitediff'
 require 'sitediff/diff'
+require 'sitediff/report'
 require 'digest/sha1'
 require 'fileutils'
 
 class SiteDiff
-  class Result < Struct.new(:path, :before, :after, :before_encoding, :after_encoding, :error, :verbose)
+  # SiteDiff Result Object.
+  class Result < Struct.new(
+    :path,
+    :before,
+    :after,
+    :before_encoding,
+    :after_encoding,
+    :error,
+    :verbose
+  )
     STATUS_SUCCESS  = 0   # Identical before and after
     STATUS_FAILURE  = 1   # Different before and after
     STATUS_ERROR    = 2   # Couldn't fetch page
-    STATUS_TEXT = %w[success failure error].freeze
+    STATUS_TEXT = %w[unchanged changed error].freeze
 
     attr_reader :status, :diff
 
+    ##
+    # Creates a Result.
     def initialize(*args)
       super
       if error
         @status = STATUS_ERROR
       else
         if !before_encoding || !after_encoding
-          @diff = Diff.binary_diffy(before, after, before_encoding, after_encoding)
+          @diff = Diff.binary_diffy(
+            before,
+            after,
+            before_encoding,
+            after_encoding
+          )
         else
           @diff = Diff.html_diffy(before, after)
         end
@@ -28,10 +45,22 @@ class SiteDiff
       end
     end
 
+    ##
+    # Whether the result has no diff.
+    #
+    # If there is a diff, it is not a success.
+    #
+    # TODO: Change "Success" to unchanged.
     def success?
       status == STATUS_SUCCESS
     end
 
+    ##
+    # Whether the result has an error.
+    def error?
+      status == STATUS_ERROR
+    end
+
     # Textual representation of the status
     def status_text
       STATUS_TEXT[status]
@@ -45,38 +74,37 @@ class SiteDiff
 
     # Filename to store diff
     def filename
-      File.join(SiteDiff::DIFFS_DIR, Digest::SHA1.hexdigest(path) + '.html')
+      File.join(Report::DIFFS_DIR, Digest::SHA1.hexdigest(path) + '.html')
     end
 
-    # Text of the link in the HTML report
-    def link
-      case status
-      when STATUS_ERROR then error
-      when STATUS_SUCCESS then status_text
-      when STATUS_FAILURE then "<a href='#{filename}'>DIFF</a>"
-      end
+    # Returns a URL to the result diff.
+    #
+    # Returns nil if the result has no diffs.
+    def diff_url(relative = false)
+      prefix = relative ? 'files/' : '/files/'
+      return prefix + filename if status == STATUS_FAILURE
     end
 
     # Log the result to the terminal
     def log(verbose = true)
       case status
-      when STATUS_SUCCESS then
-        SiteDiff.log path, :diff_success, 'UNCHANGED'
-      when STATUS_ERROR then
-        SiteDiff.log path, :warn, "ERROR (#{error})"
-      when STATUS_FAILURE then
-        SiteDiff.log path, :diff_failure, 'CHANGED'
+      when STATUS_SUCCESS
+        SiteDiff.log path, :success, 'UNCHANGED'
+      when STATUS_ERROR
+        SiteDiff.log path + " (#{error})", :warning, 'ERROR'
+      when STATUS_FAILURE
+        SiteDiff.log path, :error, 'CHANGED'
         puts Diff.terminal_diffy(before, after) if verbose
       end
     end
 
     # Dump the result to a file
-    def dump(dir)
+    def dump(dir, relative = false)
       dump_path = File.join(dir, filename)
       base = File.dirname(dump_path)
       FileUtils.mkdir_p(base) unless File.exist?(base)
       File.open(dump_path, 'w') do |f|
-        f.write(Diff.generate_diff_output(self))
+        f.write(Diff.generate_diff_output(self, relative))
       end
     end
   end

data/lib/sitediff/sanitize.rb

@@ -8,21 +8,26 @@ require 'nokogiri'
 require 'set'
 
 class SiteDiff
+  # SiteDiff Sanitizer.
   class Sanitizer
     class InvalidSanitization < SiteDiffException; end
 
     TOOLS = {
       array: %w[dom_transform sanitization],
-      scalar: %w[selector remove_spacing]
+      scalar: %w[selector remove_spacing ignore_whitespace]
     }.freeze
-    DOM_TRANSFORMS = Set.new(%w[remove unwrap_root unwrap remove_class])
+    DOM_TRANSFORMS = Set.new(%w[remove strip unwrap_root unwrap remove_class])
 
+    ##
+    # Creates a Sanitizer.
     def initialize(html, config, opts = {})
       @html = html
       @config = config
       @opts = opts
     end
 
+    ##
+    # Performs sanitization.
     def sanitize
       return '' if @html == '' # Quick return on empty input
 
@@ -56,13 +61,13 @@ class SiteDiff
     def canonicalize_rule(name)
       (rules = @config[name]) || (return nil)
 
-      if rules[0]&.respond_to?(:[]) && rules[0]['value']
-        # Already an array
+      # Already an array? Do nothing.
+      if rules[0]&.respond_to?('each') && rules[0]&.fetch('value')
+      # If it is a hash, put it in an array.
       elsif rules['value']
-        # Hash, put it in an array
         rules = [rules]
+      # If it is a scalar value, put it in an array.
       else
-        # Scalar, put it in a hash
         rules = [{ 'value' => rules }]
       end
 
@@ -99,7 +104,13 @@ class SiteDiff
       # Prevent potential UTF-8 encoding errors by removing bytes
       # Not the only solution. An alternative is to return the
       # string unmodified.
-      @html = @html.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
+      @html = @html.encode(
+        'UTF-8',
+        'binary',
+        invalid: :replace,
+        undef: :replace,
+        replace: ''
+      )
       global.each { |r| r.apply(@html) }
     end
 
@@ -151,7 +162,13 @@ class SiteDiff
       # Prevent potential UTF-8 encoding errors by removing invalid bytes.
       # Not the only solution.
       # An alternative is to return the string unmodified.
-      str = str.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
+      str = str.encode(
+        'UTF-8',
+        'binary',
+        invalid: :replace,
+        undef: :replace,
+        replace: ''
+      )
       # Remove xml declaration and <html> tags
       str.sub!(/\A<\?xml.*$\n/, '')
       str.sub!(/\A^<html>$\n/, '')
@@ -164,6 +181,10 @@ class SiteDiff
       # Remove blank lines
       str.gsub!(/^\s*$\n/, '')
 
+      # Remove DOS newlines
+      str.gsub!(/\x0D$/, '')
+      str.gsub!(/&#13;$/, '')
+
       str
     end
 

data/lib/sitediff/sanitize/dom_transform.rb

@@ -11,18 +11,25 @@ class SiteDiff
     #  * { :type => "unwrap", :selector => "div.field-item" }
     #  * { :type => "remove", :selector => "div.extra-stuff" }
     #  * { :type => "remove_class", :class => 'class1' }
+    #  * { :type => "strip", :selector => 'h1' }
     class DomTransform
-      Transforms = {}
+      # Supported dom_transform types.
+      TRANSFORMS = {}
 
+      ##
+      # Creates a DOM Transform.
       def initialize(rule)
         @rule = rule
       end
 
+      ##
       # Often an array or scalar are both ok values. Turn either into an array.
       def to_array(val)
         [val].flatten
       end
 
+      ##
+      # TODO: Document what this method does.
       def targets(node)
         selectors = to_array(@rule['selector'])
         selectors.each do |sel|
@@ -30,42 +37,70 @@ class SiteDiff
         end
       end
 
+      ##
+      # Applies the transformation to a DOM node.
       def apply(node)
         targets(node) { |t| process(t) }
       end
 
+      ##
+      # Registers a DOM Transform plugin.
       def self.register(name)
-        Transforms[name] = self
+        TRANSFORMS[name] = self
       end
 
+      ##
+      # Creates a DOM Transform as per rule.
       def self.create(rule)
         (type = rule['type']) ||
           raise(InvalidSanitization, 'DOM transform needs a type')
-        (transform = Transforms[type]) ||
+        (transform = TRANSFORMS[type]) ||
           raise(InvalidSanitization, "No DOM transform named #{type}")
         transform.new(rule)
       end
 
-      # Remove elements matching 'selector'
+      ##
+      # Remove elements matching 'selector'.
       class Remove < DomTransform
         register 'remove'
+
+        ##
+        # Processes a node.
         def process(node)
           node.remove
         end
       end
 
-      # Unwrap elements matching 'selector'
+      # Squeeze whitespace from a tag matching 'selector'.
+      class Strip < DomTransform
+        register 'strip'
+
+        ##
+        # Processes a node.
+        def process(node)
+          node.content = node.content.strip
+        end
+      end
+
+      # Unwrap elements matching 'selector'.
       class Unwrap < DomTransform
         register 'unwrap'
+
+        ##
+        # Processes a node.
         def process(node)
           node.add_next_sibling(node.children)
           node.remove
         end
       end
 
+      ##
       # Remove classes from elements matching selector
       class RemoveClass < DomTransform
         register 'remove_class'
+
+        ##
+        # Processes a node.
         def process(node)
           classes = to_array(@rule['class'])
 
@@ -77,9 +112,13 @@ class SiteDiff
         end
       end
 
-      # Unwrap the root element
+      ##
+      # Unwrap the root element.
       class UnwrapRoot < DomTransform
         register 'unwrap_root'
+
+        ##
+        # Applies the transformation to a DOM node.
         def apply(node)
           (node.children.size == 1) ||
             raise(InvalidSanitization, 'Multiple root elements in unwrap_root')