sitediff 0.0.6 → 1.2.0

data/lib/sitediff/presets/drupal.yaml

@@ -0,0 +1,63 @@
+sanitization:
+  - title: Strip Drupal.settings
+    selector: script
+    pattern: '^(<script>)?jQuery.extend\(Drupal.settings.*$'
+  - title: Strip IE CSS/JS cache IDs
+    pattern: '("[^"]*ie\d?\.(js|css))\?[a-z0-9]{6}"'
+    substitute: '\1'
+  - title: Strip form build ID
+    selector: input
+    pattern: 'name="form_build_id" value="form-[-\w]{40,43}"'
+    substitute: 'name="form_build_id" value="form-DRUPAL_FORM_BUILD_ID"'
+  - title: Strip view DOM ID
+    pattern: '(class="view .*) view-dom-id-[a-f0-9]{32}"'
+    substitute: '\1 view-dom-id-DRUPAL_VIEW_DOM_ID"'
+  - title: Strip CSS aggregation filenames
+    selector: link[rel=stylesheet]
+    pattern: '(href="[^"]*/files/css/css_)[-\w]{40,43}\.css"'
+    substitute: '\1DRUPAL_AGGREGATED_CSS.css"'
+  - title: Strip JS aggregation filenames
+    selector: script
+    pattern: '(src="[^"]*/files/js/js_)[-\w]{40,43}\.js"'
+    substitute: '\1DRUPAL_AGGREGATED_JS.js"'
+  - title: Strip CSS/JS cache IDs
+    selector: style, script
+    pattern: '("[^"]*\.(js|css))\?[a-z0-9]{6}"'
+    substitute: '\1'
+  - title: Strip Drupal JS version tags
+    selector: script
+    pattern: '(src="[^"]*/misc/\w+\.js)?v=\d+\.\d+"'
+    substitute: '\1'
+  - title: Strip domain names from absolute URLs
+    pattern: 'http:\/\/[a-zA-Z0-9.:-]+'
+    substitute: '__domain__'
+  - title: Strip form build ID
+    selector: input
+    pattern: 'autocomplete="off" data-drupal-selector="form-[-\w]{40,43}"'
+    substitute: 'autocomplete="off" data-drupal-selector="form-DRUPAL_FORM_BUILD_ID"'
+  - title: Strip form build ID 2
+    selector: input
+    pattern: 'name="form_build_id" value="form-[-\w]{40,43}"'
+    substitute: 'name="form_build_id" value="form-DRUPAL_FORM_BUILD_ID"'
+  - title: Strip Drupal CSS link queries
+    selector: link
+    pattern: '\.css\?(\w*)'
+    substitute: '\.css'
+  - title: Strip Drupal JS link queries
+    selector: script
+    pattern: '\.js\?(\w*)'
+    substitute: '\.js'
+  - title: Strip Drupal View-DOM ID
+    pattern: 'view-dom-id-\w*'
+    substitute: 'view-dom-id-_ID_'
+  - title: Strip Drupal View-DOM ID 2
+    pattern: '(views?_dom_id"?:"?)\w*'
+    substitute: '\1_ID_'
+  - title: Ignore Drupal CSS file names
+    selector: link
+    pattern: 'css_[-\w]{40,43}(\\|%5C)?\.css'
+    substitute: 'css__ID__.css'
+  - title: Ignore Drupal JS file names
+    selector: script
+    pattern: 'js_[-\w]{40,43}\\?\.js'
+    substitute: 'js__ID__.js'

data/lib/sitediff/report.rb

@@ -0,0 +1,254 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'json'
+require 'minitar'
+require 'sitediff'
+require 'sitediff/config'
+require 'zlib'
+
+class SiteDiff
+  ##
+  # SiteDiff Report Helper.
+  class Report
+    attr_reader :results, :cache
+
+    ##
+    # Directory where diffs will be generated.
+    DIFFS_DIR = 'diffs'
+
+    ##
+    # Name of file containing a list of pages with diffs.
+    FAILURES_FILE = 'failures.txt'
+
+    ##
+    # Name of file containing HTML report of diffs.
+    REPORT_FILE_HTML = 'report.html'
+
+    ##
+    # Name of file containing JSON report of diffs.
+    REPORT_FILE_JSON = 'report.json'
+
+    ##
+    # Name of file containing exported file archive.
+    REPORT_FILE_TAR = 'report.tgz'
+
+    ##
+    # Name of directory in which to build the portable report.
+    REPORT_BUILD_DIR = '_tmp_report'
+
+    ##
+    # Name of the portable report directory.
+    REPORT_DIR = 'report'
+
+    ##
+    # Path to settings used for report.
+    SETTINGS_FILE = 'settings.yaml'
+
+    ##
+    # Creates a Reporter object.
+    #
+    # @param [Config] config.
+    # @param [Cache] cache.
+    # @param [Array] results.
+    def initialize(config, cache, results)
+      @config = config
+      @cache = cache
+      @results = results
+    end
+
+    ##
+    # Generates an HTML report.
+    #
+    # @param [String] dir
+    #   The directory in which the report is to be generated.
+    def generate_html(
+      dir,
+      report_before = nil,
+      report_after = nil
+    )
+      report_before ||= @config.before_url
+      report_after ||= @config.after_url
+      @config.before_time = get_timestamp(:before)
+      @config.after_time = get_timestamp(:after)
+
+      dir = SiteDiff.ensure_dir dir
+
+      write_diffs dir
+      write_failures dir
+
+      # Prepare report.
+      report = Diff.generate_html(
+        @results,
+        report_before,
+        report_after,
+        @cache,
+        @config
+      )
+
+      # Write report.
+      report_file = dir + REPORT_FILE_HTML
+      report_file.unlink if report_file.file?
+      report_file.open('w') { |f| f.write(report) }
+
+      write_settings dir, report_before, report_after
+
+      if @config.export
+        package_report(dir)
+      else
+        SiteDiff.log "Report generated to #{report_file.expand_path}"
+      end
+    end
+
+    ##
+    # Generates a JSON report.
+    #
+    # @param dir
+    #   The directory in which the report is to be generated.
+    def generate_json(dir)
+      dir = SiteDiff.ensure_dir dir
+      write_diffs dir
+      write_failures dir
+
+      # Prepare report.
+      report = {
+        paths_compared: @results.length,
+        paths_diffs: 0,
+        paths: {}
+      }
+      @results.each do |item|
+        report[:paths_diffs] += 1 unless item.success?
+
+        item_report = {
+          path: item.path,
+          status: item.status,
+          message: item.error
+        }
+        report[:paths][item.path] = item_report
+      end
+      report = JSON report
+
+      # Write report.
+      report_file = dir + REPORT_FILE_JSON
+      report_file.unlink if report_file.file?
+      report_file.open('w') { |f| f.write(report) }
+
+      write_settings dir
+
+      SiteDiff.log "Report generated to #{report_file.expand_path}"
+    end
+
+    ##
+    # Package report for export.
+    def package_report(dir)
+      # Create temporaryreport directories.
+      temp_path = dir + REPORT_BUILD_DIR
+      temp_path.rmtree if temp_path.directory?
+      temp_path.mkpath
+      report_path = temp_path + REPORT_DIR
+      report_path.mkpath
+      files_path = "#{report_path}files"
+      files_path.mkpath
+      diffs_path = dir + DIFFS_DIR
+
+      # Move files to place.
+      FileUtils.move(dir + REPORT_FILE_HTML, report_path)
+      FileUtils.move(diffs_path, files_path) if diffs_path.directory?
+
+      # Make tar file.
+      Dir.chdir(temp_path) do
+        Minitar.pack(
+          REPORT_DIR,
+          Zlib::GzipWriter.new(File.open(REPORT_FILE_TAR, 'wb'))
+        )
+      end
+      FileUtils.move(temp_path + REPORT_FILE_TAR, dir)
+      temp_path.rmtree
+      SiteDiff.log "Archived report generated to #{dir.join(REPORT_FILE_TAR)}"
+    end
+
+    ##
+    # Creates diff files in a directory named "diffs".
+    #
+    # If "dir" is /foo/bar, then diffs will be placed in /foo/bar/diffs.
+    #
+    # @param [Pathname] dir
+    #   The directory in which a "diffs" directory is to be generated.
+    def write_diffs(dir)
+      raise Exception 'dir must be a Pathname' unless dir.is_a? Pathname
+
+      # Delete existing "diffs" dir, if exists.
+      diff_dir = dir + DIFFS_DIR
+      diff_dir.rmtree if diff_dir.exist?
+
+      # Write diffs to the diff directory.
+      @results.each { |r| r.dump(dir, relative: @config.export) if r.status == Result::STATUS_FAILURE }
+      SiteDiff.log "All diff files written to #{diff_dir.expand_path}" unless @config.export
+    end
+
+    ##
+    # Writes paths with diffs into a file.
+    #
+    # @param [Pathname] dir
+    #   The directory in which the report is to be generated.
+    def write_failures(dir)
+      raise Exception 'dir must be a Pathname' unless dir.is_a? Pathname
+
+      failures = dir + FAILURES_FILE
+      SiteDiff.log "All failures written to #{failures.expand_path}"
+      failures.open('w') do |f|
+        @results.each { |r| f.puts r.path unless r.success? }
+      end
+    end
+
+    ##
+    # Creates report settings.yaml file.
+    #
+    # TODO: Find a way to avoid having to create this file.
+    #
+    # @param [Pathname] dir
+    #   The directory in which the report is to be generated.
+    def write_settings(dir, report_before = nil, report_after = nil)
+      raise Exception 'dir must be a Pathname' unless dir.is_a? Pathname
+
+      settings = {
+        'before' => report_before,
+        'after' => report_after,
+        'cached' => %w[before after]
+      }
+      dir.+(SETTINGS_FILE).open('w') { |f| YAML.dump(settings, f) }
+    end
+
+    ##
+    # Returns CSS for HTML report.
+    def self.css
+      output = ''
+      output += File.read(File.join(SiteDiff::FILES_DIR, 'normalize.css'))
+      output += File.read(File.join(SiteDiff::FILES_DIR, 'sitediff.css'))
+      output
+    end
+
+    ##
+    # Returns JS for HTML report.
+    def self.js
+      output = ''
+      output += File.read(File.join(SiteDiff::FILES_DIR, 'jquery.min.js'))
+      output += File.read(File.join(SiteDiff::FILES_DIR, 'sitediff.js'))
+      output
+    end
+
+    private
+
+    # Get crawl timestamps
+    def get_timestamp(tag)
+      timestamp_file = File.join(@config.directory, 'snapshot', tag.to_s, SiteDiff::Cache::TIMESTAMP_FILE)
+      if File.exist? timestamp_file
+        file = File::Stat.new(timestamp_file)
+        time = file.mtime
+        time.instance_of?(Time) ? time.strftime('%Y-%m-%d %H:%M') : ''
+      else
+        'unknown'
+      end
+    end
+  end
+end

data/lib/sitediff/result.rb

@@ -2,25 +2,42 @@
 
 require 'sitediff'
 require 'sitediff/diff'
+require 'sitediff/report'
 require 'digest/sha1'
 require 'fileutils'
 
 class SiteDiff
-  class Result < Struct.new(:path, :before, :after, :before_encoding, :after_encoding, :error, :verbose)
+  # SiteDiff Result Object.
+  class Result < Struct.new(
+    :path,
+    :before,
+    :after,
+    :before_encoding,
+    :after_encoding,
+    :error,
+    :verbose
+  )
     STATUS_SUCCESS  = 0   # Identical before and after
     STATUS_FAILURE  = 1   # Different before and after
     STATUS_ERROR    = 2   # Couldn't fetch page
-    STATUS_TEXT = %w[success failure error].freeze
+    STATUS_TEXT = %w[unchanged changed error].freeze
 
     attr_reader :status, :diff
 
+    ##
+    # Creates a Result.
     def initialize(*args)
       super
       if error
         @status = STATUS_ERROR
       else
         if !before_encoding || !after_encoding
-          @diff = Diff.binary_diffy(before, after, before_encoding, after_encoding)
+          @diff = Diff.binary_diffy(
+            before,
+            after,
+            before_encoding,
+            after_encoding
+          )
         else
           @diff = Diff.html_diffy(before, after)
         end
@@ -28,10 +45,22 @@ class SiteDiff
       end
     end
 
+    ##
+    # Whether the result has no diff.
+    #
+    # If there is a diff, it is not a success.
+    #
+    # TODO: Change "Success" to unchanged.
     def success?
       status == STATUS_SUCCESS
     end
 
+    ##
+    # Whether the result has an error.
+    def error?
+      status == STATUS_ERROR
+    end
+
     # Textual representation of the status
     def status_text
       STATUS_TEXT[status]
@@ -39,44 +68,45 @@ class SiteDiff
 
     # Printable URL
     def url(tag, prefix, cache)
+      return unless prefix
+
       base = cache.read_tags.include?(tag) ? "/cache/#{tag}" : prefix
       base.to_s + path
     end
 
     # Filename to store diff
     def filename
-      File.join(SiteDiff::DIFFS_DIR, Digest::SHA1.hexdigest(path) + '.html')
+      File.join(Report::DIFFS_DIR, "#{Digest::SHA1.hexdigest(path)}.html")
     end
 
-    # Text of the link in the HTML report
-    def link
-      case status
-      when STATUS_ERROR then error
-      when STATUS_SUCCESS then status_text
-      when STATUS_FAILURE then "<a href='#{filename}'>DIFF</a>"
-      end
+    # Returns a URL to the result diff.
+    #
+    # Returns nil if the result has no diffs.
+    def diff_url(relative: false)
+      prefix = relative ? 'files/' : '/files/'
+      return prefix + filename if status == STATUS_FAILURE
     end
 
     # Log the result to the terminal
-    def log(verbose = true)
+    def log(verbose: true)
       case status
-      when STATUS_SUCCESS then
-        SiteDiff.log path, :diff_success, 'UNCHANGED'
-      when STATUS_ERROR then
-        SiteDiff.log path, :warn, "ERROR (#{error})"
-      when STATUS_FAILURE then
-        SiteDiff.log path, :diff_failure, 'CHANGED'
+      when STATUS_SUCCESS
+        SiteDiff.log path, :success, 'UNCHANGED'
+      when STATUS_ERROR
+        SiteDiff.log path + " (#{error})", :warning, 'ERROR'
+      when STATUS_FAILURE
+        SiteDiff.log path, :error, 'CHANGED'
         puts Diff.terminal_diffy(before, after) if verbose
       end
     end
 
     # Dump the result to a file
-    def dump(dir)
+    def dump(dir, relative: false)
       dump_path = File.join(dir, filename)
       base = File.dirname(dump_path)
       FileUtils.mkdir_p(base) unless File.exist?(base)
       File.open(dump_path, 'w') do |f|
-        f.write(Diff.generate_diff_output(self))
+        f.write(Diff.generate_diff_output(self, relative:))
       end
     end
   end

data/lib/sitediff/sanitize/dom_transform.rb

@@ -11,61 +11,96 @@ class SiteDiff
     #  * { :type => "unwrap", :selector => "div.field-item" }
     #  * { :type => "remove", :selector => "div.extra-stuff" }
     #  * { :type => "remove_class", :class => 'class1' }
+    #  * { :type => "strip", :selector => 'h1' }
     class DomTransform
-      Transforms = {}
+      # Supported dom_transform types.
+      TRANSFORMS = {}
 
+      ##
+      # Creates a DOM Transform.
       def initialize(rule)
         @rule = rule
       end
 
+      ##
       # Often an array or scalar are both ok values. Turn either into an array.
       def to_array(val)
         [val].flatten
       end
 
-      def targets(node)
+      ##
+      # TODO: Document what this method does.
+      def targets(node, &block)
         selectors = to_array(@rule['selector'])
         selectors.each do |sel|
-          node.css(sel).each { |n| yield n }
+          node.css(sel).each(&block)
         end
       end
 
+      ##
+      # Applies the transformation to a DOM node.
       def apply(node)
         targets(node) { |t| process(t) }
       end
 
+      ##
+      # Registers a DOM Transform plugin.
       def self.register(name)
-        Transforms[name] = self
+        TRANSFORMS[name] = self
       end
 
+      ##
+      # Creates a DOM Transform as per rule.
       def self.create(rule)
         (type = rule['type']) ||
           raise(InvalidSanitization, 'DOM transform needs a type')
-        (transform = Transforms[type]) ||
+        (transform = TRANSFORMS[type]) ||
           raise(InvalidSanitization, "No DOM transform named #{type}")
         transform.new(rule)
       end
 
-      # Remove elements matching 'selector'
+      ##
+      # Remove elements matching 'selector'.
       class Remove < DomTransform
         register 'remove'
+
+        ##
+        # Processes a node.
         def process(node)
           node.remove
         end
       end
 
-      # Unwrap elements matching 'selector'
+      # Squeeze whitespace from a tag matching 'selector'.
+      class Strip < DomTransform
+        register 'strip'
+
+        ##
+        # Processes a node.
+        def process(node)
+          node.content = node.content.strip
+        end
+      end
+
+      # Unwrap elements matching 'selector'.
       class Unwrap < DomTransform
         register 'unwrap'
+
+        ##
+        # Processes a node.
         def process(node)
           node.add_next_sibling(node.children)
           node.remove
         end
       end
 
+      ##
       # Remove classes from elements matching selector
       class RemoveClass < DomTransform
         register 'remove_class'
+
+        ##
+        # Processes a node.
         def process(node)
           classes = to_array(@rule['class'])
 
@@ -77,9 +112,13 @@ class SiteDiff
         end
       end
 
-      # Unwrap the root element
+      ##
+      # Unwrap the root element.
       class UnwrapRoot < DomTransform
         register 'unwrap_root'
+
+        ##
+        # Applies the transformation to a DOM node.
         def apply(node)
           (node.children.size == 1) ||
             raise(InvalidSanitization, 'Multiple root elements in unwrap_root')

data/lib/sitediff/sanitize/regexp.rb

@@ -2,41 +2,62 @@
 
 class SiteDiff
   class Sanitizer
+    # Regular Expression Object.
     class Regexp
+      ##
+      # Creates a RegExp object.
       def initialize(rule)
         @rule = rule
       end
 
+      ##
+      # Whether the RegExp has a selector.
       def selector?
         false
       end
 
+      ##
+      # Whether the RegExp applies to the given markup.
       def applies?(html, _node)
         applies_to_string?(html)
       end
 
+      ##
+      # Applies the RegExp to the markup.
       def apply(html)
         gsub!(html)
       end
 
+      ##
+      # Creates a RegExp object as per rule.
       def self.create(rule)
         rule['selector'] ? WithSelector.new(rule) : new(rule)
       end
 
+      ##
+      # A RegExp with selector.
       class WithSelector < Regexp
+        ##
+        # Whether the RegExp has a selector.
         def selector?
           true
         end
 
-        def contexts(node)
-          sels = @rule['selector']
-          node.css(sels).each { |e| yield(e) }
+        ##
+        # TODO: Document what this method does.
+        def contexts(node, &block)
+          selectors = @rule['selector']
+          node.css(selectors).each(&block)
         end
 
+        ##
+        # Whether the RegExp applies to the given markup.
         def applies?(_html, node)
           enum_for(:contexts, node).any? { |e| applies_to_string?(e.to_html) }
         end
 
+        ##
+        # Applies the RegExp to the markup.
         def apply(node)
           contexts(node) { |e| e.replace(gsub!(e.to_html)) }
         end