sitediff 0.0.2 → 1.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: f75892f718764c8fd2c18d7f3f7e7cf8908d60ea07c2a765510c8ef409b9f0c1
+  data.tar.gz: 3b3744eca0dda04821152aab596fb67891204a1599b4db72e13b4af484693e65
+SHA512:
+  metadata.gz: 97e9098b290742f1b3efe3c284e9392be95ffd0f7576df413a6ec612142b0573acf8b8b4d43369961c154d801db6284fcc1a8d69cea7da8ed99b64a0a1f1af75
+  data.tar.gz: c4b0e93bc4e0acb3d675c8d675d8f6235035aae72421794495f25223cb086eaa4c87d2cde63caa0eda257b0d91f374a0efbbb416ef8ee88c2f0ffde89a608831

data/bin/sitediff

@@ -1,10 +1,16 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 # when run as gem, $0 is /usr/local/bin/sitediff not this file
-if $0 == __FILE__
-  $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+if $PROGRAM_NAME == __FILE__
+  $LOAD_PATH.unshift File.expand_path('../lib', __dir__)
 end
 
 require 'sitediff/cli'
 
-SiteDiff::Cli.start
+begin
+  SiteDiff::Cli.start
+rescue Interrupt
+  puts("\n")
+  SiteDiff.log('Stopping. Interrupted by user.')
+end

data/lib/sitediff.rb

@@ -1,146 +1,220 @@
 #!/bin/env ruby
+# frozen_string_literal: true
+
 require 'sitediff/config'
+require 'sitediff/diff'
 require 'sitediff/fetch'
 require 'sitediff/result'
+require 'sitediff/report'
 require 'pathname'
 require 'rainbow'
+require 'rubygems'
 require 'yaml'
 
+# SiteDiff Object.
 class SiteDiff
-  # path to misc. static files (e.g. erb, css files)
+  attr_reader :config, :results
+
+  # SiteDiff installation directory.
+  ROOT_DIR = File.dirname(File.dirname(__FILE__))
+
+  # Path to misc files. Ex: *.erb, *.css.
   FILES_DIR = File.join(File.dirname(__FILE__), 'sitediff', 'files')
 
-  # subdirectory containing all failing diffs
-  DIFFS_DIR = 'diffs'
-
-  # files in output
-  FAILURES_FILE = 'failures.txt'
-  REPORT_FILE = 'report.html'
-  SETTINGS_FILE = 'settings.yaml'
-
-  # label will be colorized and str will not be.
-  # type dictates the color: can be :success, :error, or :failure
-  def self.log(str, type=:info, label=nil)
-    label = label ? "[sitediff] #{label}" : '[sitediff]'
-    bg = fg = nil
-    case type
-    when :info
-    when :diff_success
-      bg = :green
+  # Logs a message.
+  #
+  # Label will be colorized and message will not.
+  # Type dictates the color: can be :success, :error, or :failure.
+  #
+  # TODO: Only print :debug messages in debug mode.
+  def self.log(message, type = :info, label = nil)
+    # Prepare label.
+    label ||= type unless type == :info
+    label = label.to_s
+    unless label.empty?
+      # Colorize label.
       fg = :black
-    when :diff_failure
-      bg = :red
-    when :warn
-      bg = :yellow
-      fg = :black
-    when :error
-      bg = :red
+      bg = :blue
+
+      case type
+      when :info
+        bg = :cyan
+      when :success
+        bg = :green
+      when :error
+        bg = :red
+      when :warning
+        bg = :yellow
+      end
+
+      label = '[' + label.to_s + ']'
+      label = Rainbow(label)
+      label = label.bg(bg) if bg
+      label = label.fg(fg) if fg
+
+      # Add a space after the label.
+      label += ' '
     end
-    label = Rainbow(label)
-    label = label.bg(bg) if bg
-    label = label.fg(fg) if fg
-    puts label + ' ' + str
+
+    puts label + message
   end
 
-  attr_reader :config, :results
+  ##
+  # Returns the "before" site's URL.
+  #
+  # TODO: Remove in favor of config.before_url.
   def before
     @config.before['url']
   end
+
+  ##
+  # Returns the "after" site's URL.
+  #
+  # TODO: Remove in favor of config.after_url.
   def after
     @config.after['url']
   end
 
-  def initialize(config, cache, verbose=true)
+  # Initialize SiteDiff.
+  def initialize(config, cache, verbose = true, debug = false)
     @cache = cache
     @verbose = verbose
+    @debug = debug
 
     # Check for single-site mode
     validate_opts = {}
     if !config.before['url'] && @cache.tag?(:before)
-      raise SiteDiffException,
-        "A cached 'before' is required for single-site mode" \
-        unless @cache.read_tags.include?(:before)
+      unless @cache.read_tags.include?(:before)
+        raise SiteDiffException,
+              "A cached 'before' is required for single-site mode"
+      end
       validate_opts[:need_before] = false
     end
     config.validate(validate_opts)
-
+    # Configure diff.
+    Diff.diff_config(config)
     @config = config
   end
 
-  # Sanitize HTML
+  # Sanitize HTML.
   def sanitize(path, read_results)
-    [:before, :after].map do |tag|
+    %i[before after].map do |tag|
       html = read_results[tag].content
-      config = @config.send(tag)
-      Sanitizer.new(html, config, :path => path).sanitize
+      # TODO: See why encoding is empty while running tests.
+      #
+      # The presence of an "encoding" value used to be used to determine
+      # if the sanitizer would be called. However, encoding turns up blank
+      # during rspec tests for some reason.
+      encoding = read_results[tag].encoding
+      if encoding || html.length.positive?
+        section = @config.send(tag, true)
+        opts = { path: path }
+        opts[:output] = @config.output if @config.output
+        Sanitizer.new(html, section, opts).sanitize
+      else
+        html
+      end
     end
   end
 
-  # Process a set of read results
+  ##
+  # Process a set of read results.
+  #
+  # This is the callback that processes items fetched by the Fetcher.
   def process_results(path, read_results)
-    if error = read_results[:before].error || read_results[:after].error
-      diff = Result.new(path, nil, nil, error)
+    error = (read_results[:before].error || read_results[:after].error)
+    if error
+      diff = Result.new(path, nil, nil, nil, nil, error)
     else
-      diff = Result.new(path, *sanitize(path, read_results), nil)
+      begin
+        diff = Result.new(
+          path,
+          *sanitize(path, read_results),
+          read_results[:before].encoding,
+          read_results[:after].encoding,
+          nil
+        )
+      rescue StandardError => e
+        raise if @debug
+
+        Result.new(path, nil, nil, nil, nil, "Sanitization error: #{e}")
+      end
     end
     @results[path] = diff
 
     # Print results in order!
-    while next_diff = @results[@ordered.first]
+    while (next_diff = @results[@ordered.first])
       next_diff.log(@verbose)
       @ordered.shift
     end
   end
 
-  # Perform the comparison, populate @results and return the number of failing
-  # paths (paths with non-zero diff).
+  ##
+  # Compute diff as per config.
+  #
+  # @return [Integer]
+  #   Number of paths which have diffs.
   def run
     # Map of path -> Result object, populated by process_results
     @results = {}
     @ordered = @config.paths.dup
 
     unless @cache.read_tags.empty?
-      SiteDiff.log("Using sites from cache: " +
-        @cache.read_tags.sort.join(', '))
+      SiteDiff.log('Using sites from cache: ' + @cache.read_tags.sort.join(', '))
     end
 
-    fetcher = Fetch.new(@cache, @config.paths,
-      :before => before, :after => after)
-    fetcher.run(&self.method(:process_results))
+    # TODO: Fix this after config merge refactor!
+    # Not quite right. We are not passing @config.before or @config.after
+    # so passing this instead but @config.after['curl_opts'] is ignored.
+    curl_opts = @config.setting :curl_opts
+    config_curl_opts = @config.before['curl_opts']
+    curl_opts = config_curl_opts.clone.merge(curl_opts) if config_curl_opts
+    fetcher = Fetch.new(
+      @cache,
+      @config.paths,
+      @config.setting(:interval),
+      @config.setting(:concurrency),
+      curl_opts,
+      @debug,
+      before: @config.before_url,
+      after: @config.after_url
+    )
+
+    # Run the Fetcher with "process results" as a callback.
+    fetcher.run(&method(:process_results))
 
     # Order by original path order
-    @results = @config.paths.map { |p| @results[p] }
-    return results.map{ |r| r unless r.success? }.compact.length
+    @results = @config.paths.map { |path| @results[path] }
+    results.map { |r| r unless r.success? }.compact.length
   end
 
-  # Dump results to disk
-  def dump(dir, report_before, report_after)
-    report_before ||= before
-    report_after ||= after
-    dir = Pathname.new(dir)
-    dir.mkpath unless dir.directory?
-
-    # store diffs of each failing case, first wipe out existing diffs
-    diff_dir = dir + DIFFS_DIR
-    diff_dir.rmtree if diff_dir.exist?
-    results.each { |r| r.dump(dir) if r.status == Result::STATUS_FAILURE }
-    SiteDiff::log "All diff files were dumped inside #{dir.expand_path}"
-
-    # store failing paths
-    failures = dir + FAILURES_FILE
-    SiteDiff::log "Writing failures to #{failures.expand_path}"
-    failures.open('w') do |f|
-      results.each { |r| f.puts r.path unless r.success? }
+  ##
+  # Get a reporter object to help with report generation.
+  def report
+    if @results.nil?
+      raise SiteDiffException(
+        'No results detected. Run SiteDiff.run before SiteDiff.report.'
+      )
     end
 
-    # create report of results
-    report = Diff::generate_html_report(results, report_before, report_after,
-      @cache)
-    dir.+(REPORT_FILE).open('w') { |f| f.write(report) }
+    Report.new(@config, @cache, @results)
+  end
 
-    # serve some settings
-    settings = { 'before' => report_before, 'after' => report_after,
-       'cached' => @cache.read_tags.map { |t| t.to_s } }
-    dir.+(SETTINGS_FILE).open('w') { |f| YAML.dump(settings, f) }
+  ##
+  # Get SiteDiff gemspec.
+  def self.gemspec
+    file = ROOT_DIR + '/sitediff.gemspec'
+    Gem::Specification.load(file)
+  end
+
+  ##
+  # Ensures that a directory exists and returns a Pathname for it.
+  #
+  # @param [String] dir
+  #   path/to/directory
+  def self.ensure_dir(dir)
+    dir = Pathname.new(dir) unless dir.is_a? Pathname
+    dir.mkpath unless dir.directory?
+    dir
   end
 end

data/lib/sitediff/api.rb

@@ -0,0 +1,265 @@
+# frozen_string_literal: true
+
+require 'sitediff'
+require 'sitediff/cache'
+require 'sitediff/config'
+require 'sitediff/config/creator'
+require 'sitediff/config/preset'
+require 'sitediff/fetch'
+require 'sitediff/webserver/resultserver'
+
+class SiteDiff
+  ##
+  # Sitediff API interface.
+  class Api
+    ##
+    # Initializes new Api object.
+    def initialize(directory, config_file = nil)
+      @dir = get_dir(directory)
+      @config = SiteDiff::Config.new(config_file, @dir)
+    end
+
+    ##
+    # Intialize a SiteDiff project.
+    #
+    # Calling:
+    #     SiteDiff::Api.init(
+    #       depth: 3,
+    #       directory: 'sitediff',
+    #       concurrency: 3,
+    #       interval: 0,
+    #       include: nil,
+    #       exclude: '*.pdf',
+    #       preset: 'drupal',
+    #       curl_opts: {timeout: 60},
+    #       crawl: false
+    #     )
+    def self.init(options)
+      # Prepare a config object and write it to the file system.
+      creator = SiteDiff::Config::Creator.new(options[:debug], options[:before_url], options[:after_url])
+      include_regex = Config.create_regexp(options[:include])
+      exclude_regex = Config.create_regexp(options[:exclude])
+      creator.create(
+        depth: options[:depth],
+        directory: options[:directory],
+        concurrency: options[:concurrency],
+        interval: options[:interval],
+        include: include_regex,
+        exclude: exclude_regex,
+        preset: options[:preset],
+        curl_opts: options[:curl_opts]
+      )
+      SiteDiff.log "Created #{creator.config_file.expand_path}", :success
+
+      # TODO: implement crawl ^^^
+      # Discover paths, if enabled.
+      # if options[:crawl]
+      #   crawl(creator.config_file)
+      #   SiteDiff.log 'You can now run "sitediff diff".', :success
+      # else
+      #   SiteDiff.log 'Run "sitediff crawl" to discover paths. You should then be able to run "sitediff diff".', :info
+      # end
+    end
+
+    ##
+    # Diff the `before` and `after`.
+    #
+    # Calling:
+    #     Api.diff(
+    #       paths: options['paths'],
+    #       paths_file: options['paths-file'],
+    #       ignore_whitespace: options['ignore-whitespace'],
+    #       export: options['export'],
+    #       before: options['before'],
+    #       after: options['after'],
+    #       cached: options['cached'],
+    #       verbose: options['verbose'],
+    #       report_format: options['report-format'],
+    #       before_report: options['before-report'],
+    #       after_report: options['after-report'],
+    #       cli_mode: false
+    #     )
+    def diff(options)
+      @config.ignore_whitespace = options[:ignore_whitespace]
+      @config.export = options[:export]
+      # Apply "paths" override, if any.
+      if options[:paths]
+        @config.paths = options[:paths]
+      else
+        paths_file = options[:paths_file]
+        paths_file ||= File.join(@dir, Config::DEFAULT_PATHS_FILENAME)
+        paths_file = File.expand_path(paths_file)
+
+        paths_count = @config.paths_file_read(paths_file)
+        SiteDiff.log "Read #{paths_count} paths from: #{paths_file}"
+      end
+
+      # TODO: Why do we allow before and after override during diff?
+      @config.before['url'] = options[:before] if options[:before]
+      @config.after['url'] = options[:after] if options[:after]
+
+      # Prepare cache.
+      cache = SiteDiff::Cache.new(
+        create: options[:cached] != 'none',
+        directory: @dir
+      )
+      cache.read_tags << :before if %w[before all].include?(options[:cached])
+      cache.read_tags << :after if %w[after all].include?(options[:cached])
+      cache.write_tags << :before << :after
+
+      # Run sitediff.
+      sitediff = SiteDiff.new(
+        @config,
+        cache,
+        options[:verbose],
+        options[:debug]
+      )
+      num_failing = sitediff.run
+      exit_code = num_failing.positive? ? 2 : 0
+
+      # Generate HTML report.
+      if options[:report_format] == 'html' || @config.export
+        sitediff.report.generate_html(
+          @dir,
+          options[:before_report],
+          options[:after_report]
+        )
+      end
+
+      # Generate JSON report.
+      if options[:report_format] == 'json' && @config.export == false
+        sitediff.report.generate_json @dir
+      end
+
+      SiteDiff.log 'Run "sitediff serve" to see a report.' unless options[:export]
+    rescue Config::InvalidConfig => e
+      SiteDiff.log "Invalid configuration: #{e.message}", :error
+      SiteDiff.log e.backtrace, :error if options[:verbose]
+    rescue Config::ConfigNotFound => e
+      SiteDiff.log "Invalid configuration: #{e.message}", :error
+      SiteDiff.log e.backtrace, :error if options[:verbose]
+    else # no exception was raised
+      # Thor::Error  --> exit(1), guaranteed by exit_on_failure?
+      # Failing diff --> exit(2), populated above
+      exit(exit_code) if options[:cli_mode]
+    end
+
+    ##
+    # Crawl the `before` site to determine `paths`.
+    def crawl
+      # Prepare cache.
+      @cache = SiteDiff::Cache.new(
+        create: true,
+        directory: @dir
+      )
+      @cache.write_tags << :before << :after
+
+      # Crawl with Hydra to discover paths.
+      hydra = Typhoeus::Hydra.new(
+        max_concurrency: @config.setting(:concurrency)
+      )
+      @paths = {}
+      @config.roots.each do |tag, url|
+        Crawler.new(
+          hydra,
+          url,
+          @config.setting(:interval),
+          @config.setting(:include),
+          @config.setting(:exclude),
+          @config.setting(:depth),
+          @config.curl_opts,
+          @debug
+        ) do |info|
+          SiteDiff.log "Visited #{info.uri}, cached."
+          after_crawl(tag, info)
+        end
+      end
+      hydra.run
+
+      # Write paths to a file.
+      @paths = @paths.values.reduce(&:|).to_a.sort
+      @config.paths_file_write(@paths)
+
+      # Log output.
+      file = Pathname.new(@dir) + Config::DEFAULT_PATHS_FILENAME
+      SiteDiff.log ''
+      SiteDiff.log "#{@paths.length} page(s) found."
+      SiteDiff.log "Created #{file.expand_path}.", :success, 'done'
+    end
+
+    ##
+    # Serves SiteDiff report for accessing in the browser.
+    #
+    # Calling:
+    #     api.serve(browse: true, port: 13080)
+    def serve(options)
+      @cache = Cache.new(directory: @dir)
+      @cache.read_tags << :before << :after
+
+      SiteDiff::Webserver::ResultServer.new(
+        options[:port],
+        @dir,
+        browse: options[:browse],
+        cache: @cache,
+        config: @config
+      ).wait
+    rescue SiteDiffException => e
+      SiteDiff.log e.message, :error
+      SiteDiff.log e.backtrace, :error if options[:verbose]
+    end
+
+    ##
+    #
+    def store(options)
+      # TODO: Figure out how to remove this config.validate call.
+      @config.validate(need_before: false)
+      @config.paths_file_read
+
+      @cache = SiteDiff::Cache.new(directory: @dir, create: true)
+      @cache.write_tags << :before
+
+      base = options[:url] || @config.after['url']
+      fetcher = SiteDiff::Fetch.new(@cache,
+                                    @config.paths,
+                                    @config.setting(:interval),
+                                    @config.setting(:concurrency),
+                                    get_curl_opts(@config.settings),
+                                    options[:debug],
+                                    before: base)
+      fetcher.run do |path, _res|
+        SiteDiff.log "Visited #{path}, cached"
+      end
+    end
+
+    private
+
+    ##
+    # Ensures that the given directory exists.
+    def get_dir(directory)
+      # Create the dir. Must go before cache initialization!
+      @dir = Pathname.new(directory || '.')
+      @dir.mkpath unless @dir.directory?
+      @dir.to_s
+    end
+
+    ##
+    # Processes a crawled path.
+    def after_crawl(tag, info)
+      path = UriWrapper.canonicalize(info.relative)
+
+      # Register the path.
+      @paths[tag] = [] unless @paths[tag]
+      @paths[tag] << path
+
+      result = info.read_result
+
+      # Write result to applicable cache.
+      @cache.set(tag, path, result)
+      # If single-site, cache "after" as "before".
+      @cache.set(:before, path, result) unless @config.roots[:before]
+
+      # TODO: Restore application of rules.
+      # @rules.handle_page(tag, res.content, info.document) if @rules && !res.error
+    end
+  end
+end