sitediff 1.0.0 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1b7854497b5e81f48d810acec8106cbc66e33492d046e032e5516f76db26f142
-  data.tar.gz: a9349a79953237dd017600d49d38b8e734afc561f0ce09a1f8732e0e933530c9
+  metadata.gz: f75892f718764c8fd2c18d7f3f7e7cf8908d60ea07c2a765510c8ef409b9f0c1
+  data.tar.gz: 3b3744eca0dda04821152aab596fb67891204a1599b4db72e13b4af484693e65
 SHA512:
-  metadata.gz: 0e91f665f3c59b8a65f16c6942ec49b9cc9ee7fd12b0a777eadb844a0b9819ab1fd9485495bf2c757ca7342a6198dcccb5ae546c4ddf2682f234d015b64309b2
-  data.tar.gz: 64b7980bbbade8710b6069af19a67083678c2bd5fa99674df3360c1c6a3ddf8a15de7c5be4e8349ec298fc1c0dc27535b816089cd4f8852b8c8633861d72a178
+  metadata.gz: 97e9098b290742f1b3efe3c284e9392be95ffd0f7576df413a6ec612142b0573acf8b8b4d43369961c154d801db6284fcc1a8d69cea7da8ed99b64a0a1f1af75
+  data.tar.gz: c4b0e93bc4e0acb3d675c8d675d8f6235035aae72421794495f25223cb086eaa4c87d2cde63caa0eda257b0d91f374a0efbbb416ef8ee88c2f0ffde89a608831

data/lib/sitediff.rb

@@ -108,7 +108,9 @@ class SiteDiff
       encoding = read_results[tag].encoding
       if encoding || html.length.positive?
         section = @config.send(tag, true)
-        Sanitizer.new(html, section, path: path).sanitize
+        opts = { path: path }
+        opts[:output] = @config.output if @config.output
+        Sanitizer.new(html, section, opts).sanitize
       else
         html
       end

data/lib/sitediff/api.rb

@@ -0,0 +1,265 @@
+# frozen_string_literal: true
+
+require 'sitediff'
+require 'sitediff/cache'
+require 'sitediff/config'
+require 'sitediff/config/creator'
+require 'sitediff/config/preset'
+require 'sitediff/fetch'
+require 'sitediff/webserver/resultserver'
+
+class SiteDiff
+  ##
+  # Sitediff API interface.
+  class Api
+    ##
+    # Initializes new Api object.
+    def initialize(directory, config_file = nil)
+      @dir = get_dir(directory)
+      @config = SiteDiff::Config.new(config_file, @dir)
+    end
+
+    ##
+    # Intialize a SiteDiff project.
+    #
+    # Calling:
+    #     SiteDiff::Api.init(
+    #       depth: 3,
+    #       directory: 'sitediff',
+    #       concurrency: 3,
+    #       interval: 0,
+    #       include: nil,
+    #       exclude: '*.pdf',
+    #       preset: 'drupal',
+    #       curl_opts: {timeout: 60},
+    #       crawl: false
+    #     )
+    def self.init(options)
+      # Prepare a config object and write it to the file system.
+      creator = SiteDiff::Config::Creator.new(options[:debug], options[:before_url], options[:after_url])
+      include_regex = Config.create_regexp(options[:include])
+      exclude_regex = Config.create_regexp(options[:exclude])
+      creator.create(
+        depth: options[:depth],
+        directory: options[:directory],
+        concurrency: options[:concurrency],
+        interval: options[:interval],
+        include: include_regex,
+        exclude: exclude_regex,
+        preset: options[:preset],
+        curl_opts: options[:curl_opts]
+      )
+      SiteDiff.log "Created #{creator.config_file.expand_path}", :success
+
+      # TODO: implement crawl ^^^
+      # Discover paths, if enabled.
+      # if options[:crawl]
+      #   crawl(creator.config_file)
+      #   SiteDiff.log 'You can now run "sitediff diff".', :success
+      # else
+      #   SiteDiff.log 'Run "sitediff crawl" to discover paths. You should then be able to run "sitediff diff".', :info
+      # end
+    end
+
+    ##
+    # Diff the `before` and `after`.
+    #
+    # Calling:
+    #     Api.diff(
+    #       paths: options['paths'],
+    #       paths_file: options['paths-file'],
+    #       ignore_whitespace: options['ignore-whitespace'],
+    #       export: options['export'],
+    #       before: options['before'],
+    #       after: options['after'],
+    #       cached: options['cached'],
+    #       verbose: options['verbose'],
+    #       report_format: options['report-format'],
+    #       before_report: options['before-report'],
+    #       after_report: options['after-report'],
+    #       cli_mode: false
+    #     )
+    def diff(options)
+      @config.ignore_whitespace = options[:ignore_whitespace]
+      @config.export = options[:export]
+      # Apply "paths" override, if any.
+      if options[:paths]
+        @config.paths = options[:paths]
+      else
+        paths_file = options[:paths_file]
+        paths_file ||= File.join(@dir, Config::DEFAULT_PATHS_FILENAME)
+        paths_file = File.expand_path(paths_file)
+
+        paths_count = @config.paths_file_read(paths_file)
+        SiteDiff.log "Read #{paths_count} paths from: #{paths_file}"
+      end
+
+      # TODO: Why do we allow before and after override during diff?
+      @config.before['url'] = options[:before] if options[:before]
+      @config.after['url'] = options[:after] if options[:after]
+
+      # Prepare cache.
+      cache = SiteDiff::Cache.new(
+        create: options[:cached] != 'none',
+        directory: @dir
+      )
+      cache.read_tags << :before if %w[before all].include?(options[:cached])
+      cache.read_tags << :after if %w[after all].include?(options[:cached])
+      cache.write_tags << :before << :after
+
+      # Run sitediff.
+      sitediff = SiteDiff.new(
+        @config,
+        cache,
+        options[:verbose],
+        options[:debug]
+      )
+      num_failing = sitediff.run
+      exit_code = num_failing.positive? ? 2 : 0
+
+      # Generate HTML report.
+      if options[:report_format] == 'html' || @config.export
+        sitediff.report.generate_html(
+          @dir,
+          options[:before_report],
+          options[:after_report]
+        )
+      end
+
+      # Generate JSON report.
+      if options[:report_format] == 'json' && @config.export == false
+        sitediff.report.generate_json @dir
+      end
+
+      SiteDiff.log 'Run "sitediff serve" to see a report.' unless options[:export]
+    rescue Config::InvalidConfig => e
+      SiteDiff.log "Invalid configuration: #{e.message}", :error
+      SiteDiff.log e.backtrace, :error if options[:verbose]
+    rescue Config::ConfigNotFound => e
+      SiteDiff.log "Invalid configuration: #{e.message}", :error
+      SiteDiff.log e.backtrace, :error if options[:verbose]
+    else # no exception was raised
+      # Thor::Error  --> exit(1), guaranteed by exit_on_failure?
+      # Failing diff --> exit(2), populated above
+      exit(exit_code) if options[:cli_mode]
+    end
+
+    ##
+    # Crawl the `before` site to determine `paths`.
+    def crawl
+      # Prepare cache.
+      @cache = SiteDiff::Cache.new(
+        create: true,
+        directory: @dir
+      )
+      @cache.write_tags << :before << :after
+
+      # Crawl with Hydra to discover paths.
+      hydra = Typhoeus::Hydra.new(
+        max_concurrency: @config.setting(:concurrency)
+      )
+      @paths = {}
+      @config.roots.each do |tag, url|
+        Crawler.new(
+          hydra,
+          url,
+          @config.setting(:interval),
+          @config.setting(:include),
+          @config.setting(:exclude),
+          @config.setting(:depth),
+          @config.curl_opts,
+          @debug
+        ) do |info|
+          SiteDiff.log "Visited #{info.uri}, cached."
+          after_crawl(tag, info)
+        end
+      end
+      hydra.run
+
+      # Write paths to a file.
+      @paths = @paths.values.reduce(&:|).to_a.sort
+      @config.paths_file_write(@paths)
+
+      # Log output.
+      file = Pathname.new(@dir) + Config::DEFAULT_PATHS_FILENAME
+      SiteDiff.log ''
+      SiteDiff.log "#{@paths.length} page(s) found."
+      SiteDiff.log "Created #{file.expand_path}.", :success, 'done'
+    end
+
+    ##
+    # Serves SiteDiff report for accessing in the browser.
+    #
+    # Calling:
+    #     api.serve(browse: true, port: 13080)
+    def serve(options)
+      @cache = Cache.new(directory: @dir)
+      @cache.read_tags << :before << :after
+
+      SiteDiff::Webserver::ResultServer.new(
+        options[:port],
+        @dir,
+        browse: options[:browse],
+        cache: @cache,
+        config: @config
+      ).wait
+    rescue SiteDiffException => e
+      SiteDiff.log e.message, :error
+      SiteDiff.log e.backtrace, :error if options[:verbose]
+    end
+
+    ##
+    #
+    def store(options)
+      # TODO: Figure out how to remove this config.validate call.
+      @config.validate(need_before: false)
+      @config.paths_file_read
+
+      @cache = SiteDiff::Cache.new(directory: @dir, create: true)
+      @cache.write_tags << :before
+
+      base = options[:url] || @config.after['url']
+      fetcher = SiteDiff::Fetch.new(@cache,
+                                    @config.paths,
+                                    @config.setting(:interval),
+                                    @config.setting(:concurrency),
+                                    get_curl_opts(@config.settings),
+                                    options[:debug],
+                                    before: base)
+      fetcher.run do |path, _res|
+        SiteDiff.log "Visited #{path}, cached"
+      end
+    end
+
+    private
+
+    ##
+    # Ensures that the given directory exists.
+    def get_dir(directory)
+      # Create the dir. Must go before cache initialization!
+      @dir = Pathname.new(directory || '.')
+      @dir.mkpath unless @dir.directory?
+      @dir.to_s
+    end
+
+    ##
+    # Processes a crawled path.
+    def after_crawl(tag, info)
+      path = UriWrapper.canonicalize(info.relative)
+
+      # Register the path.
+      @paths[tag] = [] unless @paths[tag]
+      @paths[tag] << path
+
+      result = info.read_result
+
+      # Write result to applicable cache.
+      @cache.set(tag, path, result)
+      # If single-site, cache "after" as "before".
+      @cache.set(:before, path, result) unless @config.roots[:before]
+
+      # TODO: Restore application of rules.
+      # @rules.handle_page(tag, res.content, info.document) if @rules && !res.error
+    end
+  end
+end

data/lib/sitediff/cache.rb

@@ -6,6 +6,8 @@ require 'fileutils'
 class SiteDiff
   # SiteDiff Cache Handler.
   class Cache
+    TIMESTAMP_FILE = 'timestamp'
+
     attr_accessor :read_tags, :write_tags
 
     ##
@@ -17,6 +19,7 @@ class SiteDiff
       # They indicate whether we should use the cache for reading or writing.
       @read_tags = Set.new
       @write_tags = Set.new
+      @timestamp_flag = { before: false, after: false }
 
       # The directory used by the cache for storage.
       @dir = opts[:directory] || '.'
@@ -52,6 +55,7 @@ class SiteDiff
     def set(tag, path, result)
       return unless @write_tags.include? tag
 
+      save_timestamp(tag)
       filename = File.join(
         @dir,
         'snapshot',
@@ -102,5 +106,19 @@ class SiteDiff
       @dir.mkpath unless @dir.directory?
       @dir.to_s
     end
+
+    private
+
+    def save_timestamp(tag)
+      # run once
+      return if @timestamp_flag[tag]
+
+      @timestamp_flag[tag] = true
+      cache_dir = File.join(@dir, 'snapshot', tag.to_s)
+      if File.exist? cache_dir
+        file = File.join(cache_dir, TIMESTAMP_FILE)
+        FileUtils.touch(file)
+      end
+    end
   end
 end

data/lib/sitediff/cli.rb

@@ -2,16 +2,10 @@
 
 require 'thor'
 require 'sitediff'
-require 'sitediff/cache'
-require 'sitediff/config'
-require 'sitediff/config/creator'
-require 'sitediff/config/preset'
-require 'sitediff/fetch'
-require 'sitediff/webserver/resultserver'
+require 'sitediff/api'
 
 class SiteDiff
   # SiteDiff CLI.
-  # TODO: Use config.defaults to feed default values for sitediff.yaml params?
   class Cli < Thor
     class_option 'directory',
                  type: :string,
@@ -78,7 +72,6 @@ class SiteDiff
            enum: %w[html json],
            default: 'html',
            desc: 'The format in which a report should be generated.'
-    # TODO: Deprecate the parameters before-report / after-report?
     option 'before-report',
            type: :string,
            desc: 'URL to use in reports. Useful if port forwarding.',
@@ -107,82 +100,31 @@ class SiteDiff
     ##
     # Computes diffs.
     def diff(config_file = nil)
-      @dir = get_dir(options['directory'])
-      config = SiteDiff::Config.new(config_file, @dir)
-
       # Determine "paths" override based on options.
       if options['paths'] && options['paths-file']
         SiteDiff.log "Can't specify both --paths-file and --paths.", :error
         exit(-1)
       end
 
-      # Ignore whitespace option.
-      config.ignore_whitespace = options['ignore-whitespace'] if options['ignore-whitespace']
-
-      # Export report option.
-      config.export = options['export']
-
-      # Apply "paths" override, if any.
-      config.paths = options['paths'] if options['paths']
-
-      # Determine and apply "paths-file", if "paths" is not specified.
-      unless options['paths']
-        paths_file = options['paths-file']
-        paths_file ||= File.join(@dir, Config::DEFAULT_PATHS_FILENAME)
-        paths_file = File.expand_path(paths_file)
-
-        paths_count = config.paths_file_read(paths_file)
-        SiteDiff.log "Read #{paths_count} paths from: #{paths_file}"
-      end
-
-      # TODO: Why do we allow before and after override during diff?
-      config.before['url'] = options['before'] if options['before']
-      config.after['url'] = options['after'] if options['after']
-
-      # Prepare cache.
-      cache = SiteDiff::Cache.new(
-        create: options['cached'] != 'none',
-        directory: @dir
-      )
-      cache.read_tags << :before if %w[before all].include?(options['cached'])
-      cache.read_tags << :after if %w[after all].include?(options['cached'])
-      cache.write_tags << :before << :after
-
-      # Run sitediff.
-      sitediff = SiteDiff.new(
-        config,
-        cache,
-        options['verbose'],
-        options[:debug]
-      )
-      num_failing = sitediff.run
-      exit_code = num_failing.positive? ? 2 : 0
-
-      # Generate HTML report.
-      if options['report-format'] == 'html' || config.export
-        sitediff.report.generate_html(
-          @dir,
-          options['before-report'],
-          options['after-report']
+      api = Api.new(options['directory'], config_file)
+      api_options =
+        clean_keys(
+          options,
+          :paths,
+          :paths_file,
+          :ignore_whitespace,
+          :export,
+          :before,
+          :after,
+          :cached,
+          :verbose,
+          :debug,
+          :report_format,
+          :before_report,
+          :after_report
         )
-      end
-
-      # Generate JSON report.
-      if options['report-format'] == 'json' && config.export == false
-        sitediff.report.generate_json @dir
-      end
-
-      SiteDiff.log 'Run "sitediff serve" to see a report.' unless options['export']
-    rescue Config::InvalidConfig => e
-      SiteDiff.log "Invalid configuration: #{e.message}", :error
-      SiteDiff.log e.backtrace, :error if options[:verbose]
-    rescue Config::ConfigNotFound => e
-      SiteDiff.log "Invalid configuration: #{e.message}", :error
-      SiteDiff.log e.backtrace, :error if options[:verbose]
-    else # no exception was raised
-      # Thor::Error  --> exit(1), guaranteed by exit_on_failure?
-      # Failing diff --> exit(2), populated above
-      exit(exit_code)
+      api_options[:cli_mode] = true
+      api.diff(api_options)
     end
 
     option :port,
@@ -198,22 +140,9 @@ class SiteDiff
     ##
     # Serves SiteDiff report for accessing in the browser.
     def serve(config_file = nil)
-      @dir = get_dir(options['directory'])
-      config = SiteDiff::Config.new(config_file, @dir)
-
-      cache = Cache.new(directory: @dir)
-      cache.read_tags << :before << :after
-
-      SiteDiff::Webserver::ResultServer.new(
-        options[:port],
-        options['directory'],
-        browse: options[:browse],
-        cache: cache,
-        config: config
-      ).wait
-    rescue SiteDiffException => e
-      SiteDiff.log e.message, :error
-      SiteDiff.log e.backtrace, :error if options[:verbose]
+      api = Api.new(options['directory'], config_file)
+      api_options = clean_keys(options, :browse, :port)
+      api.serve(api_options)
     end
 
     option :depth,
@@ -236,19 +165,14 @@ class SiteDiff
            type: :numeric,
            default: Config::DEFAULT_CONFIG['settings']['interval'],
            desc: 'Crawling delay - interval in milliseconds.'
-    option :whitelist,
+    option :include,
            type: :string,
-           default: Config::DEFAULT_CONFIG['settings']['whitelist'],
-           desc: 'Optional whitelist for crawling.'
-    option :blacklist,
+           default: Config::DEFAULT_CONFIG['settings']['include'],
+           desc: 'Optional URL include regex for crawling.'
+    option :exclude,
            type: :string,
-           default: Config::DEFAULT_CONFIG['settings']['blacklist'],
-           desc: 'Optional blacklist for crawling.'
-    # TODO: Remove this option. Always ignore SSL errors.
-    option :insecure,
-           type: :boolean,
-           default: false,
-           desc: 'Ignore many HTTPS/SSL errors'
+           default: Config::DEFAULT_CONFIG['settings']['exclude'],
+           desc: 'Optional URL exclude regex for crawling.'
     option :curl_options,
            type: :hash,
            default: {},
@@ -261,29 +185,26 @@ class SiteDiff
         SiteDiff.log 'sitediff init requires one or two URLs', :error
         exit(2)
       end
-
-      # Prepare a config object and write it to the file system.
-      @dir = get_dir(options['directory'])
-      creator = SiteDiff::Config::Creator.new(options[:debug], *urls)
-      creator.create(
-        depth: options[:depth],
-        directory: @dir,
-        concurrency: options[:concurrency],
-        interval: options[:interval],
-        whitelist: Config.create_regexp(options['whitelist']),
-        blacklist: Config.create_regexp(options['blacklist']),
-        preset: options[:preset],
-        curl_opts: get_curl_opts(options)
-      )
-      SiteDiff.log "Created #{creator.config_file.expand_path}", :success
-
-      # Discover paths, if enabled.
-      if options[:crawl]
-        crawl(creator.config_file)
-        SiteDiff.log 'You can now run "sitediff diff".', :success
-      else
-        SiteDiff.log 'Run "sitediff crawl" to discover paths. You should then be able to run "sitediff diff".', :info
-      end
+      api_options =
+        clean_keys(
+          options,
+          :depth,
+          :concurrency,
+          :interval,
+          :include,
+          :exclude,
+          :preset,
+          :crawl
+        )
+        .merge(
+          {
+            after_url: urls.pop,
+            before_url: urls.pop, # may be nil
+            directory: get_dir(options['directory']),
+            curl_opts: get_curl_opts(options)
+          }
+        )
+      Api.init(api_options)
     end
 
     option :url,
@@ -294,26 +215,9 @@ class SiteDiff
     ##
     # Caches the current version of the site.
     def store(config_file = nil)
-      @dir = get_dir(options['directory'])
-      config = SiteDiff::Config.new(config_file, @dir)
-      # TODO: Figure out how to remove this config.validate call.
-      config.validate(need_before: false)
-      config.paths_file_read
-
-      cache = SiteDiff::Cache.new(directory: @dir, create: true)
-      cache.write_tags << :before
-
-      base = options[:url] || config.after['url']
-      fetcher = SiteDiff::Fetch.new(cache,
-                                    config.paths,
-                                    config.setting(:interval),
-                                    config.setting(:concurrency),
-                                    get_curl_opts(config.settings),
-                                    options[:debug],
-                                    before: base)
-      fetcher.run do |path, _res|
-        SiteDiff.log "Visited #{path}, cached"
-      end
+      api = Api.new(options['directory'], config_file)
+      api_options = clean_keys(options, :url, :debug)
+      api.store(api_options)
     end
 
     desc 'crawl [CONFIG-FILE]',
@@ -321,58 +225,15 @@ class SiteDiff
     ##
     # Crawls the "before" site to determine "paths".
     #
-    # TODO: Move actual crawling to sitediff.crawl(config).
-    # TODO: Switch to paths = sitediff.crawl().
     def crawl(config_file = nil)
-      # Prepare configuration.
-      @dir = get_dir(options['directory'])
-      @config = SiteDiff::Config.new(config_file, @dir)
-
-      # Prepare cache.
-      @cache = SiteDiff::Cache.new(
-        create: options['cached'] != 'none',
-        directory: @dir
-      )
-      @cache.write_tags << :before << :after
-
-      # Crawl with Hydra to discover paths.
-      hydra = Typhoeus::Hydra.new(
-        max_concurrency: @config.setting(:concurrency)
-      )
-      @paths = {}
-      @config.roots.each do |tag, url|
-        Crawler.new(
-          hydra,
-          url,
-          @config.setting(:interval),
-          @config.setting(:whitelist),
-          @config.setting(:blacklist),
-          @config.setting(:depth),
-          get_curl_opts(@config.settings),
-          @debug
-        ) do |info|
-          SiteDiff.log "Visited #{info.uri}, cached."
-          after_crawl(tag, info)
-        end
-      end
-      hydra.run
-
-      # Write paths to a file.
-      @paths = @paths.values.reduce(&:|).to_a.sort
-      @config.paths_file_write(@paths)
-
-      # Log output.
-      file = Pathname.new(@dir) + Config::DEFAULT_PATHS_FILENAME
-      SiteDiff.log ''
-      SiteDiff.log "#{@paths.length} page(s) found."
-      SiteDiff.log "Created #{file.expand_path}.", :success, 'done'
+      api = Api.new(options['directory'], config_file)
+      api.crawl
     end
 
     no_commands do
       # Generates CURL options.
       #
-      # TODO: This should be in the config class instead.
-      # TODO: Make all requests insecure and avoid custom curl-opts.
+      # TODO: Possibly move to API class.
       def get_curl_opts(options)
         # We do want string keys here
         bool_hash = { 'true' => true, 'false' => false }
@@ -381,10 +242,6 @@ class SiteDiff
                     .merge(options['curl_options'] || {})
                     .merge(options['curl_opts'] || {})
         curl_opts.each { |k, v| curl_opts[k] = bool_hash.fetch(v, v) }
-        if options[:insecure]
-          curl_opts[:ssl_verifypeer] = false
-          curl_opts[:ssl_verifyhost] = 0
-        end
         curl_opts
       end
 
@@ -398,23 +255,10 @@ class SiteDiff
       end
 
       ##
-      # Processes a crawled path.
-      def after_crawl(tag, info)
-        path = UriWrapper.canonicalize(info.relative)
-
-        # Register the path.
-        @paths[tag] = [] unless @paths[tag]
-        @paths[tag] << path
-
-        result = info.read_result
-
-        # Write result to applicable cache.
-        @cache.set(tag, path, result)
-        # If single-site, cache "after" as "before".
-        @cache.set(:before, path, result) unless @config.roots[:before]
-
-        # TODO: Restore application of rules.
-        # @rules.handle_page(tag, res.content, info.document) if @rules && !res.error
+      # Clean keys - return a subset of a hash with keys as symbols.
+      def clean_keys(hash, *keys)
+        new_hash = hash.transform_keys { |k| k.tr('-', '_').to_sym }
+        new_hash.slice(*keys)
       end
     end
   end