validate-website 1.0.5 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1f9ca28a5036d0be57d7b0bdea0a22977c3d6a0f
-  data.tar.gz: 9add05e1e19342356734344c065fa8f727de5a39
+  metadata.gz: 134cbdd3da2da6847c525ffe5a2ee68f1a380ae2
+  data.tar.gz: 678ff514b9f6f368bbb78e93a8dc42f38a35a803
 SHA512:
-  metadata.gz: 21b55b544e3db2d75598e43e7c2d41980e045b17f1ea2d9a2961770a71ee32d66f80cd3ced92ce0a36baaae33a0eccf90519f5de3cbdd972af97b496362ae8b1
-  data.tar.gz: 84a3117d4d7d1a125df96b779de04454858024269493ae519c6a301e8069bf90b14b9489b92a69dead1a5d4aadc39d224fd55a9d583ecff198eb04d517c73f34
+  metadata.gz: dd90c5dec7d0c80ea9b94abcb1a38a425ee59be32c73fe9fc97620a5a00ee4c9cf2dae52aa099509c8573f2dfca117377ed15373bce2d93ed7c25a43e6ed067e
+  data.tar.gz: 4d24ac9b1dccd744a3b7bd7ebbb18b946c05c4043a0c76898eafc543892e886715e2d99670ea91f3b9d0c3203425cf0de420445d041360c44fb67a934c6165c2

data/Rakefile

@@ -4,8 +4,8 @@ require 'rake/testtask'
 task default: [:test]
 
 RDoc::Task.new do |rd|
-  rd.main = "README.rdoc"
-  rd.rdoc_files.include("README.rdoc", "lib/**/*.rb")
+  rd.main = 'README.rdoc'
+  rd.rdoc_files.include('README.rdoc', 'lib/**/*.rb')
 end
 
 # install asciidoc libxml2-utils xmlto docbook-xsl docbook-xml
@@ -15,6 +15,6 @@ task :manpage do
 end
 
 Rake::TestTask.new do |t|
-  t.pattern = "spec/*_spec.rb"
+  t.pattern = 'spec/*_spec.rb'
 end
 task spec: :test

data/lib/validate_website.rb

@@ -1,2 +1,3 @@
 # encoding: utf-8
 require 'validate_website/core'
+require 'validate_website/version'

data/lib/validate_website/core.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 require 'set'
 require 'open-uri'
 require 'webrick/cookie'
@@ -10,7 +8,11 @@ require 'validate_website/colorful_messages'
 
 require 'spidr'
 
+# Base module ValidateWebsite
 module ValidateWebsite
+  autoload :Crawl, 'validate_website/crawl'
+  autoload :Static, 'validate_website/static'
+
   # Core class for static or website validation
   class Core
     attr_accessor :site
@@ -23,55 +25,16 @@ module ValidateWebsite
     EXIT_FAILURE_NOT_FOUND = 65
     EXIT_FAILURE_MARKUP_NOT_FOUND = 66
 
-    PING_URL = 'http://www.google.com/'
-
-    def initialize(options = {}, validation_type = :crawl)
+    def initialize(options = {}, validation_type)
       @not_founds_count = 0
       @errors_count = 0
-      @options = Parser.parse(options, validation_type)
+      @options = Parser.parse(options, validation_type).to_h
       @site = @options[:site]
-      @service_url =  @options[:'html5-validator-service-url']
+      @service_url =  @options[:html5_validator_service_url]
       Validator.html5_validator_service_url = @service_url if @service_url
       puts color(:note, "validating #{@site}\n", @options[:color])
     end
 
-    # @param [Hash] options
-    #   :color [Boolean] color output (true, false)
-    #   :exclude [String] a String used by Regexp.new
-    #   :markup [Boolean] Check the markup validity
-    #   :not_found [Boolean] Check for not found page (404)
-    #
-    def crawl(options = {})
-      @options = @options.to_hash.merge(options)
-      @options.merge!(ignore_links: @options[:exclude]) if @options[:exclude]
-      puts color(:warning, "No internet connection") unless internet_connection?
-
-      @crawler = spidr_crawler(@site, @options)
-      print_status_line(@crawler.history.size,
-                        @crawler.failures.size,
-                        @not_founds_count,
-                        @errors_count)
-    end
-
-    # @param [Hash] options
-    #
-    def crawl_static(options = {})
-      @options = @options.to_hash.merge(options)
-      @site = @options[:site]
-
-      files = Dir.glob(@options[:pattern])
-      files.each do |f|
-        next unless File.file?(f)
-
-        response = fake_httpresponse(open(f).read)
-        page = Spidr::Page.new(URI.join(@site, URI.encode(f)), response)
-
-        validate(page.doc, page.body, f) if @options[:markup]
-        check_static_not_found(page.links) if @options[:not_found]
-      end
-      print_status_line(files.size, 0, @not_founds_count, @errors_count)
-    end
-
     def errors?
       @errors_count > 0
     end
@@ -99,71 +62,33 @@ module ValidateWebsite
       end
     end
 
-    private
-
-    def internet_connection?
-      true if open(ValidateWebsite::Core::PING_URL)
-    rescue
-      false
-    end
-
-    def static_site_link(l)
-      link = URI.parse(URI.encode(l))
-      link = URI.join(@site, link) if link.host.nil?
-      link
-    end
-
-    def in_static_domain?(site, link)
-      URI.parse(site).host == link.host
-    end
-
-    # check files linked on static document
-    # see lib/validate_website/runner.rb
-    def check_static_not_found(links)
-      links.each_with_object(Set[]) do |l, result|
-        next if l.include?('#')
-        link = static_site_link(l)
-        next unless in_static_domain?(@site, link)
-        file_path = URI.parse(File.join(Dir.getwd, link.path || '/')).path
-        not_found_error(file_path) && next unless File.exist?(file_path)
-        # Check CSS url()
-        if File.extname(file_path) == '.css'
-          response = fake_httpresponse(open(file_path).read, ['text/css'])
-          css_page = Spidr::Page.new(l, response)
-          result.merge extract_urls_from_css(css_page)
-        end
-      end
-    end
-
-    def not_found_error(location)
-      puts "\n"
-      puts color(:error, "#{location} linked but not exist", @options[:color])
-      @not_founds_count += 1
-    end
-
     # Extract urls from CSS page
     #
     # @param [Spidr::Page] an Spidr::Page object
     # @return [Array] Lists of urls
     #
-    def extract_urls_from_css(page)
-      page.body.scan(/url\((['".\/\w-]+)\)/).reduce(Set[]) do |result, url|
-        url = url.first.gsub("'", "").gsub('"', '')
-        abs = page.to_absolute(URI.parse(url))
-        result << abs
+    def self.extract_urls_from_css(page)
+      page.body.scan(%r{url\((['".\/\w-]+)\)}).reduce(Set[]) do |result, url|
+        url = url.first.gsub("'", '').gsub('"', '')
+        abs = page.to_absolute(url)
+        result << abs.to_s
       end
     end
 
-    # Extract imgs urls from page
-    #
-    # @param [Spidr::Page] an Spidr::Page object
-    # @return [Array] Lists of urls
-    #
-    def extract_imgs_from_page(page)
-      page.doc.search('//img[@src]').reduce(Set[]) do |result, elem|
-        u = elem.attributes['src']
-        result << page.to_absolute(URI.parse(u))
-      end
+    private
+
+    def print_status_line(total, failures, not_founds, errors)
+      puts "\n\n"
+      puts color(:info, ["#{total} visited",
+                         "#{failures} failures",
+                         "#{not_founds} not founds",
+                         "#{errors} errors"].join(', '), options[:color])
+    end
+
+    def not_found_error(location)
+      puts "\n"
+      puts color(:error, "#{location} linked but not exist", options[:color])
+      @not_founds_count += 1
     end
 
     ##
@@ -177,65 +102,16 @@ module ValidateWebsite
       if validator.valid?
         print color(:success, '.', options[:color]) # rspec style
       else
-        @errors_count += 1
-        puts "\n"
-        puts color(:error, "* #{url}", options[:color])
-        if options[:verbose]
-          puts color(:error, validator.errors.join(', '), options[:color])
-        end
-      end
-    end
-
-    # Fake http response for Spidr static crawling
-    # see https://github.com/ruby/ruby/blob/trunk/lib/net/http/response.rb
-    #
-    # @param [String] response body
-    # @param [Array] content types
-    # @return [Net::HTTPResponse] fake http response
-    def fake_httpresponse(body, content_types = ['text/html', 'text/xhtml+xml'])
-      response = Net::HTTPResponse.new '1.1', 200, 'OK'
-      response.instance_variable_set(:@read, true)
-      response.body = body
-      content_types.each do |c|
-        response.add_field('content-type', c)
+        handle_validation_error(validator, url)
       end
-      response
     end
 
-    def print_status_line(total, failures, not_founds, errors)
-      puts "\n\n"
-      puts color(:info, ["#{total} visited",
-                         "#{failures} failures",
-                         "#{not_founds} not founds",
-                         "#{errors} errors"].join(', '), @options[:color])
-    end
-
-    def spidr_crawler(site, options)
-      @host = URI(site).host
-      Spidr.site(site, options) do |crawler|
-        crawler.cookies[@host] = default_cookies if options[:cookies]
-        crawler.every_css_page do |page|
-          extract_urls_from_css(page).each do |u|
-            crawler.enqueue(u)
-          end
-        end
-
-        crawler.every_html_page do |page|
-          extract_imgs_from_page(page).each do |i|
-            crawler.enqueue(i)
-          end
-
-          if options[:markup] && page.html?
-            validate(page.doc, page.body, page.url, options[:ignore])
-          end
-        end
-
-        if options[:not_found]
-          crawler.every_failed_url do |url|
-            not_found_error(url)
-          end
-        end
-      end
+    def handle_validation_error(validator, url)
+      @errors_count += 1
+      puts "\n"
+      puts color(:error, "* #{url}", options[:color])
+      return unless options[:verbose]
+      puts color(:error, validator.errors.join(', '), options[:color])
     end
   end
 end

data/lib/validate_website/crawl.rb

@@ -0,0 +1,78 @@
+require 'validate_website/core'
+
+module ValidateWebsite
+  # Class for http website validation
+  class Crawl < Core
+    def initialize(options = {}, validation_type = :crawl)
+      super
+    end
+
+    # @param [Hash] options
+    #   :color [Boolean] color output (true, false)
+    #   :exclude [String] a String used by Regexp.new
+    #   :markup [Boolean] Check the markup validity
+    #   :not_found [Boolean] Check for not found page (404)
+    #
+    def crawl(options = {})
+      @options = @options.merge(options)
+      @options.merge!(ignore_links: @options[:exclude]) if @options[:exclude]
+
+      @crawler = spidr_crawler(@site, @options)
+      print_status_line(@crawler.history.size,
+                        @crawler.failures.size,
+                        @not_founds_count,
+                        @errors_count)
+    end
+
+    private
+
+    # Extract imgs urls from page
+    #
+    # @param [Spidr::Page] an Spidr::Page object
+    # @return [Array] Lists of urls
+    #
+    def extract_imgs_from_page(page)
+      page.doc.search('//img[@src]').reduce(Set[]) do |result, elem|
+        u = elem.attributes['src']
+        result << page.to_absolute(URI.parse(u))
+      end
+    end
+
+    def spidr_crawler(site, options)
+      @host = URI(site).host
+      Spidr.site(site, options) do |crawler|
+        crawler.cookies[@host] = default_cookies if options[:cookies]
+        on_every_css_page(crawler)
+        on_every_html_page(crawler)
+        on_every_failed_url(crawler)
+      end
+    end
+
+    def on_every_css_page(crawler)
+      crawler.every_css_page do |page|
+        ValidateWebsite::Core.extract_urls_from_css(page).each do |u|
+          crawler.enqueue(u)
+        end
+      end
+    end
+
+    def on_every_html_page(crawler)
+      crawler.every_html_page do |page|
+        extract_imgs_from_page(page).each do |i|
+          crawler.enqueue(i)
+        end
+
+        if options[:markup] && page.html?
+          validate(page.doc, page.body, page.url, options[:ignore])
+        end
+      end
+    end
+
+    def on_every_failed_url(crawler)
+      return unless options[:not_found]
+      crawler.every_failed_url do |url|
+        not_found_error(url)
+      end
+    end
+  end
+end

data/lib/validate_website/option_parser.rb

@@ -4,38 +4,69 @@ require 'slop'
 module ValidateWebsite
   # Internal class for parse command line args
   class Parser
+    VALID_TYPES = [:crawl, :static].freeze
+
     DEFAULT_OPTIONS = {
+      site: 'http://localhost/',
+      pattern: '**/*.html',
+      exclude: nil,
+      user_agent: nil,
       markup: true,
       # crawler: log not found url (404 status code)
       # static: log not found url (not on filesystem, `pwd` considered
-      # as root « / »)
+      # as root " / ")
       not_found: false,
       file: nil,
       # regex to ignore certain validation errors
       ignore: nil,
       color: true,
       # internal verbose for ValidateWebsite
-      verbose: false,
+      verbose: false
     }
 
-    DEFAULT_OPTIONS_CRAWL = {
-      site: 'http://localhost:3000/',
-      exclude: nil,
-      user_agent: nil,
-    }.merge(DEFAULT_OPTIONS)
-
-    DEFAULT_OPTIONS_STATIC = {
-      site: 'http://www.example.com/',
-      pattern: '**/*.html',
-    }.merge(DEFAULT_OPTIONS)
-
+    # Generic parse method for crawl or static options
     def self.parse(options, type)
-      const = "DEFAULT_OPTIONS_#{type.to_s.upcase}"
-      fail ArgumentError unless const_defined?(const)
-      if Array === options
+      fail ArgumentError unless VALID_TYPES.include?(type)
+      # We are in command line (ARGV)
+      if options.is_a?(Array)
         send("command_line_parse_#{type}", options)
       else
-        const_get(const).merge(options)
+        # for testing or Ruby usage with a Hash
+        DEFAULT_OPTIONS.merge(options)
+      end
+    end
+
+    def self.default_args
+      Slop.parse do |o|
+        yield o if block_given?
+        boolean_options(o)
+        o.regexp('-i', '--ignore',
+                 'Validation errors to ignore (ex: "valign|autocorrect")')
+        o.string('-5', '--html5-validator-service-url',
+                 'Change default html5 validator service URL')
+        verbose_help_options(o)
+      end
+    end
+
+    def self.boolean_options(o)
+      o.bool('-m', '--markup',
+             "Markup validation (default: #{DEFAULT_OPTIONS[:markup]})",
+             default: DEFAULT_OPTIONS[:markup])
+      o.bool('-n', '--not-found',
+             "Log not found url (default: #{DEFAULT_OPTIONS[:not_found]})",
+             default: DEFAULT_OPTIONS[:not_found])
+      o.bool('--color',
+             "Show colored output (default: #{DEFAULT_OPTIONS[:color]})",
+             default: DEFAULT_OPTIONS[:color])
+    end
+
+    def self.verbose_help_options(o)
+      o.bool('-v', '--verbose',
+             "Show validator errors (default: #{DEFAULT_OPTIONS[:verbose]})",
+             default: DEFAULT_OPTIONS[:verbose])
+      o.on('-h', '--help', 'Display this help message.') do
+        puts o
+        exit
       end
     end
 
@@ -43,28 +74,15 @@ module ValidateWebsite
     # @params [ARGV]
     # @return [Hash]
     def self.command_line_parse_crawl(_args)
-      Slop.parse(help: true) do
-        banner 'Usage: validate-website [OPTIONS]'
-
-        on("s", "site=", "Website to crawl",
-           default: DEFAULT_OPTIONS_CRAWL[:site])
-        on(:u, :user_agent=, "Change user agent",
-           default: DEFAULT_OPTIONS_CRAWL[:user_agent])
-        on("e", "exclude=", "Url to exclude (ex: 'redirect|news')",
-           type: :regexp)
-        on("c", "cookies=", "Set defaults cookies")
-        on("m", "markup", "Markup validation",
-           default: DEFAULT_OPTIONS_CRAWL[:markup])
-        on("i", "ignore=", "Validation errors to ignore",
-           type: :regexp)
-        on(:n, :not_found, "Log not found url",
-           default: DEFAULT_OPTIONS_CRAWL[:not_found])
-        on("color", "Show colored output",
-           default: DEFAULT_OPTIONS_CRAWL[:color])
-        on("5", "html5-validator-service-url=",
-           "Change default html5 validator service URL")
-        on("v", "verbose", "Show validator errors",
-           default: DEFAULT_OPTIONS_CRAWL[:verbose])
+      default_args do |o|
+        o.string('-s', '--site',
+                 "Website to crawl (default: #{DEFAULT_OPTIONS[:site]})",
+                 default: DEFAULT_OPTIONS[:site])
+        o.string('-u', '--user-agent',
+                 'Change user agent',
+                 default: DEFAULT_OPTIONS[:user_agent])
+        o.regexp('-e', '--exclude', 'Url to exclude (ex: "redirect|news")')
+        o.string('-c', '--cookies', 'Set defaults cookies')
       end
     end
 
@@ -72,26 +90,13 @@ module ValidateWebsite
     # @params [ARGV]
     # @return [Hash]
     def self.command_line_parse_static(_args)
-      Slop.parse(help: true) do
-        banner 'Usage: validate-website-static [OPTIONS]'
-
-        on("s", "site=", "Website to crawl",
-           default: DEFAULT_OPTIONS_STATIC[:site])
-        on("p", "pattern=", "Change filenames pattern",
-           type: :regexp, default: DEFAULT_OPTIONS_STATIC[:pattern])
-        on("c", "cookies=", "Set defaults cookies")
-        on("m", "markup", "Markup validation",
-           default: DEFAULT_OPTIONS_STATIC[:markup])
-        on("i", "ignore=", "Validation errors to ignore",
-           type: :regexp)
-        on(:n, :not_found, "Log not found url",
-           default: DEFAULT_OPTIONS_STATIC[:not_found])
-        on("color", "Show colored output",
-           default: DEFAULT_OPTIONS_STATIC[:color])
-        on("5", "html5-validator-service-url=",
-           "Change default html5 validator service URL")
-        on("v", "verbose", "Show validator errors",
-           default: DEFAULT_OPTIONS_STATIC[:verbose])
+      default_args do |o|
+        o.string('-s', '--site',
+                 "Website to crawl (default: #{DEFAULT_OPTIONS[:site]})",
+                 default: DEFAULT_OPTIONS[:site])
+        o.regexp('-p', '--pattern',
+                 "Filename pattern (default: #{DEFAULT_OPTIONS[:pattern]})",
+                 default: DEFAULT_OPTIONS[:pattern])
       end
     end
   end