validate-website 0.9.5 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e99e7c0d34cdd411b03d1a8a0990ae75f029a1b8
-  data.tar.gz: 601f3a5c01cba46f473a4cead89f10908e2e4d6b
+  metadata.gz: 278cca47ee331ef1b68e0c64aaca600af25917d6
+  data.tar.gz: cdb2ba927605214d326b080d5dad5afe4a6a4930
 SHA512:
-  metadata.gz: 833b02e5cdb8f91f314042c1db35e4f5dcd1ca17347d6bce93a31ba329424b97ad80caf13d6fcfb4ccf12f78a2618bbdcf7e17a89b7b9c440bc52e4b6fdcad9d
-  data.tar.gz: 96cc9f5b765251133c82d5525fcc1a5b1232aef251c76f2a65810f5d721d20bd1ae4dc3d158159726fcc6d1a3efdfbfa0c2688c2b1b3fd6aff0f9d8bf7eb4da2
+  metadata.gz: fde7934823f6e0b41d4baa22a784758c756d104ed2a91429585c50291be9b4e3bf9da60f978fe82bdba05e22d01b5157f0bace57a6f5c3398f6492117e941775
+  data.tar.gz: bf687a37f1a933a981a1696e88ef0b00f311c46db635170b2021faf46475bb44c5612715abbedd96a14cb05e159954716d058e056e09db97ec118c221e1953ca

data/lib/validate_website/core.rb

@@ -2,6 +2,7 @@
 
 require 'set'
 require 'open-uri'
+require 'webrick/cookie'
 
 require 'validate_website/option_parser'
 require 'validate_website/validator'
@@ -13,7 +14,7 @@ module ValidateWebsite
   # Core class for static or website validation
   class Core
     attr_accessor :site
-    attr_reader :options, :crawler
+    attr_reader :options, :crawler, :errors_count, :not_founds_count, :host
 
     include ColorfulMessages
 
@@ -25,37 +26,27 @@ module ValidateWebsite
     PING_URL = 'http://www.google.com/'
 
     def initialize(options = {}, validation_type = :crawl)
-      @markup_error = nil
-      @not_found_error = nil
-
+      @not_founds_count = 0
+      @errors_count = 0
       @options = Parser.parse(options, validation_type)
-
-      @file = @options[:file]
-      if @file
-        # truncate file
-        open(@file, 'w').write('')
-      end
-
       @site = @options[:site]
+      puts color(:note, "validating #{@site}\n", @options[:color])
     end
 
-    ##
-    #
     # @param [Hash] options
-    #   :quiet [Boolean] no output (true, false)
     #   :color [Boolean] color output (true, false)
     #   :exclude [String] a String used by Regexp.new
-    #   :markup_validation [Boolean] Check the markup validity
+    #   :markup [Boolean] Check the markup validity
     #   :not_found [Boolean] Check for not found page (404)
     #
-    def crawl(opts = {})
-      opts = @options.merge(opts)
-      opts.merge!(ignore_links: Regexp.new(opts[:exclude])) if opts[:exclude]
-
-      puts color(:note, "validating #{@site}", opts[:color]) unless opts[:quiet]
+    def crawl(options = {})
+      @options = @options.to_hash.merge(options)
+      @options.merge!(ignore_links: @options[:exclude]) if @options[:exclude]
       puts color(:warning, "No internet connection") unless internet_connection?
 
-      @crawler = Spidr.site(@site, opts) do |crawler|
+      @host = URI(@site).host
+      @crawler = Spidr.site(@site, @options) do |crawler|
+        crawler.cookies[@host] = default_cookies if @options[:cookies]
         crawler.every_css_page do |page|
           extract_urls_from_css(page).each do |u|
             crawler.enqueue(u)
@@ -67,75 +58,98 @@ module ValidateWebsite
             crawler.enqueue(i)
           end
 
-          if opts[:markup_validation] && page.html?
-            validate(page.doc, page.body, page.url, opts)
+          if @options[:markup] && page.html?
+            validate(page.doc, page.body, page.url, @options[:ignore])
           end
         end
 
-        crawler.every_failed_url do |url|
-          if opts[:not_found]
-            @not_found_error = true
-            puts color(:error, "#{url} linked but not exist", opts[:color])
-            to_file(url)
+        if @options[:not_found]
+          crawler.every_failed_url do |url|
+            not_found_error(url)
           end
         end
       end
+      print_status_line(@crawler.history.size,
+                        @crawler.failures.size,
+                        @not_founds_count,
+                        @errors_count)
     end
 
-    def internet_connection?
-      true if open(ValidateWebsite::Core::PING_URL)
-    rescue
-      false
-    end
-
-    def crawl_static(opts = {})
-      opts = @options.merge(opts)
-      puts color(:note, "validating #{@site}", opts[:color])
+    # @param [Hash] options
+    #
+    def crawl_static(options = {})
+      @options = @options.to_hash.merge(options)
+      @site = @options[:site]
 
-      files = Dir.glob(opts[:pattern])
+      files = Dir.glob(@options[:pattern])
       files.each do |f|
         next unless File.file?(f)
 
-        response = fake_http_response(open(f).read)
-        page = Spidr::Page.new(URI.parse(opts[:site] + URI.encode(f)), response)
+        response = fake_httpresponse(open(f).read)
+        page = Spidr::Page.new(URI.join(@site, URI.encode(f)), response)
 
-        if opts[:markup_validation]
-          validate(page.doc, page.body, f)
-        end
-        if opts[:not_found]
-          check_static_not_found(page.links)
-        end
+        validate(page.doc, page.body, f) if @options[:markup]
+        check_static_not_found(page.links) if @options[:not_found]
       end
+      print_status_line(files.size, 0, @not_founds_count, @errors_count)
+    end
+
+    def errors?
+      @errors_count > 0
+    end
+
+    def not_founds?
+      @not_founds_count > 0
     end
 
     def exit_status
-      if @markup_error && @not_found_error
+      if errors? && not_founds?
         EXIT_FAILURE_MARKUP_NOT_FOUND
-      elsif @markup_error
+      elsif errors?
         EXIT_FAILURE_MARKUP
-      elsif @not_found_error
+      elsif not_founds?
         EXIT_FAILURE_NOT_FOUND
       else
         EXIT_SUCCESS
       end
     end
 
+    def default_cookies
+      WEBrick::Cookie.parse(@options[:cookies]).inject({}) do |hash, cookie|
+        hash[cookie.name] = cookie.value
+        hash
+      end
+    end
+
     private
 
-    def to_file(msg)
-      return unless @file
-      open(@file, 'a').write("#{msg}\n") if File.exist?(@file)
+    def internet_connection?
+      true if open(ValidateWebsite::Core::PING_URL)
+    rescue
+      false
+    end
+
+    def static_site_link(l)
+      link = URI.parse(URI.encode(l))
+      link = URI.join(@site, link) if link.host.nil?
+      link
+    end
+
+    def in_static_domain?(site, link)
+      URI.parse(site).host == link.host
     end
 
     # check files linked on static document
     # see lib/validate_website/runner.rb
     def check_static_not_found(links)
       links.each do |l|
-        file_location = URI.parse(File.join(Dir.getwd, l.path)).path
-        not_found_error and next unless File.exist?(file_location)
+        link = static_site_link(l)
+        next unless in_static_domain?(@site, link)
+        file_path = URI.parse(File.join(Dir.getwd, link.path || '/')).path
+        not_found_error(file_path) && next unless File.exist?(file_path)
         # Check CSS url()
-        if File.extname(file_location) == '.css'
-          response = fake_http_response(open(file_location).read, ['text/css'])
+        if File.extname(file_path) == '.css'
+          response = fake_httpresponse(open(file_path).read, ['text/css'])
           css_page = Spidr::Page.new(l, response)
           links.concat extract_urls_from_css(css_page)
           links.uniq!
@@ -143,10 +157,10 @@ module ValidateWebsite
       end
     end
 
-    def not_found_error
-      @not_found_error = true
-      puts color(:error, "#{file_location} linked but not exist", @options[:color])
-      to_file(file_location)
+    def not_found_error(location)
+      puts "\n"
+      puts color(:error, "#{location} linked but not exist", @options[:color])
+      @not_founds_count += 1
     end
 
     # Extract urls from CSS page
@@ -178,28 +192,19 @@ module ValidateWebsite
     # @param [Nokogiri::HTML::Document] original_doc
     # @param [String] The raw HTTP response body of the page
     # @param [String] url
-    # @param [Hash] options
-    #   :quiet no output (true, false)
-    #   :color color output (true, false)
+    # @param [Regexp] Errors to ignore
     #
-    def validate(doc, body, url, opts = {})
-      opts = @options.merge(opts)
-      validator = Validator.new(doc, body, opts)
-      msg = " well formed? #{validator.valid?}"
-      # TODO: create a formatter
+    def validate(doc, body, url, ignore = nil)
+      validator = Validator.new(doc, body, ignore)
       if validator.valid?
-        if opts[:quiet]
-          print color(:success, '.', opts[:color]) # rspec style
-        else
-          print color(:info, url, opts[:color])
-          puts color(:success, msg, opts[:color])
-        end
+        print color(:success, '.', options[:color]) # rspec style
       else
-        @markup_error = true
-        print color(:info, url, opts[:color])
-        puts color(:error, msg, opts[:color])
-        puts color(:error, validator.errors.join(', '), opts[:color]) if opts[:verbose]
-        to_file(url)
+        @errors_count += 1
+        puts "\n"
+        puts color(:error, "* #{url}", options[:color])
+        if options[:verbose]
+          puts color(:error, validator.errors.join(', '), options[:color])
+        end
       end
     end
 
@@ -209,7 +214,7 @@ module ValidateWebsite
     # @param [String] response body
     # @param [Array] content types
     # @return [Net::HTTPResponse] fake http response
-    def fake_http_response(body, content_types = ['text/html', 'text/xhtml+xml'])
+    def fake_httpresponse(body, content_types = ['text/html', 'text/xhtml+xml'])
       response = Net::HTTPResponse.new '1.1', 200, 'OK'
       response.instance_variable_set(:@read, true)
       response.body = body
@@ -218,5 +223,13 @@ module ValidateWebsite
       end
       response
     end
+
+    def print_status_line(total, failures, not_founds, errors)
+      puts "\n\n"
+      puts color(:info, ["#{total} visited",
+                         "#{failures} failures",
+                         "#{not_founds} not founds",
+                         "#{errors} errors"].join(', '), @options[:color])
+    end
   end
 end

data/lib/validate_website/option_parser.rb

@@ -1,172 +1,94 @@
 # encoding: utf-8
-require 'optparse'
+require 'slop'
 
 module ValidateWebsite
   # Internal class for parse command line args
   class Parser
-    DEFAULT_OPTS_ALL = {
-      markup_validation: true,
+    DEFAULT_OPTIONS = {
+      markup: true,
       # crawler: log not found url (404 status code)
       # static: log not found url (not on filesystem, `pwd` considered
       # as root « / »)
       not_found: false,
-      quiet: false,
       file: nil,
       # regex to ignore certain validation errors
-      ignore_errors: nil,
+      ignore: nil,
       color: true,
       # internal verbose for ValidateWebsite
       verbose: false,
     }
 
-    DEFAULT_OPTS_CRAWL = {
+    DEFAULT_OPTIONS_CRAWL = {
       site: 'http://localhost:3000/',
       exclude: nil,
-    }.merge(DEFAULT_OPTS_ALL)
+      user_agent: nil,
+    }.merge(DEFAULT_OPTIONS)
 
-    DEFAULT_OPTS_STATIC = {
+    DEFAULT_OPTIONS_STATIC = {
       site: 'http://www.example.com/',
       pattern: '**/*.html',
-    }.merge(DEFAULT_OPTS_ALL)
+    }.merge(DEFAULT_OPTIONS)
 
     def self.parse(options, type)
-      if const_defined?("DEFAULT_OPTS_#{type.to_s.upcase}")
-        @default_opts = const_get("DEFAULT_OPTS_#{type.to_s.upcase}")
-        if Array === options
-          send("command_line_parse_#{type}", options)
-        else
-          @default_opts.merge(options)
-        end
+      const = "DEFAULT_OPTIONS_#{type.to_s.upcase}"
+      fail ArgumentError unless const_defined?(const)
+      if Array === options
+        send("command_line_parse_#{type}", options)
       else
-        fail ArgumentError, "Unknown options type : #{type}"
+        const_get(const).merge(options)
       end
     end
 
-    def self.command_line_parse_crawl(args)
-      options = {}
-      opts = OptionParser.new do |o|
-        o.set_summary_indent('  ')
-        o.banner =    'Usage: validate-website [OPTIONS]'
-        o.define_head %(validate-website - Web crawler for checking the \
-validity of your documents)
-        o.separator ''
+    # Parse command line for validate-website bin
+    # @params [ARGV]
+    # @return [Hash]
+    def self.command_line_parse_crawl(_args)
+      Slop.parse(help: true) do
+        banner 'Usage: validate-website [OPTIONS]'
 
-        o.on("-s", "--site 'SITE'", String,
-             "Website to crawl (Default: #{@default_opts[:site]})") { |v|
-          options[:site] = v
-        }
-        o.on("-u", "--user-agent 'USERAGENT'", String,
-             "Change user agent") { |v|
-          options[:user_agent] = v
-        }
-        o.on("-e", "--exclude 'EXCLUDE'", String,
-             "Url to exclude (ex: 'redirect|news')") { |v|
-          options[:exclude] = v
-        }
-        o.on("-f", "--file 'FILE'", String,
-             "Save not well formed or not found urls") { |v|
-          options[:file] = v
-        }
-
-        o.on("-c", "--cookies 'COOKIES'", String,
-             "Set defaults cookies") { |v|
-          options[:cookies] = v
-        }
-
-        o.on("-m", "--[no-]markup-validation",
-             %(Markup validation \
-(Default: #{@default_opts[:markup_validation]}))) { |v|
-          options[:markup_validation] = v
-        }
-        o.on("-i", "--ignore-errors 'IGNORE'", String,
-             "Validation errors to ignore (regex)") { |v|
-          options[:ignore_errors] = v
-        }
-        o.on("-n", "--not-found",
-             "Log not found url (Default: #{@default_opts[:not_found]})") { |v|
-          options[:not_found] = v
-        }
-        o.on("--[no-]color",
-             "Show colored output (Default: #{@default_opts[:color]})") { |v|
-          options[:color] = v
-        }
-        o.on("-v", "--verbose",
-             %(Show validator errors \
-(Default: #{@default_opts[:verbose]}))) { |v|
-          options[:verbose] = v
-        }
-        o.on("-q", "--quiet",
-             "Only report errors (Default: #{@default_opts[:quiet]})") { |v|
-          options[:quiet] = v
-        }
-
-        o.separator ""
-        o.on_tail("-h", "--help", "Show this help message.") do
-          puts o
-          exit
-        end
+        on("s", "site=", "Website to crawl",
+           default: DEFAULT_OPTIONS_CRAWL[:site])
+        on(:u, :user_agent=, "Change user agent",
+           default: DEFAULT_OPTIONS_CRAWL[:user_agent])
+        on("e", "exclude=", "Url to exclude (ex: 'redirect|news')",
+           type: :regexp)
+        on("c", "cookies=", "Set defaults cookies")
+        on("m", "markup", "Markup validation",
+           default: DEFAULT_OPTIONS_CRAWL[:markup])
+        on("i", "ignore=", "Validation errors to ignore",
+           type: :regexp)
+        on(:n, :not_found, "Log not found url",
+           default: DEFAULT_OPTIONS_CRAWL[:not_found])
+        on("color", "Show colored output",
+           default: DEFAULT_OPTIONS_CRAWL[:color])
+        on("v", "verbose", "Show validator errors",
+           default: DEFAULT_OPTIONS_CRAWL[:verbose])
       end
-      command_line_parse!(opts, args, options)
     end
 
-    def self.command_line_parse_static(args)
-      options = {}
-      opts = OptionParser.new do |o|
-        o.set_summary_indent('  ')
-        o.banner =    'Usage: validate-website-static [OPTIONS]'
-        o.define_head %(validate-website-static - check the validity of \
-          your documents)
-        o.separator ''
+    # Parse command line for validate-website-static bin
+    # @params [ARGV]
+    # @return [Hash]
+    def self.command_line_parse_static(_args)
+      Slop.parse(help: true) do
+        banner 'Usage: validate-website-static [OPTIONS]'
 
-        o.on("-s", "--site 'SITE'", String,
-             %(Where static files will be hosted \
-(Default: #{@default_opts[:site]}))) { |v|
-          options[:site] = v
-        }
-        o.on("-p", "--pattern 'PATTERN'", String,
-             %(Change filenames pattern \
-(Default: #{@default_opts[:pattern]}))) { |v|
-          options[:pattern] = v.strip
-        }
-        o.on("-f", "--file 'FILE'", String,
-             "Save not well formed urls") { |v|
-          options[:file] = v
-        }
-        o.on("-i", "--ignore-errors 'IGNORE'", String,
-             "Validation errors to ignore (regex)") { |v|
-          options[:ignore_errors] = v
-        }
-
-        o.on("-m", "--[no-]markup-validation",
-             %(Markup validation \
-(Default: #{@default_opts[:markup_validation]}))) { |v|
-          options[:markup_validation] = v
-        }
-        o.on("-n", "--not-found",
-             %(Log files not on filesystem, pwd considered as root « / » \
-(Default: #{@default_opts[:not_found]}))) { |v|
-          options[:not_found] = v
-        }
-        o.on("-v", "--verbose",
-             %(Show validator errors \
-(Default: #{@default_opts[:verbose]}))) { |v|
-          options[:verbose] = v
-        }
-        o.on("-q", "--quiet",
-             "Only report errors (Default: #{@default_opts[:quiet]})") { |v|
-          options[:quiet] = v
-        }
+        on("s", "site=", "Website to crawl",
+           default: DEFAULT_OPTIONS_STATIC[:site])
+        on("p", "pattern=", "Change filenames pattern",
+           type: :regexp, default: DEFAULT_OPTIONS_STATIC[:pattern])
+        on("c", "cookies=", "Set defaults cookies")
+        on("m", "markup", "Markup validation",
+           default: DEFAULT_OPTIONS_STATIC[:markup])
+        on("i", "ignore=", "Validation errors to ignore",
+           type: :regexp)
+        on(:n, :not_found, "Log not found url",
+           default: DEFAULT_OPTIONS_STATIC[:not_found])
+        on("color", "Show colored output",
+           default: DEFAULT_OPTIONS_STATIC[:color])
+        on("v", "verbose", "Show validator errors",
+           default: DEFAULT_OPTIONS_STATIC[:verbose])
       end
-      command_line_parse!(opts, args, options)
-    end
-
-    def self.command_line_parse!(opts, args, options)
-      opts.parse!(args)
-      @default_opts.merge(options)
-    rescue OptionParser::InvalidOption, OptionParser::MissingArgument
-      puts $ERROR_INFO.to_s
-      puts opts
-      exit 128
     end
   end
 end

data/lib/validate_website/validator.rb

@@ -13,13 +13,15 @@ module ValidateWebsite
     ##
     # @param [Nokogiri::HTML::Document] original_doc
     # @param [String] The raw HTTP response body of the page
-    def initialize(original_doc, body, opts = {})
+    # @param [Regexp] Errors to ignore
+    #
+    def initialize(original_doc, body, ignore = nil)
       @original_doc = original_doc
       @body = body
-      @options = opts
+      @ignore = ignore
       @dtd = @original_doc.internal_subset
       init_namespace(@dtd)
-      find_errors
+      @errors = []
     end
 
     ##
@@ -29,12 +31,8 @@ module ValidateWebsite
     end
 
     def errors
-      if @options[:ignore_errors]
-        ignore_re = Regexp.compile @options[:ignore_errors]
-        @errors.reject { |e| ignore_re =~ e }
-      else
-        @errors
-      end
+      find_errors
+      @ignore ? @errors.reject { |e| @ignore =~ e } : @errors
     end
 
     private
@@ -48,14 +46,16 @@ module ValidateWebsite
       @namespace = File.basename(@dtd_uri.path, '.dtd')
     end
 
-    def find_errors
-      @errors = []
+    def document
+      return @document if @document
       if @dtd_uri && @body.match(@dtd_uri.to_s)
-        document = @body.sub(@dtd_uri.to_s, @namespace + '.dtd')
+        @document = @body.sub(@dtd_uri.to_s, @namespace + '.dtd')
       else
-        document = @body
+        @document = @body
       end
+    end
 
+    def find_errors
       @doc = Dir.chdir(XHTML_PATH) do
         Nokogiri::XML(document) { |cfg|
           cfg.noent.dtdload.dtdvalid

data/man/man1/validate-website-static.1

@@ -2,12 +2,12 @@
 .\"     Title: validate-website-static
 .\"    Author: [see the "AUTHOR" section]
 .\" Generator: DocBook XSL Stylesheets v1.78.1 <http://docbook.sf.net/>
-.\"      Date: 09/20/2014
+.\"      Date: 09/26/2014
 .\"    Manual: \ \&
 .\"    Source: \ \&
 .\"  Language: English
 .\"
-.TH "VALIDATE\-WEBSITE\-S" "1" "09/20/2014" "\ \&" "\ \&"
+.TH "VALIDATE\-WEBSITE\-S" "1" "09/26/2014" "\ \&" "\ \&"
 .\" -----------------------------------------------------------------
 .\" * Define some portability stuff
 .\" -----------------------------------------------------------------
@@ -48,23 +48,18 @@ http://www\&.example\&.com/)
 Change filenames pattern (Default: **/*\&.html)
 .RE
 .PP
-\fB\-i\fR, \fB\-\-ignore\-errors\fR \fIIGNORE\fR
+\fB\-i\fR, \fB\-\-ignore\fR \fIIGNORE\fR
 .RS 4
 Ignore certain validation errors (ex:
 \fIautocorrect\fR)
 .RE
 .PP
-\fB\-f\fR, \fB\-\-file\fR \fIFILE\fR
-.RS 4
-Save not well formed urls
-.RE
-.PP
-\fB\-m\fR, \fB\-\-[no\-]markup\-validation\fR
+\fB\-m\fR, \fB\-\-[no\-]markup\fR
 .RS 4
 Markup validation (Default: true)
 .RE
 .PP
-\fB\-n\fR, \fB\-\-not\-found\fR
+\fB\-n\fR, \fB\-\-notfound\fR
 .RS 4
 Log files not on filesystem, pwd considered as root \(Fo / \(Fc (Default: false)
 .RE
@@ -79,11 +74,6 @@ Show colored output (Default: true)
 Show detail of validator errors (Default: false)\&.
 .RE
 .PP
-\fB\-q\fR, \fB\-\-quiet\fR
-.RS 4
-Only report errors (Default: false)\&.
-.RE
-.PP
 \fB\-h\fR, \fB\-\-help\fR
 .RS 4
 Show help message and exit\&.
@@ -116,4 +106,4 @@ Laurent Arnoud <laurent@spkdev\&.net>
 .sp
 The MIT License
 .sp
-Copyright (c) 2009\-2011 Laurent Arnoud <laurent@spkdev\&.net>
+Copyright (c) 2009\-2014 Laurent Arnoud <laurent@spkdev\&.net>