validate-website 0.6.5 → 0.7.0

data/README.rdoc

@@ -41,6 +41,8 @@ HTML5 support with Validator.nu Web Service.
       Markup validation (Default: true)
   -n, --not-found
       Log not found url (Default: false)
+  --[no-]color
+      Show colored output (Default: true)
   -v, --verbose
       Show detail of validator errors (Default: false).
   -q, --quiet

data/lib/validate_website/colorful_messages.rb

@@ -3,26 +3,31 @@ require 'rainbow'
 
 module ValidateWebsite
   module ColorfulMessages
+    def color(type, message, colored=true)
+      return message unless colored
+      send(type, message)
+    end
+
     def error(message)
-      message.to_s.foreground(:red)
+      message.foreground(:red)
     end
 
     def warning(message)
-      message.to_s.foreground(:yellow)
+      message.foreground(:yellow)
     end
 
     def success(message)
-      message.to_s.foreground(:green)
+      message.foreground(:green)
     end
 
     alias_method :message, :success
 
     def note(message)
-      message.to_s.foreground(:magenta)
+      message.foreground(:magenta)
     end
 
     def info(message)
-      message.to_s.foreground(:blue)
+      message.foreground(:blue)
     end
   end
 end

data/lib/validate_website/core.rb

@@ -37,53 +37,24 @@ module ValidateWebsite
       @site = @options[:site]
     end
 
-    def validate(doc, body, url, opts={})
-      opts = @options.merge(opts)
-      validator = Validator.new(doc, body)
-      msg = " well formed? %s" % validator.valid?
-      if validator.valid?
-        unless opts[:quiet]
-          print info(url)
-          puts success(msg)
-        end
-      else
-        @markup_error = true
-        print info(url)
-        puts error(msg)
-        puts error(validator.errors.join(", ")) if opts[:validate_verbose]
-        to_file(url)
-      end
-    end
-
     def crawl(opts={})
       opts = @options.merge(opts)
-      puts note("Validating #{@site}") if opts[:validate_verbose]
+      puts color(:note, "validating #{@site}", opts[:color])
 
       @anemone = Anemone.crawl(@site, opts) do |anemone|
         anemone.skip_links_like Regexp.new(opts[:exclude]) if opts[:exclude]
 
         # select the links on each page to follow (iframe, link, css url)
-        anemone.focus_crawl { |p|
+        anemone.focus_crawl { |page|
           links = []
-          if p.html?
-            p.doc.css('img, script, iframe').each do |elem|
-              url = get_url(p, elem, "src")
-              links << url unless url.nil?
-            end
-            p.doc.css('link').each do |link|
-              url = get_url(p, link, "href")
-              links << url unless url.nil?
-            end
+          if page.html?
+            links.concat extract_urls_from_img_script_iframe_link(page)
           end
-          if p.content_type == 'text/css'
-            p.body.scan(/url\((['".\/\w-]+)\)/).each do |url|
-              url = url.first.gsub("'", "").gsub('"', '')
-              abs = p.to_absolute(URI(url))
-              links << abs
-            end
+          if page.content_type == 'text/css'
+            links.concat extract_urls_from_css(page)
           end
           links.uniq!
-          p.links.concat(links)
+          page.links.concat(links)
         }
 
         anemone.on_every_page { |page|
@@ -98,7 +69,7 @@ module ValidateWebsite
 
           if opts[:not_found] && page.not_found?
             @not_found_error = true
-            puts error("%s linked in %s but not exist" % [url, page.referer])
+            puts color(:error, "%s linked in %s but not exist" % [url, page.referer], opts[:color])
             to_file(url)
           end
 
@@ -108,17 +79,24 @@ module ValidateWebsite
       end
     end
 
-    # check files linked on static document
-    # see lib/validate_website/runner.rb
-    def check_static_not_found(links, opts={})
+    def crawl_static(opts={})
       opts = @options.merge(opts)
-      if opts[:not_found]
-        links.each do |l|
-          unless File.exists?(l.path)
-            @not_found_error = true
-            puts error("%s linked but not exist" % l)
-            to_file(l)
-          end
+      puts color(:note, "validating #{@site}", opts[:color])
+
+      files = Dir.glob(opts[:pattern])
+      files.each do |f|
+        next unless File.file?(f)
+
+        page = Anemone::Page.new(URI.parse(opts[:site] + f), :body => open(f).read,
+                                 :headers => {'content-type' => ['text/html', 'application/xhtml+xml']})
+
+        if opts[:markup_validation]
+          validate(page.doc, page.body, f)
+        end
+        if opts[:not_found]
+          links = page.links
+          links.concat extract_urls_from_img_script_iframe_link(page)
+          check_static_not_found(links.uniq)
         end
       end
     end
@@ -136,6 +114,7 @@ module ValidateWebsite
     end
 
     private
+
     def to_file(msg)
       if @file && File.exist?(@file)
         open(@file, 'a').write("#{msg}\n")
@@ -143,14 +122,81 @@ module ValidateWebsite
     end
 
     def get_url(page, elem, attrname)
-      u = elem.attributes[attrname] if elem.attributes[attrname]
-      return if u.nil?
-      begin
-        abs = page.to_absolute(URI(u))
-      rescue
-        abs = nil
+      u = elem.attributes[attrname].to_s
+      return if u.nil? || u.empty?
+      abs = page.to_absolute(u) rescue nil
+      abs if abs && page.in_domain?(abs)
+    end
+
+    # check files linked on static document
+    # see lib/validate_website/runner.rb
+    def check_static_not_found(links, opts={})
+      opts = @options.merge(opts)
+      links.each do |l|
+        file_location = URI.parse(File.join(Dir.getwd, l.path)).path
+        # Check CSS url()
+        if File.exists?(file_location) && File.extname(file_location) == '.css'
+          css_page = Anemone::Page.new(l, :body => File.read(file_location),
+                                       :headers => {'content-type' => ['text/css']})
+          links.concat extract_urls_from_css(css_page)
+          links.uniq!
+        end
+        unless File.exists?(file_location)
+          @not_found_error = true
+          puts color(:error, "%s linked but not exist" % file_location, opts[:color])
+          to_file(file_location)
+        end
+      end
+    end
+
+    # Extract urls from img script iframe and link element
+    #
+    # @param [Anemone::Page] an Anemone::Page object
+    # @return [Array] Lists of urls
+    #
+    def extract_urls_from_img_script_iframe_link(page)
+      links = []
+      page.doc.css('img, script, iframe').each do |elem|
+        url = get_url(page, elem, "src")
+        links << url unless url.nil? || url.to_s.empty?
+      end
+      page.doc.css('link').each do |link|
+        url = get_url(page, link, "href")
+        links << url unless url.nil? || url.to_s.empty?
+      end
+      links
+    end
+
+    # Extract urls from CSS page
+    #
+    # @param [Anemone::Page] an Anemone::Page object
+    # @return [Array] Lists of urls
+    #
+    def extract_urls_from_css(page)
+      page.body.scan(/url\((['".\/\w-]+)\)/).inject([]) do |result, url|
+        url = url.first.gsub("'", "").gsub('"', '')
+        abs = page.to_absolute(URI.parse(url))
+        result << abs
       end
-      return abs if abs && page.in_domain?(abs)
     end
+
+    def validate(doc, body, url, opts={})
+      opts = @options.merge(opts)
+      validator = Validator.new(doc, body)
+      msg = " well formed? %s" % validator.valid?
+      if validator.valid?
+        unless opts[:quiet]
+          print color(:info, url, opts[:color])
+          puts color(:success, msg, opts[:color])
+        end
+      else
+        @markup_error = true
+        print color(:info, url, opts[:color])
+        puts color(:error, msg, opts[:color])
+        puts color(:error, validator.errors.join(', '), opts[:color]) if opts[:validate_verbose]
+        to_file(url)
+      end
+    end
+
   end
 end

data/lib/validate_website/option_parser.rb

@@ -3,7 +3,6 @@ require 'optparse'
 
 module ValidateWebsite
   class Parser
-    # TODO: no color
     DEFAULT_OPTS_CRAWL = {
       :site              => 'http://localhost:3000/',
       :markup_validation => true,
@@ -20,9 +19,11 @@ module ValidateWebsite
       :cookies           => nil,
       :accept_cookies    => true,
       :redirect_limit    => 0,
+      :color             => true,
     }
 
     DEFAULT_OPTS_STATIC = {
+      :site              => 'http://www.example.com/',
       :pattern           => '**/*.html',
       :file              => nil,
       :validate_verbose  => false,
@@ -30,6 +31,7 @@ module ValidateWebsite
       :markup_validation => true,
       # log not found url (not on filesystem, pwd considered as root « / »)
       :not_found         => false,
+      :color             => true,
     }
 
     def self.parse(options, type)
@@ -84,6 +86,10 @@ module ValidateWebsite
              "Log not found url (Default: #{@@default_opts[:not_found]})") { |v|
           options[:not_found] = v
         }
+        o.on("--[no-]color",
+             "Show colored output (Default: #{@@default_opts[:color]})") { |v|
+          options[:color] = v
+        }
         o.on("-v", "--verbose",
              "Show validator errors (Default: #{@@default_opts[:validate_verbose]})") { |v|
           options[:validate_verbose] = v
@@ -100,8 +106,7 @@ module ValidateWebsite
         o.separator ""
         o.on_tail("-h", "--help", "Show this help message.") { puts o; exit }
       end
-      opts.parse!(args)
-      @@default_opts.merge(options)
+      command_line_parse!(opts, args, options)
     end
 
     def self.command_line_parse_static(args)
@@ -113,6 +118,10 @@ module ValidateWebsite
           'your documents'
         o.separator   ''
 
+        o.on("-s", "--site 'SITE'", String,
+             "Where static files will be hosted (Default: #{@@default_opts[:site]})") { |v|
+          options[:site] = v
+        }
         o.on("-p", "--pattern 'PATTERN'", String,
              "Change filenames pattern (Default: #{@@default_opts[:pattern]})") { |v|
           options[:pattern] = v.strip
@@ -138,7 +147,17 @@ module ValidateWebsite
           options[:quiet] = v
         }
       end
-      opts.parse!(args)
+      command_line_parse!(opts, args, options)
+    end
+
+    def self.command_line_parse!(opts, args, options)
+      begin
+        opts.parse!(args)
+      rescue OptionParser::InvalidOption, OptionParser::MissingArgument
+        puts $!.to_s
+        puts opts
+        exit 128
+      end
       @@default_opts.merge(options)
     end
   end

data/lib/validate_website/runner.rb

@@ -19,32 +19,7 @@ module ValidateWebsite
     def self.run_static(args)
       trap_interrupt
       validate_website = ValidateWebsite::Core.new(args, :static)
-      opts = validate_website.options
-      links = []
-      files = Dir.glob(opts[:pattern])
-      files.each do |f|
-        next unless File.file?(f)
-
-        body = open(f).read
-        doc = Nokogiri::HTML(body)
-
-        # TODO: check css url for static files
-        if opts[:not_found]
-          doc.search("//a[@href]").each do |a|
-            u = a['href']
-            next if u.nil? || u.empty?
-            next if u.match(/^https?:\/\//)
-            abs = URI(File.join(Dir.getwd, u)) rescue next
-            links << abs if abs.host.nil?
-          end
-          links.uniq!
-        end
-
-        if opts[:markup_validation]
-          validate_website.validate(doc, body, f)
-        end
-      end
-      validate_website.check_static_not_found(links)
+      validate_website.crawl_static
       validate_website.exit_status
     end
   end

data/man/man1/validate-website-static.1

@@ -37,6 +37,12 @@ validate-website-static \- check the validity of your documents
 validate\-website\-static check the markup validity of your local documents with XML Schema / DTD\&. HTML5 support with Validator\&.nu Web Service\&.
 .SH "OPTIONS"
 .PP
+\fB\-s\fR, \fB\-\-site\fR \fISITE\fR
+.RS 4
+Where static files will be hosted (Default:
+http://www\&.example\&.com/)
+.RE
+.PP
 \fB\-p\fR, \fB\-\-pattern\fR \fIPATTERN\fR
 .RS 4
 Change filenames pattern (Default: **/*\&.html)
@@ -57,6 +63,11 @@ Markup validation (Default: true)
 Log files not on filesystem, pwd considered as root \(Fo / \(Fc (Default: false)
 .RE
 .PP
+\fB\-\-[no\-]color\fR
+.RS 4
+Show colored output (Default: true)
+.RE
+.PP
 \fB\-v\fR, \fB\-\-verbose\fR
 .RS 4
 Show detail of validator errors (Default: false)\&.

data/man/man1/validate-website.1

@@ -74,6 +74,11 @@ Markup validation (Default: true)
 Log not found url (Default: false)
 .RE
 .PP
+\fB\-\-[no\-]color\fR
+.RS 4
+Show colored output (Default: true)
+.RE
+.PP
 \fB\-v\fR, \fB\-\-verbose\fR
 .RS 4
 Show detail of validator errors (Default: false)\&.

data/spec/core_spec.rb

@@ -5,7 +5,7 @@ module ValidateWebsite
 
     before(:each) do
       FakeWeb.clean_registry
-      @validate_website = ValidateWebsite::Core.new
+      @validate_website = ValidateWebsite::Core.new(:color => false)
     end
 
     context('html') do
@@ -17,7 +17,7 @@ module ValidateWebsite
                             :content_type => 'text/html')
         @validate_website.site = page.url
         @validate_website.crawl(:quiet => true)
-        @validate_website.anemone.should have(3).pages
+        @validate_website.anemone.should have(5).pages
       end
     end
 

data/spec/data/xhtml1-strict.html

@@ -15,5 +15,8 @@
 			<li><a href="/my-url2" title="title">my url</a></li>
 			<li><a href="/my-url1" title="title">my url</a></li>
 		</ul>
+        <p><img src="http://test.com/img.png" alt="non local img" /></p>
+        <p><img src="http://www.example.com/img1.png" alt="local img with absolute uri" /></p>
+        <p><img src="/img2.png" alt="local img with non absolute uri" /></p>
 	</body>
 </html>

metadata

@@ -1,91 +1,128 @@
---- !ruby/object:Gem::Specification
+--- !ruby/object:Gem::Specification 
 name: validate-website
-version: !ruby/object:Gem::Version
-  version: 0.6.5
+version: !ruby/object:Gem::Version 
+  hash: 3
   prerelease: 
+  segments: 
+  - 0
+  - 7
+  - 0
+  version: 0.7.0
 platform: ruby
-authors:
+authors: 
 - Laurent Arnoud
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-06-04 00:00:00.000000000Z
-dependencies:
-- !ruby/object:Gem::Dependency
+
+date: 2011-06-06 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
   name: anemone
-  requirement: &22543340 !ruby/object:Gem::Requirement
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 5
+        segments: 
+        - 0
+        - 6
+        - 1
         version: 0.6.1
   type: :runtime
-  prerelease: false
-  version_requirements: *22543340
-- !ruby/object:Gem::Dependency
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
   name: rainbow
-  requirement: &22542860 !ruby/object:Gem::Requirement
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 17
+        segments: 
+        - 1
+        - 1
+        - 1
         version: 1.1.1
   type: :runtime
-  prerelease: false
-  version_requirements: *22542860
-- !ruby/object:Gem::Dependency
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
   name: multipart_body
-  requirement: &22542400 !ruby/object:Gem::Requirement
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 21
+        segments: 
+        - 0
+        - 2
+        - 1
         version: 0.2.1
   type: :runtime
-  prerelease: false
-  version_requirements: *22542400
-- !ruby/object:Gem::Dependency
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
   name: rake
-  requirement: &22541940 !ruby/object:Gem::Requirement
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 49
+        segments: 
+        - 0
+        - 8
+        - 7
         version: 0.8.7
   type: :development
-  prerelease: false
-  version_requirements: *22541940
-- !ruby/object:Gem::Dependency
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
   name: rspec
-  requirement: &22541480 !ruby/object:Gem::Requirement
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 23
+        segments: 
+        - 2
+        - 6
+        - 0
         version: 2.6.0
   type: :development
-  prerelease: false
-  version_requirements: *22541480
-- !ruby/object:Gem::Dependency
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
   name: fakeweb
-  requirement: &22541020 !ruby/object:Gem::Requirement
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 27
+        segments: 
+        - 1
+        - 3
+        - 0
         version: 1.3.0
   type: :development
-  prerelease: false
-  version_requirements: *22541020
-description: validate-website is a web crawler for checking the markup validity with
-  XML Schema / DTD and not found urls.
+  version_requirements: *id006
+description: validate-website is a web crawler for checking the markup validity with XML Schema / DTD and not found urls.
 email: laurent@spkdev.net
-executables:
+executables: 
 - validate-website
 - validate-website-static
 extensions: []
+
 extra_rdoc_files: []
-files:
+
+files: 
 - README.rdoc
 - Rakefile
 - LICENSE
@@ -199,25 +236,32 @@ files:
 - bin/validate-website
 - bin/validate-website-static
 homepage: http://github.com/spk/validate-website
-licenses:
+licenses: 
 - MIT
 post_install_message: 
 rdoc_options: []
-require_paths:
+
+require_paths: 
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement
+required_ruby_version: !ruby/object:Gem::Requirement 
   none: false
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
-      version: '0'
-required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
-      version: '0'
-requirements:
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: 
 - anemone
 - rainbow
 - multipart_body
@@ -226,7 +270,7 @@ rubygems_version: 1.7.2
 signing_key: 
 specification_version: 3
 summary: Web crawler for checking the validity of your documents
-test_files:
+test_files: 
 - spec/spec_helper.rb
 - spec/data/html5.html
 - spec/data/html4-strict.html
@@ -236,3 +280,4 @@ test_files:
 - spec/core_spec.rb
 - spec/fakeweb_helper.rb
 - spec/validator_spec.rb
+has_rdoc: