validate-website 0.5.7 → 0.6.0

data/README.rdoc

@@ -12,14 +12,16 @@
 
 == SYNOPSIS
 
-validate-website [OPTIONS]
+  validate-website [OPTIONS]
+  validate-website-static [OPTIONS]
 
 == DESCRIPTION
 
 validate-website is a web crawler for checking the markup validity and not
 found urls.
+validate-website-static check the markup validity of your local documents.
 
-== OPTIONS
+== VALIDATE WEBSITE OPTIONS
 
   -s, --site SITE
       Website to crawl (Default: http://localhost:3000/)
@@ -56,9 +58,8 @@ found urls.
 
 == REQUIREMENTS
 
-* spk-anemone, '>= 0.4.0'
+* anemone, '>= 0.5.0'
 * rainbow, '>= 1.1'
-* html5, '= 0.10.0'
 
 == CREDITS
 

data/Rakefile

@@ -7,10 +7,10 @@ require "rspec/core/rake_task" # RSpec 2.0
 # Globals
 
 PKG_NAME = 'validate-website'
-PKG_VERSION = '0.5.7'
+PKG_VERSION = '0.6.0'
 
 PKG_FILES = ['README.rdoc', 'Rakefile', 'LICENSE']
-Find.find('bin/', 'lib/', 'man/', 'spec/') do |f|
+Find.find('bin/', 'lib/', 'man/', 'spec/', 'share/') do |f|
   if FileTest.directory?(f) and f =~ /\.svn|\.git/
     Find.prune
   else
@@ -64,7 +64,7 @@ end
 
 desc 'Update manpage from asciidoc file'
 task :manpage do
-  system('a2x -f manpage -D man/man1 doc/validate-website.txt')
+  system('find doc/ -type f -exec a2x -f manpage -D man/man1 {} \;')
 end
 
 # RSpec 2.0

data/bin/validate-website

@@ -1,16 +1,14 @@
 #!/usr/bin/env ruby
 # encoding: utf-8
 
-lib_dir = File.join(File.dirname(__FILE__), '..', 'lib')
-$:.unshift(File.expand_path(lib_dir))
-
 developer_mode = false
 developer_mode = true if __FILE__ == $0
-require 'rubygems' if developer_mode
-
-require 'validate_website'
-
-validate_website = ValidateWebsite.new(ARGV)
-validate_website.crawl
-
-exit(validate_website.exit_status)
+if developer_mode
+  lib_dir = File.join(File.dirname(__FILE__), '..', 'lib')
+  $:.unshift(File.expand_path(lib_dir))
+  require 'rubygems'
+end
+
+require 'validate_website/runner'
+exit_status = ValidateWebsite::Runner.run_crawl(ARGV)
+exit(exit_status)

data/bin/validate-website-static

@@ -1,25 +1,14 @@
 #!/usr/bin/env ruby
 # encoding: utf-8
 
-lib_dir = File.join(File.dirname(__FILE__), '..', 'lib')
-$:.unshift(File.expand_path(lib_dir))
-
 developer_mode = false
 developer_mode = true if __FILE__ == $0
-require 'rubygems' if developer_mode
-
-require 'validate_website'
-
-validate_website = ValidateWebsite.new(ARGV)
-
-files = Dir.glob(File.join("**", "*.html"))
-files.each do |f|
-  next unless File.file?(f)
-
-  body = open(f).read
-  doc = Nokogiri::HTML(body)
-
-  validate_website.validate(doc, body, f)
+if developer_mode
+  lib_dir = File.join(File.dirname(__FILE__), '..', 'lib')
+  $:.unshift(File.expand_path(lib_dir))
+  require 'rubygems'
 end
 
-exit(validate_website.exit_status)
+require 'validate_website/runner'
+exit_status = ValidateWebsite::Runner.run_static(ARGV)
+exit(exit_status)

data/lib/validate_website.rb

@@ -1,211 +1,2 @@
 # encoding: utf-8
-
-require 'optparse'
-require 'open-uri'
-
-require 'validator'
-require 'colorful_messages'
-
-require 'anemone'
-
-class ValidateWebsite
-
-  attr_accessor :site
-  attr_reader :options, :anemone
-
-  include ColorfulMessages
-
-  EXIT_SUCCESS = 0
-  EXIT_FAILURE_MARKUP = 64
-  EXIT_FAILURE_NOT_FOUND = 65
-  EXIT_FAILURE_MARKUP_NOT_FOUND = 66
-
-  def initialize(args=[], validation_type = :crawl)
-    @markup_error = nil
-    @not_found_error = nil
-
-    @options_crawl = {
-      :site              => 'http://localhost:3000/',
-      :markup_validation => true,
-      :exclude           => nil,
-      :file              => nil,
-      # log not found url (404 status code)
-      :not_found         => false,
-      # internal verbose for ValidateWebsite
-      :validate_verbose  => false,
-      :quiet             => false,
-
-      # Anemone options see anemone/lib/anemone/core.rb
-      :verbose           => false,
-      :user_agent        => Anemone::Core::DEFAULT_OPTS[:user_agent],
-      :cookies           => nil,
-      :accept_cookies    => true,
-      :redirect_limit    => 0,
-    }
-    send("parse_#{validation_type}_options", args)
-
-    @file = @options[:file]
-    if @file
-      # truncate file
-      open(@file, 'w').write('')
-    end
-
-    @site = @options[:site]
-  end
-
-  def parse_crawl_options(args)
-    @options = @options_crawl
-
-    opts = OptionParser.new do |o|
-      o.set_summary_indent('  ')
-      o.banner =    'Usage: validate-website [OPTIONS]'
-      o.define_head 'validate-website - Web crawler for checking the validity'+
-        ' of your documents'
-      o.separator   ''
-
-      o.on("-s", "--site 'SITE'", String,
-           "Website to crawl (Default: #{@options[:site]})") { |v|
-        @options[:site] = v
-      }
-      o.on("-u", "--user-agent 'USERAGENT'", String,
-           "Change user agent (Default: #{@options[:user_agent]})") { |v|
-        @options[:user_agent] = v
-      }
-      o.on("-e", "--exclude 'EXCLUDE'", String,
-           "Url to exclude (ex: 'redirect|news')") { |v|
-        @options[:exclude] = v
-      }
-      o.on("-f", "--file 'FILE'", String,
-           "Save not well formed or not found urls") { |v| @options[:file] = v }
-
-      o.on("-c", "--cookies 'COOKIES'", String,
-           "Set defaults cookies") { |v| @options[:cookies] = v }
-
-      o.on("-m", "--[no-]markup-validation",
-           "Markup validation (Default: #{@options[:markup_validation]})") { |v|
-        @options[:markup_validation] = v
-      }
-      o.on("-n", "--not-found",
-           "Log not found url (Default: #{@options[:not_found]})") { |v|
-        @options[:not_found] = v
-      }
-      o.on("-v", "--verbose",
-           "Show validator errors (Default: #{@options[:validate_verbose]})") { |v|
-        @options[:validate_verbose] = v
-      }
-      o.on("-q", "--quiet",
-           "Only report errors (Default: #{@options[:quiet]})") { |v|
-        @options[:quiet] = v
-      }
-      o.on("-d", "--debug",
-           "Show anemone log (Default: #{@options[:verbose]})") { |v|
-        @options[:verbose] = v
-      }
-
-      o.separator ""
-      o.on_tail("-h", "--help", "Show this help message.") { puts o; exit }
-    end
-    opts.parse!(args)
-  end
-
-  def validate(doc, body, url, opts={})
-    opts = @options.merge(opts)
-    validator = Validator.new(doc, body)
-    msg = " well formed? %s" % validator.valid?
-    if validator.valid?
-      unless opts[:quiet]
-        print info(url)
-        puts success(msg)
-      end
-    else
-      @markup_error = true
-      print info(url)
-      puts error(msg)
-      puts error(validator.errors.join(", ")) if opts[:validate_verbose]
-      to_file(url)
-    end
-  end
-
-  def crawl(opts={})
-    opts = @options.merge(opts)
-    puts note("Validating #{@site}") if opts[:validate_verbose]
-
-    @anemone = Anemone.crawl(@site, opts) do |anemone|
-      anemone.skip_links_like Regexp.new(opts[:exclude]) if opts[:exclude]
-
-      # select the links on each page to follow (iframe, link, css url)
-      anemone.focus_crawl { |p|
-        links = []
-        if p.html?
-          p.doc.css('img, script, iframe').each do |elem|
-            url = get_url(p, elem, "src")
-            links << url unless url.nil?
-          end
-          p.doc.css('link').each do |link|
-            url = get_url(p, link, "href")
-            links << url unless url.nil?
-          end
-        end
-        if p.content_type == 'text/css'
-          p.body.scan(/url\((['".\/\w-]+)\)/).each do |url|
-            url = url.first.gsub("'", "").gsub('"', '')
-            abs = p.to_absolute(URI(url))
-            links << abs
-          end
-        end
-        links.uniq!
-        p.links.concat(links)
-      }
-
-      anemone.on_every_page { |page|
-        url = page.url.to_s
-
-        if opts[:markup_validation]
-          # validate html/html+xml
-          if page.html? && page.fetched?
-            validate(page.doc, page.body, url, opts)
-          end
-        end
-
-        if opts[:not_found] && page.not_found?
-          @not_found_error = true
-          puts error("%s linked in %s but not exist" % [url, page.referer])
-          to_file(url)
-        end
-
-        # throw away the page (hope this saves memory)
-        page = nil
-      }
-    end
-  end
-
-  def exit_status
-    if @markup_error && @not_found_error
-      EXIT_FAILURE_MARKUP_NOT_FOUND
-    elsif @markup_error
-      EXIT_FAILURE_MARKUP
-    elsif @not_found_error
-      EXIT_FAILURE_NOT_FOUND
-    else
-      EXIT_SUCCESS
-    end
-  end
-
-  private
-  def to_file(msg)
-    if @file && File.exist?(@file)
-      open(@file, 'a').write("#{msg}\n")
-    end
-  end
-
-  def get_url(page, elem, attrname)
-    u = elem.attributes[attrname] if elem.attributes[attrname]
-    return if u.nil?
-    begin
-      abs = page.to_absolute(URI(u))
-    rescue
-      abs = nil
-    end
-    return abs if abs && page.in_domain?(abs)
-  end
-end
+require 'validate_website/core'

data/lib/validate_website/colorful_messages.rb

@@ -0,0 +1,28 @@
+# encoding: utf-8
+require 'rainbow'
+
+module ValidateWebsite
+  module ColorfulMessages
+    def error(message)
+      message.to_s.foreground(:red)
+    end
+
+    def warning(message)
+      message.to_s.foreground(:yellow)
+    end
+
+    def success(message)
+      message.to_s.foreground(:green)
+    end
+
+    alias_method :message, :success
+
+    def note(message)
+      message.to_s.foreground(:magenta)
+    end
+
+    def info(message)
+      message.to_s.foreground(:blue)
+    end
+  end
+end

data/lib/validate_website/core.rb

@@ -0,0 +1,141 @@
+# encoding: utf-8
+
+require 'open-uri'
+
+require 'validate_website/option_parser'
+require 'validate_website/validator'
+require 'validate_website/colorful_messages'
+
+require 'anemone'
+
+module ValidateWebsite
+
+  class Core
+
+    attr_accessor :site
+    attr_reader :options, :anemone
+
+    include ColorfulMessages
+
+    EXIT_SUCCESS = 0
+    EXIT_FAILURE_MARKUP = 64
+    EXIT_FAILURE_NOT_FOUND = 65
+    EXIT_FAILURE_MARKUP_NOT_FOUND = 66
+
+    def initialize(options={}, validation_type = :crawl)
+      @markup_error = nil
+      @not_found_error = nil
+
+      @options = Parser.parse(options, validation_type)
+
+      @file = @options[:file]
+      if @file
+        # truncate file
+        open(@file, 'w').write('')
+      end
+
+      @site = @options[:site]
+    end
+
+    def validate(doc, body, url, opts={})
+      opts = @options.merge(opts)
+      validator = Validator.new(doc, body)
+      msg = " well formed? %s" % validator.valid?
+      if validator.valid?
+        unless opts[:quiet]
+          print info(url)
+          puts success(msg)
+        end
+      else
+        @markup_error = true
+        print info(url)
+        puts error(msg)
+        puts error(validator.errors.join(", ")) if opts[:validate_verbose]
+        to_file(url)
+      end
+    end
+
+    def crawl(opts={})
+      opts = @options.merge(opts)
+      puts note("Validating #{@site}") if opts[:validate_verbose]
+
+      @anemone = Anemone.crawl(@site, opts) do |anemone|
+        anemone.skip_links_like Regexp.new(opts[:exclude]) if opts[:exclude]
+
+        # select the links on each page to follow (iframe, link, css url)
+        anemone.focus_crawl { |p|
+          links = []
+          if p.html?
+            p.doc.css('img, script, iframe').each do |elem|
+              url = get_url(p, elem, "src")
+              links << url unless url.nil?
+            end
+            p.doc.css('link').each do |link|
+              url = get_url(p, link, "href")
+              links << url unless url.nil?
+            end
+          end
+          if p.content_type == 'text/css'
+            p.body.scan(/url\((['".\/\w-]+)\)/).each do |url|
+              url = url.first.gsub("'", "").gsub('"', '')
+              abs = p.to_absolute(URI(url))
+              links << abs
+            end
+          end
+          links.uniq!
+          p.links.concat(links)
+        }
+
+        anemone.on_every_page { |page|
+          url = page.url.to_s
+
+          if opts[:markup_validation]
+            # validate html/html+xml
+            if page.html? && page.fetched?
+              validate(page.doc, page.body, url, opts)
+            end
+          end
+
+          if opts[:not_found] && page.not_found?
+            @not_found_error = true
+            puts error("%s linked in %s but not exist" % [url, page.referer])
+            to_file(url)
+          end
+
+          # throw away the page (hope this saves memory)
+          page = nil
+        }
+      end
+    end
+
+    def exit_status
+      if @markup_error && @not_found_error
+        EXIT_FAILURE_MARKUP_NOT_FOUND
+      elsif @markup_error
+        EXIT_FAILURE_MARKUP
+      elsif @not_found_error
+        EXIT_FAILURE_NOT_FOUND
+      else
+        EXIT_SUCCESS
+      end
+    end
+
+    private
+    def to_file(msg)
+      if @file && File.exist?(@file)
+        open(@file, 'a').write("#{msg}\n")
+      end
+    end
+
+    def get_url(page, elem, attrname)
+      u = elem.attributes[attrname] if elem.attributes[attrname]
+      return if u.nil?
+      begin
+        abs = page.to_absolute(URI(u))
+      rescue
+        abs = nil
+      end
+      return abs if abs && page.in_domain?(abs)
+    end
+  end
+end