validate-website 0.4.1 → 0.5.0

data/LICENSE

@@ -0,0 +1,22 @@
+The MIT License
+
+Copyright (c) 2009-2010 Laurent Arnoud <laurent@spkdev.net>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -1,8 +1,4 @@
-== validate-website
-
-== DESCRIPTION
-
-  Web crawler for checking the validity of your documents.
+= validate-website - Web crawler for checking the validity of your documents
 
 == INSTALLATION
 
@@ -10,48 +6,60 @@
 
 == SYNOPSIS
 
-  validate-website --help
-  validate-website -s "http://localhost:4567/"
-  -u "Mozilla 5.0" \ # user agent
-  -f not-well-formed.txt \ # log file
-  --auth=user,pass \ # http auth
-  -e 'redirect|news' \ # exclude regex
-  -n # log not found (404)
-  -c "name=val;name2=val2"
-  -v # verbose: show detail of validator errors
-  -d # debug: show anemone log
+validate-website [OPTIONS]
+
+== DESCRIPTION
+
+validate-website is a web crawler for checking the markup validity and not
+found urls.
+
+== OPTIONS
+
+  -s, --site SITE
+      Website to crawl (Default: http://localhost:3000/)
+  -u, --user-agent USERAGENT
+      Change user agent (Default: Anemone/VERSION)
+  -e, --exclude EXCLUDE
+      Url to exclude (ex: redirect|news)
+  -f, --file FILE
+      Save not well formed or not found (with -n used) urls
+  -a, --authorization USER,PASS
+      Basic http authentification
+  -c, --cookies COOKIES
+      Set defaults cookies
+  -m, --[no-]markup-validation
+      Markup validation (Default: true)
+  -n, --not-found
+      Log not found url (Default: false)
+  -v, --verbose
+      Show detail of validator errors (Default: false).
+  -d, --debug
+      Show anemone log (Default: false)
+  -h, --help
+      Show help message and exit.
 
-== REQUIREMENTS:
+== EXIT STATUS
+0::
+  Markup is valid and no 404 found.
+64::
+  Not valid markup found.
+65::
+  There are pages not found.
+66::
+  There are not valid markup and pages not found.
 
-  spk-anemone >= 0.2.4 (forked from chriskite/anemone,
-                        add auth support and html body)
-  rainbow >= 1.0.4 (for console color)
+== REQUIREMENTS
 
-== CREDITS:
+* spk-anemone, '>= 0.4.0'
+* rainbow, '>= 1.1'
+* html5, '= 0.10.0'
 
-  Thanks tenderlove for Nokogiri,
-  this library is inspired from markup_validity
+== CREDITS
+
+* Thanks tenderlove for Nokogiri, this tool is inspired from markup_validity.
+* And Chris Kite for Anemone web-spider framework.
 
 == LICENSE
-(The MIT License)
-
-Copyright (c) 2009 Laurent Arnoud (spk)
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-'Software'), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+The MIT License
+
+Copyright (c) 2009-2010 Laurent Arnoud <laurent@spkdev.net>

data/Rakefile

@@ -2,14 +2,15 @@ require 'rake/packagetask'
 require 'rake/rdoctask'
 require 'rake'
 require 'find'
+require "rspec/core/rake_task" # RSpec 2.0
 
 # Globals
 
 PKG_NAME = 'validate-website'
-PKG_VERSION = '0.4.1'
+PKG_VERSION = '0.5.0'
 
-PKG_FILES = ['README.rdoc', 'Rakefile']
-Find.find('lib/', 'bin/', 'spec/') do |f|
+PKG_FILES = ['README.rdoc', 'Rakefile', 'LICENSE']
+Find.find('bin/', 'lib/', 'man/', 'spec/') do |f|
   if FileTest.directory?(f) and f =~ /\.svn|\.git/
     Find.prune
   else
@@ -52,19 +53,32 @@ spec = Gem::Specification.new do |s|
   s.summary = 'Web crawler for checking the validity of your documents'
   s.name = PKG_NAME
   s.version = PKG_VERSION
-  s.requirements << 'spk-anemone' << 'rainbow'
+  s.requirements << 'spk-anemone' << 'rainbow' << 'html5'
   s.add_dependency('spk-anemone', '>= 0.4.0')
   s.add_dependency('rainbow', '>= 1.1')
-  s.add_development_dependency('rspec', '>= 1.3.0')
+  s.add_dependency('html5', '= 0.10.0')
+  s.add_development_dependency('rspec', '>= 2.0.0')
   s.add_development_dependency('fakeweb', '>= 1.3.0')
   s.require_path = 'lib'
   s.bindir = 'bin'
   s.executables << 'validate-website'
   s.files = PKG_FILES
-  s.description = 'Web crawler for checking the validity of your documents'
+  s.description = 'validate-website is a web crawler for checking the markup' +
+    'validity and not found urls.'
 end
 
 Rake::GemPackageTask.new(spec) do |pkg|
   pkg.need_zip = true
   pkg.need_tar = true
 end
+
+desc 'Update manpage from asciidoc file'
+task :manpage do
+  system('a2x -f manpage -D man/man1 doc/validate-website.txt')
+end
+
+# RSpec 2.0
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = 'spec/*_spec.rb'
+  spec.rspec_opts = ['--backtrace']
+end

data/bin/validate-website

@@ -10,16 +10,16 @@ require 'validate_website'
 validate_website = ValidateWebsite.new(ARGV)
 options = validate_website.options
 
-exit_code = validate_website.crawl options[:site],
-              :markup_validation => options[:markup_validation],
-              :user_agent        => options[:useragent],
-              :exclude           => options[:exclude],
-              :file              => options[:file],
-              :authorization     => options[:auth],
-              :not_found         => options[:not_found],
-              :cookies           => options[:cookies],
-              :accept_cookies    => options[:accept_cookies],
-              :verbose           => options[:debug],
-              :error_verbose     => options[:verbose]
+validate_website.crawl options[:site],
+  :markup_validation => options[:markup_validation],
+  :user_agent        => options[:user_agent],
+  :exclude           => options[:exclude],
+  :file              => options[:file],
+  :authorization     => options[:authorization],
+  :not_found         => options[:not_found],
+  :cookies           => options[:cookies],
+  :accept_cookies    => options[:accept_cookies],
+  :verbose           => options[:debug],
+  :error_verbose     => options[:verbose]
 
-exit(exit_code)
+exit(validate_website.exit_status)

data/lib/validate_website.rb

@@ -10,14 +10,22 @@ class ValidateWebsite
 
   include ColorfulMessages
 
+  EXIT_SUCCESS = 0
+  EXIT_FAILURE_MARKUP = 64
+  EXIT_FAILURE_NOT_FOUND = 65
+  EXIT_FAILURE_MARKUP_NOT_FOUND = 66
+
   def initialize(args=[])
+    @markup_error = nil
+    @not_found_error = nil
+
     @options = {
       :site              => 'http://localhost:3000/',
       :markup_validation => true,
-      :useragent         => Anemone::Core::DEFAULT_OPTS[:user_agent],
+      :user_agent        => Anemone::Core::DEFAULT_OPTS[:user_agent],
       :exclude           => nil,
       :file              => nil,
-      :auth              => nil,
+      :authorization     => nil,
       # log not found url (404 status code)
       :not_found         => false,
       :cookies           => nil,
@@ -36,30 +44,47 @@ class ValidateWebsite
   def parse(args)
     opts = OptionParser.new do |o|
       o.set_summary_indent('  ')
-      o.banner =    "Usage: validate-website [OPTIONS]"
-      o.define_head "validate-website - Web crawler for checking the validity of your documents"
-      o.separator   ""
-
-      o.on("-s", "--site=val", String,
-           "Default: #{@options[:site]}") { |v| @options[:site] = v }
-      o.on("-u", "--useragent=val", String,
-           "Default: #{@options[:useragent]}") { |v| @options[:useragent] = v }
-      o.on("-e", "--exclude=val", String,
-           "Url to exclude") { |v| @options[:exclude] = v }
-      o.on("-f", "--file=val", String,
-           "Save not well formed urls") { |v| @options[:file] = v }
-      o.on("--auth=[user,pass]", Array,
-           "Basic http authentification") { |v| @options[:auth] = v }
-      o.on("-c", "--cookies=val", "Set defaults cookies") { |v| @options[:cookies] = v }
-
+      o.banner =    'Usage: validate-website [OPTIONS]'
+      o.define_head 'validate-website - Web crawler for checking the validity'+
+        ' of your documents'
+      o.separator   ''
+
+      o.on("-s", "--site 'SITE'", String,
+           "Website to crawl (Default: #{@options[:site]})") { |v|
+        @options[:site] = v
+      }
+      o.on("-u", "--user-agent 'USERAGENT'", String,
+           "Change user agent (Default: #{@options[:user_agent]})") { |v|
+        @options[:user_agent] = v
+      }
+      o.on("-e", "--exclude 'EXCLUDE'", String,
+           "Url to exclude (ex: 'redirect|news')") { |v|
+        @options[:exclude] = v
+      }
+      o.on("-f", "--file 'FILE'", String,
+           "Save not well formed or not found urls") { |v| @options[:file] = v }
+      o.on("-a", "--authorization 'USER,PASS'", Array,
+           "Basic http authentification") { |v|
+        @options[:authorization] = v
+      }
+      o.on("-c", "--cookies 'COOKIES'", String,
+           "Set defaults cookies") { |v| @options[:cookies] = v }
       o.on("-m", "--[no-]markup-validation",
-           "Markup validation (Default: #{@options[:markup_validation]})") { |v| @options[:markup_validation] = v }
+           "Markup validation (Default: #{@options[:markup_validation]})") { |v|
+        @options[:markup_validation] = v
+      }
       o.on("-n", "--not-found",
-           "Log not found url (Default: #{@options[:not_found]})") { |v| @options[:not_found] = v }
+           "Log not found url (Default: #{@options[:not_found]})") { |v|
+        @options[:not_found] = v
+      }
       o.on("-v", "--verbose",
-           "Show detail of validator errors (Default: #{@options[:verbose]})") { |v| @options[:verbose] = v }
+           "Show validator errors (Default: #{@options[:verbose]})") { |v|
+        @options[:verbose] = v
+      }
       o.on("-d", "--debug",
-           "Show anemone log (Default: #{@options[:debug]})") { |v| @options[:debug] = v }
+           "Show anemone log (Default: #{@options[:debug]})") { |v|
+        @options[:debug] = v
+      }
 
       o.separator ""
       o.on_tail("-h", "--help", "Show this help message.") { puts o; exit }
@@ -79,8 +104,6 @@ class ValidateWebsite
   end
 
   def crawl(site, opts={})
-    exit_code = 0
-
     @anemone = Anemone.crawl(site, opts) do |anemone|
       anemone.skip_links_like Regexp.new(opts[:exclude]) if opts[:exclude]
 
@@ -119,22 +142,33 @@ class ValidateWebsite
             if validator.valid?
               puts success(msg)
             else
-              exit_code = 1
+              @markup_error = true
               puts error(msg)
-              puts error(validator.errors) if opts[:error_verbose]
+              puts error(validator.errors.join(", ")) if opts[:error_verbose]
               to_file(url)
             end
           end
         end
 
         if opts[:not_found] && page.not_found?
-          exit_code = 1
+          @not_found_error = true
           puts error("%s linked in %s but not exist" % [url, page.referer])
           to_file(url)
         end
       }
     end
-    exit_code
+  end
+
+  def exit_status
+    if @markup_error && @not_found_error
+      EXIT_FAILURE_MARKUP_NOT_FOUND
+    elsif @markup_error
+      EXIT_FAILURE_MARKUP
+    elsif @not_found_error
+      EXIT_FAILURE_NOT_FOUND
+    else
+      EXIT_SUCCESS
+    end
   end
 
   private

data/lib/validator.rb

@@ -6,14 +6,17 @@ class Validator
   def initialize(page)
     @page = page
     @dtd = @page.doc.internal_subset
+    init_namespace(@dtd)
+    @errors = []
 
-    if @dtd.system_id
-      @dtd_uri = URI.parse(@dtd.system_id)
-      # http://www.w3.org/TR/xhtml1/#dtds
-      @namespace = File.basename(@dtd_uri.path, '.dtd')
-      fixed_dtd = @page.body.sub(@dtd_uri.to_s, @namespace + '.dtd')
+    if @namespace
+      if @dtd_uri && @page.body.match(@dtd_uri.to_s)
+        document = @page.body.sub(@dtd_uri.to_s, @namespace + '.dtd')
+      else
+        document = @page.body
+      end
       @doc = Dir.chdir(XHTML_PATH) do
-        Nokogiri::XML(fixed_dtd) { |cfg|
+        Nokogiri::XML(document) { |cfg|
           cfg.noent.dtdload.dtdvalid
         }
       end
@@ -24,15 +27,46 @@ class Validator
           Nokogiri::XML::Schema(File.read(@namespace + '.xsd'))
         end
       end
-      @errors = @xsd ? @xsd.validate(@doc) : ["Don't have this xsd"]
+
+      if @xsd
+        # have the xsd so use it
+        @errors = @xsd.validate(@doc)
+      else
+        # dont have xsd fall back to dtd
+        @doc = Dir.chdir(XHTML_PATH) do
+          Nokogiri::HTML.parse(document)
+        end
+        @errors = @doc.errors
+      end
+    elsif @page.body =~ /^\<!DOCTYPE html\>/i
+      # html5 doctype
+      # http://dev.w3.org/html5/spec/Overview.html#the-doctype
+      require 'html5'
+      require 'html5/filters/validator'
+      html5_parser = HTML5::HTMLParser.new(:tokenizer => HTMLConformanceChecker)
+      html5_parser.parse(@page.body)
+      @errors = html5_parser.errors.collect{|e| e[1] }
     else
-      @errors = ['DTD unknown']
+      @errors << 'Unknown Document'
     end
   rescue Nokogiri::XML::SyntaxError => e
-    @errors = [e]
+    # http://nokogiri.org/tutorials/ensuring_well_formed_markup.html
+    @errors << e
   end
 
   def valid?
     @errors.length == 0
   end
+
+  private
+  def init_namespace(dtd)
+    if dtd.system_id
+      dtd_uri = URI.parse(dtd.system_id)
+      if dtd.system_id && dtd_uri.path
+        @dtd_uri = dtd_uri
+        # http://www.w3.org/TR/xhtml1/#dtds
+        @namespace = File.basename(@dtd_uri.path, '.dtd')
+      end
+    end
+  end
 end

data/man/man1/validate-website.1

@@ -0,0 +1,124 @@
+'\" t
+.\"     Title: validate-website
+.\"    Author: [see the "AUTHOR" section]
+.\" Generator: DocBook XSL Stylesheets v1.75.2 <http://docbook.sf.net/>
+.\"      Date: 10/31/2010
+.\"    Manual: \ \&
+.\"    Source: \ \&
+.\"  Language: English
+.\"
+.TH "VALIDATE\-WEBSITE" "1" "10/31/2010" "\ \&" "\ \&"
+.\" -----------------------------------------------------------------
+.\" * Define some portability stuff
+.\" -----------------------------------------------------------------
+.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.\" http://bugs.debian.org/507673
+.\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html
+.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.ie \n(.g .ds Aq \(aq
+.el       .ds Aq '
+.\" -----------------------------------------------------------------
+.\" * set default formatting
+.\" -----------------------------------------------------------------
+.\" disable hyphenation
+.nh
+.\" disable justification (adjust text to left margin only)
+.ad l
+.\" -----------------------------------------------------------------
+.\" * MAIN CONTENT STARTS HERE *
+.\" -----------------------------------------------------------------
+.SH "NAME"
+validate-website \- Web crawler for checking the validity of your documents
+.SH "SYNOPSIS"
+.sp
+\fBvalidate\-website\fR [\fIOPTIONS\fR]
+.SH "DESCRIPTION"
+.sp
+validate\-website is a web crawler for checking the markup validity and not found urls\&.
+.SH "OPTIONS"
+.PP
+\fB\-s\fR, \fB\-\-site\fR \fISITE\fR
+.RS 4
+Website to crawl (Default:
+http://localhost:3000/)
+.RE
+.PP
+\fB\-u\fR, \fB\-\-user\-agent\fR \fIUSERAGENT\fR
+.RS 4
+Change user agent (Default: Anemone/VERSION)
+.RE
+.PP
+\fB\-e\fR, \fB\-\-exclude\fR \fIEXCLUDE\fR
+.RS 4
+Url to exclude (ex:
+\fIredirect|news\fR)
+.RE
+.PP
+\fB\-f\fR, \fB\-\-file\fR \fIFILE\fR
+.RS 4
+Save not well formed or not found (with \-n used) urls
+.RE
+.PP
+\fB\-a\fR, \fB\-\-authorization\fR \fIUSER,PASS\fR
+.RS 4
+Basic http authentification
+.RE
+.PP
+\fB\-c\fR, \fB\-\-cookies\fR \fICOOKIES\fR
+.RS 4
+Set defaults cookies
+.RE
+.PP
+\fB\-m\fR, \fB\-\-[no\-]markup\-validation\fR
+.RS 4
+Markup validation (Default: true)
+.RE
+.PP
+\fB\-n\fR, \fB\-\-not\-found\fR
+.RS 4
+Log not found url (Default: false)
+.RE
+.PP
+\fB\-v\fR, \fB\-\-verbose\fR
+.RS 4
+Show detail of validator errors (Default: false)\&.
+.RE
+.PP
+\fB\-d\fR, \fB\-\-debug\fR
+.RS 4
+Show anemone log (Default: false)
+.RE
+.PP
+\fB\-h\fR, \fB\-\-help\fR
+.RS 4
+Show help message and exit\&.
+.RE
+.SH "EXIT STATUS"
+.PP
+0
+.RS 4
+Markup is valid and no 404 found\&.
+.RE
+.PP
+64
+.RS 4
+Not valid markup found\&.
+.RE
+.PP
+65
+.RS 4
+There are pages not found\&.
+.RE
+.PP
+66
+.RS 4
+There are not valid markup and pages not found\&.
+.RE
+.SH "AUTHOR"
+.sp
+Laurent Arnoud <laurent@spkdev\&.net>
+.SH "LICENSE"
+.sp
+The MIT License
+.sp
+Copyright (c) 2009\-2010 Laurent Arnoud <laurent@spkdev\&.net>

data/spec/data/html4-strict.html

@@ -0,0 +1,239 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
+<html lang="fr">
+<head>
+  <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+  <title>Debian -- Le système d'exploitation universel </title>
+  <link rev="made" href="mailto:webmaster@debian.org">
+  <link rel="shortcut icon" href="favicon.ico">
+  <meta name="Keywords" content="debian, GNU, linux, unix, open source, libre, DFSG">
+  <meta name="Description" content="Debian GNU/Linux est une distribution libre du système d'exploitation GNU/Linux. Elle est développée et mise à jour grâce au travail de nombreux utilisateurs qui offrent leur temps et leurs efforts.">
+  <meta name="Generator" content="WML 2.0.11 (19-Aug-2006)">
+  <meta name="Modified" content="2010-10-22 23:28:09">
+<link rel="alternate" type="application/rss+xml"
+ title="Annonces de sécurité Debian (titres seulement)" href="security/dsa">
+<link rel="alternate" type="application/rss+xml"
+ title="Annonces de sécurité Debian (résumés)" href="security/dsa-long">
+<link href="./debian.css" rel="stylesheet" type="text/css">
+  <link href="./debian-fr.css" rel="stylesheet" type="text/css" media="all">
+</head>
+<body>
+<div id="header">
+   <div id="upperheader">
+   <div id="logo">
+  <a href="./"><img src="./logos/openlogo-nd-50.png" width="50" height="61" alt=""></a>
+  <a href="./" rel="start"><img src="Pics/debian.png" width="179" height="61" alt="Projet Debian"></a>
+  </div> <!-- end logo -->
+</div> <!-- end upperheader -->
+<!--UdmComment-->
+<div id="navbar">
+<p class="hidecss"><a href="#inner">Sauter le menu</a></p>
+<ul>
+   <li><a href="intro/about">À propos de Debian</a></li>
+   <li><a href="./News/">Actualités</a></li>
+   <li><a href="distrib/">Obtenir Debian</a></li>
+   <li><a href="./support">Assistance</a></li>
+   <li><a href="./devel/">Le&nbsp;coin&nbsp;du&nbsp;développeur</a></li>
+   <li><a href="./sitemap">Plan du site</a></li>
+   <li><a href="http://search.debian.org/">Recherche</a></li>
+</ul>
+</div> <!-- end navbar -->
+</div> <!-- end header -->
+<!--/UdmComment-->
+<div id="outer">
+<div id="inner">
+<div id="leftcol">
+<!--UdmComment-->
+    <ul>
+    <li><a href="intro/about">À&nbsp;propos&nbsp;de&nbsp;Debian</a>
+      <ul>
+      <li><a href="./social_contract">Notre&nbsp;contrat&nbsp;social</a></li>
+      <li><a href="./intro/free">Logiciel&nbsp;libre</a></li>
+      <li><a href="./partners/">Partenaires</a></li>
+      <li><a href="./donations">Dons</a></li>
+      <li><a href="./contact">Nous&nbsp;contacter</a></li>
+      </ul>
+    </li>
+    <li><a href="./News/">Actualités</a>
+      <ul>
+      <li><a href="./News/project/">Nouvelles du projet</a></li>
+      <li><a href="./events/">Événements</a></li>
+      </ul>
+    </li>
+    <li><a href="distrib/">Obtenir Debian</a>
+      <ul>
+      <li><a href="CD/vendors/">Vendeurs de CD</a></li>
+      <li><a href="CD/">Images ISO de CD</a></li>
+      <li><a href="distrib/netinst">Installation par le réseau</a></li>
+      <li><a href="distrib/pre-installed">Préinstallée</a></li>
+      </ul>
+    </li>
+    <li><a href="distrib/packages">Paquets&nbsp;Debian</a></li>
+    <li><a href="doc/">Documentation</a>
+      <ul>
+      <li><a href="./releases/">Dernière&nbsp;version</a></li>
+      <li><a href="./releases/stable/installmanual">Manuel&nbsp;d'installation</a></li>
+      <li><a href="doc/books">Livres&nbsp;Debian</a></li>
+      </ul>
+    </li>
+    <li><a href="./support">Assistance</a>
+      <ul>
+      <li><a href="./international/">Debian&nbsp;et&nbsp;l'international</a></li>
+      <li><a href="./security/">Informations&nbsp;sur&nbsp;la&nbsp;sécurité</a></li>
+      <li><a href="Bugs/">Rapports de bogues</a></li>
+      <li><a href="MailingLists/">Listes&nbsp;de&nbsp;diffusion</a></li>
+      <li><a href="http://lists.debian.org/">Archives&nbsp;des&nbsp;listes&nbsp;de&nbsp;diffusion</a></li>
+      <li><a href="./ports/">Portages/Architectures</a></li>
+      </ul>
+    </li>
+    <li><a href="misc/">Divers</a></li>
+    <li><a href="./intro/help">Aider Debian</a></li>
+    <li><a href="./devel/">Le&nbsp;coin&nbsp;du&nbsp;développeur</a></li>
+    <li><a href="./sitemap">Plan du site</a></li>
+    <li><a href="http://search.debian.org/">Recherche</a></li>
+    </ul>
+    <form method="get" action="http://search.debian.org/">
+    <p>
+    <input type="text" name="q" size="12">
+    </p>
+    </form>
+    <p>
+    <a href="./sponsor.html"><img src="sponsor_img.jpg" alt="Le site de notre sponsor" width="102" height="60"></a>
+    </p>
+    <p>
+    <a href="http://validator.w3.org/check/referer"><img src="http://www.w3.org/Icons/valid-html401" alt="Valid HTML 4.01!" width="88" height="31"></a>
+    </p>
+    <p>
+    <img src="http://jigsaw.w3.org/css-validator/images/vcss"
+        alt="Valid CSS!" width="88" height="31">
+    </p>
+<!--/UdmComment-->
+</div> <!-- end leftcol -->
+<div id="maincol">
+<a href="./News/2009/20090214"><img src="Pics/lennybanner_indexed.png" alt="Debian 5.0 - Le système d'exploitation universel" width="380" height="310" style="margin-right: 10px; float: left;"></a>
+<h2>Qu'est-ce que Debian&nbsp;?</h2>
+<p><a href="http://www.debian.org/">Debian</a> est un système d'exploitation
+<a href="intro/free">libre</a> pour votre ordinateur. Un système d'exploitation
+est la suite des programmes de base et des utilitaires qui permettent à un
+ordinateur de fonctionner. Debian utilise le noyau
+<a href="http://www.kernel.org/">Linux</a> (le c&oelig;ur d'un système d'exploitation),
+mais la plupart des outils de base du système proviennent du
+<a href="http://www.gnu.org/">projet GNU</a>&nbsp;; d'où le nom GNU/Linux.</p>
+<p>Debian GNU/Linux est bien plus qu'un simple système d'exploitation&nbsp;:
+il contient plus de 25000
+<a href="distrib/packages">paquets</a>&nbsp;; les paquets sont des composants
+logiciels précompilés conçus pour s'installer facilement sur votre machine.</p>
+<p><a href="intro/about">Suite...</a>
+<hr>
+<h2>Pour commencer</h2>
+<p>La <a href="releases/stable/">dernière version stable de Debian</a> est
+la 5.0. La dernière mise à jour de cette version a été publiée
+le 4 septembre 2010. Vous pouvez aussi accéder aux
+<a href="releases/">autres versions disponibles de Debian</a>.</p>
+<p>Si vous souhaitez commencer à utiliser Debian, vous pouvez facilement
+<a href="distrib/">en obtenir une copie</a>, et ensuite suivre les
+<a href="releases/stable/installmanual">instructions d'installation</a>
+pour l'installer.</p>
+<p>Si vous mettez à niveau votre système depuis une ancienne version vers
+la dernière version stable publiée, veuillez lire les
+<a href="releases/stable/releasenotes">notes de publication</a>
+avant de commencer.</p>
+<p>Pour obtenir de l'aide concernant l'utilisation ou la configuration
+de Debian, consultez nos pages sur la <a href="doc/">documentation</a>
+et l'<a href="support">assistance</a>.</p>
+<p>Les utilisateurs qui parlent une langue autre que l'anglais peuvent
+consulter la section sur l'<a href="international/">international</a>.</p>
+<p>Les personnes ayant un autre système qu'Intel x86 peuvent
+consulter la section sur les <a href="ports/">portages</a>.</p>
+<hr>
+<h2>Actualités</h2>
+<p><tt>[19 octobre 2010]</tt> <strong><a href="News/2010/20101019">Debian sur le point d'accueillir officiellement les contributeurs non empaqueteurs</a></strong><br>
+<tt>[7 octobre 2010]</tt> <strong><a href="News/2010/20101007">Debian à la rencontre de la <q>Society for Neuroscience</q></a></strong><br>
+<tt>[8 septembre 2010]</tt> <strong><a href="News/2010/20100908">Paris Mini-DebConf 2010</a></strong><br>
+<tt>[5 septembre 2010]</tt> <strong><a href="News/2010/20100905">Le service de rétroportages (« backports ») devient officiel</a></strong><br>
+<tt>[4 septembre 2010]</tt> <strong><a href="News/2010/20100904">Publication de la mise à jour de Debian GNU/Linux 5.0.6</a></strong><br>
+<tt>[3 septembre 2010]</tt> <strong><a href="News/2010/20100903">Conférence 2010 de la communauté Debian italienne - du 17 au 19 septembre à Pérouse, Italie</a></strong><br>
+</p>
+<p>Pour les communiqués plus anciens, consultez la suite de la page <a href="./News/">actualités</a>.
+Si vous voulez recevoir un courrier (en anglais) à chaque fois qu'un communiqué paraît, abonnez-vous
+à la <a href="MailingLists/debian-announce">liste de diffusion debian-announce</a>.</p>
+<hr>
+<h2>Annonces de sécurité</h2>
+<p><tt>[22 octobre 2010]</tt> <strong><a href="security/2010/dsa-2122">DSA-2122 glibc</a></strong> - missing input sanitization <br>
+<tt>[19 octobre 2010]</tt> <strong><a href="security/2010/dsa-2121">DSA-2121 typo3-src</a></strong> - several vulnerabilities <br>
+<tt>[12 octobre 2010]</tt> <strong><a href="security/2010/dsa-2120">DSA-2120 postgresql-8.3</a></strong> - privilege escalation <br>
+<tt>[12 octobre 2010]</tt> <strong><a href="security/2010/dsa-2119">DSA-2119 poppler</a></strong> - several vulnerabilities <br>
+<tt>[8 octobre 2010]</tt> <strong><a href="security/2010/dsa-2118">DSA-2118 subversion</a></strong> - logic flaw <br>
+<tt>[4 octobre 2010]</tt> <strong><a href="security/2010/dsa-2117">DSA-2117 apr-util</a></strong> - denial of service <br>
+<tt>[4 octobre 2010]</tt> <strong><a href="security/2010/dsa-2116">DSA-2116 freetype</a></strong> - integer overflow <br>
+<tt>[29 septembre 2010]</tt> <strong><a href="security/2010/dsa-2115">DSA-2115 moodle</a></strong> - several vulnerabilities <br>
+<tt>[26 septembre 2010]</tt> <strong><a href="security/2010/dsa-2114">DSA-2114 git-core</a></strong> - buffer overflow <br>
+<tt>[20 septembre 2010]</tt> <strong><a href="security/2010/dsa-2113">DSA-2113 drupal6</a></strong> - several vulnerabilities <br>
+<tt>[20 septembre 2010]</tt> <strong><a href="security/2010/dsa-2112">DSA-2112 bzip2</a></strong> - integer overflow <br>
+</p>
+<p>Pour les annonces de sécurité, consultez la <a href="./security/">
+page sécurité</a>.
+Si vous voulez recevoir les annonces de sécurité (en anglais) dès leur parution, abonnez-vous
+à la <a href="http://lists.debian.org/debian-security-announce/">liste de diffusion debian-security-announce</a>.</p>
+</div> <!-- end maincol -->
+<div class="clr"></div>
+</div> <!-- end inner -->
+<div id="footer">
+  <hr class="hidecss">
+  <!--UdmComment-->
+<p>
+Cette page est aussi disponible dans les langues suivantes&nbsp;:
+</p><p class="navpara">
+ <a href="index.ar.html" title="arabe" hreflang="ar" lang="ar" rel="alternate">&#1593;&#1585;&#1576;&#1610;&#1577;&nbsp;(Arabiya)</a>
+ <a href="index.bg.html" title="bulgare" hreflang="bg" lang="bg" rel="alternate">&#1041;&#1098;&#1083;&#1075;&#1072;&#1088;&#1089;&#1082;&#1080;&nbsp;(B&#601;lgarski)</a>
+ <a href="index.ca.html" title="catalan" hreflang="ca" lang="ca" rel="alternate">catal&agrave;</a>
+ <a href="index.cs.html" title="tchèque" hreflang="cs" lang="cs" rel="alternate">&#269;esky</a>
+ <a href="index.da.html" title="danois" hreflang="da" lang="da" rel="alternate">dansk</a>
+ <a href="index.de.html" title="allemand" hreflang="de" lang="de" rel="alternate">Deutsch</a>
+ <a href="index.el.html" title="grec" hreflang="el" lang="el" rel="alternate">&#917;&#955;&#955;&#951;&#957;&#953;&#954;&#940;&nbsp;(Ellinika)</a>
+ <a href="index.en.html" title="anglais" hreflang="en" lang="en" rel="alternate">English</a>
+ <a href="index.es.html" title="espagnol" hreflang="es" lang="es" rel="alternate">espa&ntilde;ol</a>
+ <a href="index.eo.html" title="espéranto" hreflang="eo" lang="eo" rel="alternate">Esperanto</a>
+ <a href="index.ko.html" title="coréen" hreflang="ko" lang="ko" rel="alternate">&#54620;&#44397;&#50612;&nbsp;(Hangul)</a>
+ <a href="index.hy.html" title="arménien" hreflang="hy" lang="hy" rel="alternate">&#1344;&#1377;&#1397;&#1381;&#1408;&#1381;&#1398;&nbsp;(hayeren)</a>
+ <a href="index.hr.html" title="croate" hreflang="hr" lang="hr" rel="alternate">hrvatski</a>
+ <a href="index.it.html" title="italien" hreflang="it" lang="it" rel="alternate">Italiano</a>
+ <a href="index.he.html" title="hébreu" hreflang="he" lang="he" rel="alternate">&#1506;&#1489;&#1512;&#1497;&#1514;&nbsp;(ivrit)</a>
+ <a href="index.lt.html" title="lituanien" hreflang="lt" lang="lt" rel="alternate">Lietuvi&#371;</a>
+ <a href="index.hu.html" title="hongrois" hreflang="hu" lang="hu" rel="alternate">magyar</a>
+ <a href="index.nl.html" title="néerlandais" hreflang="nl" lang="nl" rel="alternate">Nederlands</a>
+ <a href="index.ja.html" title="japonais" hreflang="ja" lang="ja" rel="alternate">&#26085;&#26412;&#35486;&nbsp;(Nihongo)</a>
+ <a href="index.nb.html" title="norvégien" hreflang="nb" lang="nb" rel="alternate">norsk&nbsp;(bokm&aring;l)</a>
+ <a href="index.pl.html" title="polonais" hreflang="pl" lang="pl" rel="alternate">polski</a>
+ <a href="index.pt.html" title="portugais" hreflang="pt" lang="pt" rel="alternate">Portugu&ecirc;s</a>
+ <a href="index.ro.html" title="roumain" hreflang="ro" lang="ro" rel="alternate">rom&acirc;n&#259;</a>
+ <a href="index.ru.html" title="russe" hreflang="ru" lang="ru" rel="alternate">&#1056;&#1091;&#1089;&#1089;&#1082;&#1080;&#1081;&nbsp;(Russkij)</a>
+ <a href="index.sk.html" title="slovaque" hreflang="sk" lang="sk" rel="alternate">slovensky</a>
+ <a href="index.fi.html" title="finnois" hreflang="fi" lang="fi" rel="alternate">suomi</a>
+ <a href="index.sv.html" title="suédois" hreflang="sv" lang="sv" rel="alternate">svenska</a>
+ <a href="index.ta.html" title="tamoul" hreflang="ta" lang="ta" rel="alternate">&#2980;&#2990;&#3007;&#2996;&#3021;&nbsp;(Tamil)</a>
+ <a href="index.tr.html" title="turc" hreflang="tr" lang="tr" rel="alternate">T&uuml;rk&ccedil;e</a>
+ <a href="index.uk.html" title="ukrainien" hreflang="uk" lang="uk" rel="alternate">&#1091;&#1082;&#1088;&#1072;&#1111;&#1085;&#1089;&#1100;&#1082;&#1072;&nbsp;(ukrajins'ka)</a>
+ <a href="index.zh-cn.html" title="chinois (Chine)" hreflang="zh-CN" lang="zh-CN" rel="alternate">&#20013;&#25991;(&#31616;)</a>
+ <a href="index.zh-hk.html" title="chinois (Hong Kong)" hreflang="zh-HK" lang="zh-HK" rel="alternate">&#20013;&#25991;(HK)</a>
+ <a href="index.zh-tw.html" title="chinois (Taïwan)" hreflang="zh-TW" lang="zh-TW" rel="alternate">&#20013;&#25991;(&#32321;)</a>
+</p><p>
+Comment configurer la <a href="./intro/cn">langue par défaut du document</a>
+</p>
+<!--/UdmComment-->
+<hr>
+<!--UdmComment-->
+<div id="fineprint">
+  <p>Pour signaler un problème sur le site web, envoyez un courriel en anglais à <a href="mailto:debian-www@lists.debian.org">debian-www@lists.debian.org</a> ou en français à <a href="mailto:debian-l10n-french@lists.debian.org">debian-l10n-french@lists.debian.org</a>. Pour obtenir d'autres informations, référez-vous à la <a href="./contact">page contact</a> de Debian.</p>
+<p>
+Dernière modification&nbsp;: vendredi 22 octobre 2010 23:28:09 UTC
+  <br>
+  Copyright &copy; 1997-2010
+ <a href="http://www.spi-inc.org/">SPI</a>; voir <a href="./license" rel="copyright">les termes de la licence</a>.<br>
+  Debian est une <a href="./trademark">marque déposée</a> de Software in the Public Interest, Inc.
+</p>
+</div>
+<!--/UdmComment-->
+</div> <!-- end footer -->
+</div> <!-- end outer -->
+</body>
+</html>

data/spec/data/html5.html

@@ -1,9 +1,8 @@
 <!DOCTYPE html>
-<html lang="en">
+<html>
 	<head>
 		<meta charset="utf-8" />
 		<title>title</title>
-		<!-- meta -->
 		<meta name="description" content="" />
 		<meta name="keywords" content="" />
 		<meta name="author" content="" />

data/spec/validate_website_spec.rb

@@ -4,6 +4,7 @@ describe ValidateWebsite do
 
   before(:each) do
     FakeWeb.clean_registry
+    @validate_website = ValidateWebsite.new
   end
 
   context('html') do
@@ -13,9 +14,8 @@ describe ValidateWebsite do
       page = FakePage.new(name,
                           :body => open(file).read,
                           :content_type => 'text/html')
-      validate_website = ValidateWebsite.new
-      validate_website.crawl(page.url)
-      validate_website.anemone.should have(3).pages
+      @validate_website.crawl(page.url)
+      @validate_website.anemone.should have(3).pages
     end
   end
 
@@ -27,27 +27,24 @@ describe ValidateWebsite do
                                     .tests {background-image: url(/image/pouet_42.png)}
                                     .tests {background-image: url(/image/pouet)}",
                           :content_type => 'text/css')
-      validate_website = ValidateWebsite.new
-      validate_website.crawl(page.url)
-      validate_website.anemone.should have(5).pages
+      @validate_website.crawl(page.url)
+      @validate_website.anemone.should have(5).pages
     end
 
     it "should extract url with single quote" do
       page = FakePage.new('test.css',
                             :body => ".test {background-image: url('pouet');}",
                             :content_type => 'text/css')
-      validate_website = ValidateWebsite.new
-      validate_website.crawl(page.url)
-      validate_website.anemone.should have(2).pages
+      @validate_website.crawl(page.url)
+      @validate_website.anemone.should have(2).pages
     end
 
     it "should extract url with double quote" do
       page = FakePage.new('test.css',
                           :body => ".test {background-image: url(\"pouet\");}",
                           :content_type => 'text/css')
-      validate_website = ValidateWebsite.new
-      validate_website.crawl(page.url)
-      validate_website.anemone.should have(2).pages
+      @validate_website.crawl(page.url)
+      @validate_website.anemone.should have(2).pages
     end
   end
 end

data/spec/validator_spec.rb

@@ -21,7 +21,6 @@ describe Validator do
     end
 
     it "html5 should be valid" do
-      pending("need to work on html5")
       name = 'html5'
       file = File.join('spec', 'data', "#{name}.html")
       page = FakePage.new(name,
@@ -31,4 +30,15 @@ describe Validator do
       validator = Validator.new(@html5_page)
       validator.should be_valid
     end
+
+    it 'should validate html4' do
+      name = 'html4-strict'
+      file = File.join('spec', 'data', "#{name}.html")
+      page = FakePage.new(name,
+                          :body => open(file).read,
+                          :content_type => 'text/html')
+      @html4_strict_page = @http.fetch_page(page.url)
+      validator = Validator.new(@html4_strict_page)
+      validator.should be_valid
+    end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: validate-website
 version: !ruby/object:Gem::Version 
-  hash: 13
+  hash: 11
   prerelease: false
   segments: 
   - 0
-  - 4
-  - 1
-  version: 0.4.1
+  - 5
+  - 0
+  version: 0.5.0
 platform: ruby
 authors: 
 - Laurent Arnoud
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-10-24 00:00:00 +02:00
+date: 2010-11-01 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -50,25 +50,41 @@ dependencies:
   type: :runtime
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  name: rspec
+  name: html5
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        hash: 55
+        segments: 
+        - 0
+        - 10
+        - 0
+        version: 0.10.0
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 27
+        hash: 15
         segments: 
-        - 1
-        - 3
+        - 2
         - 0
-        version: 1.3.0
+        - 0
+        version: 2.0.0
   type: :development
-  version_requirements: *id003
+  version_requirements: *id004
 - !ruby/object:Gem::Dependency 
   name: fakeweb
   prerelease: false
-  requirement: &id004 !ruby/object:Gem::Requirement 
+  requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -80,8 +96,8 @@ dependencies:
         - 0
         version: 1.3.0
   type: :development
-  version_requirements: *id004
-description: Web crawler for checking the validity of your documents
+  version_requirements: *id005
+description: validate-website is a web crawler for checking the markupvalidity and not found urls.
 email: laurent@spkdev.net
 executables: 
 - validate-website
@@ -92,6 +108,8 @@ extra_rdoc_files: []
 files: 
 - README.rdoc
 - Rakefile
+- LICENSE
+- bin/validate-website
 - lib/validator.rb
 - lib/colorful_messages.rb
 - lib/xhtml/xhtml-blkphras-1.xsd
@@ -185,9 +203,10 @@ files:
 - lib/xhtml/xhtml-attribs-1.xsd
 - lib/xhtml/xhtml-ruby-1.xsd
 - lib/validate_website.rb
-- bin/validate-website
+- man/man1/validate-website.1
 - spec/validator_spec.rb
 - spec/spec_helper.rb
+- spec/data/html4-strict.html
 - spec/data/html5.html
 - spec/data/xhtml1-strict.html
 - spec/fakeweb_helper.rb
@@ -222,6 +241,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: 
 - spk-anemone
 - rainbow
+- html5
 rubyforge_project: 
 rubygems_version: 1.3.7
 signing_key: