validate-website 0.1

data/README

@@ -0,0 +1,39 @@
+== validate-website
+
+== DESCRIPTION
+
+  Web crawler that print if the page is valid with the dtd.
+  compatible ruby 1.9
+
+== SYNOPSIS
+
+  validate-website --help
+  validate-website -s "http://localhost:4567/" -u "Mozilla 5.0" -f not-well-formed.txt --auth=user,pass -e 'redirect|news'
+
+== REQUIREMENTS:
+
+  libxml-ruby >= 1.1.3
+
+== LICENSE
+(The MIT License)
+
+Copyright (c) 2009 spk
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,71 @@
+require 'rake/testtask'
+require 'rake/packagetask'
+require 'rake/rdoctask'
+require 'rake'
+require 'find'
+
+# Globals
+
+PKG_NAME = 'validate-website'
+PKG_VERSION = '0.1'
+
+PKG_FILES = ['README', 'Rakefile']
+Find.find('lib/', 'bin/') do |f|
+  if FileTest.directory?(f) and f =~ /\.svn|\.git/
+    Find.prune
+  else
+    PKG_FILES << f
+  end
+end
+
+# Tasks
+
+task :default => [:clean, :repackage]
+
+#Rake::TestTask.new do |t|
+  #t.libs << "test"
+  #t.test_files = FileList['test/tc_*.rb']
+#end
+
+Rake::RDocTask.new do |rd|
+  f = []
+  require 'find'
+  Find.find('lib/') do |file|
+    # Skip hidden files (.svn/ directories and Vim swapfiles)
+    if file.split(/\//).last =~ /^\./
+      Find.prune
+    else
+      f << file if not FileTest.directory?(file)
+    end
+  end
+  rd.rdoc_files.include(f)
+  rd.options << '--all'
+end
+
+Rake::PackageTask.new(PKG_NAME, PKG_VERSION) do |p|
+  p.need_tar = true
+  p.package_files = PKG_FILES
+end
+
+# "Gem" part of the Rakefile
+require 'rake/gempackagetask'
+
+spec = Gem::Specification.new do |s|
+  s.author = 'spk'
+  s.email = 'spk@tuxfamily.org'
+  s.platform = Gem::Platform::RUBY
+  s.summary = "Web crawler for testing webpage validity"
+  s.name = PKG_NAME
+  s.version = PKG_VERSION
+  s.requirements << 'libxml-ruby'
+  s.require_path = 'lib'
+  s.bindir = 'bin'
+  s.executables << 'validate-website'
+  s.files = PKG_FILES
+  s.description = "Web crawler that print if the page is valid with the dtd"
+end
+
+Rake::GemPackageTask.new(spec) do |pkg|
+  pkg.need_zip = true
+  pkg.need_tar = true
+end

data/bin/validate-website

@@ -0,0 +1,77 @@
+#!/usr/bin/env ruby
+$:.unshift '../lib'
+require 'spkspider'
+require 'colorful_messages'
+require 'open-uri'
+require 'xml'
+require 'optparse'
+
+include ColorfulMessages
+
+XML.default_validity_checking = true
+XML.default_load_external_dtd = true
+
+# default options
+OPTIONS = {
+  :site        => 'http://localhost:3000/',
+  :useragent   => '',
+  :exclude     => nil,
+  :file        => nil,
+  :auth        => nil,
+}
+
+ARGV.options do |o|
+  script_name = File.basename($0)
+  o.set_summary_indent('  ')
+  o.banner =    "Usage: #{script_name} [OPTIONS]"
+  o.define_head "validate website"
+  o.separator   ""
+
+  o.on("-s", "--site=val", String,
+       "Default: #{OPTIONS[:site]}") { |OPTIONS[:site]| }
+  o.on("-u", "--useragent=val", String,
+       "Default: #{OPTIONS[:useragent]}") { |OPTIONS[:useragent]| }
+  o.on("-e", "--exclude=val", String,
+       "Url to exclude") { |OPTIONS[:exclude]| }
+  o.on("-f", "--file=val", String,
+       "save not well formed urls") { |OPTIONS[:file]| }
+  o.on("--auth=[user,pass]", Array,
+       "Basic http authentification") { |OPTIONS[:auth]| }
+
+  o.separator ""
+  o.on_tail("-h", "--help", "Show this help message.") { puts o; exit }
+  o.parse!
+end
+
+spider = SpkSpider.new(OPTIONS[:site])
+spider.user_agent = OPTIONS[:useragent]
+spider.exclude = Regexp.new(OPTIONS[:exclude]) if OPTIONS[:exclude]
+spider.basic_auth = OPTIONS[:auth]
+
+if OPTIONS[:file]
+  file = OPTIONS[:file]
+  open(file, 'w').write('')
+end
+
+spider.crawl do |url, document|
+  begin
+    xp = XML::Parser.string(document)
+    exception = nil
+    XML::Error.set_handler do |error|
+      exception = error
+    end
+
+    doc = xp.parse
+
+    msg = " well formed? %s" % xp.context.well_formed?
+    if xp.context.well_formed?
+      print success(msg)
+    else
+      print error(msg)
+      open(file, 'a').write(url+"\n") if OPTIONS[:file]
+    end
+  rescue
+    print error(msg)
+    open(file, 'a').write(url+"\n") if OPTIONS[:file]
+  end
+end

data/lib/colorful_messages.rb

@@ -0,0 +1,30 @@
+module ColorfulMessages
+
+  # red
+  def error(message)
+    "\033[1;31m#{message}\033[0m"
+  end
+
+  # yellow
+  def warning(message)
+    "\033[1;33m#{message}\033[0m"
+  end
+
+  # green
+  def success(message)
+    "\033[1;32m#{message}\033[0m"
+  end
+
+  alias_method :message, :success
+
+  # magenta
+  def note(message)
+    "\033[1;35m#{message}\033[0m"
+  end
+
+  # blue
+  def info(message)
+    "\033[1;34m#{message}\033[0m"
+  end
+
+end

data/lib/spkspider.rb

@@ -0,0 +1,147 @@
+# encoding: utf-8
+require 'open-uri'
+# SpkSpider is a ruby crawler
+
+class SpkSpider
+  VERSION = '0.0.5'
+
+  attr_accessor :links_to_visit, :site, :user_agent, :basic_auth
+  attr_accessor :parser, :exclude
+  attr_reader :visited_links, :external_links, :errors
+
+  # initialize method take the site to crawl in argument
+  def initialize(site)
+    puts "SpkSpider #{VERSION} initializing..."
+    @site = URI.parse(site) || raise("You didn't give me a site to crawl")
+    @user_agent = "SpkSpr/#{VERSION}"
+    @links_to_visit = Array.new
+    @visited_links = Array.new
+    @external_links = Array.new
+    @errors = Hash.new
+    @links_to_visit << site
+    @parser = 'xml'
+    puts "Ready to crawl"
+  end
+
+  def init_xml_parser(doc)
+    require 'xml'
+    xp = XML::HTMLParser.string(doc, {:options => XML::HTMLParser::Options::RECOVER | XML::HTMLParser::Options::NOERROR | XML::HTMLParser::Options::NOWARNING })
+    XML::Error.set_handler do |error|
+      exception = error
+    end
+    document = xp.parse
+    links = document.find("//a[@href]")
+  end
+
+  def fetch_links(doc)
+    case @parser
+    when 'xml'
+      init_xml_parser(doc)
+    when 'hpricot'
+      require 'hpricot'
+      Hpricot.buffer_size = 204800
+      Hpricot(doc).search("//a[@href]")
+    else
+      init_xml_parser(doc)
+    end
+  rescue
+    init_xml_parser(doc)
+  end
+
+  # download the document
+  def fetch_html(url)
+    uri = URI.parse(url)
+    print "Visiting: #{url}"
+    begin
+      @document = uri.read('User-Agent' => @user_agent, 'Referer' => url, :http_basic_authentication => @basic_auth)
+    rescue
+      # OpenURI::HTTPError
+    end
+    @visited_links << url
+    @document
+  end
+
+  # reading the document and extract the urls
+  def read_document(document, url)
+    if document
+      case document.content_type
+      when "text/html"
+        link_extractor(document, url)
+      else
+        print " ... not text/html, skipping ..."
+      end
+    else
+      print " ... document does not exist, skipping ..."
+    end
+  end
+
+  # extract the link and un-relative
+  def link_extractor(document, document_url)
+    links = fetch_links(document)
+    links.each do |link|
+      href = link.attributes['href']
+      if href && href.length > 0 && (@exclude && !href.match(@exclude) || @exclude.nil?)
+        begin
+          url = href
+          uri = URI.parse(url)
+          document_uri = URI.parse(document_url)
+        rescue
+          #print " #{url} skip this link"
+          next
+        end
+      else
+        #print " skip this link"
+        next
+      end
+
+      # Derelativeize links if necessary
+      if uri.relative?
+        url = document_uri.merge(url).to_s if url[0,1] == '?'
+        url = @site.merge(url).to_s
+        uri = URI.parse(url)
+      end
+
+      # skip anchor link
+      if url.include?('#')
+        #print '... Anchor link found, skipping ...'
+        next
+      end
+
+      # Check domain, if in same domain, keep link, else trash it
+      if uri.host != @site.host
+        @external_links << url
+        @external_links.uniq!
+        next
+      end
+
+      # Find out if we've seen this link already
+      if (@visited_links.include? url) || (@links_to_visit.include? url)
+        next
+      end
+
+      @links_to_visit << url
+    end
+  end
+
+  # lunch the crawling
+  def crawl
+    while !@links_to_visit.empty?
+      # get the first element of the links_to_visit
+      url = @links_to_visit.shift
+      document = fetch_html(url)
+      read_document(document, url)
+      if block_given?
+        yield(url, document)
+      end
+      puts ' done!'
+    end
+  end
+end
+
+if __FILE__ == $0
+  site = 'http://localhost:4567/'
+  site = ARGV[0] if ARGV[0]
+  spider = SpkSpider.new(site)
+  spider.user_agent = ''
+  spider.crawl
+end

metadata

@@ -0,0 +1,59 @@
+--- !ruby/object:Gem::Specification 
+name: validate-website
+version: !ruby/object:Gem::Version 
+  version: "0.1"
+platform: ruby
+authors: 
+- spk
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-10-24 00:00:00 +02:00
+default_executable: 
+dependencies: []
+
+description: Web crawler that print if the page is valid with the dtd
+email: spk@tuxfamily.org
+executables: 
+- validate-website
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- README
+- Rakefile
+- lib/colorful_messages.rb
+- lib/spkspider.rb
+- bin/validate-website
+has_rdoc: true
+homepage: 
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: 
+- libxml-ruby
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: Web crawler for testing webpage validity
+test_files: []
+