simplecrawler 0.1.0

data/README

@@ -0,0 +1,4 @@
+SimpleCrawler - a web crawling library for Ruby
+
+see http://www.peterkrantz.com/simplecrawler/wiki/ for more information
+

data/examples/accessibility_report.rb

@@ -0,0 +1,23 @@
+require '../lib/simplecrawler.rb'
+require 'raakt'
+require 'ruport'
+
+# Set up a new crawler
+sc = SimpleCrawler::Crawler.new(ARGV[0])
+sc.skip_patterns = ["\\.doc$", "\\.pdf$", "\\.xls$", "\\.pdf$", "\\.zip$"]
+sc.maxcount = 10
+
+report_data = Ruport::Data::Table.new :column_names => ["Url", "Error count"]
+
+sc.crawl { |document|
+
+	# Run basic accessibility check
+	raakt = Raakt::Test.new(document.data)
+	result = raakt.all
+	puts "#{result.length}\t#{document.uri}"
+	if result.length > 0
+		report_data << [document.uri, result.length]
+	end
+}
+
+puts report_data

data/examples/crawl.rb

@@ -0,0 +1,12 @@
+require '../lib/simplecrawler.rb'
+
+# Set up a new crawler
+sc = SimpleCrawler::Crawler.new(ARGV[0])
+sc.skip_patterns = ["\\.doc$", "\\.pdf$", "\\.xls$", "\\.pdf$", "\\.zip$"]
+
+sc.crawl { |document|
+
+	# Print links for entire site
+	puts document.uri
+
+}

data/examples/list_site_links.rb

@@ -0,0 +1,11 @@
+require '../lib/simplecrawler.rb'
+
+# Set up a new crawler
+sc = SimpleCrawler::Crawler.new(ARGV[0])
+sc.skip_patterns = ["\\.doc$", "\\.pdf$", "\\.xls$", "\\.pdf$", "\\.zip$"]
+
+sc.crawl { |document|
+	# List links for entire site
+	puts document.uri
+}
+

data/lib/simplecrawler.rb

@@ -0,0 +1,173 @@
+# == Simple Crawler 
+# :title: SimpleCrawler - a generic web crawler library in Ruby
+# Author::    Peter Krantz (http://www.peterkrantz.com)
+# License::   LGPL (See LICENSE file)
+#
+# The SimpleCrawler module is a library for crawling web sites. The crawler provides comprehensive data from the page crawled which can be used for page analysis, indexing, accessibility checks etc. Restrictions can be specified to limit crawling of binary files.
+#
+# == Output
+# The SimpleCrawler::Crawler class yields a SimpleCrawler::Document object instance. This object contains information about a specific URI such as http headers and response data etc.
+#
+# == Contributions
+# None yet :-) Why don't you go ahead and be first?
+#
+# == Example usage
+# See the "Simple Crawler wiki"[http://www.peterkrantz.com/simplecrawler/wiki/].
+
+module SimpleCrawler
+
+	require 'uri'
+	require 'rubygems'
+	require 'hpricot'
+	require 'open-uri'
+
+	MARKUP_MIME_TYPES = ["text/html", "text/xml", "application/xml", "application/xhtml+xml"]
+
+	VERSION = "0.1.0"
+
+	class Document
+		attr_accessor :uri, :data, :headers, :fetched_at
+
+		def to_s
+			puts "Document"
+			puts " .uri:\t\t#{uri}"
+			puts " .fetched_at:\t#{fetched_at}"
+			puts " .headers:"
+			for header in headers
+				puts "   #{header[0]}: #{header[1]}"
+			end
+			puts " .data.length:\t#{(data.length)}"   
+		end
+	end
+
+
+
+
+	class Crawler
+
+		attr_accessor :user_agent, :skip_patterns, :load_binary_data, :queue, :visited, :site_uri, :maxcount, :current_count
+
+		def initialize(url)
+			@load_binary_data = false #default, skip loading of pagedata for binary files into Document.data
+			@site_uri = URI.parse(url)
+			@site_uri.path = "/" if @site_uri.path == ""
+			@visited = Hash.new
+			@queue = Array.new
+			@current_count = 0
+			add_uri(@site_uri)
+		end
+
+
+		# Check if a path should be ignored because it matches a skip pattern or is already visited.
+		def skip_uri?(uri)
+
+			#Check if maxcount is reached
+			if @maxcount
+				if @current_count >= @maxcount
+					return true
+				end
+			end
+
+			#Check if path belongs to site
+			unless (uri.relative? or uri.host == @site_uri.host)
+				return true
+			end
+
+			#Check if fragment identifier (e.g. #content)
+			if uri.path.length == 0 and uri.fragment.length > 0
+				return true
+			end
+
+			#Check if uri already visited in this crawl or if it is queued for crawling
+			if @visited.has_key?(uri.path) or @queue.include?(uri.path)
+				return true
+			end
+			
+			#Check if uri is in a skip pattern
+			if @skip_patterns
+				for skip_pattern in @skip_patterns
+					re = Regexp.new(skip_pattern)
+					if re.match(uri.path) 
+						return true
+					end
+				end
+			end
+
+			return false
+		end
+
+
+		def add_uri(uri)
+
+			if uri.class == String
+				uri = URI.parse(uri.strip)
+			end
+			
+			unless skip_uri?(uri)
+				@queue.push uri.path
+				@current_count = @current_count + 1
+				@visited[uri.path] = false
+			end
+
+		end
+
+
+		def get_doc(path)
+			doc = Document.new
+			begin
+				uri = @site_uri.clone
+				uri.path = uri.path + path if path != "/"
+				doc.uri = uri
+
+				file = open(uri)
+
+				mime_type = file.meta["content-type"].split(";")[0] if file.meta["content-type"]
+				
+				if MARKUP_MIME_TYPES.include?(mime_type.downcase) or @load_binary_data
+					doc.data = file.read
+				else
+					doc.data = nil
+				end
+
+				doc.headers = file.meta
+				doc.fetched_at = Time.now
+			rescue Exception
+				puts "Error: #{$!}"
+				return ""
+			end
+			return doc 
+		end
+
+
+		def queue_local_links(doc)
+			return if doc.data == nil
+			Hpricot.buffer_size = 262144 #Allow for asp.net bastard-sized viewstate attributes...
+			doc = Hpricot(doc.data)
+			links = doc.search("a[@href]")
+			for link in links
+				if link.attributes["href"].length > 0 then
+					begin
+						uri = URI.parse(link.attributes["href"])
+						add_uri(uri)
+					rescue
+						#skip this link
+					end
+				end
+			end
+			doc = nil
+		end
+
+
+		# Initiate crawling.
+		def crawl()
+			while (!@queue.empty?)
+				uri = @queue.shift
+				current_doc = get_doc(uri)
+				yield current_doc
+				queue_local_links(current_doc)
+				@visited[uri] = true
+			end
+		end
+
+	end
+end

data/tests/simplecrawler_test.rb

@@ -0,0 +1,101 @@
+require File.dirname(__FILE__) + '/../lib/simplecrawler'
+require 'test/unit'
+require 'uri'
+
+class SimpleCrawlerTest < Test::Unit::TestCase
+  
+  def setup
+	  @simplecrawler = SimpleCrawler::Crawler.new("http://www.example.com/")
+  end
+  
+
+  def test_initialize_crawler
+	  @crawler = SimpleCrawler::Crawler.new("http://www.example.com/")
+	  assert @crawler.queue.length == 1
+  end
+
+
+  def test_initialize_crawler_without_uri_path
+	  @crawler = SimpleCrawler::Crawler.new("http://www.example.com")
+	  assert @crawler.queue.length == 1
+
+	  # Default path (/) should be appended
+	  assert @crawler.queue[0][-1..-1] == "/"
+  end
+
+
+
+  def test_maxcount_limit
+	  @simplecrawler.maxcount = 2
+	  @simplecrawler.add_uri("http://www.example.com/second/")
+	  @simplecrawler.add_uri("http://www.example.com/third/")
+	  assert_equal 2, @simplecrawler.queue.length
+  end
+
+  def test_maxcount_unlimited
+	  @simplecrawler.add_uri("http://www.example.com/second/")
+	  @simplecrawler.add_uri("http://www.example.com/third/")
+	  assert @simplecrawler.queue.length == 3
+  end
+
+  def test_skip_uri
+	  @simplecrawler.skip_patterns = ["\\.doc$"]
+	  assert @simplecrawler.skip_uri?(URI.parse("http://www.example.com/word.doc"))
+	  assert_equal false, @simplecrawler.skip_uri?(URI.parse("http://www.example.com/doc.htm"))
+  end
+  
+
+  def test_addded_paths_shuld_be_distinct
+	  @simplecrawler.add_uri("http://www.example.com/") # This path is already in the queue
+	  assert_equal 1, @simplecrawler.queue.length
+  end
+
+  def test_add_uri
+	  @simplecrawler.add_uri("http://www.example.com/new/")
+
+	  # The queue should now contain the initial base url and the newly added path
+	  assert_equal 2, @simplecrawler.queue.length
+  end
+
+
+  def test_add_uri_with_space
+	  @simplecrawler.add_uri("http://www.example.com/new/    ")
+
+	  # The queue should now contain the initial base url and the newly added path without spaces
+	  assert_equal 2, @simplecrawler.queue.length
+	  assert @simplecrawler.queue[1][-1..-1] != " "
+  end
+
+
+
+  def test_queue_local_link
+	  doc = SimpleCrawler::Document.new
+	  doc.data = "<html><head></head><body><a href=\"http://www.example.com/new/\">Test</a></body></html>"
+	  @simplecrawler.queue_local_links(doc)
+	  assert_equal 2, @simplecrawler.queue.length
+  end
+
+
+  def test_queue_local_fragment_identifier_skipped
+	  doc = SimpleCrawler::Document.new
+	  doc.data = "<html><head></head><body><a href=\"#new\">Test</a></body></html>"
+	  @simplecrawler.queue_local_links(doc)
+	  assert_equal 1, @simplecrawler.queue.length
+  end
+
+
+  def test_queue_local_links_for_empty_doc
+	  doc = SimpleCrawler::Document.new
+	  doc.data = ""
+	  @simplecrawler.queue_local_links(doc)
+	  assert_equal 1, @simplecrawler.queue.length
+  end
+
+
+  def test_queue_local_links_for_nil_doc
+	  doc = SimpleCrawler::Document.new
+	  doc.data = nil
+	  @simplecrawler.queue_local_links(doc)
+	  assert_equal 1, @simplecrawler.queue.length
+  end
+end

metadata

@@ -0,0 +1,59 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.9.2
+specification_version: 1
+name: simplecrawler
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+date: 2007-08-27 00:00:00 +02:00
+summary: A generic library for web crawling.
+require_paths: 
+- lib
+email: peter.krantzNODAMNSPAM@gmail.com
+homepage: http://www.peterkrantz.com/simplecrawler/wiki/
+rubyforge_project: simplecrawler
+description: 
+autorequire: simplecrawler
+default_executable: 
+bindir: bin
+has_rdoc: true
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: 1.8.2
+  version: 
+platform: ruby
+signing_key: 
+cert_chain: 
+post_install_message: 
+authors: 
+- Peter Krantz
+files: 
+- README
+- lib/simplecrawler.rb
+- tests/simplecrawler_test.rb
+- examples/accessibility_report.rb
+- examples/crawl.rb
+- examples/list_site_links.rb
+test_files: 
+- tests/simplecrawler_test.rb
+rdoc_options: []
+
+extra_rdoc_files: []
+
+executables: []
+
+extensions: []
+
+requirements: []
+
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: hpricot
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Version::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0.5"
+    version: