simplecrawler 0.1.0 → 0.1.1

data/examples/accessibility_report.rb

@@ -1,3 +1,8 @@
+# == Basic accessibility report - SimpleCrawler example 
+# Author::    Peter Krantz (http://www.peterkrantz.com)
+#
+# This is an example of how SimpleCrawler can be used together with Raakt and Ruport to check basic accessibility of an entire website. For details on the error message id:s generated in the report see http://www.peterkrantz.com/raakt/wiki/error-message-ids
+#
 require '../lib/simplecrawler.rb'
 require 'raakt'
 require 'ruport'
@@ -5,9 +10,9 @@ require 'ruport'
 # Set up a new crawler
 sc = SimpleCrawler::Crawler.new(ARGV[0])
 sc.skip_patterns = ["\\.doc$", "\\.pdf$", "\\.xls$", "\\.pdf$", "\\.zip$"]
-sc.maxcount = 10
+sc.maxcount = 100
 
-report_data = Ruport::Data::Table.new :column_names => ["Url", "Error count"]
+report_data = Ruport::Data::Table.new :column_names => ["Url", "Error"]
 
 sc.crawl { |document|
 
@@ -15,8 +20,11 @@ sc.crawl { |document|
 	raakt = Raakt::Test.new(document.data)
 	result = raakt.all
 	puts "#{result.length}\t#{document.uri}"
+
 	if result.length > 0
-		report_data << [document.uri, result.length]
+		for error in result
+			report_data << [document.uri, error.eid.to_s]
+		end
 	end
 }
 

data/examples/find_pdfs.rb

@@ -0,0 +1,20 @@
+# == Find PDF documents - SimpleCrawler example 
+# Author::    Peter Krantz (http://www.peterkrantz.com)
+#
+# This is an example of how SimpleCrawler can be used to find dcuments of a specific type on a website.
+#
+require '../lib/simplecrawler.rb'
+require 'raakt'
+require 'ruport'
+
+# Set up a new crawler
+sc = SimpleCrawler::Crawler.new(ARGV[0])
+sc.maxcount = 200 #Only crawl 200 pages
+
+sc.crawl { |document|
+
+	if document.headers["content-type"] == "application/pdf"
+		puts document.uri
+	end
+
+}

data/lib/document.rb

@@ -0,0 +1,16 @@
+module SimpleCrawler
+	class Document
+		attr_accessor :uri, :data, :headers, :fetched_at, :http_status
+
+		def to_s
+			puts "Document"
+			puts " .uri:\t\t#{uri}"
+			puts " .fetched_at:\t#{fetched_at}"
+			puts " .headers:"
+			for header in headers
+				puts "   #{header[0]}: #{header[1]}"
+			end
+			puts " .data.length:\t#{(data.length)}"   
+		end
+	end
+end

data/lib/simplecrawler.rb

@@ -20,28 +20,10 @@ module SimpleCrawler
 	require 'rubygems'
 	require 'hpricot'
 	require 'open-uri'
+	require File.dirname(__FILE__) + '/document'
 
 	MARKUP_MIME_TYPES = ["text/html", "text/xml", "application/xml", "application/xhtml+xml"]
-
-	VERSION = "0.1.0"
-
-	class Document
-		attr_accessor :uri, :data, :headers, :fetched_at
-
-		def to_s
-			puts "Document"
-			puts " .uri:\t\t#{uri}"
-			puts " .fetched_at:\t#{fetched_at}"
-			puts " .headers:"
-			for header in headers
-				puts "   #{header[0]}: #{header[1]}"
-			end
-			puts " .data.length:\t#{(data.length)}"   
-		end
-	end
-
-
-
+	VERSION = "0.1.1"
 
 	class Crawler
 
@@ -58,6 +40,11 @@ module SimpleCrawler
 		end
 
 
+		# Override this method for your own logging needs.
+		def log(message)
+			puts message
+		end
+
 		# Check if a path should be ignored because it matches a skip pattern or is already visited.
 		def skip_uri?(uri)
 
@@ -119,6 +106,8 @@ module SimpleCrawler
 				uri.path = uri.path + path if path != "/"
 				doc.uri = uri
 
+				log("Trying #{uri}")
+
 				file = open(uri)
 
 				mime_type = file.meta["content-type"].split(";")[0] if file.meta["content-type"]
@@ -132,8 +121,8 @@ module SimpleCrawler
 				doc.headers = file.meta
 				doc.fetched_at = Time.now
 			rescue Exception
-				puts "Error: #{$!}"
-				return ""
+				log("Error fetching [#{uri}]: #{$!}")
+				return doc
 			end
 			return doc 
 		end

data/tests/simplecrawler_test.rb

@@ -98,4 +98,5 @@ class SimpleCrawlerTest < Test::Unit::TestCase
 	  @simplecrawler.queue_local_links(doc)
 	  assert_equal 1, @simplecrawler.queue.length
   end
+
 end

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.2
 specification_version: 1
 name: simplecrawler
 version: !ruby/object:Gem::Version 
-  version: 0.1.0
-date: 2007-08-27 00:00:00 +02:00
+  version: 0.1.1
+date: 2007-08-30 00:00:00 +02:00
 summary: A generic library for web crawling.
 require_paths: 
 - lib
@@ -30,10 +30,12 @@ authors:
 - Peter Krantz
 files: 
 - README
+- lib/document.rb
 - lib/simplecrawler.rb
 - tests/simplecrawler_test.rb
 - examples/accessibility_report.rb
 - examples/crawl.rb
+- examples/find_pdfs.rb
 - examples/list_site_links.rb
 test_files: 
 - tests/simplecrawler_test.rb