simplecrawler 0.1.6 → 0.1.7

data/examples/riksdagen.txt

@@ -0,0 +1,66 @@
+   Added http://www.riksdagen.se/
+Opening http://www.riksdagen.se/
+Loading data from http://www.riksdagen.se/
+3	http://www.riksdagen.se/
+Queuing links for http://www.riksdagen.se/
+   Added http://www.riksdagen.se/om
+   Added http://www.riksdagen.se/innehall
+   Added http://www.riksdagen.se/fragor
+   Added http://www.riksdagen.se/anvander
+   Added http://www.riksdagen.se/kontakt
+   Added /default____4.aspx
+   Added /templates/R_Page____283.aspx
+   Added /templates/R_OptionPage____285.aspx
+   Added /shopping/R_Default____2508.aspx
+   Added /webbnav/index.aspx?nid=53
+   Added /templates/R_Page____275.aspx
+   Added /templates/R_Page____2161.aspx
+   Added /templates/R_Page____273.aspx
+   Added /templates/R_Page____3618.aspx
+   Added /templates/R_Page____271.aspx
+   Added /templates/R_SubStartPage____5029.aspx
+   Added /templates/R_SubStartPage____6694.aspx
+   Added /templates/R_SubStartPage____272.aspx
+   Added /templates/R_SubStartPage____282.aspx
+   Added /templates/R_SubStartPage____238.aspx
+   Added /templates/R_Page____1068.aspx
+   Added /templates/R_Page____6531.aspx
+   Added /templates/R_CustomCalendar____1765.aspx
+   Added /Webbnav/index.aspx?nid=7800
+   Added /templates/R_SubStartPage____4935.aspx
+   Added /templates/R_Page____4335.aspx
+   Added /templates/R_Page____13951.aspx
+   Added /templates/R_Page____1595.aspx
+   Added /templates/R_Page____16886.aspx
+   Added /templates/R_Page____14748.aspx
+   Added /templates/R_Page____12861.aspx
+   Added #
+   Added /templates/R_ExternalPage____3383.aspx?op=search&search_freetext=&search_parlamentary_session=2008%2F09&search_type=Interpellationsdebatt&search_committee=&search_speaker=&search_party=&search_sdate1=2009%2D01%2D16&search_sdate2=2009%2D01%2D16&rpage=0
+   Added http://www.riksdagen.se/templates/R_HtmlCallPage____17356.aspx
+   Added http://www.riksdagen.se/templates/R_Page____17944.aspx
+   Added http://www.riksdagen.se/templates/R_Page____17958.aspx
+   Added http://www.riksdagen.se/templates/R_HtmlCallPage____17361.aspx
+   Added /templates/R_LopsedelArkiv____3388.aspx
+   Added /templates/R_Page____1977.aspx
+   Added /templates/R_LLSubStartPage____4303.aspx
+   Added /default____56.aspx
+   Added /templates/R_Page____1928.aspx
+   Added /templates/R_Page____6546.aspx
+   Added /templates/R_SubStartPage____257.aspx
+   Added /templates/R_SubStartPage____448.aspx
+   Added /templates/R_SubStartPage____4492.aspx
+   Added /templates/R_Page____498.aspx
+   Added /templates/R_Page____734.aspx
+Opening http://www.riksdagen.se/om
+Loading data from http://www.riksdagen.se/om
+3	http://www.riksdagen.se/om
+Queuing links for http://www.riksdagen.se/om
+   Added /templates/R_Page____735.aspx
+   Added /templates/R_Page____5915.aspx
+   Added /templates/R_Page____1791.aspx
+   Added /templates/R_Page____1788.aspx
+   Added /templates/R_PageFull____6580.aspx
+   Added /templates/R_Page____6558.aspx
+   Added /templates/R_Page____5674.aspx
+Opening http://www.riksdagen.se/innehall
+Loading data from http://www.riksdagen.se/innehall

data/lib/simplecrawler.rb

@@ -23,7 +23,7 @@ module SimpleCrawler
 	require File.dirname(__FILE__) + '/document'
 
 	MARKUP_MIME_TYPES = ["text/html", "text/xml", "application/xml", "application/xhtml+xml"]
-	VERSION = "0.1.6"
+	VERSION = "0.1.7"
 
 	class Crawler
 
@@ -107,6 +107,7 @@ module SimpleCrawler
 				@queue.push uri.path
 				@current_count = @current_count + 1
 				@visited[uri.path] = false
+				log("   Added #{uri}")
 			end
 
 		end
@@ -118,6 +119,7 @@ module SimpleCrawler
 				uri = @site_uri.clone
 				uri.path = path if path != "/"
 				doc.uri = uri
+				doc.fetched_at = Time.now
 
 				log("Opening #{uri}")
 
@@ -135,10 +137,15 @@ module SimpleCrawler
 
 				doc.headers = file.meta
 				doc.http_status = file.status
-				doc.fetched_at = Time.now
-			rescue Exception
-				log("Error fetching [#{uri}]: #{$!}")
-				return doc
+
+			rescue => error
+				log("Error fetching #{uri}: #{error.message}")
+				if error.message[0..2] =~ /\d\d\d/ then
+					doc.http_status = [error.message[0..2], error.message[3..-1]]
+					return doc
+				else
+					raise error
+				end
 			end
 			return doc 
 		end
@@ -147,7 +154,7 @@ module SimpleCrawler
 		def queue_local_links(doc)
 			return if doc.data == nil
 			log("Queuing links for #{doc.uri}")
-			Hpricot.buffer_size = 262144 #Allow for asp.net bastard-sized viewstate attributes...
+			Hpricot.buffer_size = 524288 #Allow for asp.net bastard-sized viewstate attributes...
 			doc = Hpricot(doc.data)
 			links = doc.search("a[@href]")
 			for link in links
@@ -155,7 +162,6 @@ module SimpleCrawler
 					begin
 						uri = URI.parse(link.attributes["href"])
 						add_uri(uri)
-						log("   Added #{uri}")
 					rescue
 						#skip this link
 					end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: simplecrawler
 version: !ruby/object:Gem::Version 
-  version: 0.1.6
+  version: 0.1.7
 platform: ruby
 authors: 
 - Peter Krantz
@@ -9,7 +9,7 @@ autorequire: simplecrawler
 bindir: bin
 cert_chain: []
 
-date: 2008-11-28 00:00:00 +01:00
+date: 2009-05-04 00:00:00 +02:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -37,8 +37,11 @@ files:
 - tests/simplecrawler_test.rb
 - examples/accessibility_report.rb
 - examples/crawl.rb
+- examples/find_broken_links.rb
 - examples/find_pdfs.rb
 - examples/list_site_links.rb
+- examples/result.htm
+- examples/riksdagen.txt
 has_rdoc: true
 homepage: http://www.peterkrantz.com/simplecrawler/wiki/
 post_install_message: