simplecrawler 0.1.1 → 0.1.3

data/lib/document.rb

@@ -6,6 +6,7 @@ module SimpleCrawler
 			puts "Document"
 			puts " .uri:\t\t#{uri}"
 			puts " .fetched_at:\t#{fetched_at}"
+			puts " .http_status:\t#{http_status}"
 			puts " .headers:"
 			for header in headers
 				puts "   #{header[0]}: #{header[1]}"

data/lib/simplecrawler.rb

@@ -23,7 +23,7 @@ module SimpleCrawler
 	require File.dirname(__FILE__) + '/document'
 
 	MARKUP_MIME_TYPES = ["text/html", "text/xml", "application/xml", "application/xhtml+xml"]
-	VERSION = "0.1.1"
+	VERSION = "0.1.3"
 
 	class Crawler
 
@@ -119,6 +119,7 @@ module SimpleCrawler
 				end
 
 				doc.headers = file.meta
+				doc.http_status = file.status
 				doc.fetched_at = Time.now
 			rescue Exception
 				log("Error fetching [#{uri}]: #{$!}")
@@ -130,6 +131,7 @@ module SimpleCrawler
 
 		def queue_local_links(doc)
 			return if doc.data == nil
+			log("Queuing links for #{doc.uri}")
 			Hpricot.buffer_size = 262144 #Allow for asp.net bastard-sized viewstate attributes...
 			doc = Hpricot(doc.data)
 			links = doc.search("a[@href]")

metadata

@@ -1,33 +1,34 @@
 --- !ruby/object:Gem::Specification 
-rubygems_version: 0.9.2
-specification_version: 1
 name: simplecrawler
 version: !ruby/object:Gem::Version 
-  version: 0.1.1
-date: 2007-08-30 00:00:00 +02:00
-summary: A generic library for web crawling.
-require_paths: 
-- lib
-email: peter.krantzNODAMNSPAM@gmail.com
-homepage: http://www.peterkrantz.com/simplecrawler/wiki/
-rubyforge_project: simplecrawler
-description: 
-autorequire: simplecrawler
-default_executable: 
-bindir: bin
-has_rdoc: true
-required_ruby_version: !ruby/object:Gem::Version::Requirement 
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      version: 1.8.2
-  version: 
+  version: 0.1.3
 platform: ruby
-signing_key: 
-cert_chain: 
-post_install_message: 
 authors: 
 - Peter Krantz
+autorequire: simplecrawler
+bindir: bin
+cert_chain: []
+
+date: 2008-01-26 00:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: hpricot
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0.5"
+    version: 
+description: 
+email: peter.krantzNODAMNSPAM@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
 files: 
 - README
 - lib/document.rb
@@ -37,25 +38,31 @@ files:
 - examples/crawl.rb
 - examples/find_pdfs.rb
 - examples/list_site_links.rb
-test_files: 
-- tests/simplecrawler_test.rb
+has_rdoc: true
+homepage: http://www.peterkrantz.com/simplecrawler/wiki/
+post_install_message: 
 rdoc_options: []
 
-extra_rdoc_files: []
-
-executables: []
-
-extensions: []
-
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: 1.8.2
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
 requirements: []
 
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: hpricot
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Version::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: "0.5"
-    version: 
+rubyforge_project: simplecrawler
+rubygems_version: 1.0.0
+signing_key: 
+specification_version: 2
+summary: A generic library for web crawling.
+test_files: 
+- tests/simplecrawler_test.rb