rcrawl 0.4.7 → 0.5.0

data/Rakefile

@@ -18,7 +18,7 @@ end
 
 spec = Gem::Specification.new do |s|
   s.name         = "rcrawl"
-  s.version      = "0.4.7"
+  s.version      = "0.5.0"
   s.author       = "Digital Duckies"
   s.email        = "rcrawl@digitalduckies.net"
   s.homepage     = "http://digitalduckies.net"

data/TODO

@@ -3,5 +3,3 @@ Add max connections and max connections/second code
 Add referer code
 Add proxy code?  Is this high up on anyone's list, or can it be put off for now?
 Logging code
-Store page headers and page metadata
-

data/lib/rcrawl/crawler.rb

@@ -4,7 +4,7 @@ module Rcrawl
 		
 		attr_accessor :links_to_visit, :site, :user_agent
 		attr_reader :visited_links, :external_links, :raw_html, :rules, :sites,
-						:errors
+						:errors, :meta
 		# Initializes various variables when a new Crawler object is instantiated
 		def initialize(site)
 			puts "Rcrawl Version #{VERSION} initializing..."
@@ -16,6 +16,7 @@ module Rcrawl
 			@user_agent = "Rcrawl/#{VERSION} (http://rubyforge.org/projects/rcrawl/)"
 			@sites = Hash.new
 			@errors = Hash.new
+			@meta = Hash.new
 			@site = URI.parse(site) || raise("You didn't give me a site to crawl")
 			@links_to_visit << site
 			puts "Ready to crawl #{site}"
@@ -78,6 +79,7 @@ module Rcrawl
 				when "text/html"
 					link_extractor(document)
 					process_html(document)
+					page_meta(document)
 				else
 					print "... not HTML, skipping..."
 			end
@@ -133,6 +135,10 @@ module Rcrawl
 				
 		end
 		
+		def page_meta(document)
+			@meta[@url] = document.meta
+		end
+		
 		# robots.txt parsing
 		def robot_safe?(url)
 			uri = URI.parse(url)

data/lib/rcrawl/version.rb

@@ -1,5 +1,5 @@
 module Rcrawl
 	class Crawler
-		VERSION = "0.4.6"
+		VERSION = "0.5.0"
 	end
 end

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.0
 specification_version: 1
 name: rcrawl
 version: !ruby/object:Gem::Version 
-  version: 0.4.7
-date: 2006-09-27 00:00:00 -05:00
+  version: 0.5.0
+date: 2006-10-02 00:00:00 -05:00
 summary: A web crawler written in ruby
 require_paths: 
 - lib
@@ -29,8 +29,8 @@ post_install_message:
 authors: 
 - Digital Duckies
 files: 
-- lib/rcrawl.rb
 - lib/rcrawl
+- lib/rcrawl.rb
 - lib/rcrawl/robot_rules.rb
 - lib/rcrawl/crawler.rb
 - lib/rcrawl/version.rb