rcrawl 0.2.6 → 0.3.0

data/README

@@ -9,21 +9,23 @@ The structure of the crawling process was inspired by the specs of the Mercator
 == The Rcrawl process
 1. Remove an absolute URL from the URL Server.
 2. Download corresponding document from the internet, grabbing and processing robots.txt first, if available.
-3. Feed the document into a rewing input stream(ris) to be read/re-read as needed.  Based on MIME type, invoke the process method of
-4. Processing module associated with that MIME type.  For example, a link extractor or tag counter module for text/html MIME types, of a gif stats module for image/gif.  By default, all text/html MIME types will pass through the link extractor.  Each link will be converted to an absolute URL and tested against a (ideally user-supplied) URL filter to determine if it should be downloaded.
+3. Feed the document into a rewind input stream(ris) to be read/re-read as needed.  Based on MIME type, invoke the process method of the
+4. processing module associated with that MIME type.  For example, a link extractor or tag counter module for text/html MIME types, or a gif stats module for image/gif.  By default, all text/html MIME types will pass through the link extractor.  Each link will be converted to an absolute URL and tested against a (ideally user-supplied) URL filter to determine if it should be downloaded.
 5. If the URL passes the filter (currently hard coded as Same Domain?), then call the URL-seen? test.
-6. Has the URL been seen before?  Namely, is it in the URL Server of has it been downloaded already?  If the URL is new, it is added to the URL Server.
+6. Has the URL been seen before?  Namely, is it in the URL Server or has it been downloaded already?  If the URL is new, it is added to the URL Server.
 7. Back to step 1, repeat until the URL Server is empty.
 
 == Examples
+
 # Instantiate a new Rcrawl object
-	crawler = Rcrawl.new(url)
+	crawler = Rcrawl::Crawler.new(url)
 
 
 # Begin the crawl process
 	crawler.crawl
 
 == After the crawler is done crawling
+
 # Returns an array of visited links
 	crawler.visited_links
 
@@ -42,8 +44,8 @@ The structure of the crawling process was inspired by the specs of the Mercator
 	crawler.external_links
 
 == License
-Copyright © 2006 Shawn Hansen, under MIT License
+Copyright © 2006 Digital Duckies, LLC, under MIT License
 
-Developed for http://denomi.net
+Developed for http://digitalduckies.net
 
-News, code, and documentation at http://blog.denomi.net
+News, code, and documentation at http://digitalduckies.net

data/Rakefile

@@ -9,7 +9,7 @@ require 'rake/gempackagetask'
 desc "Generate documentation"
 Rake::RDocTask.new(:rdoc) do |rdoc|
   rdoc.rdoc_dir = "rdoc"
-  rdoc.title    = "Crawler"
+  rdoc.title    = "Rcrawl"
   rdoc.options << "--line-numbers"
   rdoc.options << "--inline-source"
   rdoc.rdoc_files.include("README")
@@ -18,10 +18,10 @@ end
 
 spec = Gem::Specification.new do |s|
   s.name         = "rcrawl"
-  s.version      = "0.2.6"
-  s.author       = "Shawn Hansen"
-  s.email        = "shawn.hansen@gmail.com"
-  s.homepage     = "http://blog.denomi.net"
+  s.version      = "0.3.0"
+  s.author       = "Digital Duckies"
+  s.email        = "rcrawl@digitalduckies.net"
+  s.homepage     = "http://digitalduckies.net"
   s.platform     = Gem::Platform::RUBY
   s.summary      = "A web crawler written in ruby"
   s.files        = FileList["{test,lib}/**/*", "README", "MIT-LICENSE", "Rakefile", "TODO"].to_a

data/lib/rcrawl/crawler.rb

@@ -0,0 +1,4 @@
+module Rcrawl
+	class Crawler
+	end
+end

data/lib/rcrawl/process/html.rb

@@ -0,0 +1,8 @@
+module Rcrawl
+	module Process
+		module HTML
+			def link_extractor
+			end
+		end
+	end
+end

data/lib/rcrawl/robot_rules.rb

@@ -0,0 +1,82 @@
+#!/usr/bin/env ruby
+
+# robot_rules.rb
+#
+#  Created by James Edward Gray II on 2006-01-31.
+#  Copyright 2006 Gray Productions. All rights reserved.
+#  Included with rcrawl by permission from James Edward Gray II
+
+require "uri"
+
+module Rcrawl
+
+	# Based on Perl's WWW::RobotRules module, by Gisle Aas.
+	class RobotRules
+		def initialize( user_agent )
+		  @user_agent = user_agent.scan(/\S+/).first.sub(%r{/.*},
+																	"").downcase
+		  @rules      = Hash.new { |rules, rule| rules[rule] = Array.new }
+		end
+
+		def parse( text_uri, robots_data )
+		  uri      = URI.parse(text_uri)
+		  location = "#{uri.host}:#{uri.port}"
+		  @rules.delete(location)
+
+		  rules      = robots_data.split(/[\015\012]+/).
+											map { |rule| rule.sub(/\s*#.*$/, "") }
+		  anon_rules = Array.new
+		  my_rules   = Array.new
+		  current    = anon_rules
+		  rules.each do |rule|
+			 case rule
+			 when /^\s*User-Agent\s*:\s*(.+?)\s*$/i
+				break unless my_rules.empty?
+
+				current = if $1 == "*"
+				  anon_rules
+				elsif $1.downcase.index(@user_agent)
+				  my_rules
+				else
+				  nil
+				end
+			 when /^\s*Disallow\s*:\s*(.*?)\s*$/i
+				next if current.nil?
+
+				if $1.empty?
+				  current << nil
+				else
+				  disallow = URI.parse($1)
+
+				  next unless disallow.scheme.nil? or disallow.scheme == uri.scheme
+				  next unless disallow.port.nil?   or disallow.port == uri.port
+				  next unless disallow.host.nil?   or
+								  disallow.host.downcase == uri.host.downcase
+
+				  disallow = disallow.path
+				  disallow = "/"            if disallow.empty?
+				  disallow = "/#{disallow}" unless disallow[0] == ?/
+
+				  current << disallow
+				end
+			 end
+		  end
+
+		  @rules[location] = if my_rules.empty?
+			 anon_rules.compact
+		  else
+			 my_rules.compact
+		  end
+		end
+
+		def allowed?( text_uri )
+		  uri      = URI.parse(text_uri)
+		  location = "#{uri.host}:#{uri.port}"
+		  path     = uri.path
+
+		  return true unless %w{http https}.include?(uri.scheme)
+
+		  not @rules[location].any? { |rule| path.index(rule) == 0 }
+		end
+	end
+end

data/lib/rcrawl.rb

@@ -1,178 +1,181 @@
 #!/usr/bin/env ruby
-# rcrawl/0.2.0
 
 require 'rubygems'
 require 'open-uri'
 require 'scrapi'
-require 'robot_rules'
-
-# Rcrawl will retrieve an entire website, one page at a time,
-# parsing the page using whatever modules you pass it to.
-class Rcrawl
-
-	# Initializes various variables when a new Rcrawl object is instantiated
-	def initialize(site)
-		@links_to_visit = Array.new
-		@visited_links = Array.new
-		@external_links = Array.new
-		@raw_html = Hash.new
-		@rules = RobotRules.new("Rcrawl")
-		@sites = Hash.new
-		@site = URI.parse(site)
-		@links_to_visit << site
-		@errors = Hash.new
-		puts "Site is #{site}"
-	end
-
-	# Coordinates the whole crawling process
-	def crawl
-		until @links_to_visit.empty? do
-			begin
-				# Get link
-				url_server
-				next unless robot_safe? @url
-				# Parse robots.txt, then download document if robot_safe
-				fetch_http(@url)
-				# Store raw HTML in variable to read/reread as needed
-				# Then call any processing modules you need for the current document
-				ris(@document)
-			rescue
-				puts ""
-				puts "I died on #{@url}"
-				$stderr.puts $!
-				@errors[@url] = $!
-				next
-			ensure
-				# Stuff you want to make sure gets printed out
-				puts " done!"
-			end
+require 'rcrawl/robot_rules'
+require 'rcrawl/process/html'
+
+module Rcrawl
+
+	# Crawler will retrieve an entire website, one page at a time,
+	# parsing the page using whatever modules you pass it to.
+	class Crawler
+
+		# Initializes various variables when a new Rcrawl object is instantiated
+		def initialize(site)
+			@links_to_visit = Array.new
+			@visited_links = Array.new
+			@external_links = Array.new
+			@raw_html = Hash.new
+			@rules = RobotRules.new("Rcrawl")
+			@sites = Hash.new
+			@site = URI.parse(site)
+			@links_to_visit << site
+			@errors = Hash.new
+			puts "Site is #{site}"
 		end
 
-		puts "Visited #{@visited_links.size} links."
-	end
+		# Coordinates the whole crawling process
+		def crawl
+			until @links_to_visit.empty? do
+				begin
+					# Get link
+					url_server
+					next unless robot_safe? @url
+					# Parse robots.txt, then download document if robot_safe
+					fetch_http(@url)
+					# Store raw HTML in variable to read/reread as needed
+					# Then call any processing modules you need for the current document
+					ris(@document)
+				rescue
+					puts ""
+					puts "I died on #{@url}"
+					$stderr.puts $!
+					@errors[@url] = $!
+					next
+				ensure
+					# Stuff you want to make sure gets printed out
+					puts " done!"
+				end
+			end
 
-	# Authoritative list of URLs to be processed by Rcrawl
-	def url_server
-		unless @links_to_visit.empty?
-			@url = @links_to_visit.pop
+			puts "Visited #{@visited_links.size} links."
 		end
-	end
 
-	# Download the document
-   def fetch_http(url)
-      # Make sure robots.txt has been parsed for this site first,
-		# if not, parse robots.txt then grab document.
-		uri = URI.parse(url)
-		print "Visiting: #{url}"
-		@document = uri.read
-		@visited_links << url
-   end
-
-   # Rewind Input Stream, for storing and reading of raw HTML
-   def ris(document)
-		print "."
-      # Store raw HTML into local variable
-      # Based on MIME type, invoke the proper processing modules
-		case document.content_type
-			when "text/html"
-				link_extractor(document)
-				process_html(document)
-			else
-				print "... not HTML, skipping..."
-		end
-   end
-
-	# HTML processing module for extracting links
-	def link_extractor(document)
-		print "."
-
-		# Parse all links from HTML into an array
-		# Set up the scrAPI (http://labnotes.org)
-		links = Scraper.define do
-			array :urls
-			process "a[href]", :urls => "@href"
-			result :urls
+		# Authoritative list of URLs to be processed by Rcrawl
+		def url_server
+			unless @links_to_visit.empty?
+				@url = @links_to_visit.pop
+			end
 		end
-		
-		urls = links.scrape(document)
 
-		urls.each { |url|
+		# Download the document
+		def fetch_http(url)
+			# Make sure robots.txt has been parsed for this site first,
+			# if not, parse robots.txt then grab document.
 			uri = URI.parse(url)
+			print "Visiting: #{url}"
+			@document = uri.read
+			@visited_links << url
+		end
 
-			# Derelativeize links if necessary
-			if uri.relative?
-				url = @site.merge(url).to_s
-				uri = URI.parse(url)
+		# Rewind Input Stream, for storing and reading of raw HTML
+		def ris(document)
+			print "."
+			# Store raw HTML into local variable
+			# Based on MIME type, invoke the proper processing modules
+			case document.content_type
+				when "text/html"
+					link_extractor(document)
+					process_html(document)
+				else
+					print "... not HTML, skipping..."
 			end
+		end
 
-			# Check domain, if in same domain, keep link, else trash it
-			if uri.host != @site.host
-				@external_links << url
-				@external_links.uniq!
-				next
-			end
+		# HTML processing module for extracting links
+		def link_extractor(document)
+			print "."
 
-			# Find out if we've seen this link already
-			if (@visited_links.include? url) || (@links_to_visit.include? url)
-				next
+			# Parse all links from HTML into an array
+			# Set up the scrAPI (http://labnotes.org)
+			links = Scraper.define do
+				array :urls
+				process "a[href]", :urls => "@href"
+				result :urls
 			end
+			
+			urls = links.scrape(document)
 
-			@links_to_visit << url
-		}
+			urls.each { |url|
+				uri = URI.parse(url)
 
-	end
+				# Derelativeize links if necessary
+				if uri.relative?
+					url = @site.merge(url).to_s
+					uri = URI.parse(url)
+				end
+
+				# Check domain, if in same domain, keep link, else trash it
+				if uri.host != @site.host
+					@external_links << url
+					@external_links.uniq!
+					next
+				end
+
+				# Find out if we've seen this link already
+				if (@visited_links.include? url) || (@links_to_visit.include? url)
+					next
+				end
+
+				@links_to_visit << url
+			}
 
-	# HTML processing module for raw HTML storage
-	def process_html(document)
-		# Add link and raw HTML to a hash as key/value
-		# for later storage in database
-		unless @raw_html.has_value?(document)
-		  print "."
-		  @raw_html[document.base_uri] = document
 		end
-	end
 
-	# robots.txt parsing
-	def robot_safe?(url)
-		uri = URI.parse(url)
-		location = "#{uri.host}:#{uri.port}"
+		# HTML processing module for raw HTML storage
+		def process_html(document)
+			# Add link and raw HTML to a hash as key/value
+			# for later storage in database
+			unless @raw_html.has_value?(document)
+			  print "."
+			  @raw_html[document.base_uri] = document
+			end
+		end
 
-		return true unless %w{http https}.include?(uri.scheme)
+		# robots.txt parsing
+		def robot_safe?(url)
+			uri = URI.parse(url)
+			location = "#{uri.host}:#{uri.port}"
 
-		unless @sites.include? location
-			@sites[location] = true
+			return true unless %w{http https}.include?(uri.scheme)
 
-			robot_url = "http://#{location}/robots.txt"
-			begin
-				robot_file = open(robot_url) { |page| page.read }
-			rescue
-				return true
+			unless @sites.include? location
+				@sites[location] = true
+
+				robot_url = "http://#{location}/robots.txt"
+				begin
+					robot_file = open(robot_url) { |page| page.read }
+				rescue
+					return true
+				end
+				@rules.parse(robot_url, robot_file)
 			end
-			@rules.parse(robot_url, robot_file)
+
+			@rules.allowed? url
 		end
 
-		@rules.allowed? url
-	end
+		# Returns array of links visited during crawl 
+		def visited_links
+			return @visited_links
+		end
 
-   # Returns array of links visited during crawl 
-	def visited_links
-		return @visited_links
-	end
+		# Returns array of external links
+		def external_links
+			return @external_links
+		end
+		
+		# Returns a hash where {key => URL, value => HTML} from all pages crawled 
+		def dump
+			return @raw_html
+		end
 
-	# Returns array of external links
-	def external_links
-		return @external_links
-	end
-	
-	# Returns a hash where {key => URL, value => HTML} from all pages crawled 
-	def dump
-		return @raw_html
-	end
+		# Returns a hash where {key => URL, value => "Error message"} from any
+		# errors encountered during the crawl
+		def errors
+			return @errors
+		end
 
-   # Returns a hash where {key => URL, value => "Error message"} from any
-   # errors encountered during the crawl
-	def errors
-		return @errors
 	end
-
 end

metadata

@@ -3,13 +3,13 @@ rubygems_version: 0.9.0
 specification_version: 1
 name: rcrawl
 version: !ruby/object:Gem::Version 
-  version: 0.2.6
-date: 2006-09-21 00:00:00 -05:00
+  version: 0.3.0
+date: 2006-09-22 00:00:00 -05:00
 summary: A web crawler written in ruby
 require_paths: 
 - lib
-email: shawn.hansen@gmail.com
-homepage: http://blog.denomi.net
+email: rcrawl@digitalduckies.net
+homepage: http://digitalduckies.net
 rubyforge_project: rcrawl
 description: 
 autorequire: rcrawl.rb
@@ -27,10 +27,14 @@ signing_key:
 cert_chain: 
 post_install_message: 
 authors: 
-- Shawn Hansen
+- Digital Duckies
 files: 
-- lib/robot_rules.rb
 - lib/rcrawl.rb
+- lib/rcrawl
+- lib/rcrawl/crawler.rb
+- lib/rcrawl/robot_rules.rb
+- lib/rcrawl/process
+- lib/rcrawl/process/html.rb
 - README
 - MIT-LICENSE
 - Rakefile

data/lib/robot_rules.rb

@@ -1,79 +0,0 @@
-#!/usr/bin/env ruby
-
-# robot_rules.rb
-#
-#  Created by James Edward Gray II on 2006-01-31.
-#  Copyright 2006 Gray Productions. All rights reserved.
-#  Included with rcrawl by permission from James Edward Gray II
-
-require "uri"
-
-# Based on Perl's WWW::RobotRules module, by Gisle Aas.
-class RobotRules
-   def initialize( user_agent )
-     @user_agent = user_agent.scan(/\S+/).first.sub(%r{/.*},
-																"").downcase
-     @rules      = Hash.new { |rules, rule| rules[rule] = Array.new }
-   end
-
-   def parse( text_uri, robots_data )
-     uri      = URI.parse(text_uri)
-     location = "#{uri.host}:#{uri.port}"
-     @rules.delete(location)
-
-     rules      = robots_data.split(/[\015\012]+/).
-                              map { |rule| rule.sub(/\s*#.*$/, "") }
-     anon_rules = Array.new
-     my_rules   = Array.new
-     current    = anon_rules
-     rules.each do |rule|
-       case rule
-       when /^\s*User-Agent\s*:\s*(.+?)\s*$/i
-         break unless my_rules.empty?
-
-         current = if $1 == "*"
-           anon_rules
-         elsif $1.downcase.index(@user_agent)
-           my_rules
-         else
-           nil
-         end
-       when /^\s*Disallow\s*:\s*(.*?)\s*$/i
-         next if current.nil?
-
-         if $1.empty?
-           current << nil
-         else
-           disallow = URI.parse($1)
-
-           next unless disallow.scheme.nil? or disallow.scheme == uri.scheme
-           next unless disallow.port.nil?   or disallow.port == uri.port
-           next unless disallow.host.nil?   or
-                       disallow.host.downcase == uri.host.downcase
-
-           disallow = disallow.path
-           disallow = "/"            if disallow.empty?
-           disallow = "/#{disallow}" unless disallow[0] == ?/
-
-           current << disallow
-         end
-       end
-     end
-
-     @rules[location] = if my_rules.empty?
-       anon_rules.compact
-     else
-       my_rules.compact
-     end
-   end
-
-   def allowed?( text_uri )
-     uri      = URI.parse(text_uri)
-     location = "#{uri.host}:#{uri.port}"
-     path     = uri.path
-
-     return true unless %w{http https}.include?(uri.scheme)
-
-     not @rules[location].any? { |rule| path.index(rule) == 0 }
-   end
-end