rspider 0.8.4

data/lib/rspider/ThreadPool.rb

@@ -0,0 +1,69 @@
+=begin rdoc
+Author:: aragorn(xurenlu@gmail.com)
+URL::	http://www.162cm.com/
+Version:: 1.0.0
+License:: LGPL
+=end
+
+require 'set'
+require "thread"
+module Rspider
+=begin rdoc
+	This class acts as a manager and control threads to run in order
+and avoid to make CPU load heavy.
+=== Examples
+*	tp=ThreadPool.new(3)
+*
+*	1.upto(30) { |j|
+*		t=tp.dispatch(j) {|i|
+*			puts "thread ##{i} start"
+*			sleep rand
+*			puts "thread ##{i} end"
+*		}
+*	}
+*	tp.shutdown
+=end
+	class ThreadPool
+		#initialize method
+		#Param:	max_size:	Max threads can be active at same time
+		def initialize(max_size)
+			@pool=[]
+			@max_size=max_size
+			@pool_mutex=Mutex.new
+			@pool_cv=ConditionVariable.new
+		end
+		#Add a new thread to the pool
+		#
+		def dispatch(*args)	
+			Thread.new do
+				@pool_mutex.synchronize do
+					while @pool.size >=@max_size
+						print "pool full;waiting run #{args.join(',')}...\n" if $DEBUG
+						@pool_cv.wait(@pool_mutex)
+					end
+				end
+				@pool << Thread.current
+				begin 
+					yield(* args)
+				rescue => e
+					exception(self,e,*args)
+				ensure
+					@pool_mutex.synchronize do
+						@pool.delete(Thread.current)
+						@pool_cv.signal
+					end	
+				end
+			end
+		end	
+		#wait all the threads to exit
+		def shutdown
+			@pool_mutex.synchronize {
+				@pool_cv.wait(@pool_mutex) until @pool.empty?
+			}
+		end
+		#we got an error 
+		def exception (thread,exception,*original_args)
+			puts "Exception in thread #{thread}:#{exception}"
+		end
+	end
+end

data/lib/rspider/UrlDispatcher.rb

@@ -0,0 +1,59 @@
+=begin rdoc
+Author:: aragorn(xurenlu@gmail.com)
+URL::	http://www.162cm.com/
+Version:: 1.0.0
+License:: LGPL
+=end
+
+
+require "socket"
+module Rspider
+=begin rdoc
+	Base class of UrlDispatchers
+=end
+	class UrlDispatcher
+		#@param host:: Host ,like '127.0.0.1'
+		#@param port:: Port ,like 1099
+		def initialize(host,port,source)
+			@host=host
+			@port=port
+			@source=source
+			@visited=[]
+			@urlsToVisit=[]
+		end
+		def AddUrl(url,site)
+		end
+		def FetchUnvisitedUrl(site)
+		end
+		def ReportWhenVisited(url,site)
+		end	
+		def IfUrlVisited(url,site)
+		end
+		def getResponse(msg)
+			socket = TCPSocket.new(@host,@port)
+			socket.puts(msg)
+			line    = socket.gets
+			return	line
+		end
+	end
+
+	class UrlDispatcherClient < UrlDispatcher
+		
+		def <<(url)
+			getResponse("add #{url} #{@source}")	
+		end	
+		def pop()
+			u=getResponse("fetch #{@source}")
+			return u.strip unless u.nil?
+		end		
+		def visited(url)
+			getResponse("mark_visited #{url} #{@source}")
+		end
+		def visited?(url)
+				
+		end	
+	end
+
+	class UrlDispatcherServer < UrlDispatcher
+	end
+end

data/lib/rspider/UrlScorer.rb

@@ -0,0 +1,44 @@
+=begin rdoc
+Author:: aragorn(xurenlu@gmail.com)
+URL::	http://www.162cm.com/
+Version:: 1.0.0
+License:: LGPL
+=end
+module Rspider
+	class UrlScorer
+		def initialize
+		end
+#caculate the score of url
+		def UrlScorer.score(url)
+			UrlScorer.scoreDomain(url) + UrlScorer.scoreExt(url) + UrlScorer.scoreDepth(url)
+		end
+#caculate the score of domain
+		def UrlScorer.scoreDomain(url)
+			u=url.gsub(/http:\/\/([^\/]+)\/(.*)/,'\1')
+			return 5 if u =~ /org$/
+			return 4 if u =~ /org\.cn$/
+			return 2 if u =~ /cn$/
+			return 3
+		end
+#caculate the score of the extension name of file
+		def UrlScorer.scoreExt(url)
+			ext=File.extname(url).split("?").shift.to_s
+			if (ext == ".jsp" or ext == ".asp" or ext == ".php" or ext == ".aspx" or ext == ".cgi" or ext == ".pl")
+				return	2
+			elsif (ext==".html" or ext == ".html" or ext == ".shtml")
+				return 3
+			elsif (ext == ".doc" or ext == ".ppt" or ext == ".zip" )
+				return 4
+			else
+				return 2
+			end
+		end
+#caculate the score of the url's file depth
+		def UrlScorer.scoreDepth(url)
+			u=url.gsub("http://","")
+			depth=u.split("/").length
+			return 1 if depth > 4
+			return 5-depth
+		end
+	end
+end

data/lib/rspider/UrlStorage.rb

@@ -0,0 +1,44 @@
+module Rspider
+	class UrlStorage
+		attr_accessor :urls,:visitedUrls
+		def initialize()
+			@visitedUrls=[]
+			@urls=[]
+		end
+		def <<(u)
+			#puts "add u:#{u}"
+			return nil if @visitedUrls.include?(u)
+			return nil if @urls.include?(u)
+			@urls<< u
+		end
+		def visited?(u)
+			@visitedUrls.include?(u)	
+		end
+		def pop()
+			@urls[rand(@urls.length-1)]
+		end
+		def visited(u)
+			@urls.delete(u) { puts "not found when delete #{u}"}
+			@visitedUrls<<u
+		end	
+		def error(url)
+			@urls.delete(url)
+		end
+	end
+	class UrlStorageInMemcache
+		def initialize()
+			@cache=MemCache.new "localhost:11211",:namespace=>"hel"
+		end
+		def <<(u)
+			@cache.set("#{u}","N",86400,true)
+		end
+		def pop()
+		end
+		def visited(u)
+			@cache.set("#{u}","Y",86400,true)
+		end
+		def visited?(u)
+			@cache.get("#{u}",true) == "Y"	
+		end
+	end
+end

data/lib/rspider/browser.rb

@@ -0,0 +1,127 @@
+#
+require "net/http"
+require "open-uri"
+module Net #:nodoc:
+  class HTTPResponse #:nodoc:
+    def success?; false; end 
+    def redirect?; false; end 
+  end 
+  class HTTPSuccess #:nodoc:
+    def success?; true; end 
+  end 
+  class HTTPRedirection #:nodoc:
+    def redirect?; true; end 
+  end 
+end
+#  httpal/lib/httpal/browser.rb
+#
+#  Created by Bryce Kerley on 2006-12-23.
+#  Copyright (c) 2006-2007 Bryce Kerley. All rights reserved.
+#  This software is licensed under the GNU LGPL 2.1 .
+#  See COPYING in the httpal root for the full license.
+#
+#  Subversion info:
+#  $Id: browser.rb 28 2007-03-23 15:41:07Z bkerley $
+module HTTPal
+	class Browser
+		attr_accessor :read_timeout,:open_timeout
+		def initialize(ua="Mozilla/Firefox 2.0.11",max_len=2048)
+			@cookies = []
+			@referer = nil
+			@userAgent=ua
+			@max_len=max_len-1
+			@open_timeout=5
+			@read_timeout=5
+		end
+	
+		def use(&block)
+			instance_eval(&block)
+		end
+		
+		def get(uri)
+			uri,path = parseuri(uri)
+			req = Net::HTTP::Get.new(path)
+			send_request(req, uri)
+		end
+		
+		def post(uri, fields)
+			uri,path = parseuri(uri)
+			req = Net::HTTP::Post.new(path)
+			req.set_form_data fields
+			send_request(req, uri)
+		end
+		
+		private
+		
+		def parseuri(uri)
+			uri = URI.parse(uri) unless uri.is_a? URI::HTTP
+			path = uri.path.empty? ? '/' : uri.path
+			path << '?' << uri.query if uri.query
+			return [uri, path]
+		end
+		
+		def send_request(req, uri)
+			req['cookie'] = get_cookies_for_uri(uri)
+			req['referer'] =get_referer
+			req["User-Agent"]=@userAgent
+			req["Range"]="bytes=0-#{@max_len}"
+            default_port = uri.scheme == "https" ? "443" : "80"
+            http = Net::HTTP.new(uri.host, uri.port ? uri.port : default_port)
+			http.open_timeout=@open_timeout
+			http.read_timeout=@read_timeout
+            http.use_ssl = true if uri.scheme == "https"
+      
+            res = http.start {
+            	|h| 
+            	h.request(req)
+            }
+			set_cookie_for_uri(uri, res['set-cookie'])
+			set_referer(uri)
+			
+			return res
+		end
+		
+		def get_referer
+			@referer if @referer
+			""
+		end
+	
+		def set_referer(referer)
+			@referer = referer.to_s
+		end
+	
+		def get_cookies_for_uri(uri)
+			hs = uri.host.split('.')
+			hm = (1..hs.length).inject([]) do |c,n|
+				c << hs[(-n)..-1].join('.')
+			end
+			ps = uri.path.split('/')
+			pm = ['/']
+			ps.length.times do |n|
+				pm << ps[0..n].join('/') 
+			end
+			pm.delete ''
+			
+			hostmatch = @cookies.inject([]) do |set, cur|
+				set << cur if hm.include? cur.domain
+				set
+			end
+			
+			# TODO: PATH-MATCH COOKIES
+		end
+	
+		def set_cookie_for_uri(uri, setcookie)
+			return unless setcookie
+			newcookies = CookieMonster.parse_set_cookie(setcookie)
+			@cookies.instance_eval do
+				newcookies.each do |c|
+					c.domain = uri.host unless c.domain
+					self[index(c)] = c if include?(c)
+					self << c unless include?(c)
+				end
+			end
+		
+		end
+	
+	end
+end

data/lib/rspider/cookie.rb

@@ -0,0 +1,113 @@
+#
+#  httpal/lib/httpal/cookie.rb
+#
+#  Created by Bryce Kerley on 2006-12-23.
+#  Copyright (c) 2006-2007 Bryce Kerley. All rights reserved.
+#  This software is licensed under the GNU LGPL 2.1 .
+#  See COPYING in the httpal root for the full license.
+#
+#  Subversion info:
+#  $Id: cookie.rb 20 2007-02-08 02:13:16Z bkerley $
+module HTTPal
+	class CookieMonster
+		def self.parse_set_cookie(sc)
+			r = []
+			# need to figure out how to do the multi-cookie split
+			# since the comma separates both cookies and dates
+		
+			# TODO: IMPLEMENT A LESS BAD SOLUTION
+			sc = sc.gsub(/expires=(\w\w\w)\,/,'expires=\1')
+		
+			sc.split(',').each do |str|
+				str.chomp!
+				r << Cookie.new(str)
+			end
+			return r
+		end
+	end
+        class CookieManager
+		def initialize
+			@cookies=[]
+		end
+		def get_cookies_for_uri(uri)
+			hs = uri.host.split('.')
+			hm = (1..hs.length).inject([]) do |c,n|
+				c << hs[(-n)..-1].join('.')
+			end
+			ps = uri.path.split('/')
+			pm = ['/']
+			ps.length.times do |n|
+				pm << ps[0..n].join('/') 
+			end
+			pm.delete ''
+			
+			hostmatch = @cookies.inject([]) do |set, cur|
+				set << cur if hm.include? cur.domain
+				set
+			end
+			
+			# TODO: PATH-MATCH COOKIES
+		end
+	
+		def set_cookie_for_uri(uri, setcookie)
+			return unless setcookie
+			newcookies = CookieMonster.parse_set_cookie(setcookie)
+			@cookies.instance_eval do
+				newcookies.each do |c|
+					c.domain = uri.host unless c.domain
+					self[index(c)] = c if include?(c)
+					self << c unless include?(c)
+				end
+			end
+		
+		end
+	end
+	class Cookie
+		@@fields = [:path, :domain, :expires]
+		attr_accessor :name, :value
+		attr_accessor *@@fields
+		def initialize(string)
+			components = string.split(';')
+			firstpair = nil
+			components.each do |p|
+				sp = p.split('=',2)
+				k = sp[0]
+				v = sp[1]
+				k.strip!
+				v.strip!
+				if firstpair == nil
+					send(:name=, k)
+					send(:value=, v)
+					firstpair = true
+				else
+					next unless @@fields.include? k.downcase.to_sym
+					v = v.gsub(/^\./,'') if k.downcase.to_sym == :domain
+					send((k+'=').downcase.to_sym, v)
+				end
+			end
+		end
+	
+		def to_s
+			r = "#{name}=#{value}"
+			@@fields.each do |f|
+				r << "; #{f.to_s}=#{send(f)}" if send(f)
+			end
+			return r
+		end
+	
+		def inspect
+			"\#<Cookie #{name}=#{value} from #{domain}#{path} until #{expires}>"
+		end
+	
+		include Comparable
+		def <=> (anOther)
+			#sort by domain, then path, then name
+			d = self.domain.<=>(anOther.domain)
+			p = self.path.<=>(anOther.path)
+			n = self.name.<=>(anOther.name)
+			return d unless d == 0
+			return p unless p == 0
+			return n
+		end
+	end
+end