rspider 0.8.4

data/lib/rspider/DataWasher.rb

@@ -0,0 +1,129 @@
+module Rspider
+	class DataWasher
+		#根据文章diff的结果取回标题和内容.
+		def getDiffRows(exampleFile,dataFile)
+			diff_res=%x{diff #{exampleFile} #{dataFile}}
+			lines=diff_res.split("\n")
+			row_id=1
+			rows=[]
+			cache=""
+			lines.each{|l|
+				if	(l[0,2] == "--")
+				elsif(l[0,1] == "<")
+				elsif(l[0,1] == ">")
+					cache= cache + l[1,l.length]+"\n"
+				else
+					rows.push cache
+					cache=""
+				end
+			}
+			rows.push cache
+			rows	
+		end
+		
+		def parseDir(srcDir,destDir)
+			puts "now:parse Directory:#{srcDir}"
+			files=[]
+			Dir.foreach(srcDir){|f|
+				files <<  f unless f == "." or f == ".."	
+			}	
+			l=files.length
+			first=files[0]
+			last=files[l-1]
+			k=0
+			if $ENV == "PRO" 
+				files.each{|f|
+					if (k==0)
+						w=parseText(srcDir+last,srcDir+f) unless File.file?(destDir+f)
+						open(destDir+f,"w+").puts  w unless w.nil?
+					else
+						w=parseText(srcDir+first,srcDir+f) unless File.file?(destDir+f)
+						open(destDir+f,"w+").puts  w unless w.nil?
+					end
+					k=k+1
+				}	
+			else
+				i=0
+				files.each{|f|
+					if (i>3)
+						break
+					end
+					diffRows=[]
+					if (k==0)
+						diffRows= getDiffRows(srcDir+last,srcDir+f)
+					else
+						diffRows= getDiffRows(srcDir+first,srcDir+f)
+					end
+					puts "\n\n\n ==============Diff Rows[#{i}]================\n"
+					x=0
+					diffRows.each{ |l|
+						puts "\n+ rows[#{x}]:\n"
+						puts l
+						x=x+1
+					}
+					k=k+1
+					i=i+1
+				}	
+			end	
+		end
+
+		#根据文章diff的结果取回标题和内容(针对和讯理财)
+		def parseTextHexun(exampleFile,dataFile)
+			rows=getDiffRows(exampleFile,dataFile)
+			i=0
+			cur=0
+			contents=[]
+			rows.each{ |l|
+				if l =~ %r{\s*进入.*吧} 
+					puts "got the end of content;#{l}"
+					break
+				end
+				if l =~ %r{^\s*[\d]{1}\*}
+					next
+				end
+				if l =~ %r{^\s*上一页\s*}
+					next
+				end
+				if l =~ %r{^\s*下一页\s*}
+					next
+				end
+				#if l =~ %r{^\s*第[\d]页} and l.length()<25 
+				#	next
+				#end
+				contents.push l if i>3 
+				i = i+1
+			}
+			returns=""
+			returns << rows[1].sub("-理财频道-和讯网","")
+			returns << "::==++\n"
+			returns << contents.join("\n") 
+			returns 
+		end
+		#根据文章diff的结果取回标题和内容.
+		def parseText(exampleFile,dataFile)
+			if $_SOURCE == "hexun"
+				return parseTextHexun(exampleFile,dataFile)
+			else
+				puts "not hexun"
+			end
+			rows=getDiffRows(exampleFile,dataFile)
+			i=0
+			cur=0
+			rows.each{ |l|
+				if(l.length>150 )
+					cur=i
+					break
+				end
+				i=i+1
+			}
+			if cur==0 
+				return nil
+			end
+			returns=""
+			returns << rows[1]
+			returns << "::==++\n"
+			returns << rows[cur]
+			returns 
+		end
+	end
+end

data/lib/rspider/Document.rb

@@ -0,0 +1,100 @@
+module Rspider
+	class Document < Hash
+	end
+	class HTMLDocument < Hash
+		#得到头条之类的大纲
+		def get_headlines(html)
+			r=Regexp.compile('<h[0-9][^>]*>(.*?)<\/h[0-9]>',Regexp::IGNORECASE|Regexp::MULTILINE)
+			lines=[]
+			ms=html.scan(r)
+			return nil if ms.nil?
+			ms.each{|m|
+				lines.push m[0]	
+			}
+			lines.join("\n")
+		end
+		#得到head区的内容
+		def get_head_data(html)
+			r=Regexp.compile('<head[^>]*>(.*?)<\/head>',Regexp::IGNORECASE|Regexp::MULTILINE)
+			m=html.scan(r)
+			return nil if m.nil?
+			return nil if m[0].nil?
+			head={}
+			head[:title]=""
+			head[:keywords]=""
+			head[:robots]=""
+			head[:description]=""
+			head[:nofollow]=false
+			head[:noindex]=false
+			head[:base]=""
+
+			
+			h=m[0][0]
+			begin
+				r_robots=/<meta +name *=[\"']?robots[\"']? *content=[\"']?([^<>'\"]+)[\"']?/im
+				robots=h.scan(r_robots)[0][0]
+				head[:robots]=robots
+			rescue
+			end	
+			begin
+				r_desc=/<meta +name *=[\"']?description[\"']? *content=[\"']?([^<>'\"]+)[\"']?/im
+				head[:description]=h.scan(r_desc)[0][0]
+			rescue
+			end	
+			begin
+				r_keys=/<meta +name *=[\"']?keywords[\"']? *content=[\"']?([^<>'\"]+)[\"']?/im
+				head[:keywords]=h.scan(r_keys)[0][0]
+			rescue
+			end
+
+			begin
+				r_charset=/<meta +http\-equiv*=[\"']?Content-Type[\"']? *content=[\"']?([^<>'\"]+)[\"']?/im
+				head[:charset]=h.scan(r_charset)[0][0].split("=").pop
+			rescue
+			end
+
+			begin
+				r_base=/<base +href *= *[\"']?([^<>'\"]+)[\"']?/im
+				head[:base]=h.scan(r_base)[0][0]
+			rescue
+			end
+			begin
+				r_title=/<title *>(.*?)<\/title*>/im
+				head[:title]=h.scan(r_title)[0][0].gsub("\n","")
+			rescue
+			end
+
+			begin
+				archives=[]
+				r_archives=/<link +rel*=[\"']?archives[\"']? *[^\>]+href=[\"']?([^<>'\"]+)[\"']?/im
+				h.scan(r_archives).each{ |l|
+					archives.push l[0]
+				}
+				head[:archives]=archives
+			rescue
+			end
+			begin
+				links=[]
+				r_alternates=/<link +rel*=[\"']?alternate[\"']? *[^\>]+href=[\"']?([^<>'\"]+)[\"']?/im
+				h.scan(r_alternates).each{ |l|
+					links.push l[0]
+				}
+				head[:rss_links]=links
+			rescue
+			end
+			robots.downcase.split(",").each{ |j|
+				head[:noindex]=true if j=="noindex"
+				head[:nofollow]=true if j=="nofollow"
+			}	unless robots.nil?
+			head
+		end
+	end
+	class RSSItemDocument < Hash
+	end
+	class RSSParser 
+		def initialize(rss)
+		end
+	end
+	class PDFDocument < Hash
+	end
+end

data/lib/rspider/DocumentExtractor.rb

@@ -0,0 +1,21 @@
+=begin rdoc
+Author:: aragorn(xurenlu@gmail.com)
+URL::	http://www.162cm.com/
+Version:: 1.0.0
+License:: LGPL
+=end
+module Rspider
+#These classes Extract a document from PDF,HTML,Mail,Doc,XML files
+	class DocumentExtractor
+		def initialize
+		end
+	end
+	class PDFDocumentExtractor < DocumentExtractor
+	end
+	class DocDocumentExtractor < DocumentExtractor
+	end
+	class TEXTDocumentExtractor < DocumentExtractor
+	end
+	class XMLDocumentExtractor < DocumentExtractor
+	end
+end

data/lib/rspider/HtmlTidy.rb

@@ -0,0 +1,34 @@
+=begin rdoc
+Author:: aragorn(xurenlu@gmail.com)
+URL::	http://www.162cm.com/
+Version:: 1.0.0
+License:: LGPL
+=end
+require "iconv"
+module Rspider
+#this class guesses the encoding of a html document,convert it from various encoding to UTF-8
+	class HtmlTidy
+
+		def initialize()
+			#@defaultCharset="UTF-8"
+		end
+		# guess the encoding of html document
+		def guess_encoding(resp)
+			resp[0..400].scan(/content="(.*)"/i).flatten.join("\t").scan(/charset=([a-z0-9\-]+)/i).flatten.join("\t")
+		end
+		#convert document from one encoding to another
+		def iconv(from,to,text)
+			Iconv.new(to.upcase+"//IGNORE",from.upcase+"//IGNORE").iconv(text)
+		end
+		#guess the encoding of the document and convert it to UTF-8
+		def tidy(html)
+			encoding = guess_encoding(html).upcase
+			return html if encoding == "UTF-8"
+			encoding="GBK" if encoding =~ /gb2312/
+			encoding="GBK" if encoding =~ /gbk/
+			iconv(encoding,"UTF-8",html)
+		end
+		def strip_tags()
+		end	
+	end
+end

data/lib/rspider/Logger.rb

@@ -0,0 +1,49 @@
+#! /usr/bin/ruby
+=begin rdoc
+	Simple Logger
+	Written by aragorn(xurenlu@gmail.com)
+Author:: aragorn(xurenlu@gmail.com)
+Homepage:: http://www.162cm.com/
+MSN:: helloasp@hotmail.com		
+=end
+
+module Rspider
+=begin rdoc
+This class is written for Logging messages when program running.
+
+Examples:
+
+* $LOGGER=Logger.new("/var/run/log/cns.log")
+* $LOGGER.log_msg("Program inited!")
+* ....
+* $LOGGER.log_msg("Sockets inited")
+* $LOGGER.flush_msg();
+=end
+	class Logger
+		attr_accessor :max,:msgs
+
+		#path: the file path to storage the logs	
+		def initialize(path)
+			@max=10
+			@log_path=path
+			@msgs=[]
+			puts "logs created :#{path}" if $DEBUG
+		end
+		#add a new log 
+		#the log may not be sync to disk
+		#The Logger syncs every 100 messages automaticly
+		def log_msg(log,level="NOTE")
+			@msgs.push(level+":"+Time.now.to_i.to_s+":"+log)
+			flush_msg if (@msgs.length>@max) or level=="ERROR"
+		end
+		#sync the logs to hard disk
+		def flush_msg()
+			open(@log_path,"a+") do |f|
+#				f.puts "Generated at :"+Time.now.to_s+"\n"
+				f.puts @msgs.join("\n")
+			end
+			@msgs=[]
+			puts "logs flushed ." if $DEBUG
+		end
+	end
+end

data/lib/rspider/MysqlUrlRelationStorage.rb

@@ -0,0 +1,31 @@
+#!/usr/bin/ruby -w
+require "digest/md5"
+require "zlib"
+module Rspider
+	#The class MysqlUrlRelationStorage store url relations in Mysql database
+	#For better performance, we create an UrlStorage object to cache urls in memory
+	class MysqlUrlRelationStorage
+		def initialize(hash,source="default")
+			@seed=1024
+			@source=source
+			@my=Mysql::new(hash["host"],hash["user"],hash["pass"],hash["db"])
+			raise MysqlException if @my.nil?
+		end
+		#get the MD5 hash of string param "string"
+		def md5(string)
+			t=Digest::MD5.new 
+			t << string
+			t.to_s
+		end
+		def save(referer,url)
+			url_crc=Zlib::crc32(url,@seed)
+			referer_crc=Zlib::crc32(referer,@seed)
+			sql="INSERT INTO url_relations (url,referer,url_crc32,referer_crc32) values('#{@my.quote(url)}','#{@my.quote(referer)}','#{url_crc}','#{referer_crc}')"
+			begin
+			@my.query(sql)
+			rescue Mysql::Error
+			rescue
+			end
+		end
+	end
+end

data/lib/rspider/MysqlUrlStorage.rb

@@ -0,0 +1,107 @@
+#!/usr/bin/ruby -w
+require "digest/md5"
+require "zlib"
+module Rspider
+	#we re-defined the UrlStorage Class and add the urlStored? method
+	class UrlStorageCache < UrlStorage
+		MAX=512
+		STEP=128
+		@seed=1024
+		def urlStored?(url)
+			STEP.times do 
+				@urls.shift
+			end if @urls.length>MAX
+			STEP.times do 
+				@visitedUrls.shift
+			end if @visitedUrls.length>MAX
+			@visitedUrls.include?(url) or @urls.include?(url)
+		end
+	end
+	#The class MysqlUrlStorage store urls in Mysql database
+	#For better performance, we create an UrlStorage object to cache urls in memory
+	class MysqlUrlStorage
+		attr_accessor :cache
+		#Param hash is a hash includes mysql-host,mysql-databasename,mysql-user,mysql-pass
+		#Param source is the name of cralwering task
+		def initialize(hash,source="default")
+			@source=source
+			@my=Mysql::new(hash["host"],hash["user"],hash["pass"],hash["db"])
+			raise MysqlException if @my.nil?
+			@cache=UrlStorageCache.new
+		end
+		#get the MD5 hash of string param "string"
+		def md5(string)
+			t=Digest::MD5.new 
+			t << string
+			t.to_s
+		end
+		#asking if the url has been visited?
+		def visited?(url)
+			return true if @cache.visited?(url)
+			ukey=md5(url)+@source
+			crc=Zlib::crc32(url)
+			sql="SELECT visited FROM `urls` WHERE  AND url_crc32=#{crc} AND `ukey`='"+ukey+"'  LIMIT 1"
+			begin 
+				rs=@my.query(sql)
+				rs.each do |r|
+					return true if r[0].to_i>0
+				end
+				return nil
+			rescue Mysql::Error => e
+				return nil
+			end
+		end
+		#we discover a new url and record it
+		def <<(url)
+			return nil if @cache.urlStored?(url)	
+			ukey=md5(url)+@source
+			crc=Zlib::crc32(url,@seed)
+			sql="INSERT INTO `urls` (`url`,`source`,`added`,`visited`,`ukey`,`score`,`url_crc32`) VALUES ('"+url+"','"+@source+"','"+Time.now().to_i.to_s+"','0','"+ukey+"','"+Rspider::UrlScorer.score(url).to_s+"','"+crc.to_s+"')";
+			begin 
+				@my.query(sql)
+				@cache.<<(url)
+			rescue Mysql::Error,StandardError,Exception => e
+			else
+			end
+		end
+		#got a url to cralwer
+		def pop()
+			#sql="SELECT url FROM `urls` WHERE visited='0' AND `source`='"+@source+"'  ORDER BY errors asc,score desc,RAND() LIMIT 1"
+			url=@cache.pop
+			return url unless url.nil?
+			sql="SELECT url FROM `urls` WHERE visited='0' AND `source`='"+@source+"'  ORDER BY RAND() LIMIT 1"
+			begin 
+				rs=@my.query(sql)
+				rs.each do |r|
+					return	r[0]
+				end
+			rescue Mysql::Error
+				return nil
+			end
+		end
+		#we have cralwered an url ,so we recored it
+		def visited(url)
+			@cache.visited(url)
+			ukey=md5(url)+@source
+			crc=Zlib::crc32(url)
+			sql="UPDATE `urls` SET visited='"+Time.now.to_i.to_s+"' WHERE url_crc32='#{crc}' AND  `ukey`='"+ukey+"' LIMIT 1"
+			begin 
+				@my.query(sql)
+			rescue Mysql::Error
+				return false
+			end
+			return true
+		end
+		#we meet an error,so we log it
+		def error(url)
+			@cache.error(url)	
+			ukey=md5(url)+@source
+			crc=Zlib::crc32(url)
+			sql="UPDATE `urls` SET score=score-3,errors=errors+1 WHERE url_crc32=#{crc} AND `ukey`='"+ukey+"' LIMIT 1"
+			@my.query(sql)
+		end
+		def close
+			@my.close
+		end	
+	end
+end