baidu 0.2.3 → 0.2.4

data/lib/baidu.rb

@@ -1,61 +1,127 @@
 #coding:UTF-8
 require 'mechanize'
 class Baidu
-	def initialize(offset100=false)
-		@a = Mechanize.new {|agent| agent.user_agent_alias = 'Linux Mozilla'}
-		@a.max_history = 1
-		@perpage = 10
-		@perpage = 100 if offset100==true
-		@baseuri = "http://www.baidu.com/s?rn=#{@perpage}&wd=" 
-	end
-	def query(query)
-		query = "#{query}"
-		@uri = @baseuri+URI.encode(query.encode('GBK'))
-		@page = @a.get @uri
-		#File.open('/tmp/testpage','w'){|f|f.puts @page.body} if query=='1458 Italia'
-		self.clean
-		@number = self.how_many
-		@maxpage = (@number / @perpage.to_f).round
-		@maxpage =10 if @maxpage>10
-		@currpage =0
-	end
-	def how_many_pages(uri)
-		self.query("site:#{uri}")
-		return self.how_many
-	end
-	def how_many_links(uri)
-		self.query("domain:\"#{uri}\"")
-		return self.how_many
-	end
-	def how_many_pages_with(url,string)
-		self.query("site:#{url} inurl:#{string}")
-		return self.how_many
-	end
-	def rank(host)
-		@page.search("//table[@class=\"result\"]").each do |table|
-			href = @page.search("//table[@id=\"#{table['id']}\"]//a").first['href']
-			begin
-				return table['id'] if host==URI.parse(URI.encode(href)).host
-			rescue URI::InvalidURIError
-				puts "invalid uri:#{href}"
-			end
-		end
-		return false
-	end
-	def how_many
-		return false if @page.search("//span[@class='nums']").first.nil?
-		return @page.search("//span[@class='nums']").first.content.gsub(/\D/,'').to_i
-	end
-	def next
-		nextbtn = @page.link_with(:text=>/下一页/)
-		return false if (nextbtn.nil? or @currpage >= @maxpage)
-		@page = @a.click(nextbtn)
-		self.clean
-		return true
-	end
-	def clean
-		@page.body.force_encoding('GBK')
-		@page.body.encode!('UTF-8',:invalid => :replace, :undef => :replace, :replace => "")
-		@page.body.gsub! ("[\U0080-\U2C77]+") #mechanize will be confuzed without removing the few characters
-	end
+    attr_accessor :perpage,:pagenumber,:debug
+    attr_reader :page,:wd,:data
+    BaseUri = 'http://www.baidu.com/s?'
+    def initialize
+        @a = Mechanize.new {|agent| agent.user_agent_alias = 'Linux Mozilla'}
+        @a.idle_timeout = 2
+        @a.max_history = 1
+        @perpage = 100
+        @page = nil
+        @debug = false
+        @data = Hash.new
+        #@baseuri = "http://www.baidu.com/s?rn=#{@perpage}&wd="
+    end
+
+    public
+    def query(wd)
+        @data.clear
+        @wd = wd
+        @data.clear
+        q = Array.new
+        q << "wd=#{wd}"
+        q << "rn=#{@perpage}"
+        queryStr = q.join("&")
+        uri = URI.encode((BaseUri + queryStr).encode('GBK'))
+        begin
+        @page = @a.get uri
+        rescue SocketError => e
+            puts e
+        end
+        clean
+        @number = self.how_many
+        @maxpage = (@number / @perpage.to_f).round
+        @currpage =0
+=begin
+        query = "#{query}"
+        @uri = @baseuri+URI.encode(query.encode('GBK'))
+        @page = @a.get @uri
+        self.clean
+        @number = self.how_many
+        @maxpage = (@number / @perpage.to_f).round
+        @maxpage =10 if @maxpage>10
+        @currpage =0
+=end
+    end
+
+    #site:xxx.yyy.com
+    def how_many_pages(host)
+        return @data['how_many']if @data.has_key?'how_many'
+        query("site:#{host}")
+        return how_many
+    end
+
+    #domain:xxx.yyy.com/path/file.html
+    def how_many_links(uri)
+        return @data['how_many']if @data.has_key?'how_many'
+        query("domain:\"#{uri}\"")
+        return how_many
+    end
+
+    #site:xxx.yyy.com inurl:zzz
+    def how_many_pages_with(host,string)
+        return @data['how_many']if @data.has_key?'how_many'
+        query("site:#{host} inurl:#{string}")
+        return how_many
+    end
+    ########################################################################################################################
+    #look up a word and get the rank of a uri with $host
+    def rank(host)#on base of ranks
+        return @data['rank'][host] if @data.has_key?'rank' and @data['rank'].has_key?host
+        ranks.each_with_index do |uri,index|
+            if URI.parse(URI.encode(uri).host)
+                @data << {'rank'=>{host=>index+1}}
+                return index+1
+            end
+        end
+=begin
+        @page.search("//table[@class=\"result\"]").each do |table|
+            href = @page.search("//table[@id=\"#{table['id']}\"]//a").first['href']
+            begin
+                return table['id'] if host==URI.parse(URI.encode(href)).host
+            rescue URI::InvalidURIError
+                puts "invalid uri:#{href}" if @debug
+            end
+        end
+        return false
+=end
+    end
+
+    def ranks#(keyword=false)
+        return @data['ranks'] if @data.has_key?'ranks'
+        raise StandardError,'wrong with @page' unless @page.instance_of? Mechanize::Page
+        #self.query(keyword) if keyword
+        ranks = Array.new
+        @page.search("//table[@class=\"result\"]").each do |table|
+            ranks << @page.search("//table[@id=\"#{table['id']}\"]//a").first['href']
+        end
+        @data['ranks'] = ranks
+        return ranks
+    end
+
+    def how_many
+        return @data['how_many'] if @data.has_key?'how_many'
+        raise StandardError,'wrong with @page' unless @page.instance_of? Mechanize::Page
+        numSpan = @page.search("//span[@class='nums']").first
+        return false if numSpan.nil?
+        return numSpan.content.gsub(/\D/,'').to_i
+        #return false if @page.search("//span[@class='nums']").first.nil?
+        #return @page.search("//span[@class='nums']").first.content.gsub(/\D/,'').to_i
+    end
+
+    def next
+        nextbtn = @page.link_with(:text=>/下一页/)
+        return false if (nextbtn.nil? or @currpage >= @maxpage)
+        @page = @a.click(nextbtn)
+        self.clean
+        return true
+    end
+    private
+    def clean
+        @page.body.force_encoding('GBK')
+        @page.body.encode!('UTF-8',:invalid => :replace, :undef => :replace, :replace => "")
+        @page.body.gsub! ("[\U0080-\U2C77]+") #mechanize will be confuzed without removing the few characters
+    end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: baidu
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.4
   prerelease: 
 platform: ruby
 authors:
@@ -39,7 +39,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.15
+rubygems_version: 1.8.17
 signing_key: 
 specification_version: 3
 summary: to get data from www.baidu.com. this is built by a newbie, so please be careful