baidu 1.1.4 → 1.1.5

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6e5e2be5751728aa5aab6cc36fc9552a74aadba1
+  data.tar.gz: 28a8bb12fad2e0d3164908a61f13a9fc1d1d03e2
+SHA512:
+  metadata.gz: d2b99a90386ebad73fac60db3331d0ea3e8021c657ef7d6d4c0b703023df86f8b3bfc2df434862a07dcc7b1496f026779bcbd592b9159e11ae2790ccdb9d0481
+  data.tar.gz: ea3191864319521de14ab735a770e08204a64fc40e23d836be4e196669f24e2b0f1c994f2ab9ed2e5f924ebb650f8dba6f6ae318a472bdc46d64c27b3afb684d

data/lib/baidu.rb

@@ -1,5 +1,4 @@
-#coding:UTF-8
-require 'mechanize'
+# encoding: utf-8
 require 'nokogiri'
 require 'json'
 require 'addressable/uri'
@@ -10,7 +9,7 @@ class SearchEngine
         URI(url)
         result = query(url)
         return result.has_result?
-    end 
+    end
 end
 class SearchResult
     def initialize(body,baseuri,pagenumber=nil)
@@ -141,7 +140,9 @@ class MbaiduResult < SearchResult
             href,text,host,is_mobile = '','','',false
             a = result.search("a").first
             is_mobile = true unless a.search("img").empty?
-            host = result.search('[@class="site"]').first.text
+            host = result.search('[@class="site"]').first
+            next if host.nil?
+            host = host.text
             href = a['href']
             text = a.text
             id = href.scan(/&order=(\d+)&/)
@@ -237,7 +238,7 @@ class Baidu < SearchEngine
     def extend(words,level=3,sleeptime=1)
         level = level.to_i - 1
         words = [words] unless words.respond_to? 'each'
-        
+
         extensions = Array.new
         words.each do |word|
             self.query(word)
@@ -254,7 +255,7 @@ class Baidu < SearchEngine
     def popular?(wd)
         return @a.get("http://index.baidu.com/main/word.php?word=#{URI.encode(wd.encode("GBK"))}").body.include?"boxFlash"
     end
-    
+
     def query(wd)
         q = Array.new
         q << "wd=#{wd}"
@@ -263,8 +264,9 @@ class Baidu < SearchEngine
         #uri = URI.encode((BaseUri + queryStr).encode('GBK'))
         uri = URI.encode((BaseUri + queryStr))
         begin
-            @page = @a.get uri
-            BaiduResult.new(@page)
+            # @page = @a.get uri
+            @page = HTTParty.get uri
+            BaiduResult.new(@page,uri)
         rescue Net::HTTP::Persistent::Error
             warn "[timeout] #{uri}"
             return false
@@ -307,23 +309,28 @@ class Baidu < SearchEngine
 end
 
 class BaiduResult < SearchResult
-    def initialize(page)
-        raise ArgumentError 'should be Mechanize::Page' unless page.class == Mechanize::Page
-        @page = page
+    def initialize(page,baseuri,pagenumber=1)
+        File.open('/tmp/file','w'){|f|f.puts page}
+        @page = Nokogiri::HTML page
+        @baseuri = baseuri
+        @pagenumber = pagenumber
+        # raise ArgumentError 'should be Mechanize::Page' unless page.class == Mechanize::Page
+        # @page = page
     end
-    
+
     def ranks
         return @ranks unless @ranks.nil?
         @ranks = Hash.new
-        @page.search("//table[@class=\"result\"]").each do |table|
+        @page.search("//table[@class=\"result\"]|//table[@class=\"result-op\"]").each do |table|
             id = table['id']
             @ranks[id] = Hash.new
-            url = @page.search("//table[@id=\"#{table['id']}\"]//span[@class=\"g\"]").first
-            a = @page.search("//table[@id=\"#{table['id']}\"]//h3/a")
+            url = table.search("[@class=\"g\"]").first
+            url = url.text unless url.nil?
+            a = table.search("a").first
             @ranks[id]['text'] = a.text
-            @ranks[id]['href'] = a.first['href'].sub('http://www.baidu.com/link?url=','').strip
+            @ranks[id]['href'] = url #a.first['href'].sub('http://www.baidu.com/link?url=','').strip
             unless url.nil?
-                url = url.text.strip
+                url = url.strip
                 @ranks[id]['host'] = Addressable::URI.parse(URI.encode("http://#{url}")).host
             else
                 @ranks[id]['host'] = nil
@@ -332,7 +339,49 @@ class BaiduResult < SearchResult
         #@page.search("//table[@class=\"result\"]").map{|table|@page.search("//table[@id=\"#{table['id']}\"]//span[@class=\"g\"]").first}.map{|rank|URI(URI.encode('http://'+rank.text.strip)).host unless rank.nil?}
         @ranks
     end
-    
+
+    def ads_top
+        ads = {}
+        id=0
+        @page.search("//table[@class='EC_mr15']|//table[@class='ec_pp_f']").each do |table|
+            table_id = table['id']
+            next if table_id.nil?
+            id += 1
+            href = table.search("font[@color='#008000']").text.split(/\s/).first.strip
+            title = table.search("a").first.text.strip
+            ads[id.to_s]= {'title'=>title,'href' => href,'host'=>href}
+        end
+        ads
+    end
+    def ads_bottom
+        ads = {}
+        @page.search("//table[@class='EC_mr15']|//table[@class='ec_pp_f']").each do |table|
+            id = table['id']
+            next unless id.nil?
+            id = id[-1,1]
+            href = table.search("font[@color='#008000']").text.split(/\s/).first.strip
+            title = table.search("a").first.text.strip
+            ads[id]= {'title'=>title,'href' => href,'host'=>href}
+        end
+        ads
+    end
+    def ads_right
+        ads = {}
+        @page.search("//div[@id='ec_im_container']").each do |table|
+            table.search("div[@id]").each do |div|
+                id = div['id'][-1,1].to_i+1
+                title = div.search("a").first
+                next if title.nil?
+                title = title.text
+                url = div.search("font[@color='#008000']").first
+                next if url.nil?
+                url = url.text
+                ads[id.to_s] = {'title'=>title,'href'=>url,'host'=>url}
+            end
+        end
+        ads
+    end
+
     #return the top rank number from @ranks with the input host
     # def rank(host)#on base of ranks
     #     ranks.each do |id,line|
@@ -353,13 +402,19 @@ class BaiduResult < SearchResult
     def related_keywords
         @related_keywords ||= @page.search("//div[@id=\"rs\"]//tr//a").map{|keyword| keyword.text}
     end
-    
+
     def next
-        @page = BaiduResult.new(Mechanize.new.click(@page.link_with(:text=>/下一页/))) unless @page.link_with(:text=>/下一页/).nil?
+        url = @page.xpath('//a[text()="下一页>"]').first
+        return if url.nil?
+        url = url['href']
+        url = URI.join(@baseuri,url).to_s
+        body = HTTParty.get(url)
+        return BaiduResult.new(body,url,@pagenumber+1)
+        # @page = BaiduResult.new(Mechanize.new.click(@page.link_with(:text=>/下一页/))) unless @page.link_with(:text=>/下一页/).nil?
     end
 
     def has_result?
         @page.search('//div[@class="nors"]').empty?
     end
-    
+
 end

metadata

@@ -1,8 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: baidu
 version: !ruby/object:Gem::Version
-  version: 1.1.4
-  prerelease: 
+  version: 1.1.5
 platform: ruby
 authors:
 - seoaqua
@@ -10,8 +9,50 @@ autorequire:
 bindir: bin
 cert_chain: []
 date: 2012-06-13 00:00:00.000000000 Z
-dependencies: []
-description: to get keyword ranking,related queries and popularity from baidu.com.
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: to get keyword ranking,related queries and popularity from www.baidu.com,www.so.com,m.baidu.com.
   this is built by a newbie, so please be careful. welcome to check my homepage, http://seoaqua.com
 email: seoaqua@qq.com
 executables: []
@@ -21,27 +62,26 @@ files:
 - lib/baidu.rb
 homepage: https://github.com/seoaqua/ruby-baidu
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 2.0.3
 signing_key: 
-specification_version: 3
-summary: to get keyword ranking,related queries and popularity from baidu.com. this
-  is built by a newbie, so please be careful. welcome to check my homepage, http://seoaqua.com
+specification_version: 4
+summary: to get keyword ranking,related queries and popularity from www.baidu.com,www.so.com,m.baidu.com.
+  this is built by a newbie, so please be careful. welcome to check my homepage, http://seoaqua.com
 test_files: []