baiduserp 2.1.1 → 2.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c411628080652da333ee3833dc986e0ede74639c
-  data.tar.gz: 555ff4417a419c469459883781a2e01fc7e2c5e4
+  metadata.gz: 333ba9624a535b422d93a318e5b5db9f50681fab
+  data.tar.gz: fcc4bce281984552a81f1ecf138b91146ce3042d
 SHA512:
-  metadata.gz: a7aa377b0ad8b77909eb916394554994bb3e5649b215ae68bf8ff5e794c1297313e9fef5464ece6f85db6af0638358d141f2e749d119dcd574b4f6dccfbe8d27
-  data.tar.gz: 7b1a2a7e8f3c56bd60eef8e148c15aebd913ee27bae78fcf8719923973932f2eda10bd46656818ac10118ab05afd8ebed53eab405d08c9abeb1b5308e1daf7ed
+  metadata.gz: ad3edca21e77a528e13d2231737d8d78dd0fa4c221d6107a5e7ea933e0c51ee1e785c5fda5ffb79306e914b687a04dd6aa48ee8a34911474219d4bb6d0b81514
+  data.tar.gz: 5702393d3254700fd2ff721363ab68a7a394b7f1e3cae00a54adfbd4bff18a8cf4715f977729a90531c7b29977a9bfe1825980b4e8244d5d366092baa57e5d54

data/lib/baiduserp.rb

@@ -6,6 +6,10 @@ module Baiduserp
     Parser.new.search(keyword,page)
   end
 
+  def self.get_search_html(keyword,page=1)
+    Parser.new.get_search_html(keyword,page)
+  end
+
   def self.parse(html)
     Parser.new.parse html
   end

data/lib/baiduserp/client.rb

@@ -23,7 +23,7 @@ module Baiduserp
           sleep(10)
           retry
         end
-        if response.code == 301
+        if response.code != 200
           sleep(rand(60)+60)
           response = self.get_serp(url,retries - 1)
         end

data/lib/baiduserp/parser.rb

@@ -40,10 +40,16 @@ module Baiduserp
     end
 
     def search(keyword,page=1)
+      html = get_search_html(keyword,page)
+      parse html
+    end
+
+    def get_search_html(keyword,page=1)
       keyword = keyword.gsub(" ","+")
       page = page.to_i > 1 ? "&pn=#{page.to_i-1}0" : ""
       serp_url = URI.escape("http://www.baidu.com/s?wd=#{keyword}#{page}&ie=utf-8")
-      parse_file(serp_url)
+p serp_url
+      Client.get_serp(serp_url)
     end
 
     def parse_file(file_path)

data/lib/baiduserp/version.rb

@@ -1,3 +1,3 @@
 module Baiduserp
-  VERSION = "2.1.1"
+  VERSION = "2.1.5"
 end

data/lib/parsers/ads_top.rb

@@ -1,29 +1,33 @@
 class Baiduserp::Parser
   def _parse_ads_top(file)
     result = []
-    file[:doc].search('div.ec_pp_f').each do |div|
+    rank = 0
+
+    file[:doc].search('div#content_left').first.children.each do |div|
       id = div['id'].to_i
-      next unless id >= 3000
-      r = {rank: id}
+      break if id > 0 && id < 3000
+      next unless div['class'].to_s.include?('ec_pp_f')
+      rank += 1
 
-      r[:title] = Baiduserp::Helper.get_content_safe(div.search('div.ec_title'))
+      if div.name == 'div'
+        r = {rank: rank, id: id}
 
-      r[:content] = Baiduserp::Helper.get_content_safe(div.search('div.ec_desc'))
+        r[:title] = Baiduserp::Helper.get_content_safe(div.search('div.ec_title'))
 
-      r[:site] = Baiduserp::Helper.get_content_safe(div.search('span.ec_url'))
+        r[:content] = Baiduserp::Helper.get_content_safe(div.search('div.ec_desc'))
 
-      result << r
-    end
+        r[:site] = Baiduserp::Helper.get_content_safe(div.search('span.ec_url'))
+
+        result << r
 
-    if result.empty?
-      file[:doc].search('table.ec_pp_f').each_with_index do |table,i|
-        r = {rank: i + 1}
+      else # div.name == 'table'
+        r = {rank: rank, id: id}
 
-        r[:title] = Baiduserp::Helper.get_content_safe(table.search('td.EC_header/a'))
+        r[:title] = Baiduserp::Helper.get_content_safe(div.search('td.EC_header/a'))
 
-        r[:content] = Baiduserp::Helper.get_content_safe(table.search('a.EC_desc'))
+        r[:content] = Baiduserp::Helper.get_content_safe(div.search('a.EC_desc'))
 
-        r[:site] = Baiduserp::Helper.get_content_safe(table.search('a.EC_url'))
+        r[:site] = Baiduserp::Helper.get_content_safe(div.search('a.EC_url'))
 
         result << r
 

data/lib/parsers/ranks.rb

@@ -5,8 +5,19 @@ class Baiduserp::Parser
       next if table.nil?
       id = table['id'].to_i
       next unless id > 0 && id < 3000
+
       r = {:rank => id}
 
+      r[:result_op] = table['class'].to_s.include?('result-op')
+
+      r[:fk] = table['fk']
+
+      r[:srcid] = table['srcid']
+
+      r[:tpl] = table['tpl']
+
+      r[:mu] = table['mu']
+
       url = table.search('h3/a').first
       unless url.nil?
         url = url['href'] 
@@ -23,8 +34,6 @@ class Baiduserp::Parser
 
       r[:content] = Baiduserp::Helper.get_content_safe(table.search('div.c-abstract'))
 
-      r[:mu] = table['mu']
-
       table.search('a').each do |link|
         r[:baiduopen] = true if link['href'].to_s.include?('open.baidu.com')
       end

data/lib/parsers/zhixin.rb

@@ -2,7 +2,11 @@ class Baiduserp::Parser
   def _parse_zhixin(file)
     result = []
     file[:doc].search("div#content_left .result-zxl").each do |zxl|
-      result << {:id => zxl['id'], :tpl => zxl['tpl'], :mu => zxl['mu'] }
+      result << {:id => zxl['id'], 
+        :srcid => zxl['srcid'],
+        :fk => zxl['fk'],
+        :tpl => zxl['tpl'], 
+        :mu => zxl['mu'] }
     end
     result
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: baiduserp
 version: !ruby/object:Gem::Version
-  version: 2.1.1
+  version: 2.1.5
 platform: ruby
 authors:
 - MingQian Zhang
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-05 00:00:00.000000000 Z
+date: 2013-11-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri