baiduserp 2.1.6 → 2.1.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f9ab6771775f8898485a11610cbb943f18c2104f
-  data.tar.gz: dcc6a0c371330b1cc9fffcd675b5f54f185a02bf
+  metadata.gz: d5fe173a0b067ff22f37a68ab8cd4d633b556630
+  data.tar.gz: e42a0a457467435480fd7cf7e82dfbfac88b9691
 SHA512:
-  metadata.gz: 8c3589e23d461f3629dd9419d3b595f17c6cab74262bd9764f105d719dc3be39c25f9bae3ad0be55a2dd820cac68eac5fb1000dbe187f2b2b3c98cf57dc5470b
-  data.tar.gz: bb1609eaffcf0f201f52956c6883927127383d8f19ab41edbd90fe6204bebbbd3a97ef6a2ce4908ab6fea07dcaf4c0091e69c4baa41f47addc205a7dba9b3aa5
+  metadata.gz: c4d75fa5fb5429aaa2e9a0293203dd624b4724aab1a1a7e5f3b39dba74b40a9c2bcbd82ec21c6318bc8f8955fa70a7d8a17c7dfef57049ba4e8e8f8366d6ab55
+  data.tar.gz: 90f9f95d9f822d278f0c755ce0f4048b6201f089f999ebf9b3976981d504e35b3f1b09ddf5d85a698e002f56fd30f769689f0e8d56c737f58d815a31a53156ca

data/lib/baiduserp/client.rb

@@ -11,23 +11,48 @@ module Baiduserp
     include HTTParty
     base_uri 'www.baidu.com'
     follow_redirects false
-    headers "User-Agent" => self.rand_ua
+    headers "User-Agent" => self.rand_ua, "Referer" => 'http://www.baidu.com/'
 
-    def self.get_serp(url, retries = 6)
+    def self.get_serp(url,retries = 3)
+      self.new.get_serp(url,retries)
+    end
+
+    def get_serp(url, retries = 3)
       if retries > 0
         begin
-          response = self.get(url)
-        rescue Timeout::Error => e
+          response = self.class.get(url)
+        rescue StandardError => e
           puts e.class
           puts e.message
           sleep(10)
           retry
         end
+
         if response.code != 200
-          sleep(rand(60)+60)
-          response = self.get_serp(url,retries - 1)
+          puts response
+          puts "Retry on URL: #{url}"
+          sleep(rand(60)+1200)
+          response = self.class.get_serp(url,retries - 1)
+        end
+
+        if response.nil?
+          puts "Still error after 3 tries, sleep 3600s now."
+          sleep(3600)
+          response = self.class.get_serp(url)
         end
-        return response.body
+
+        if response.headers['Content-Length'].to_i != response.body.bytesize
+          issue_file = "/tmp/baiduserp_crawler_issue_#{Time.now.strftime("%Y%m%d%H%M%S")}.html"
+          open(issue_file,'w').puts(response.body)
+          puts "Notice:"
+          puts "Baiduserp get an error when crawl SERP: response size (#{response.headers['Content-Length']}) not match body size."
+          puts "Please see file #{issue_file} for body content."
+          puts "Sleep 10s and retry"
+          sleep(10)
+          response = self.class.get_serp(url)
+        end
+
+        return response
       else
         return nil
       end

data/lib/baiduserp/helper.rb

@@ -8,7 +8,14 @@ module Baiduserp
         noko.first.content.strip
       end
 
-
+      def parse_data_click(str)
+        JSON.parse(str
+                     .gsub("'",'"')
+                     .gsub(/({|,)([a-zA-Z0-9_]+):/, '\1"\2":')
+                     #.gsub(/'*([a-zA-Z0-9_]+)'*:/, '"\1":')
+                     #.gsub(/:'([^(',\")]*)'(,|})/,':"\1"\2')
+                   )
+      end
     end
   end
 end

data/lib/baiduserp/parser.rb

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 require 'nokogiri'
 require 'uri'
+require 'json'
 require 'baiduserp/client'
 require 'baiduserp/helper'
 require 'baiduserp/result'
@@ -47,15 +48,16 @@ module Baiduserp
     def get_search_html(keyword,page=1)
       keyword = keyword.gsub(" ","+")
       page = page.to_i > 1 ? "&pn=#{page.to_i-1}0" : ""
-      serp_url = URI.escape("http://www.baidu.com/s?wd=#{keyword}#{page}&ie=utf-8")
-      Client.get_serp(serp_url)
+      serp_url = URI.escape("http://www.baidu.com/s?wd=#{keyword}#{page}&rsv_bp=0&ch=&tn=baidu&bar=&rsv_spt=3&ie=utf-8&rsv_sug3=2&rsv_sug=0&rsv_sug1=2&rsv_sug4=24&inputT=#{1000+rand(1000)}")
+      # serp_url = URI.escape("http://www.baidu.com/s?wd=#{keyword}#{page}&ie=utf-8")
+      Client.get_serp(serp_url).body
     end
 
     def parse_file(file_path)
       if File.exists? file_path
         html = open(file_path).read
       else
-        html = Client.get_serp(file_path)
+        html = Client.get_serp(file_path).body
       end
       parse html
     end

data/lib/baiduserp/result.rb

@@ -1,3 +1,5 @@
+require 'domainatrix'
+
 module Baiduserp
   class Result < Hash
     def seo_urls
@@ -9,19 +11,23 @@ module Baiduserp
       self[:ranks].each do |rank| 
         url = rank[:url].to_s
         next if url.empty?
-        result << URI(URI.escape(rank[:url])).host.downcase
+        result << Addressable::URI.parse(rank[:url]).host
       end
       result
     end
 
-    def sem_sites
+    def sem_urls
       result = []
       (self[:ads_top] + self[:ads_right]).each do |ad| 
         site = ad[:site].to_s
         next if site.empty?
-        result << ad[:site].downcase
+        result << ad[:site]
       end
       result
     end
+
+    def sem_sites
+      sem_urls
+    end
   end
 end

data/lib/baiduserp/version.rb

@@ -1,3 +1,3 @@
 module Baiduserp
-  VERSION = "2.1.6"
+  VERSION = "2.1.14"
 end

data/lib/parsers/con_ar.rb

@@ -0,0 +1,14 @@
+class Baiduserp::Parser
+  def _parse_con_ar(file)
+    result = []
+    divs = file[:doc].search("div#content_right div#con-ar").first
+    return [] if divs.nil?
+    divs.children.each do |div|
+      next unless div['class'].to_s.include?('result-op')
+      result << {:tpl => div['tpl'], 
+        :data_click => Baiduserp::Helper.parse_data_click(div['data-click'])
+      }
+    end
+    result
+  end
+end

data/lib/parsers/zhixin.rb

@@ -6,7 +6,9 @@ class Baiduserp::Parser
         :srcid => zxl['srcid'],
         :fk => zxl['fk'],
         :tpl => zxl['tpl'], 
-        :mu => zxl['mu'] }
+        :mu => zxl['mu'],
+        :data_click => Baiduserp::Helper.parse_data_click(zxl['data-click'])
+      }
     end
     result
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: baiduserp
 version: !ruby/object:Gem::Version
-  version: 2.1.6
+  version: 2.1.14
 platform: ruby
 authors:
 - MingQian Zhang
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-07 00:00:00.000000000 Z
+date: 2013-11-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -38,6 +38,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: domainatrix
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Parse Baidu SERP result page.
 email:
 - zmingqian@qq.com
@@ -54,6 +68,7 @@ files:
 - lib/baiduserp.rb
 - lib/parsers/ads_right.rb
 - lib/parsers/ads_top.rb
+- lib/parsers/con_ar.rb
 - lib/parsers/pinpaizhuanqu.rb
 - lib/parsers/ranks.rb
 - lib/parsers/related_keywords.rb