baiduserp 2.5.2 → 2.5.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 89ad01defa5be250a6f32c0e32d2ad9cb64044ac
-  data.tar.gz: 06d8a6914183c1630f10b036d0f1dacc70f67899
+  metadata.gz: 8c607f8765a54bd7ad35fb07d98b99e9fd81c525
+  data.tar.gz: e942c9538d47efe080a438401c7dc71e713b184b
 SHA512:
-  metadata.gz: 0c52c60cd473b2b7dba88f0c1039311d1a811d6f27690c6c0ceb4350c5e608c995e56d5bc69e94982a43404aaf10dad756749bcad96af2163258f08818a3a3c2
-  data.tar.gz: f1b21e87c779f24d2b9bd5780e68330071c8f59463d5dfebd369831036f8f909532442092dc656074140540bd5b69c2c6d9e078f2b7347ba698601685b724f75
+  metadata.gz: 196029dde69973fd778e821b416a6f2a70de085923f91c6c0d9d951d1be0c23ceddf4780383ae84edb6c545f29c5eaff5ed9ad502ccfcfa89e7ab9b536bd844e
+  data.tar.gz: 041858553ce31abf38933f568b385c4ac0adf53add4dba35fd9c66301fe96d5f7aba1d2d1623f48c194b77dd0d41455179d876617bbe9301ee0c8744f340acb0

data/lib/baiduserp/analyser.rb

@@ -61,11 +61,10 @@ module Baiduserp
     def search(date=Date.today)
       htmls = model_htmls(date)
       serps = model_serps(date)
-      p = ProgressBar.create(:title => "Searching Keywords", :total => @keywords.all.count)
+      p = ProgressBar.create(:title => "Searching Keywords", :total => @keywords.all.count, :format => '%t (%c/%C) %a %E |%w')
       @keywords.each do |k|
         htmls.find_or_create(:keyword_id => k[:id]) {|r| r.content = Baiduserp.get_search_html(k[:term]) }
         serps.find_or_create(:keyword_id => k[:id]) {|r| r.content = YAML.dump(Baiduserp.parse(htmls.where(:keyword_id => k[:id]).first[:content])) }
-        p.log k.to_hash
         p.increment
       end
     end
@@ -73,21 +72,20 @@ module Baiduserp
     def regenerate_serps(date=Date.today)
       htmls = model_htmls(date)
       serps = model_serps(date)
-      p = ProgressBar.create(:title => "ReGenerating SERPS", :total => htmls.count)
+      p = ProgressBar.create(:title => "ReGenerating SERPS", :total => htmls.count, :format => '%t (%c/%C) %a %E |%w')
       htmls.each do |html|
         keyword_id = html[:keyword_id]
         html = html[:content]
         r = serps.find_or_create(:keyword_id => keyword_id) 
         r.update(:content => YAML.dump(Baiduserp.parse(html)))
 
-        p.log keyword_id
         p.increment
       end
     end
 
     def generate_weights(date=Date.today)
       serps = model_serps(date)
-      p = ProgressBar.create(:title => "Generating Weights", :total => serps.count)
+      p = ProgressBar.create(:title => "Generating Weights", :total => serps.count, :format => '%t (%c/%C) %a %E |%w')
       serps.each do |s|
         keyword_id = s[:keyword_id]
         serp = YAML.load(s[:content])
@@ -115,7 +113,6 @@ module Baiduserp
             r.normalized_weight = normalized_weight
           end
         end
-        p.log keyword_id
         p.increment
       end
     end

data/lib/baiduserp/client.rb

@@ -57,24 +57,26 @@ module Baiduserp
           response = self.class.get_serp(url)
         end
 
-        if response.headers['Content-Length'].nil?
-          response = self.class.get_serp(url,retries)
-        end
-
-        if response.headers['Content-Length'].to_i != response.body.bytesize
-          issue_file = "/tmp/baiduserp_crawler_issue_#{Time.now.strftime("%Y%m%d%H%M%S")}.html"
-          open(issue_file,'w').puts(response.body)
-          puts "Notice:"
-          puts "Baiduserp get an error when crawl SERP: response size (#{response.headers['Content-Length']}) not match body size."
-          puts "Please see file #{issue_file} for body content."
-          puts "Sleep 10s and retry"
-          sleep(10)
-          response = self.class.get_serp(url)
-        end
+        ##Baidu Stopped response Content-Length in headers...
+        #if response.headers['Content-Length'].nil?
+        #  puts "Can't read Content-Length from response, retry."
+        #  response = self.class.get_serp(url,retries-1)
+        #end
+        #
+        #if response.headers['Content-Length'].to_i != response.body.bytesize
+        #  issue_file = "/tmp/baiduserp_crawler_issue_#{Time.now.strftime("%Y%m%d%H%M%S")}.html"
+        #  open(issue_file,'w').puts(response.body)
+        #  puts "Notice:"
+        #  puts "Baiduserp get an error when crawl SERP: response size (#{response.headers['Content-Length']}) not match body size."
+        #  puts "Please see file #{issue_file} for body content."
+        #  puts "Sleep 10s and retry"
+        #  sleep(10)
+        #  response = self.class.get_serp(url)
+        #end
 
-        return response
+        response
       else
-        return nil
+        nil
       end
     end
 

data/lib/baiduserp/parser.rb

@@ -48,7 +48,7 @@ module Baiduserp
     def get_search_html(keyword,page=1)
       keyword = keyword.gsub(" ","+")
       page = page.to_i > 1 ? "&pn=#{page.to_i-1}0" : ""
-      serp_url = URI.escape("http://www.baidu.com/s?wd=#{keyword}#{page}&rsv_bp=0&ch=&tn=baidu&bar=&rsv_spt=3&ie=utf-8&rsv_sug3=2&rsv_sug=0&rsv_sug1=2&rsv_sug4=24&inputT=#{1000+rand(1000)}")
+      serp_url = URI.escape("http://www.baidu.com/s?wd=#{keyword}#{page}&ie=utf-8&inputT=#{1000+rand(1000)}")
       # serp_url = URI.escape("http://www.baidu.com/s?wd=#{keyword}#{page}&ie=utf-8")
       Client.get_serp(serp_url).body
     end

data/lib/baiduserp/version.rb

@@ -1,3 +1,3 @@
 module Baiduserp
-  VERSION = "2.5.2"
+  VERSION = "2.5.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: baiduserp
 version: !ruby/object:Gem::Version
-  version: 2.5.2
+  version: 2.5.5
 platform: ruby
 authors:
 - MingQian Zhang
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-17 00:00:00.000000000 Z
+date: 2014-01-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri