baiduserp 2.3.5 → 2.3.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c4f925a3ac00d8faedd0ce3c596b957cd76f2323
-  data.tar.gz: 4d3f836b54ed03bfd70603cb82425f78e96c3cb1
+  metadata.gz: 2886ad6a97fe01a9cc1ba3ab6f9893a7b37495d3
+  data.tar.gz: 3e78db82105ef932cf65253655fc49f01f2f7118
 SHA512:
-  metadata.gz: abf62c6dcb480558c39f58b8e4f7b27b7146ccfe2d380225d7541ac32def7575f37628b1df86ba0d73da574ecb040a758169736dc3ceddb29167c9ea83b89fa8
-  data.tar.gz: 5192e5a8175459b121f46413d3d6701b5ffba45f615e9dcfbee360af151daf82b9b68e4b930c99639eb465553f45e49aaba3d4d73129d2452263c178d4cfeac0
+  metadata.gz: 83192c563fb0ca280cbfccf647c52474bc0e4b286923e4d06060de574a146260476162b126cc98a006357928e8b4f2d96ab4607f5cef7fcf774427d21a244f56
+  data.tar.gz: d99d993c78e32fe16f66768c684cb441123250a27f774296e4dcf4189db1bc0b04ac87a7fc57b847ae20ee964e1bf2ce13b59063f0b0a763cf66979032e268e3

data/lib/baiduserp/analyser.rb

@@ -2,6 +2,7 @@ require 'sequel'
 require 'csv'
 require 'date'
 require 'yaml'
+require 'ruby-progressbar'
 
 module Baiduserp
   class Analyser
@@ -54,26 +55,36 @@ module Baiduserp
     end
 
     def search(date=Date.today)
+      p = ProgressBar.create(:title => "Search Keywords", :total => @keywords.all.count)
       @keywords.each do |k|
-        next if @htmls.where(:date => date, :keyword_id => k[:id]).count > 0
-        puts k.to_hash
+        if @htmls.where(:date => date, :keyword_id => k[:id]).count > 0
+          p.increment
+          next
+        end
+        p.log k.to_hash
         html = Baiduserp.get_search_html(k[:term])
-        serp = Baiduserp.parse(html)
         @htmls.find_or_create(:keyword_id => k[:id], :date => date) {|r| r.content = html}
-        @serps.find_or_create(:keyword_id => k[:id], :date => date) {|r| r.content = YAML.dump(serp)}
+        p.increment
       end
     end
 
     def generate_serps(date=Date.today)
-      @htmls.where(:date => date).each do |html|
+      htmls = @htmls.where(:date => date)
+      p = ProgressBar.create(:title => "Generating SERPS", :total => htmls.count)
+      htmls.each do |html|
         keyword_id = html[:keyword_id]
         html = html[:content]
         @serps.find_or_create(:date => date, :keyword_id => keyword_id) {|r| r.content = YAML.dump(Baiduserp.parse(html))}
+
+        p.log keyword_id
+        p.increment
       end
     end
 
     def generate_weights(date=Date.today)
-      @serps.where(:date => date).each do |s|
+      serps = @serps.where(:date => date)
+      p = ProgressBar.create(:title => "Generating Weights", :total => serps.count)
+      serps.where(:date => date).each do |s|
         keyword_id = s[:keyword_id]
         serp = YAML.load(s[:content])
 
@@ -100,6 +111,8 @@ module Baiduserp
             r.normalized_weight = normalized_weight
           end
         end
+        p.log keyword_id
+        p.increment
       end
     end
 

data/lib/baiduserp/version.rb

@@ -1,3 +1,3 @@
 module Baiduserp
-  VERSION = "2.3.5"
+  VERSION = "2.3.7"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: baiduserp
 version: !ruby/object:Gem::Version
-  version: 2.3.5
+  version: 2.3.7
 platform: ruby
 authors:
 - MingQian Zhang
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-11 00:00:00.000000000 Z
+date: 2013-12-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -80,6 +80,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Parse Baidu SERP result page.
 email:
 - zmingqian@qq.com