baiduserp 2.3.7 → 2.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2886ad6a97fe01a9cc1ba3ab6f9893a7b37495d3
-  data.tar.gz: 3e78db82105ef932cf65253655fc49f01f2f7118
+  metadata.gz: 89ad01defa5be250a6f32c0e32d2ad9cb64044ac
+  data.tar.gz: 06d8a6914183c1630f10b036d0f1dacc70f67899
 SHA512:
-  metadata.gz: 83192c563fb0ca280cbfccf647c52474bc0e4b286923e4d06060de574a146260476162b126cc98a006357928e8b4f2d96ab4607f5cef7fcf774427d21a244f56
-  data.tar.gz: d99d993c78e32fe16f66768c684cb441123250a27f774296e4dcf4189db1bc0b04ac87a7fc57b847ae20ee964e1bf2ce13b59063f0b0a763cf66979032e268e3
+  metadata.gz: 0c52c60cd473b2b7dba88f0c1039311d1a811d6f27690c6c0ceb4350c5e608c995e56d5bc69e94982a43404aaf10dad756749bcad96af2163258f08818a3a3c2
+  data.tar.gz: f1b21e87c779f24d2b9bd5780e68330071c8f59463d5dfebd369831036f8f909532442092dc656074140540bd5b69c2c6d9e078f2b7347ba698601685b724f75

data/lib/baiduserp/analyser.rb

@@ -9,72 +9,76 @@ module Baiduserp
     # Dir[File.expand_path('../analyser/*.rb', __FILE__)].each{|f| require f}
 
     def initialize(name,attrs={})
-      @db_file = name + ".db"
+      @name = name
+      Dir.mkdir @name unless Dir.exists? @name # store htmls and serps data under the dir
+
+      @db_file = @name + ".sqlite"
       @attrs = attrs
       @keywords_imported = File.exists?(@db_file)
 
-      @db = Sequel.connect("sqlite://" + @db_file)
+      @db = Sequel.sqlite(@db_file)
 
       migrate!
 
-      @keywords = Class.new(Sequel::Model) do
-        set_dataset :keywords
-      end
-
-      @htmls = Class.new(Sequel::Model) do
-        set_dataset :htmls
-      end
-
-      @serps = Class.new(Sequel::Model) do
-        set_dataset :serps
-      end
-      
-      @weights = Class.new(Sequel::Model) do
-        set_dataset :weights
-      end
+      @keywords = Class.new(Sequel::Model(@db[:keywords]))
+      @weights = Class.new(Sequel::Model(@db[:weights]))
 
       import_keywords unless @keywords_imported
     end
 
-    def run
-      
+    def run(date=Date.today)
+      search(date)
+      generate_weights(date)
     end
 
-    def migrate!
+    def migrate!(db = @db, schema = 'weights')
       Sequel.extension :migration, :core_extensions
-      Sequel::Migrator.apply(@db, File.expand_path('../migrations/',__FILE__))
+      Sequel::Migrator.apply(db, File.expand_path("../migrations/#{schema}/",__FILE__))
     end
 
     def import_keywords(file=@attrs[:keywords])
       CSV.foreach(file) do |l|
         @keywords.find_or_create(:term => l[0]) do |r| 
-          r.weight = l[1]
+          r.search_volume = l[1]
           r.category = l[2]
         end
       end
     end
 
+    def model_htmls(date=Date.today)
+      db = Sequel.sqlite("#{@name}/htmls_#{date}.sqlite")
+      migrate!(db, 'htmls')
+      Class.new(Sequel::Model(db[:htmls]))
+    end
+
+    def model_serps(date=Date.today)
+      db = Sequel.sqlite("#{@name}/serps_#{date}.sqlite")
+      migrate!(db, 'serps')
+      Class.new(Sequel::Model(db[:serps]))
+    end
+
+    # Search Keywords -> Store Html -> Parse SERP
     def search(date=Date.today)
-      p = ProgressBar.create(:title => "Search Keywords", :total => @keywords.all.count)
+      htmls = model_htmls(date)
+      serps = model_serps(date)
+      p = ProgressBar.create(:title => "Searching Keywords", :total => @keywords.all.count)
       @keywords.each do |k|
-        if @htmls.where(:date => date, :keyword_id => k[:id]).count > 0
-          p.increment
-          next
-        end
+        htmls.find_or_create(:keyword_id => k[:id]) {|r| r.content = Baiduserp.get_search_html(k[:term]) }
+        serps.find_or_create(:keyword_id => k[:id]) {|r| r.content = YAML.dump(Baiduserp.parse(htmls.where(:keyword_id => k[:id]).first[:content])) }
         p.log k.to_hash
-        html = Baiduserp.get_search_html(k[:term])
-        @htmls.find_or_create(:keyword_id => k[:id], :date => date) {|r| r.content = html}
         p.increment
       end
     end
 
-    def generate_serps(date=Date.today)
-      htmls = @htmls.where(:date => date)
-      p = ProgressBar.create(:title => "Generating SERPS", :total => htmls.count)
+    def regenerate_serps(date=Date.today)
+      htmls = model_htmls(date)
+      serps = model_serps(date)
+      p = ProgressBar.create(:title => "ReGenerating SERPS", :total => htmls.count)
       htmls.each do |html|
         keyword_id = html[:keyword_id]
         html = html[:content]
-        @serps.find_or_create(:date => date, :keyword_id => keyword_id) {|r| r.content = YAML.dump(Baiduserp.parse(html))}
+        r = serps.find_or_create(:keyword_id => keyword_id) 
+        r.update(:content => YAML.dump(Baiduserp.parse(html)))
 
         p.log keyword_id
         p.increment
@@ -82,9 +86,9 @@ module Baiduserp
     end
 
     def generate_weights(date=Date.today)
-      serps = @serps.where(:date => date)
+      serps = model_serps(date)
       p = ProgressBar.create(:title => "Generating Weights", :total => serps.count)
-      serps.where(:date => date).each do |s|
+      serps.each do |s|
         keyword_id = s[:keyword_id]
         serp = YAML.load(s[:content])
 

data/lib/baiduserp/client.rb

@@ -17,6 +17,22 @@ module Baiduserp
       self.new.get_serp(url,retries)
     end
 
+    def self.get_rank_url(url)
+      self.new.get_rank_url(url)
+    end
+
+    def get_rank_url(url)
+      begin
+        response = self.class.get(url)
+      rescue StandardError => e
+        puts e.class
+        puts e.message
+        sleep(10)
+        retry
+      end
+      response
+    end
+
     def get_serp(url, retries = 3)
       if retries > 0
         begin

data/lib/baiduserp/migrations/{002_create_htmls_table.rb → htmls/001_create_htmls_table.rb}

@@ -2,11 +2,10 @@ Sequel.migration do
   up do
     create_table :htmls do
       primary_key :id
-      foreign_key :keyword_id, :keywords
-      Date :date
+      Integer :keyword_id
       String :content, :text => true
       
-      index :date
+      index :keyword_id
     end
   end
 

data/lib/baiduserp/migrations/{003_create_serps_table.rb → serps/001_create_serps_table.rb}

@@ -2,11 +2,10 @@ Sequel.migration do
   up do
     create_table :serps do
       primary_key :id
-      foreign_key :keyword_id, :keywords
-      Date :date
+      Integer :keyword_id
       String :content, :text => true
-      
-      index :date
+
+      index :keyword_id
     end
   end
 

data/lib/baiduserp/migrations/{001_create_keywords_table.rb → weights/001_create_keywords_table.rb}

@@ -3,7 +3,7 @@ Sequel.migration do
     create_table :keywords do
       primary_key :id
       String :term
-      Integer :weight
+      Integer :search_volume
       String :category
 
       index :term

data/lib/baiduserp/migrations/{004_create_weights_table.rb → weights/002_create_weights_table.rb}

@@ -9,6 +9,8 @@ Sequel.migration do
       String :type
       String :name
       String :site
+      String :subdomain
+      String :url
       Integer :side_rank
       Float :weight
       Float :normalized_weight

data/lib/baiduserp/parser/ranks.rb

@@ -24,12 +24,8 @@ class Baiduserp::Parser
       url = table.search('h3/a').first
       unless url.nil?
         url = url['href'] 
-        begin
-          url = Baiduserp::Client.get(url).headers['location'] if url.include?('http://www.baidu.com/link?')
-        rescue Exception
-          sleep(10)
-          retry
-        end
+        sleep(rand)
+        url = Baiduserp::Client.get_rank_url(url).headers['location'] if url.include?('http://www.baidu.com/link?')
       end
       r[:url] = url
 

data/lib/baiduserp/result.rb

@@ -92,6 +92,7 @@ module Baiduserp
         side_rank += 1
 
         url = ad[:url].to_s
+        url = ad[:site].to_s if url.empty? # patch to campatible with older versions of baiduserp
         type = 'SEM'
         name = ''
         site = Baiduserp::Helper.parse_site(url)
@@ -108,6 +109,7 @@ module Baiduserp
         side_rank += 1
 
         url = ad[:url].to_s
+        url = ad[:site].to_s if url.empty? # patch to compatible with older versions of baiduserp
         type = 'SEM'
         name = ''
         site = Baiduserp::Helper.parse_site(url)

data/lib/baiduserp/version.rb

@@ -1,3 +1,3 @@
 module Baiduserp
-  VERSION = "2.3.7"
+  VERSION = "2.5.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: baiduserp
 version: !ruby/object:Gem::Version
-  version: 2.3.7
+  version: 2.5.2
 platform: ruby
 authors:
 - MingQian Zhang
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-12 00:00:00.000000000 Z
+date: 2013-12-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -105,11 +105,10 @@ files:
 - lib/baiduserp/analyser.rb
 - lib/baiduserp/client.rb
 - lib/baiduserp/helper.rb
-- lib/baiduserp/migrations/001_create_keywords_table.rb
-- lib/baiduserp/migrations/002_create_htmls_table.rb
-- lib/baiduserp/migrations/003_create_serps_table.rb
-- lib/baiduserp/migrations/004_create_weights_table.rb
-- lib/baiduserp/migrations/005_add_subdomain_url_to_weights.rb
+- lib/baiduserp/migrations/htmls/001_create_htmls_table.rb
+- lib/baiduserp/migrations/serps/001_create_serps_table.rb
+- lib/baiduserp/migrations/weights/001_create_keywords_table.rb
+- lib/baiduserp/migrations/weights/002_create_weights_table.rb
 - lib/baiduserp/parser/ads_right.rb
 - lib/baiduserp/parser/ads_top.rb
 - lib/baiduserp/parser/con_ar.rb

data/lib/baiduserp/migrations/005_add_subdomain_url_to_weights.rb

@@ -1,11 +0,0 @@
-Sequel.migration do
-  up do
-    add_column :weights, :subdomain, String
-    add_column :weights, :url, String
-  end
-
-  down do
-    drop_column :weights, :subdomain
-    drop_column :weights, :url
-  end
-end