baiduserp 2.2.9 → 2.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1d1c2a61259d5e3134d0566e002ba551a791b06c
-  data.tar.gz: 8b3f78ad2190a17a2f530019c0b0719bb9abfb93
+  metadata.gz: fefc5d8f8028b48f1557fdc0bacc9ea655fb5140
+  data.tar.gz: b078c04a225b413fb1e6f70d01d89c63b285c930
 SHA512:
-  metadata.gz: 859d3c01765741cfe50eed92a05f5cbec93c68c003d13d2a2dad2e6b85e22d63d2171ded85fa3cd2271d7b830c522bfa95b3b27f91a269af621f811f23c3a87a
-  data.tar.gz: 5847dd8a81f4c61a0f72d2d8a2e6cd91e4aa1e9eed32f3c495453e31fe006a5c8162f902ebf2e831adb16731114fcb6b7d4a4867ed8ce575aecc39e2feca14c4
+  metadata.gz: 9e56e95afb223c96499f99ee38a5bb209954361944ebfea43550a93a802c416ad7198b8a672b01d4ff99accc0c1f4591de69c0db58013f4691d967e2cd21ad77
+  data.tar.gz: 1f476da432c0efca2536001b79e5cdd88748da2fd0bc4d7ace4aff97c39fdf79009dfd161f80292b5af45014ca42c5a168e022be0d0d20060681455e15bbbdf9

data/lib/baiduserp/analyser.rb

@@ -27,6 +27,10 @@ module Baiduserp
       @serps = Class.new(Sequel::Model) do
         set_dataset :serps
       end
+      
+      @weights = Class.new(Sequel::Model) do
+        set_dataset :weights
+      end
 
       import_keywords unless @keywords_imported
     end
@@ -37,12 +41,15 @@ module Baiduserp
 
     def migrate!
       Sequel.extension :migration, :core_extensions
-      Sequel::Migrator.apply(@db, File.expand_path('../analyser-migrations/',__FILE__))
+      Sequel::Migrator.apply(@db, File.expand_path('../migrations/',__FILE__))
     end
 
     def import_keywords(file=@attrs[:keywords])
       CSV.foreach(file) do |l|
-        @keywords.insert(:term => l[0], :weight => l[1], :category => l[2])
+        @keywords.find_or_create(:term => l[0]) do |r| 
+          r.weight = l[1]
+          r.category = l[2]
+        end
       end
     end
 
@@ -52,8 +59,43 @@ module Baiduserp
         puts k.to_hash
         html = Baiduserp.get_search_html(k[:term])
         serp = Baiduserp.parse(html)
-        @htmls.insert(:keyword_id => k[:id], :date => date, :content => html)
-        @serps.insert(:keyword_id => k[:id], :date => date, :content => YAML.dump(serp))
+        @htmls.find_or_create(:keyword_id => k[:id], :date => date) {|r| r.content = html}
+        @serps.find_or_create(:keyword_id => k[:id], :date => date) {|r| r.content = YAML.dump(serp)}
+      end
+    end
+
+    def generate_serps(date=Date.today)
+      @htmls.where(:date => date).each do |html|
+        keyword_id = html[:keyword_id]
+        html = html[:content]
+        @serps.find_or_create(:date => date, :keyword_id => keyword_id) {|r| r.content = YAML.dump(Baiduserp.parse(html))}
+      end
+    end
+
+    def generate_weights(date=Date.today)
+      @serps.where(:date => date).each do |s|
+        keyword_id = s[:keyword_id]
+        serp = YAML.load(s[:content])
+
+        serp.weights.each do |w|
+          type = w[:type]
+          name = w[:name]
+          site = w[:site]
+          side_rank = w[:side_rank]
+          weight = w[:weight]
+          side = w[:side]
+          part = w[:part]
+          normalized_weight = w[:normalized_weight]
+
+          @weights.find_or_create(:date => date, :keyword_id => keyword_id, :side => side, :side_rank => side_rank) do |r|
+            r.type = type
+            r.name = name
+            r.site = site
+            r.weight = weight
+            r.part = part
+            r.normalized_weight = normalized_weight
+          end
+        end
       end
     end
 

data/lib/baiduserp/helper.rb

@@ -1,3 +1,5 @@
+require 'domainatrix'
+
 module Baiduserp
   module Helper
     class << self
@@ -8,6 +10,8 @@ module Baiduserp
         noko.first.content.strip
       end
 
+      # parse data click value from baidu div property,
+      # which is a JSON like format
       def parse_data_click(str)
         JSON.parse(str
                      .gsub("'",'"')
@@ -16,6 +20,23 @@ module Baiduserp
                      #.gsub(/:'([^(',\")]*)'(,|})/,':"\1"\2')
                    )
       end
+
+      # normalize weight of given data,
+      # the data must be a hash array structure.
+      # for example : [{a: 1, b: 2}, {a: 2, b: 3}]
+      def normalize(data,weight_col=:weight,normalized_col=:normalized_weight)
+        total_weight = data.reduce(0.0) {|sum,d| sum += d[weight_col].to_f}
+        data.each do|d|
+          d[normalized_col] = d[weight_col].to_f/total_weight
+        end
+        data
+      end
+
+      def parse_site(url)
+        url = Domainatrix.parse(url.to_s)
+        url.domain + '.' + url.public_suffix
+      end
+
     end
   end
 end

data/lib/baiduserp/migrations/004_create_weights_table.rb

@@ -0,0 +1,23 @@
+Sequel.migration do
+  up do
+    create_table :weights do
+      primary_key :id
+      foreign_key :keyword_id, :keywords
+      Date :date
+      String :side
+      String :part
+      String :type
+      String :name
+      String :site
+      Integer :side_rank
+      Float :weight
+      Float :normalized_weight
+
+      index [:date, :keyword_id, :side, :side_rank]
+    end
+  end
+
+  down do
+    drop_table :weights
+  end
+end

data/lib/baiduserp/result.rb

@@ -1,7 +1,143 @@
-require 'domainatrix'
-
 module Baiduserp
   class Result < Hash
+    def initialize(*args)
+      @config = {
+        :left_parts => [:ads_top,
+                        :zhixin,
+                        :ranks
+                       ],
+
+        :right_parts => [:con_ar,
+                         :ads_right
+                        ],
+
+        :left_part_weight => 8,
+        
+        :right_part_weight => 2,
+        
+        :zhixin_weight => 3.5,
+
+        :baiduopen_weight => 3,
+
+        :rank_special_weight => 2,
+
+        :con_ar_weight => 2
+      }
+
+      super
+    end
+
+    def weights
+      result = []
+      [:left,:right].each do |side|
+        side_rank = 0
+
+        @config["#{side}_parts".to_sym].each do |part|
+          rs,side_rank = self.send("weight_of_#{part}",side_rank)
+
+          rs.each do |r|
+            r[:side] = side.to_s
+            r[:part] = part
+
+            r[:weight] = r[:weight].to_f * @config["#{side}_part_weight".to_sym].to_f
+            result << r
+          end
+        end
+      end
+      Baiduserp::Helper.normalize(result)
+    end
+
+    # weight_of_*** functions
+    # return a hash array
+    # each hash includes: type, name, site, weight
+
+    def weight_of_ranks(side_rank)
+      result = []
+      self[:ranks].each do |rank|
+        side_rank += 1
+
+        type = 'SEO'
+        type = 'Special' if rank[:baiduopen]
+        type = 'Special' unless rank[:mu].to_s.empty?
+
+        name = rank[:tpl].to_s
+
+        site = Baiduserp::Helper.parse_site(rank[:url])
+
+        weight = 1.0/side_rank.to_f
+        if type == 'Special'
+          if rank[:baiduopen]
+            weight = weight * @config[:baiduopen_weight].to_f
+          else
+            weight = weight * @config[:rank_special_weight].to_f
+          end
+        end
+
+        result << {type: type, name: name, site: site, side_rank: side_rank, weight: weight}
+      end
+      [result, side_rank]
+    end
+
+    # def weight_of_pinpaizhuanqu(side_rank)
+    #   [[{ type: 'Special', name: 'pinpaizhuanqu', site: '', side_rank: side_rank, weight: 1.0}], (side_rank + 1)]
+    # end
+
+    def weight_of_ads_top(side_rank)
+      result = []
+      self[:ads_top].each do |ad|
+        side_rank += 1
+
+        type = 'SEM'
+        name = ''
+        site = Baiduserp::Helper.parse_site(ad[:site])
+        weight = 1.0/side_rank.to_f
+        result << {type: type, name: name, site: site, side_rank: side_rank, weight: weight}
+      end
+      [result, side_rank]
+    end
+
+    def weight_of_ads_right(side_rank)
+      result = []
+      self[:ads_right].each do |ad|
+        side_rank += 1
+
+        type = 'SEM'
+        name = ''
+        site = Baiduserp::Helper.parse_site(ad[:site])
+        weight = 1.0/side_rank.to_f
+        result << {type: type, name: name, site: site, side_rank: side_rank, weight: weight}
+      end
+      [result, side_rank]
+    end
+
+    def weight_of_con_ar(side_rank)
+      result = []
+      self[:con_ar].each do |con|
+        side_rank += 1
+
+        type = 'Special'
+        name = con[:tpl]
+        site = Baiduserp::Helper.parse_site(con[:data_click]['mu'])
+        weight = 1.0 * @config[:con_ar_weight]
+        result << {type: type, name: name, site: site, side_rank: side_rank, weight: weight}
+      end
+      [result, side_rank]
+    end
+
+    def weight_of_zhixin(side_rank)
+      result = []
+      self[:zhixin].each do |zhixin|
+        side_rank += 1
+
+        type = 'Special'
+        name = zhixin[:tpl]
+        site = Baiduserp::Helper.parse_site(zhixin[:mu])
+        weight = 1.0 * @config[:zhixin_weight]
+        result << {type: type, name: name, site: site, side_rank: side_rank, weight: weight}
+      end
+      [result, side_rank]
+    end
+
     def seo_urls
       self[:ranks].reduce([]) {|result,rank| result << rank[:url]}
     end
@@ -29,5 +165,6 @@ module Baiduserp
     def sem_sites
       sem_urls
     end
+
   end
 end

data/lib/baiduserp/version.rb

@@ -1,3 +1,3 @@
 module Baiduserp
-  VERSION = "2.2.9"
+  VERSION = "2.3.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: baiduserp
 version: !ruby/object:Gem::Version
-  version: 2.2.9
+  version: 2.3.3
 platform: ruby
 authors:
 - MingQian Zhang
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-02 00:00:00.000000000 Z
+date: 2013-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -88,12 +88,13 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- lib/baiduserp/analyser-migrations/001_create_keywords_table.rb
-- lib/baiduserp/analyser-migrations/002_create_htmls_table.rb
-- lib/baiduserp/analyser-migrations/003_create_serps_table.rb
 - lib/baiduserp/analyser.rb
 - lib/baiduserp/client.rb
 - lib/baiduserp/helper.rb
+- lib/baiduserp/migrations/001_create_keywords_table.rb
+- lib/baiduserp/migrations/002_create_htmls_table.rb
+- lib/baiduserp/migrations/003_create_serps_table.rb
+- lib/baiduserp/migrations/004_create_weights_table.rb
 - lib/baiduserp/parser/ads_right.rb
 - lib/baiduserp/parser/ads_top.rb
 - lib/baiduserp/parser/con_ar.rb