baiduserp 2.3.3 → 2.3.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fefc5d8f8028b48f1557fdc0bacc9ea655fb5140
-  data.tar.gz: b078c04a225b413fb1e6f70d01d89c63b285c930
+  metadata.gz: c4f925a3ac00d8faedd0ce3c596b957cd76f2323
+  data.tar.gz: 4d3f836b54ed03bfd70603cb82425f78e96c3cb1
 SHA512:
-  metadata.gz: 9e56e95afb223c96499f99ee38a5bb209954361944ebfea43550a93a802c416ad7198b8a672b01d4ff99accc0c1f4591de69c0db58013f4691d967e2cd21ad77
-  data.tar.gz: 1f476da432c0efca2536001b79e5cdd88748da2fd0bc4d7ace4aff97c39fdf79009dfd161f80292b5af45014ca42c5a168e022be0d0d20060681455e15bbbdf9
+  metadata.gz: abf62c6dcb480558c39f58b8e4f7b27b7146ccfe2d380225d7541ac32def7575f37628b1df86ba0d73da574ecb040a758169736dc3ceddb29167c9ea83b89fa8
+  data.tar.gz: 5192e5a8175459b121f46413d3d6701b5ffba45f615e9dcfbee360af151daf82b9b68e4b930c99639eb465553f45e49aaba3d4d73129d2452263c178d4cfeac0

data/lib/baiduserp/analyser.rb

@@ -81,6 +81,8 @@ module Baiduserp
           type = w[:type]
           name = w[:name]
           site = w[:site]
+          subdomain = w[:subdomain]
+          url = w[:url]
           side_rank = w[:side_rank]
           weight = w[:weight]
           side = w[:side]
@@ -91,6 +93,8 @@ module Baiduserp
             r.type = type
             r.name = name
             r.site = site
+            r.subdomain = subdomain
+            r.url = url
             r.weight = weight
             r.part = part
             r.normalized_weight = normalized_weight

data/lib/baiduserp/helper.rb

@@ -33,8 +33,31 @@ module Baiduserp
       end
 
       def parse_site(url)
-        url = Domainatrix.parse(url.to_s)
-        url.domain + '.' + url.public_suffix
+        begin
+          url = Domainatrix.parse(url.to_s)
+          site = url.domain + '.' + url.public_suffix
+        rescue Exception => e
+          puts "parse_site from url error:"
+          puts url
+          puts e.class
+          puts e.message
+          site = ''
+        end
+        site
+      end
+
+      def parse_subdomain(url)
+        begin
+          url = Domainatrix.parse(url.to_s)
+          subdomain = url.subdomain
+        rescue Exception => e
+          puts "parse_site from url error:"
+          puts url
+          puts e.class
+          puts e.message
+          subdomain = ''
+        end
+        subdomain
       end
 
     end

data/lib/baiduserp/migrations/005_add_subdomain_url_to_weights.rb

@@ -0,0 +1,11 @@
+Sequel.migration do
+  up do
+    add_column :weights, :subdomain, String
+    add_column :weights, :url, String
+  end
+
+  down do
+    drop_column :weights, :subdomain
+    drop_column :weights, :url
+  end
+end

data/lib/baiduserp/parser/ads_right.rb

@@ -10,7 +10,7 @@ class Baiduserp::Parser
 
       r[:content] = Baiduserp::Helper.get_content_safe(div.search('a.EC_desc/font'))
 
-      r[:site] = Baiduserp::Helper.get_content_safe(div.search('font.EC_url'))
+      r[:url] = Baiduserp::Helper.get_content_safe(div.search('font.EC_url'))
 
       result << r
     end

data/lib/baiduserp/parser/ads_top.rb

@@ -19,7 +19,7 @@ class Baiduserp::Parser
 
         r[:content] = Baiduserp::Helper.get_content_safe(div.search('div.ec_desc'))
 
-        r[:site] = Baiduserp::Helper.get_content_safe(div.search('span.ec_url'))
+        r[:url] = Baiduserp::Helper.get_content_safe(div.search('span.ec_url'))
 
         result << r
 
@@ -30,7 +30,7 @@ class Baiduserp::Parser
 
         r[:content] = Baiduserp::Helper.get_content_safe(div.search('a.EC_desc'))
 
-        r[:site] = Baiduserp::Helper.get_content_safe(div.search('a.EC_url'))
+        r[:url] = Baiduserp::Helper.get_content_safe(div.search('a.EC_url'))
 
         result << r
 

data/lib/baiduserp/result.rb

@@ -56,13 +56,21 @@ module Baiduserp
       self[:ranks].each do |rank|
         side_rank += 1
 
+        url = rank[:url].to_s
+        mu = rank[:mu].to_s
+
         type = 'SEO'
         type = 'Special' if rank[:baiduopen]
-        type = 'Special' unless rank[:mu].to_s.empty?
 
-        name = rank[:tpl].to_s
+        unless mu.empty?
+          url = mu
+          type = 'Special'
+        end
 
-        site = Baiduserp::Helper.parse_site(rank[:url])
+        site = Baiduserp::Helper.parse_site(url)
+        subdomain = Baiduserp::Helper.parse_subdomain(url)
+
+        name = rank[:tpl].to_s
 
         weight = 1.0/side_rank.to_f
         if type == 'Special'
@@ -73,25 +81,23 @@ module Baiduserp
           end
         end
 
-        result << {type: type, name: name, site: site, side_rank: side_rank, weight: weight}
+        result << {type: type, name: name, site: site, subdomain: subdomain, url: url, mu: mu, side_rank: side_rank, weight: weight}
       end
       [result, side_rank]
     end
 
-    # def weight_of_pinpaizhuanqu(side_rank)
-    #   [[{ type: 'Special', name: 'pinpaizhuanqu', site: '', side_rank: side_rank, weight: 1.0}], (side_rank + 1)]
-    # end
-
     def weight_of_ads_top(side_rank)
       result = []
       self[:ads_top].each do |ad|
         side_rank += 1
 
+        url = ad[:url].to_s
         type = 'SEM'
         name = ''
-        site = Baiduserp::Helper.parse_site(ad[:site])
+        site = Baiduserp::Helper.parse_site(url)
+        subdomain = Baiduserp::Helper.parse_subdomain(url)
         weight = 1.0/side_rank.to_f
-        result << {type: type, name: name, site: site, side_rank: side_rank, weight: weight}
+        result << {type: type, name: name, site: site, subdomain: subdomain, url: url, side_rank: side_rank, weight: weight}
       end
       [result, side_rank]
     end
@@ -101,11 +107,13 @@ module Baiduserp
       self[:ads_right].each do |ad|
         side_rank += 1
 
+        url = ad[:url].to_s
         type = 'SEM'
         name = ''
-        site = Baiduserp::Helper.parse_site(ad[:site])
+        site = Baiduserp::Helper.parse_site(url)
+        subdomain = Baiduserp::Helper.parse_subdomain(url)
         weight = 1.0/side_rank.to_f
-        result << {type: type, name: name, site: site, side_rank: side_rank, weight: weight}
+        result << {type: type, name: name, site: site, subdomain: subdomain, url: url, side_rank: side_rank, weight: weight}
       end
       [result, side_rank]
     end
@@ -115,11 +123,13 @@ module Baiduserp
       self[:con_ar].each do |con|
         side_rank += 1
 
+        url = con[:data_click]['mu'].to_s
         type = 'Special'
         name = con[:tpl]
-        site = Baiduserp::Helper.parse_site(con[:data_click]['mu'])
+        site = Baiduserp::Helper.parse_site(url)
+        subdomain = Baiduserp::Helper.parse_subdomain(url)
         weight = 1.0 * @config[:con_ar_weight]
-        result << {type: type, name: name, site: site, side_rank: side_rank, weight: weight}
+        result << {type: type, name: name, site: site, subdomain: subdomain, url: url, side_rank: side_rank, weight: weight}
       end
       [result, side_rank]
     end
@@ -129,11 +139,13 @@ module Baiduserp
       self[:zhixin].each do |zhixin|
         side_rank += 1
 
+        url = zhixin[:mu].to_s
         type = 'Special'
         name = zhixin[:tpl]
-        site = Baiduserp::Helper.parse_site(zhixin[:mu])
+        site = Baiduserp::Helper.parse_site(url)
+        subdomain = Baiduserp::Helper.parse_subdomain(url)
         weight = 1.0 * @config[:zhixin_weight]
-        result << {type: type, name: name, site: site, side_rank: side_rank, weight: weight}
+        result << {type: type, name: name, site: site, subdomain: subdomain, url: url, side_rank: side_rank, weight: weight}
       end
       [result, side_rank]
     end

data/lib/baiduserp/version.rb

@@ -1,3 +1,3 @@
 module Baiduserp
-  VERSION = "2.3.3"
+  VERSION = "2.3.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: baiduserp
 version: !ruby/object:Gem::Version
-  version: 2.3.3
+  version: 2.3.5
 platform: ruby
 authors:
 - MingQian Zhang
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-10 00:00:00.000000000 Z
+date: 2013-12-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -95,6 +95,7 @@ files:
 - lib/baiduserp/migrations/002_create_htmls_table.rb
 - lib/baiduserp/migrations/003_create_serps_table.rb
 - lib/baiduserp/migrations/004_create_weights_table.rb
+- lib/baiduserp/migrations/005_add_subdomain_url_to_weights.rb
 - lib/baiduserp/parser/ads_right.rb
 - lib/baiduserp/parser/ads_top.rb
 - lib/baiduserp/parser/con_ar.rb