site_classifier 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1a6185e3f8b41f26e38ba1787d9a73ec864960ae
-  data.tar.gz: d53cf863639a4841920e85ed1d68c1e1a6367c2e
+  metadata.gz: bdaa0867f7090d45492ae06ae60405f6cbb4af14
+  data.tar.gz: fb2a4af89c55d876d0a0786b2244ce64bdd06c09
 SHA512:
-  metadata.gz: 5b1e6a8e583c46f23dfdf35c2ce8448ca7539154775acd905a9cf2b212e3a6136b485c6b806ba2dae57ab8727e546b31c82f7cd2e7c879c8d23e002e8ba4f90c
-  data.tar.gz: e7ff3a1003a2baecf41486efcab6768c3ef5ee41fc24a558bdbdaadf65a46da6bbe31f121235ab1851934831937130f1e3d3b09f3fa4ea1f4d2c1ec91e55ee9e
+  metadata.gz: f3b5313c03e1b98496bb8e06ca886e8156afec5c4138090e4a4a43586b83b1e47db4f040e805d865ec05f5d3dbde2c1aef02410ee6d7f621216cfb7c355b2fb7
+  data.tar.gz: 38e2777f8386bdc950996d9d6dac82a394e5e2d7d2b9cb443719ce9d4b788d95aa24460dd5d26aba0f4e413406e8936a2013fdfc5027a394e17bef45a23541c7

data/lib/site_classifier.rb

@@ -22,4 +22,8 @@ module SiteClassifier
   def self.configuration
     @setup ||= SiteClassifier::Configuration.new
   end
+
+  def self.extract!(url)
+    SiteClassifier::Extractor.parse_site(url)
+  end
 end

data/lib/site_classifier/extractor.rb

@@ -30,29 +30,30 @@ module SiteClassifier
     # Extract most significant tags
     def most_significant
       most_sig = []
-      # if !description.nil?
-      #   if tags.any?
-      #     most_sig = tags.select {|tag| self.description.downcase.include?(tag)}.collect {|tag| tag.singularize }
-      #   else
-      #     most_sig = word_frequency.keys.select {|tag| self.description.downcase.include?(tag)}.collect {|tag| tag.singularize }
-      #   end
-      # end
 
-      description.to_s.split.each do |word|
-        self.word_frequency[word] ||= 0
-        self.word_frequency[word] += 1
-      end
+      # TODO: replace with NLP.
+      # description.to_s.split.each do |word|
+      #   self.word_frequency[word] ||= 0
+      #   self.word_frequency[word] += 1
+      # end
 
       if most_sig.empty?
-        most_sig = self.word_frequency.reject {|k,v| v < 3}.keys
-        most_sig.flatten!
-      end
+        total = self.word_frequency.values.count
+        
+        sum = 0
+        self.word_frequency.values.each do |counter|
+          sum += counter
+        end
 
-      if description && tags.any?
-        tags.each do |tag|
-          if description.include?(tag)
-            most_sig << tag.singularize
-          end
+        avg = (sum.to_f / total.to_f).floor
+
+        self.tags.each do |tag|
+          self.word_frequency[tag] ||= 0
+          self.word_frequency[tag] += (avg * 2.0)
+        end
+        most_sig = self.word_frequency.reject {|k,v| v < (avg * 2.0).floor }.to_a.uniq.sort_by {|_key, v| v}.reverse.collect(&:first).collect(&:downcase)
+        if most_sig.empty?
+          most_sig = self.word_frequency.reject {|k,v| v < (avg * 0.5).floor }.to_a.uniq.sort_by {|_key, v| v}.reverse.collect(&:first).collect(&:downcase)
         end
       end
       
@@ -89,7 +90,8 @@ module SiteClassifier
 
       debug("getting #{url}")
       html = Nokogiri::HTML(self.get(url).parsed_response)
-
+      html.search("//style").remove
+      html.search("//script").remove
       tags = []
       description = nil
       word_hash = {}
@@ -144,25 +146,37 @@ module SiteClassifier
         end
       end
 
-      if tags.empty?
-        debug("no tags, parsing body")
-        word_hash = Hash.new(0)
-        all_text = []
-        # all_text = html.search("p").collect {|p| p.text.strip }.collect {|text| text.split.collect(&:strip)}.flatten.reject {|word| word.size < 4}
-        # debug("p's extracts - #{all_text.inspect}")
-        if all_text.empty?
-          all_text = html.search("div").collect {|p| p.text.strip }.collect {|text| text.split.collect(&:strip)}.flatten.reject {|word| word.size < 4}
-          debug("divs extracts - #{all_text.inspect}")
-        end
-        all_text += description.to_s.split
+      debug("no tags, parsing body")
+      word_hash = Hash.new(0)
+      all_text = []
+      # all_text = html.search("p").collect {|p| p.text.strip }.collect {|text| text.split.collect(&:strip)}.flatten.reject {|word| word.size < 4}
+      # debug("p's extracts - #{all_text.inspect}")
+      if all_text.empty?
+        all_text = html.search("div").collect {|p| p.text.strip }.collect {|text| text.split.collect(&:strip)}.flatten.reject {|word| word.size < 4}
+        debug("divs extracts - #{all_text.inspect}")
+      end
+      all_text += description.to_s.split
 
-        all_text.flatten.each do |word|
-          word_hash[word] += 1
-        end
-        debug("final word hash - #{word_hash.inspect}")
-        word_hash.reject! {|k,v| v < 3 || k.size == 1 || k.include?(".") || k.include?("'") || k.include?("(") || k.include?(":") || k.include?("]")}
-        
+      all_text.flatten.each do |word|
+        word_hash[word] += 1
       end
+      debug("final word hash - #{word_hash.inspect}")
+      word_hash.reject! {|k,v| v < 3 || k.size == 1 || k.include?(".") || k.include?("'") || k.include?("(") || k.include?(":") || k.include?("]")}
+
+      # max_score = word_hash.values.max.to_i
+      # dmoz = HTTParty.get("http://www.dmoz.org/search", query: {q: url})
+      # dmoz_res = Nokogiri::HTML(dmoz.parsed_response)
+
+      # dmoz_classification = dmoz_res.search('//*[@id="bd-cross"]//ol[1]//li//strong').collect(&:text)
+      # if dmoz_classification.any?
+      #   dmoz_classification.each do |dmoz_class|
+      #     dmoz_class.split(": ").reject {|v| ["World"].include?(v)}.each do |tag|
+      #       tag.split(" ").each do |plain_tag|
+      #         word_hash[plain_tag] = max_score + 5
+      #       end
+      #     end
+      #   end
+      # end
       self.new(url, tags, word_hash, description, page_lang)
     end
   end

data/lib/site_classifier/version.rb

@@ -1,3 +1,3 @@
 module SiteClassifier
-  VERSION = "0.0.5"
+  VERSION = "0.0.6"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: site_classifier
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - Elad Meidar
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-01 00:00:00.000000000 Z
+date: 2013-10-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler